jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 12 Packages Wiki Activity Actions

Storage, safety, and correctness hardening

Browse source
This commit is contained in:
Jeremy Karst 2026-08-20 02:34:08 -04:00
parent 65080cdf17
commit 2e2936b249
65 changed files with 4177 additions and 838 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

340
Cargo.lock generated
View file

@ -86,7 +86,7 @@ dependencies = [
"android-properties", "android-properties",
"bitflags 2.13.1", "bitflags 2.13.1",
"cc", "cc",
"jni 0.22.4", "jni",
"libc", "libc",
"log", "log",
"ndk", "ndk",
@ -501,9 +501,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.6.0" version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]] [[package]]
name = "calloop" name = "calloop"
@ -577,12 +577,6 @@ dependencies = [
"shlex", "shlex",
] ]
[[package]]
name = "cesu8"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
[[package]] [[package]]
name = "cfb" name = "cfb"
version = "0.7.3" version = "0.7.3"
@ -745,16 +739,6 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "core-foundation"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]] [[package]]
name = "core-foundation-sys" name = "core-foundation-sys"
version = "0.8.6" version = "0.8.6"
@ -768,7 +752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081"
dependencies = [ dependencies = [
"bitflags 1.3.2", "bitflags 1.3.2",
"core-foundation 0.9.4", "core-foundation",
"core-graphics-types", "core-graphics-types",
"foreign-types 0.5.0", "foreign-types 0.5.0",
"libc", "libc",
@ -781,7 +765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf"
dependencies = [ dependencies = [
"bitflags 1.3.2", "bitflags 1.3.2",
"core-foundation 0.9.4", "core-foundation",
"libc", "libc",
] ]
@ -814,9 +798,9 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-channel" name = "crossbeam-channel"
version = "0.5.13" version = "0.5.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
dependencies = [ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
@ -1261,11 +1245,10 @@ dependencies = [
[[package]] [[package]]
name = "event-listener" name = "event-listener"
version = "5.3.1" version = "5.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2"
dependencies = [ dependencies = [
"concurrent-queue",
"parking", "parking",
"pin-project-lite", "pin-project-lite",
] ]
@ -1429,15 +1412,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.30" version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.30" version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
[[package]] [[package]]
name = "futures-lite" name = "futures-lite"
@ -1454,26 +1437,26 @@ dependencies = [
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.30" version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 3.0.3",
] ]
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.30" version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.30" version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-io", "futures-io",
@ -1481,7 +1464,6 @@ dependencies = [
"futures-task", "futures-task",
"memchr", "memchr",
"pin-project-lite", "pin-project-lite",
"pin-utils",
"slab", "slab",
] ]
@ -1764,21 +1746,23 @@ dependencies = [
[[package]] [[package]]
name = "icu_collections" name = "icu_collections"
version = "1.5.0" version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
dependencies = [ dependencies = [
"displaydoc", "displaydoc",
"potential_utf",
"utf8_iter",
"yoke", "yoke",
"zerofrom", "zerofrom",
"zerovec", "zerovec",
] ]
[[package]] [[package]]
name = "icu_locid" name = "icu_locale_core"
version = "1.5.0" version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
dependencies = [ dependencies = [
"displaydoc", "displaydoc",
"litemap", "litemap",
@ -1787,109 +1771,81 @@ dependencies = [
"zerovec", "zerovec",
] ]
[[package]]
name = "icu_locid_transform"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
dependencies = [
"displaydoc",
"icu_locid",
"icu_locid_transform_data",
"icu_provider",
"tinystr",
"zerovec",
]
[[package]]
name = "icu_locid_transform_data"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
[[package]] [[package]]
name = "icu_normalizer" name = "icu_normalizer"
version = "1.5.0" version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"
dependencies = [ dependencies = [
"displaydoc",
"icu_collections", "icu_collections",
"icu_normalizer_data", "icu_normalizer_data",
"icu_properties", "icu_properties",
"icu_provider", "icu_provider",
"smallvec", "smallvec",
"utf16_iter",
"utf8_iter",
"write16",
"zerovec", "zerovec",
] ]
[[package]] [[package]]
name = "icu_normalizer_data" name = "icu_normalizer_data"
version = "1.5.0" version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
[[package]] [[package]]
name = "icu_properties" name = "icu_properties"
version = "1.5.0" version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"
dependencies = [ dependencies = [
"displaydoc", "displaydoc",
"icu_collections", "icu_collections",
"icu_locid_transform", "icu_locale_core",
"icu_properties_data", "icu_properties_data",
"icu_provider", "icu_provider",
"tinystr", "zerotrie",
"zerovec", "zerovec",
] ]
[[package]] [[package]]
name = "icu_properties_data" name = "icu_properties_data"
version = "1.5.0" version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
[[package]] [[package]]
name = "icu_provider" name = "icu_provider"
version = "1.5.0" version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
dependencies = [ dependencies = [
"displaydoc", "displaydoc",
"icu_locid", "icu_locale_core",
"icu_provider_macros",
"stable_deref_trait",
"tinystr",
"writeable", "writeable",
"yoke", "yoke",
"zerofrom", "zerofrom",
"zerotrie",
"zerovec", "zerovec",
] ]
[[package]] [[package]]
name = "icu_provider_macros" name = "idna"
version = "1.5.0" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
dependencies = [ dependencies = [
"proc-macro2", "idna_adapter",
"quote", "smallvec",
"syn 2.0.66", "utf8_iter",
] ]
[[package]] [[package]]
name = "idna" name = "idna_adapter"
version = "1.0.0" version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
dependencies = [ dependencies = [
"icu_normalizer", "icu_normalizer",
"icu_properties", "icu_properties",
"smallvec",
"utf8_iter",
] ]
[[package]] [[package]]
@ -1980,22 +1936,6 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "jni"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"
dependencies = [
"cesu8",
"cfg-if",
"combine",
"jni-sys 0.3.0",
"log",
"thiserror 1.0.61",
"walkdir",
"windows-sys 0.45.0",
]
[[package]] [[package]]
name = "jni" name = "jni"
version = "0.22.4" version = "0.22.4"
@ -2191,9 +2131,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]] [[package]]
name = "litemap" name = "litemap"
version = "0.7.3" version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
[[package]] [[package]]
name = "litrs" name = "litrs"
@ -2964,12 +2904,6 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
[[package]]
name = "pin-utils"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]] [[package]]
name = "piper" name = "piper"
version = "0.2.3" version = "0.2.3"
@ -3045,6 +2979,15 @@ version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306"
[[package]]
name = "potential_utf"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
dependencies = [
"zerovec",
]
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.17" version = "0.2.17"
@ -3087,15 +3030,6 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
[[package]]
name = "quick-xml"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "quick-xml" name = "quick-xml"
version = "0.41.0" version = "0.41.0"
@ -3107,7 +3041,7 @@ dependencies = [
[[package]] [[package]]
name = "quicksearch-core" name = "quicksearch-core"
version = "1.1.0" version = "1.1.2"
dependencies = [ dependencies = [
"argon2", "argon2",
"cfb", "cfb",
@ -3124,7 +3058,7 @@ dependencies = [
"mime_guess", "mime_guess",
"notify", "notify",
"pdf-extract", "pdf-extract",
"quick-xml 0.31.0", "quick-xml",
"regex", "regex",
"rtf-parser", "rtf-parser",
"rusqlite", "rusqlite",
@ -3140,7 +3074,7 @@ dependencies = [
[[package]] [[package]]
name = "quicksearch-gui" name = "quicksearch-gui"
version = "1.1.0" version = "1.1.2"
dependencies = [ dependencies = [
"ashpd", "ashpd",
"chrono", "chrono",
@ -3164,9 +3098,9 @@ dependencies = [
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.36" version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
] ]
@ -3864,9 +3798,9 @@ dependencies = [
[[package]] [[package]]
name = "tinystr" name = "tinystr"
version = "0.7.6" version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
dependencies = [ dependencies = [
"displaydoc", "displaydoc",
"zerovec", "zerovec",
@ -4058,9 +3992,9 @@ checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]] [[package]]
name = "url" name = "url"
version = "2.5.1" version = "2.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
dependencies = [ dependencies = [
"form_urlencoded", "form_urlencoded",
"idna", "idna",
@ -4074,12 +4008,6 @@ version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf16_iter"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
[[package]] [[package]]
name = "utf8_iter" name = "utf8_iter"
version = "1.0.4" version = "1.0.4"
@ -4305,7 +4233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quick-xml 0.41.0", "quick-xml",
"quote", "quote",
] ]
@ -4343,15 +4271,15 @@ dependencies = [
[[package]] [[package]]
name = "webbrowser" name = "webbrowser"
version = "1.0.6" version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00f1243ef785213e3a32fa0396093424a3a6ea566f9948497e5a2309261a4c97" checksum = "ef62a3d5f7b2411119a11b6f62570dbff91d7105e011a20fb83fbf8f5761c40f"
dependencies = [ dependencies = [
"core-foundation 0.10.1", "jni",
"jni 0.21.1",
"log", "log",
"ndk-context", "ndk-context",
"objc2 0.6.4", "objc2 0.6.4",
"objc2-app-kit 0.3.2",
"objc2-foundation 0.3.2", "objc2-foundation 0.3.2",
"url", "url",
"web-sys", "web-sys",
@ -4550,15 +4478,6 @@ dependencies = [
"windows-link", "windows-link",
] ]
[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
"windows-targets 0.42.2",
]
[[package]] [[package]]
name = "windows-sys" name = "windows-sys"
version = "0.48.0" version = "0.48.0"
@ -4604,21 +4523,6 @@ dependencies = [
"windows-link", "windows-link",
] ]
[[package]]
name = "windows-targets"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
dependencies = [
"windows_aarch64_gnullvm 0.42.2",
"windows_aarch64_msvc 0.42.2",
"windows_i686_gnu 0.42.2",
"windows_i686_msvc 0.42.2",
"windows_x86_64_gnu 0.42.2",
"windows_x86_64_gnullvm 0.42.2",
"windows_x86_64_msvc 0.42.2",
]
[[package]] [[package]]
name = "windows-targets" name = "windows-targets"
version = "0.48.5" version = "0.48.5"
@ -4667,12 +4571,6 @@ dependencies = [
"windows_x86_64_msvc 0.53.1", "windows_x86_64_msvc 0.53.1",
] ]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
[[package]] [[package]]
name = "windows_aarch64_gnullvm" name = "windows_aarch64_gnullvm"
version = "0.48.5" version = "0.48.5"
@ -4691,12 +4589,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
version = "0.48.5" version = "0.48.5"
@ -4715,12 +4607,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
version = "0.48.5" version = "0.48.5"
@ -4751,12 +4637,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
version = "0.48.5" version = "0.48.5"
@ -4775,12 +4655,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
version = "0.48.5" version = "0.48.5"
@ -4799,12 +4673,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
[[package]] [[package]]
name = "windows_x86_64_gnullvm" name = "windows_x86_64_gnullvm"
version = "0.48.5" version = "0.48.5"
@ -4823,12 +4691,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
version = "0.48.5" version = "0.48.5"
@ -4862,7 +4724,7 @@ dependencies = [
"calloop 0.13.0", "calloop 0.13.0",
"cfg_aliases", "cfg_aliases",
"concurrent-queue", "concurrent-queue",
"core-foundation 0.9.4", "core-foundation",
"core-graphics", "core-graphics",
"cursor-icon", "cursor-icon",
"dpi", "dpi",
@ -4922,17 +4784,11 @@ version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "write16"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
[[package]] [[package]]
name = "writeable" name = "writeable"
version = "0.5.5" version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
[[package]] [[package]]
name = "x11-dl" name = "x11-dl"
@ -5009,11 +4865,10 @@ checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f"
[[package]] [[package]]
name = "yoke" name = "yoke"
version = "0.7.4" version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
dependencies = [ dependencies = [
"serde",
"stable_deref_trait", "stable_deref_trait",
"yoke-derive", "yoke-derive",
"zerofrom", "zerofrom",
@ -5021,9 +4876,9 @@ dependencies = [
[[package]] [[package]]
name = "yoke-derive" name = "yoke-derive"
version = "0.7.4" version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -5115,18 +4970,18 @@ dependencies = [
[[package]] [[package]]
name = "zerofrom" name = "zerofrom"
version = "0.1.4" version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
dependencies = [ dependencies = [
"zerofrom-derive", "zerofrom-derive",
] ]
[[package]] [[package]]
name = "zerofrom-derive" name = "zerofrom-derive"
version = "0.1.4" version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -5155,10 +5010,21 @@ dependencies = [
] ]
[[package]] [[package]]
name = "zerovec" name = "zerotrie"
version = "0.10.2" version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
dependencies = [
"displaydoc",
"yoke",
"zerofrom",
]
[[package]]
name = "zerovec"
version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
dependencies = [ dependencies = [
"yoke", "yoke",
"zerofrom", "zerofrom",
@ -5167,13 +5033,13 @@ dependencies = [
[[package]] [[package]]
name = "zerovec-derive" name = "zerovec-derive"
version = "0.10.2" version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 3.0.3",
] ]
[[package]] [[package]]

2
Cargo.toml
View file

@ -31,7 +31,7 @@ exclude = ["vendor/pdf-extract"]
pdf-extract = { path = "vendor/pdf-extract" } pdf-extract = { path = "vendor/pdf-extract" }
[workspace.package] [workspace.package]
version = "1.1.1" version = "1.1.2"
edition = "2021" edition = "2021"
license = "GPL-3.0-or-later" license = "GPL-3.0-or-later"
authors = ["Jeremy <jeremy@karsttech.com>"] authors = ["Jeremy <jeremy@karsttech.com>"]

38
README.md
View file

@ -223,7 +223,16 @@ inside that folder.
### GUI ### GUI
`quicksearch` with no query arguments opens the app: `quicksearch` with no query arguments opens the app. **One window at a
time**: a second launch reports that QuickSearch is already running and
exits, because two processes indexing one database corrupt it. Terminal
search (`quicksearch <query>`) only reads and keeps working while the
window is open. The guard is a kernel lock on `<database_path>.lock`, held
by the running process — so a crash or a power cut releases it, and the
leftover file never locks you out. The lock follows `database_path`: point
Settings at a different index and it moves with you, and if that index
belongs to another instance the change is refused rather than written, so a
path you cannot open can never end up in the config file.
- **Search**: results appear as you type; every keystroke cancels the - **Search**: results appear as you type; every keystroke cancels the
previous search. One checkbox enables the two fuzzy passes, and once a previous search. One checkbox enables the two fuzzy passes, and once a
@ -491,7 +500,14 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
`wal_checkpoint(TRUNCATE)` every `processing.maximum_wal_size` bytes of `wal_checkpoint(TRUNCATE)` every `processing.maximum_wal_size` bytes of
log, because SQLite's own autocheckpoint can only reset the log at an log, because SQLite's own autocheckpoint can only reset the log at an
instant no reader holds it — and a run keeps a reader per root querying instant no reader holds it — and a run keeps a reader per root querying
throughout, so left alone the log grows for the whole run. `files` holds throughout, so left alone the log grows for the whole run. That threshold
is lowered to fit the volume when free space is short, and a run stops
with an error rather than fill the disk: SQLite reaches the wal-index
through an mmap, and a page fault the filesystem cannot back arrives as
**SIGBUS**, which no `Result` can catch. The index's own files are, for
the same reason, never walked into — hashing one means opening it, and on
POSIX closing any descriptor on an inode cancels every advisory lock the
process holds on it, SQLite's documented corruption hazard. `files` holds
metadata (name, path, size, mtime, hash, MIME/type bitmask, content metadata (name, path, size, mtime, hash, MIME/type bitmask, content
state); `searchabletext` is a *contentless* FTS5 table over one column, state); `searchabletext` is a *contentless* FTS5 table over one column,
the document body (postings only, configurable tokenizer, trigram by the document body (postings only, configurable tokenizer, trigram by
@ -572,7 +588,7 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
tokenizing can do; the scheduling shares it fairly and keeps the walk tokenizing can do; the scheduling shares it fairly and keeps the walk
first, it does not raise it. Every run ends — whether first, it does not raise it. Every run ends — whether
it completed or was stopped — with an optimize pass on its own connection: it completed or was stopped — with an optimize pass on its own connection:
checkpoint, VACUUM if the file has at least 10% slack to reclaim, `PRAGMA checkpoint, VACUUM if the file has at least 20% slack to reclaim, `PRAGMA
optimize`, checkpoint again. Progress streams through a polled optimize`, checkpoint again. Progress streams through a polled
`IndexingStatus`, which reads `Optimizing` for the duration of that pass — `IndexingStatus`, which reads `Optimizing` for the duration of that pass —
and `Preparing` for everything a run does before its first file is walked: and `Preparing` for everything a run does before its first file is walked:
@ -674,13 +690,15 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
to that by construction, because a buffer allocated and freed inside one loop to that by construction, because a buffer allocated and freed inside one loop
iteration never moves RSS. `DocDecoder` must therefore never fall back to a iteration never moves RSS. `DocDecoder` must therefore never fall back to a
per-row allocating decode, and there is a trap waiting there: per-row allocating decode, and there is a trap waiting there:
`zstd::encode_all`, which the indexer writes with, is *stream*-based and so the indexer writes through `DocEncoder`, i.e. `zstd::bulk::Compressor` and
records no content size in the frame header, meaning so `ZSTD_compress2`, which is handed the whole document at once and records
`get_frame_content_size` returns `None` for every row this ever sees. The its length in the frame header. `get_frame_content_size` therefore returns
"cannot happen" branch is the only branch. Sizing the buffer from the header `Some` for every row this writer produced, the reservation is normally
and handing the `None` case to `zstd::decode_all` looks obviously right and exact, and the growth loop below it is the fallback for a frame written by
costs ~2.4 MiB per document, because `decode_all` builds a streaming decoder a *stream* encoder rather than the only branch. What must not come back is
per call — 27 of the 30 GiB a fuzzy search moved. Growing this buffer and handing the `None` case to `zstd::decode_all`, which looks obviously right
and costs ~2.4 MiB per document because it builds a streaming decoder per
call — 27 of the 30 GiB a fuzzy search moved. Growing this buffer and
keeping it is what makes decoding a row allocate nothing at all. keeping it is what makes decoding a row allocate nothing at all.
Measured over the same 77k-file index, per query: Measured over the same 77k-file index, per query:
`cascade` 582 → 14 MiB, `function` 6.0 GiB → 29 MiB, `--fuzzy cascade` `cascade` 582 → 14 MiB, `function` 6.0 GiB → 29 MiB, `--fuzzy cascade`

12
config_example.toml
View file

@ -74,6 +74,11 @@ content_extensions = []
# The Windows defaults add: "$RECYCLE.BIN", "System Volume Information", # The Windows defaults add: "$RECYCLE.BIN", "System Volume Information",
# "pagefile.sys", "hiberfil.sys", "swapfile.sys", "Thumbs.db", # "pagefile.sys", "hiberfil.sys", "swapfile.sys", "Thumbs.db",
# "desktop.ini". # "desktop.ini".
#
# The index itself (database_path and its -wal/-shm/-journal/.lock
# siblings) is always skipped and needs no pattern here — not as a
# convenience but because merely opening one of those files to hash it
# would cancel SQLite's locks on it for the whole process.
ignore_patterns = [".git", "node_modules", "*.tmp", ".venv", "venv"] ignore_patterns = [".git", "node_modules", "*.tmp", ".venv", "venv"]
# Worth adding by hand if you index a whole Windows drive rather than just # Worth adding by hand if you index a whole Windows drive rather than just
@ -124,6 +129,13 @@ fts_update_batch_size = 1000
# unattended the log grows for the whole run and can end up larger than the # unattended the log grows for the whole run and can end up larger than the
# index. Set to 0 to disable forced checkpoints; any other value below # index. Set to 0 to disable forced checkpoints; any other value below
# 16777216 is raised to it. # 16777216 is raised to it.
#
# This is a stall-frequency knob, not a safety one: when the volume holding
# the index is short on space the indexer checkpoints sooner than asked —
# including when this is 0 — and stops the run with an error rather than
# fill the disk. Running out is not a clean failure; SQLite reaches its
# wal-index through a memory map, and a write the filesystem cannot back
# kills the process with SIGBUS instead of returning an error.
maximum_wal_size = 536870912 maximum_wal_size = 536870912
# FTS5 tokenizer: 'trigram' (substring matching, the default; gets # FTS5 tokenizer: 'trigram' (substring matching, the default; gets
# remove_diacritics 1 appended), 'unicode61', 'porter', or a full FTS5 # remove_diacritics 1 appended), 'unicode61', 'porter', or a full FTS5

6
crates/quicksearch-core/Cargo.toml
View file

@ -28,7 +28,7 @@ walkdir = "2.5.0"
# containers (docx/xlsx/pptx/odt/ods/odp), whose entries are always deflate or # containers (docx/xlsx/pptx/odt/ods/odp), whose entries are always deflate or
# stored, so zstd/bzip2/aes-crypto are all dead weight here. # stored, so zstd/bzip2/aes-crypto are all dead weight here.
zip = { version = "0.6", default-features = false, features = ["deflate"] } zip = { version = "0.6", default-features = false, features = ["deflate"] }
quick-xml = "0.31" quick-xml = "0.41"
# OLE2 compound-file reader, for the pre-2007 binary Office formats # OLE2 compound-file reader, for the pre-2007 binary Office formats
# (.doc/.xls/.ppt) whose text lives in named streams rather than a zip. Already # (.doc/.xls/.ppt) whose text lives in named streams rather than a zip. Already
# in the lockfile transitively via infer, so naming it directly compiles # in the lockfile transitively via infer, so naming it directly compiles
@ -101,6 +101,10 @@ windows-sys = { version = "0.52", features = [
# the directory-count path passes null for it. # the directory-count path passes null for it.
"Win32_Security", "Win32_Security",
"Win32_Storage_FileSystem", "Win32_Storage_FileSystem",
# OVERLAPPED, which `LockFileEx` takes for the index's instance lock.
# `LockFileEx` and `GetDiskFreeSpaceExW` themselves are in
# Win32_Storage_FileSystem above.
"Win32_System_IO",
"Win32_System_Threading", "Win32_System_Threading",
"Win32_System_WindowsProgramming", "Win32_System_WindowsProgramming",
] } ] }

2
crates/quicksearch-core/examples/walkprobe.rs
View file

@ -80,7 +80,7 @@ fn serial(root: &str, config: &Config, existing: &DirRows) -> (usize, usize) {
for entry in filtered_walk(root, false, false, &ignore, &UnreadableDirs::default()) { for entry in filtered_walk(root, false, false, &ignore, &UnreadableDirs::default()) {
seen += 1; seen += 1;
// Same rule as the real walk: a name that is not valid UTF-8 cannot be // Same rule as the real walk: a name that is not valid UTF-8 cannot be
// stored in `files.path` and reopened by it, so it is skipped before // stored in the index and reopened by its path, so it is skipped before
// anything tries to hash it. Counted as seen, never prepared. // anything tries to hash it. Counted as seen, never prepared.
let Some(path) = entry.path().to_str().map(str::to_owned) else { let Some(path) = entry.path().to_str().map(str::to_owned) else {
continue; continue;

54
crates/quicksearch-core/src/cli.rs
View file

@ -103,17 +103,23 @@ impl SizeReport {
/// failure reason comes from `failed_files`, the one place it is written. /// failure reason comes from `failed_files`, the one place it is written.
pub fn status_for_path(db_path: &str, path: &str) -> Result<FileStatus, String> { pub fn status_for_path(db_path: &str, path: &str) -> Result<FileStatus, String> {
let conn = open_existing(db_path, false)?; let conn = open_existing(db_path, false)?;
let row: Option<(i64, Option<String>)> = conn let split = crate::file_handling::split_db_path(path);
.query_row( let row: Option<(i64, Option<String>)> = match split {
"SELECT f.content_state, ff.reason \ // Not a path a row could ever have been stored under, so it is not
// indexed — the same answer as a miss.
None => None,
Some((parent, name)) => conn
.query_row(
"SELECT f.content_state, ff.reason \
FROM files f \ FROM files f \
LEFT JOIN failed_files ff ON ff.file_id = f.id \ LEFT JOIN failed_files ff ON ff.file_id = f.id \
WHERE f.path = ?1", WHERE f.parent = ?1 AND f.name = ?2",
params![path], params![parent, name],
|r| Ok((r.get(0)?, r.get(1)?)), |r| Ok((r.get(0)?, r.get(1)?)),
) )
.optional() .optional()
.map_err(|e| format!("status_for_path({}): {}", path, e))?; .map_err(|e| format!("status_for_path({}): {}", path, e))?,
};
Ok(match row { Ok(match row {
None => FileStatus { None => FileStatus {
path: path.to_string(), path: path.to_string(),
@ -138,7 +144,7 @@ pub fn list_failed(db_path: &str, limit: Option<u32>) -> Result<Vec<FailedEntry>
None => String::new(), None => String::new(),
}; };
let sql = format!( let sql = format!(
"SELECT ff.file_id, f.path, ff.reason, ff.ts \ "SELECT ff.file_id, f.parent, f.name, ff.reason, ff.ts \
FROM failed_files ff \ FROM failed_files ff \
JOIN files f ON f.id = ff.file_id \ JOIN files f ON f.id = ff.file_id \
ORDER BY ff.ts DESC{}", ORDER BY ff.ts DESC{}",
@ -149,11 +155,13 @@ pub fn list_failed(db_path: &str, limit: Option<u32>) -> Result<Vec<FailedEntry>
.map_err(|e| format!("list_failed prepare: {}", e))?; .map_err(|e| format!("list_failed prepare: {}", e))?;
let rows = stmt let rows = stmt
.query_map([], |r| { .query_map([], |r| {
let parent: String = r.get(1)?;
let name: String = r.get(2)?;
Ok(FailedEntry { Ok(FailedEntry {
file_id: r.get(0)?, file_id: r.get(0)?,
path: r.get(1)?, path: format!("{}{}", parent, name),
reason: r.get(2)?, reason: r.get(3)?,
ts: r.get(3)?, ts: r.get(4)?,
}) })
}) })
.map_err(|e| format!("list_failed query: {}", e))?; .map_err(|e| format!("list_failed query: {}", e))?;
@ -252,8 +260,7 @@ mod tests {
&tx, &tx,
&NewFile { &NewFile {
name: "a.txt", name: "a.txt",
path: "/tmp/a.txt", parent: "/tmp/",
parent: "/tmp",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -269,8 +276,7 @@ mod tests {
&tx, &tx,
&NewFile { &NewFile {
name: "b.bin", name: "b.bin",
path: "/tmp/b.bin", parent: "/tmp/",
parent: "/tmp",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: None, mime: None,
@ -343,8 +349,7 @@ mod tests {
&tx, &tx,
&NewFile { &NewFile {
name: "c.txt", name: "c.txt",
path: "/tmp/c.txt", parent: "/tmp/",
parent: "/tmp",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -361,8 +366,7 @@ mod tests {
&tx, &tx,
&NewFile { &NewFile {
name: "d.bin", name: "d.bin",
path: "/tmp/d.bin", parent: "/tmp/",
parent: "/tmp",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: None, mime: None,
@ -422,8 +426,7 @@ mod tests {
&tx, &tx,
&NewFile { &NewFile {
name: "big.txt", name: "big.txt",
path: "/tmp/big.txt", parent: "/tmp/",
parent: "/tmp",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -477,13 +480,12 @@ mod tests {
{ {
let mut conn = open_or_recreate(dbp, "unicode61").unwrap(); let mut conn = open_or_recreate(dbp, "unicode61").unwrap();
let tx = conn.transaction().unwrap(); let tx = conn.transaction().unwrap();
for (name, path) in [("a.txt", "/tmp/a.txt"), ("b.txt", "/tmp/b.txt")] { for name in ["a.txt", "b.txt"] {
insert_file( insert_file(
&tx, &tx,
&NewFile { &NewFile {
name, name,
path, parent: "/tmp/",
parent: "/tmp",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: Some("text/plain"), mime: Some("text/plain"),

2
crates/quicksearch-core/src/config/diff.rs
View file

@ -15,7 +15,7 @@ use super::*;
/// app was closed. See [`crate::scope`] for the pass that applies it. /// app was closed. See [`crate::scope`] for the pass that applies it.
#[derive(Debug, Default, Clone, PartialEq, Eq)] #[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct IndexWork { pub struct IndexWork {
/// Roots that are no longer configured, in `files.path` spelling. Every /// Roots that are no longer configured, in stored spelling. Every
/// row beneath one is deleted; no filesystem access is involved, so a /// row beneath one is deleted; no filesystem access is involved, so a
/// root whose folder is gone is handled the same as one that still /// root whose folder is gone is handled the same as one that still
/// exists. /// exists.

227
crates/quicksearch-core/src/config/mod.rs
View file

@ -567,13 +567,37 @@ impl Config {
// open again. `rename` is atomic on both platforms, and the `sync_all` // open again. `rename` is atomic on both platforms, and the `sync_all`
// before it means the bytes are on the disk before the name points at // before it means the bytes are on the disk before the name points at
// them. // them.
let tmp = path.with_extension("toml.tmp"); let tmp = write_private_temp(&path, content.as_bytes()).map_err(|e| {
write_private(&tmp, content.as_bytes()) format!(
.map_err(|e| format!("Failed to write config file {}: {}", tmp.display(), e))?; "Failed to write config file beside {}: {}",
path.display(),
e
)
})?;
fs::rename(&tmp, &path).map_err(|e| { fs::rename(&tmp, &path).map_err(|e| {
let _ = fs::remove_file(&tmp); let _ = fs::remove_file(&tmp);
format!("Failed to replace config file {}: {}", path.display(), e) format!("Failed to replace config file {}: {}", path.display(), e)
})?; })?;
// The rename is atomic but not durable: it is a directory entry, and
// the directory has its own dirty state. Without this a power cut just
// after can leave *neither* name, which is the salt loss the whole
// dance exists to prevent. Best-effort, and on Windows it is a no-op
// every time rather than only on an exotic filesystem: opening a
// directory as a file needs `FILE_FLAG_BACKUP_SEMANTICS`, which
// `fs::File::open` does not pass, so this fails and is skipped. NTFS
// journals the rename itself, which is the guarantee this is reaching
// for; on Unix it has to be asked for.
if let Some(dir) = path.parent() {
if let Ok(handle) = fs::File::open(dir) {
let _ = handle.sync_all();
}
// Leftovers from a save that died between `create_new` and the
// rename. The old fixed `config.toml.tmp` overwrote itself, so
// there was never more than one; a unique name per attempt is what
// makes the write safe (see `write_private_temp`) and what makes
// them accumulate, so they are swept here instead.
sweep_stale_temps(dir, &path);
}
Ok(()) Ok(())
} }
@ -598,8 +622,72 @@ impl Config {
} }
} }
/// Whether `path` is the index itself — the database, one of SQLite's
/// `-wal`/`-shm`/`-journal` sidecars, or the instance lock.
///
/// For the paths that arrive from a `files` row rather than from a walk:
/// once the walk stops indexing the index
/// ([`crate::file_handling::index_file_set`], which explains why opening
/// one is fatal), such rows are swept away, but a row written by an older
/// build survives until the sweep reaches it and can still be opened from
/// a result list in the meantime.
///
/// The name is compared first and the directory only on a hit, so the
/// overwhelmingly common miss costs one string comparison rather than the
/// `canonicalize` the directory check needs.
///
/// Names are folded where the filesystem folds them
/// ([`crate::platform::PATHS_ARE_CASE_INSENSITIVE`]): a stored row spelling
/// the database `Index.sqlite` names the same file as a config spelling it
/// `index.sqlite`, and opening it has the same consequence. ASCII only,
/// matching `PATH_COLLATION` and [`IgnoreSet`].
pub fn is_index_file(&self, path: &Path) -> bool {
let db = self.resolved_database_path();
let (Some(db_name), Some(name)) = (
db.file_name().and_then(|s| s.to_str()),
path.file_name().and_then(|s| s.to_str()),
) else {
return false;
};
let same_name = |a: &str, b: &str| {
if crate::platform::PATHS_ARE_CASE_INSENSITIVE {
a.eq_ignore_ascii_case(b)
} else {
a == b
}
};
// `str::get`, not `name[..cut]`: the cut is `db_name`'s *byte* length
// and the two names are unrelated strings, so it can land inside a
// multi-byte character — `€xyz` against a two-byte `db_name` is six
// bytes either way — and indexing there panics. A non-boundary is
// simply not a match.
let name_matches = same_name(name, db_name)
|| crate::file_handling::INDEX_SIDECAR_SUFFIXES
.iter()
.any(|s| {
let cut = db_name.len();
name.len() == cut + s.len()
&& name.get(..cut).is_some_and(|head| same_name(head, db_name))
&& name.get(cut..).is_some_and(|tail| same_name(tail, s))
});
if !name_matches {
return false;
}
let same_dir = |a: &Path, b: &Path| {
a == b
|| a.canonicalize().unwrap_or_else(|_| a.to_path_buf())
== b.canonicalize().unwrap_or_else(|_| b.to_path_buf())
};
match (path.parent(), db.parent()) {
(Some(a), Some(b)) => same_dir(a, b),
// Both at a filesystem root, or neither: the name match stands.
(None, None) => true,
_ => false,
}
}
/// `resolved_indexing_paths` canonicalized and spelled the way /// `resolved_indexing_paths` canonicalized and spelled the way
/// `files.path` prefixes them. /// stored parents are prefixed with them.
/// ///
/// The form roots must be compared in: `~/docs`, `docs` in a portable /// The form roots must be compared in: `~/docs`, `docs` in a portable
/// config and `/home/me/docs` are one root under three spellings, and a /// config and `/home/me/docs` are one root under three spellings, and a
@ -631,30 +719,121 @@ impl Config {
} }
} }
/// Write `bytes` to `path`, owner-readable only, and flush them to the disk /// Write `bytes` to a fresh temporary file beside `target`, owner-only, with
/// before returning. /// its contents flushed to the disk, and return its path for the caller to
/// rename into place.
/// ///
/// `O_NOFOLLOW` on Unix: the config directory is not always somewhere only /// `create_new` and a unique name, not a fixed one: `O_NOFOLLOW` refuses a
/// this user can write — a portable install can sit in a shared or removable /// symlink but says nothing about a *regular* file or a hardlink that is
/// directory — and a symlink left at the config's name would otherwise /// already sitting at the name we were going to use. The config directory is
/// redirect this write onto whatever it points at. /// not always somewhere only this user can write — a portable install can sit
fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> { /// in a shared or removable directory — and an attacker who pre-creates the
/// temp file as a hardlink to a file they can read would otherwise be handed
/// `[security].salt`. `mode(0o600)` only applies to a file this call creates,
/// which is the same reason.
fn write_private_temp(target: &Path, bytes: &[u8]) -> std::io::Result<PathBuf> {
use std::io::Write; use std::io::Write;
let mut opts = fs::OpenOptions::new(); let dir = target.parent().unwrap_or_else(|| Path::new("."));
opts.write(true).create(true).truncate(true); let stem = target
#[cfg(unix)] .file_name()
{ .map(|n| n.to_string_lossy().into_owned())
use std::os::unix::fs::OpenOptionsExt; .unwrap_or_else(|| "config.toml".to_string());
opts.custom_flags(libc::O_NOFOLLOW);
opts.mode(0o600); // A handful of attempts, then give up rather than spin: if something is
// racing us for every name we pick, failing the save is the honest
// outcome — the caller treats a failed save as fatal to the change.
let mut last_err = None;
for _ in 0..8 {
let tmp = dir.join(format!("{}.{}.tmp", stem, unique_suffix()));
let mut opts = fs::OpenOptions::new();
opts.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.custom_flags(libc::O_NOFOLLOW);
opts.mode(0o600);
}
match opts.open(&tmp) {
Ok(mut f) => {
let wrote = f.write_all(bytes).and_then(|()| f.sync_all());
return match wrote {
Ok(()) => Ok(tmp),
Err(e) => {
let _ = fs::remove_file(&tmp);
Err(e)
}
};
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => last_err = Some(e),
Err(e) => return Err(e),
}
} }
let mut f = opts.open(path)?; Err(last_err.unwrap_or_else(|| {
f.write_all(bytes)?; std::io::Error::new(
// The rename that follows is atomic with respect to the *directory*, not std::io::ErrorKind::AlreadyExists,
// to the file's contents: without this, a crash can leave the new name "could not create a temporary config file",
// pointing at a block of zeroes. )
f.sync_all() }))
}
/// How old an abandoned temp file must be before [`sweep_stale_temps`] takes
/// it. A save is a serialize, a write and a rename — milliseconds — so an hour
/// is far past any doubt, while still short enough that leftovers do not
/// accumulate across a run of crashes.
const TEMP_SWEEP_AGE: std::time::Duration = std::time::Duration::from_secs(60 * 60);
/// Delete abandoned `<config name>.*.tmp` files beside the config.
///
/// [`write_private_temp`] must use a *unique* name — a fixed one can be
/// pre-created by someone else as a hardlink — and a unique name is one that
/// nothing later overwrites, so a process that dies between `create_new` and
/// the rename leaves its temp file behind for good. The old fixed
/// `config.toml.tmp` was reused by the next save and so never accumulated;
/// this is what replaces that property.
///
/// Age is the discriminator, not the recorded PID: a temp file created seconds
/// ago may belong to a save running *right now* in another process, and
/// deleting that would destroy the very write this whole dance protects.
/// Entirely best-effort — a directory that cannot be listed is not a reason to
/// fail a save that already succeeded.
fn sweep_stale_temps(dir: &Path, target: &Path) {
let Some(stem) = target.file_name().and_then(|n| n.to_str()) else {
return;
};
// The trailing dot matters: without it the config file itself would be a
// prefix match.
let prefix = format!("{}.", stem);
let Ok(entries) = fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name();
let Some(name) = name.to_str() else { continue };
if !name.starts_with(&prefix) || !name.ends_with(".tmp") {
continue;
}
let abandoned = entry
.metadata()
.and_then(|m| m.modified())
.is_ok_and(|t| t.elapsed().is_ok_and(|age| age >= TEMP_SWEEP_AGE));
if abandoned {
let _ = fs::remove_file(entry.path());
}
}
}
/// A short, non-guessable-enough suffix for the temp name.
///
/// This does not need to be unpredictable to an attacker — `create_new` is
/// what makes the write safe — only unlikely to collide with a leftover from
/// an interrupted save, so the process id and a clock reading are plenty.
fn unique_suffix() -> String {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.subsec_nanos())
.unwrap_or(0);
format!("{}-{}", std::process::id(), nanos)
} }
/// Reserved `content_extensions` entry standing for "files with no /// Reserved `content_extensions` entry standing for "files with no

52
crates/quicksearch-core/src/config/tests.rs
View file

@ -1043,3 +1043,55 @@ fn the_search_table_ships_without_size_or_modified() {
assert_eq!(older.search.display_limit, 500); assert_eq!(older.search.display_limit, 500);
assert!(older.search.live_results, "live results default to on"); assert!(older.search.live_results, "live results default to on");
} }
/// The guard the live watcher and duplicate verification consult before
/// opening a path that came from a `files` row. See
/// [`crate::file_handling::index_file_set`] for what opening one costs.
#[test]
fn the_index_and_every_sidecar_are_recognised() {
let mut c = Config::default();
c.paths.database_path = "/var/lib/qs/index.sqlite".to_string();
for name in [
"index.sqlite",
"index.sqlite-wal",
"index.sqlite-shm",
"index.sqlite-journal",
"index.sqlite.lock",
] {
let p = PathBuf::from(format!("/var/lib/qs/{}", name));
assert!(c.is_index_file(&p), "{} should be recognised", name);
}
// A near miss on the name, and the right name in the wrong directory.
for p in [
"/var/lib/qs/index.sqlite-walrus",
"/var/lib/qs/notes-wal",
"/var/lib/qs/index.sqlite2",
"/var/lib/other/index.sqlite-wal",
"/var/lib/qs/sub/index.sqlite",
] {
assert!(
!c.is_index_file(Path::new(p)),
"{} is not part of the index",
p
);
}
}
/// `database_path` is resolved the same way everywhere else resolves it, so a
/// `~` or a relative spelling has to compare equal to the absolute file.
#[test]
fn a_tilde_database_path_still_matches_the_real_file() {
let Some(home) = crate::platform::home_dir() else {
return;
};
let mut c = Config::default();
c.paths.database_path = "~/.local/share/quicksearch/index.sqlite".to_string();
let absolute = PathBuf::from(home).join(".local/share/quicksearch/index.sqlite");
assert!(c.is_index_file(&absolute));
let wal = absolute.with_file_name("index.sqlite-wal");
assert!(c.is_index_file(&wal));
assert!(!c.is_index_file(&absolute.with_file_name("other.sqlite")));
}

5
crates/quicksearch-core/src/content.rs
View file

@ -396,8 +396,7 @@ mod tests {
&tx, &tx,
&NewFile { &NewFile {
name: f.file_name().unwrap().to_str().unwrap(), name: f.file_name().unwrap().to_str().unwrap(),
path: f.to_str().unwrap(), parent: &crate::file_handling::dir_to_db_parent(&d),
parent: d.to_str().unwrap(),
size: std::fs::metadata(&f).unwrap().len(), size: std::fs::metadata(&f).unwrap().len(),
mtime: 1, mtime: 1,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -494,7 +493,7 @@ mod tests {
.lock() .lock()
.unwrap() .unwrap()
.query_row( .query_row(
"SELECT content_state FROM files WHERE path = ?1", "SELECT content_state FROM files WHERE parent || name = ?1",
rusqlite::params![p.to_str().unwrap()], rusqlite::params![p.to_str().unwrap()],
|r| r.get(0), |r| r.get(0),
) )

2
crates/quicksearch-core/src/coordinator.rs
View file

@ -234,6 +234,8 @@ impl IndexCoordinator {
last_event_at: None, last_event_at: None,
pending_since: None, pending_since: None,
needs_full_run: false, needs_full_run: false,
run_retry_at: None,
run_retry_delay: inner::RUN_RETRY_BASE,
pending_work: None, pending_work: None,
reconcile_done: None, reconcile_done: None,
reconcile_cut_short: false, reconcile_cut_short: false,

181
crates/quicksearch-core/src/coordinator/inner.rs
View file

@ -3,6 +3,32 @@
use super::*; use super::*;
/// First wait after a full run is refused or fails to start.
pub(super) const RUN_RETRY_BASE: Duration = Duration::from_secs(30);
/// Ceiling for that wait. A misconfiguration that is never fixed then costs
/// one attempt every few minutes instead of one per second.
const RUN_RETRY_MAX: Duration = Duration::from_secs(300);
/// Individually-logged occurrences of each refusal before it goes quiet. The
/// message is identical every time, and the log ring is the only record of
/// what *else* happened; these are reset the moment a run starts.
static NO_ROOTS: crate::log::Throttle = crate::log::Throttle::new(3);
static NESTED_ROOTS: crate::log::Throttle = crate::log::Throttle::new(3);
static START_FAILURES: crate::log::Throttle = crate::log::Throttle::new(3);
/// Why a full run is being started, which is what decides how a refusal is
/// reported. See [`Inner::refuse`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum RunTrigger {
/// Something the user did: *Index now*, *Rebuild index*, or a settings
/// change wide enough to need a walk. A refusal owes them an answer.
Requested,
/// The periodic timer, or repair the coordinator scheduled for itself. A
/// refusal here is worth one log line and then quiet.
Scheduled,
}
pub(super) struct Inner { pub(super) struct Inner {
pub(super) config: Config, pub(super) config: Config,
pub(super) indexing: Arc<IndexingService>, pub(super) indexing: Arc<IndexingService>,
@ -40,6 +66,18 @@ pub(super) struct Inner {
/// defer application past `pending_max_defer`. /// defer application past `pending_max_defer`.
pub(super) pending_since: Option<Instant>, pub(super) pending_since: Option<Instant>,
pub(super) needs_full_run: bool, pub(super) needs_full_run: bool,
/// Earliest time another full run may be *attempted*, when the last one
/// was refused or failed. Without it a config with no roots, or with
/// nested roots, or an index on a full disk, retries every tick forever:
/// two stderr writes and two ring pushes a second, which evicts the whole
/// 5000-line log ring in under an hour and takes the diagnostics for the
/// actual problem with it — and in the failed-run case re-walks the tree
/// each time.
pub(super) run_retry_at: Option<Instant>,
/// How long the next refusal waits, doubling to [`RUN_RETRY_MAX`] so a
/// permanent misconfiguration costs almost nothing, and reset once a run
/// actually starts.
pub(super) run_retry_delay: Duration,
/// Reconciliation owed to a config change, part-applied across ticks. /// Reconciliation owed to a config change, part-applied across ticks.
/// Unlike `needs_full_run`, this is acted on in manual mode too. /// Unlike `needs_full_run`, this is acted on in manual mode too.
pub(super) pending_work: Option<WorkCursor>, pub(super) pending_work: Option<WorkCursor>,
@ -95,7 +133,7 @@ impl Inner {
CoordCmd::SetMode(IndexMode::ManualStopped) => self.enter_manual_stopped(), CoordCmd::SetMode(IndexMode::ManualStopped) => self.enter_manual_stopped(),
// ManualRunning isn't directly settable; ReindexNow is the verb. // ManualRunning isn't directly settable; ReindexNow is the verb.
CoordCmd::SetMode(IndexMode::ManualRunning) | CoordCmd::ReindexNow => { CoordCmd::SetMode(IndexMode::ManualRunning) | CoordCmd::ReindexNow => {
self.start_full_run(); self.start_full_run(RunTrigger::Requested);
if self.mode != IndexMode::Auto { if self.mode != IndexMode::Auto {
self.mode = IndexMode::ManualRunning; self.mode = IndexMode::ManualRunning;
} }
@ -139,10 +177,18 @@ impl Inner {
self.pending_work = None; self.pending_work = None;
self.reconcile_done = None; self.reconcile_done = None;
if let Err(e) = self.indexing.delete_index_for_rebuild(&db) { if let Err(e) = self.indexing.delete_index_for_rebuild(&db) {
// Not a warning to step over: a run started now would
// reopen the *old* index and present itself as the rebuild
// the user asked for. Worse after a password change, where
// the config already claims a protection the surviving
// file does not have. Report it and leave the index alone.
crate::log_warn!("coordinator: rebuild: {}", e); crate::log_warn!("coordinator: rebuild: {}", e);
self.indexing
.report_error(format!("could not rebuild the index: {}", e));
return;
} }
self.clear_root_counts(); self.clear_root_counts();
self.start_full_run(); self.start_full_run(RunTrigger::Requested);
if self.mode != IndexMode::Auto { if self.mode != IndexMode::Auto {
self.mode = IndexMode::ManualRunning; self.mode = IndexMode::ManualRunning;
} }
@ -174,7 +220,7 @@ impl Inner {
// delivers anything else, so nothing downstream checks, and // delivers anything else, so nothing downstream checks, and
// a file renamed *out* of every root would otherwise be // a file renamed *out* of every root would otherwise be
// written into the index at its new home. Roots in the same // written into the index at its new home. Roots in the same
// spelling `files.path` uses — the caller's paths are. // spelling the index uses — the caller's paths are.
let prefixes: Vec<String> = self let prefixes: Vec<String> = self
.config .config
.normalized_indexing_paths() .normalized_indexing_paths()
@ -224,6 +270,12 @@ impl Inner {
if self.saw_running { if self.saw_running {
self.saw_running = false; self.saw_running = false;
self.was_busy = true; self.was_busy = true;
// A run that errored never stamps `last_full_index`, so
// `periodic_due` stays true and the next tick starts another one —
// a fresh whole-tree walk per second against, say, a full disk.
if matches!(status, IndexingStatus::Error(_)) {
self.defer_runs();
}
self.refresh_last_full_index(); self.refresh_last_full_index();
// Eager re-read: the run just changed the number on screen. // Eager re-read: the run just changed the number on screen.
self.files_at = None; self.files_at = None;
@ -263,8 +315,9 @@ impl Inner {
worked = true; worked = true;
} }
if self.needs_full_run || self.periodic_due() { let deferred = self.run_retry_at.is_some_and(|at| Instant::now() < at);
self.start_full_run(); if (self.needs_full_run || self.periodic_due()) && !deferred {
self.start_full_run(RunTrigger::Scheduled);
worked = true; worked = true;
} }
@ -408,7 +461,7 @@ impl Inner {
// Widening adds files only a walk can produce; mirrors `ReindexNow`, // Widening adds files only a walk can produce; mirrors `ReindexNow`,
// including the manual-mode round trip back to stopped. // including the manual-mode round trip back to stopped.
if cursor.reindex() { if cursor.reindex() {
self.start_full_run(); self.start_full_run(RunTrigger::Requested);
if self.mode != IndexMode::Auto { if self.mode != IndexMode::Auto {
self.mode = IndexMode::ManualRunning; self.mode = IndexMode::ManualRunning;
} }
@ -555,6 +608,9 @@ impl Inner {
Err(e) => { Err(e) => {
crate::log_warn!("coordinator: targeted update unavailable: {}", e); crate::log_warn!("coordinator: targeted update unavailable: {}", e);
self.targeted.clear(); self.targeted.clear();
// As in `start_full_run`: a resume point outlives its event
// here too, and would misapply to the next one for that path.
self.resume_from.retain(|p, _| self.pending.contains_key(p));
return; return;
} }
}; };
@ -633,10 +689,15 @@ impl Inner {
} }
fn periodic_due(&self) -> bool { fn periodic_due(&self) -> bool {
// `.max(1)`: at zero every tick is "due", so a full run would start
// the moment the previous one finished, forever. There is no setting
// that means "reindex continuously" — manual mode is how you say
// "never", so zero is a typo rather than an intent.
let interval_secs = self let interval_secs = self
.config .config
.indexing .indexing
.reindex_interval_minutes .reindex_interval_minutes
.max(1)
.saturating_mul(60); .saturating_mul(60);
let last = crate::lock_ok(&self.shared).last_full_index; let last = crate::lock_ok(&self.shared).last_full_index;
match last { match last {
@ -651,7 +712,20 @@ impl Inner {
} }
} }
fn start_full_run(&mut self) { fn start_full_run(&mut self, trigger: RunTrigger) {
// A run already in progress is not a refusal: whoever called this
// wanted a run and there is one. Returning here rather than falling
// through to `start_indexing`'s "already running" error keeps an
// *Index now* pressed mid-run from backing off future runs and from
// replacing the progress the user is watching with an error. It also
// covers `Optimizing`, which `start_indexing` does not reject even
// though it holds a write transaction over the whole file.
if !matches!(
self.indexing.get_status(),
IndexingStatus::Idle | IndexingStatus::Error(_)
) {
return;
}
let roots: Vec<String> = self let roots: Vec<String> = self
.config .config
.resolved_indexing_paths() .resolved_indexing_paths()
@ -659,20 +733,23 @@ impl Inner {
.map(|p| p.to_string_lossy().into_owned()) .map(|p| p.to_string_lossy().into_owned())
.collect(); .collect();
if roots.is_empty() { if roots.is_empty() {
crate::log_warn!("coordinator: no indexing roots configured"); self.refuse(trigger, &NO_ROOTS, "no indexing roots are configured");
return; return;
} }
// Backstop for hand-edited configs; the GUI rejects nested roots // Backstop for hand-edited configs; the GUI rejects nested roots
// itself. // itself.
let nested = crate::config::nested_roots(&roots); let nested = crate::config::nested_roots(&roots);
if !nested.is_empty() { if !nested.is_empty() {
for (child, parent) in &nested { let detail = nested
crate::log_warn!( .iter()
"coordinator: refusing to index: root {} is nested under {}", .map(|(child, parent)| format!("{} is nested under {}", child, parent))
child, .collect::<Vec<_>>()
parent .join("; ");
); self.refuse(
} trigger,
&NESTED_ROOTS,
&format!("refusing to index nested roots: {}", detail),
);
return; return;
} }
// The full run owns the DB (and may wipe/rebuild the file). // The full run owns the DB (and may wipe/rebuild the file).
@ -683,6 +760,12 @@ impl Inner {
// unreadable directory (`unreadable.covers`) or of an aliased symlink // unreadable directory (`unreadable.covers`) or of an aliased symlink
// target (`aliased_paths`), and those rows would leak until a rebuild. // target (`aliased_paths`), and those rows would leak until a rebuild.
self.pending.retain(|_, ev| is_removal(ev)); self.pending.retain(|_, ev| is_removal(ev));
// Resume points describe events that no longer exist. Left behind, the
// next `Create` for that path would `skip` the first n entries of a
// walk that has nothing to do with the one the count came from, and
// those files would have no row until a later full run.
self.resume_from
.retain(|p, _| self.pending.contains_key(p) || self.targeted.contains_key(p));
if self.pending.is_empty() { if self.pending.is_empty() {
self.last_event_at = None; self.last_event_at = None;
self.pending_since = None; self.pending_since = None;
@ -691,15 +774,58 @@ impl Inner {
.indexing .indexing
.start_indexing(roots, self.db_path(), self.config.clone()) .start_indexing(roots, self.db_path(), self.config.clone())
{ {
crate::log_warn!("coordinator: start indexing: {}", e); self.refuse(
trigger,
&START_FAILURES,
&format!("could not start indexing: {}", e),
);
return; return;
} }
// A run started, so whatever was wrong is not wrong any more.
self.run_retry_at = None;
self.run_retry_delay = RUN_RETRY_BASE;
NO_ROOTS.reset();
NESTED_ROOTS.reset();
START_FAILURES.reset();
// `start_indexing` claims Running before returning, so there is no // `start_indexing` claims Running before returning, so there is no
// window in which this thread believes the service idle and writes to // window in which this thread believes the service idle and writes to
// a database the run is about to reopen. // a database the run is about to reopen.
self.saw_running = true; self.saw_running = true;
} }
/// Refuse a full run: back off, and say so at a volume the trigger earns.
///
/// The throttles exist for the *scheduled* retry, which repeats the same
/// sentence every few minutes for as long as the configuration stays
/// broken and would otherwise evict the log ring. They must not silence a
/// run somebody asked for: after three refusals a fourth press of *Index
/// now* would log nothing, show nothing and change nothing, which reads as
/// a button that does not work. So a requested run resets the throttle —
/// it is a fresh occurrence in the user's eyes — and publishes the reason
/// where they are already looking, the same way a failed rebuild does.
fn refuse(&mut self, trigger: RunTrigger, throttle: &crate::log::Throttle, reason: &str) {
self.defer_runs();
if trigger == RunTrigger::Requested {
throttle.reset();
self.indexing.report_error(reason.to_string());
}
if throttle.allow() {
crate::log_warn!("coordinator: {}", reason);
}
}
/// Hold off further full runs for a while, doubling the wait each time.
///
/// Called wherever a run is refused or fails to start. `needs_full_run` is
/// cleared with it: the two refusal paths sit above the point that clears
/// it, so without this the flag stays set and `tick` retries immediately,
/// every second, for as long as the configuration stays broken.
fn defer_runs(&mut self) {
self.needs_full_run = false;
self.run_retry_at = Some(Instant::now() + self.run_retry_delay);
self.run_retry_delay = (self.run_retry_delay * 2).min(RUN_RETRY_MAX);
}
fn enter_auto(&mut self) { fn enter_auto(&mut self) {
self.mode = IndexMode::Auto; self.mode = IndexMode::Auto;
self.config.indexing.auto_index = true; self.config.indexing.auto_index = true;
@ -811,6 +937,29 @@ impl Inner {
let Some(mut reason) = w.degraded_reason() else { let Some(mut reason) = w.degraded_reason() else {
return; return;
}; };
// An overflow is not a capacity problem: the watcher is still good,
// it just cannot tell us what it missed. Schedule the run that finds
// out, rather than disabling live updates for the session over
// something a busy minute can cause.
//
// Not a restart. `inotify` and `ReadDirectoryChangesW` both keep
// delivering after their queue overflows — the rescan flag means "you
// missed some", not "this watch is broken" — so tearing down and
// re-registering would re-`stat` every one of `max_watched_dirs`
// (128k) directories, generate events of its own, and do it again the
// next time a busy tree overflowed the queue, which is exactly when
// the machine can least afford it.
//
// Consuming the reason is what makes this a one-shot: the slot is
// never otherwise cleared, so a standing `Overflowed` would re-arm
// `needs_full_run` on every tick and hide a `KernelLimit` recorded
// afterwards.
if matches!(reason, WatchError::Overflowed) {
w.clear_degraded();
crate::log_warn!("watcher: {}", reason);
self.needs_full_run = true;
return;
}
// The async notify callback can't know the count; fill it in here. // The async notify callback can't know the count; fill it in here.
if let WatchError::KernelLimit { registered } = &mut reason { if let WatchError::KernelLimit { registered } = &mut reason {
if *registered == 0 { if *registered == 0 {

2
crates/quicksearch-core/src/coordinator/tests.rs
View file

@ -512,7 +512,7 @@ impl Fixture {
fn stored_mtime(&self, path: &std::path::Path) -> Option<i64> { fn stored_mtime(&self, path: &std::path::Path) -> Option<i64> {
let conn = db::open_existing(&self.db.to_string_lossy(), false).ok()?; let conn = db::open_existing(&self.db.to_string_lossy(), false).ok()?;
conn.query_row( conn.query_row(
"SELECT mtime FROM files WHERE path = ?1", "SELECT mtime FROM files WHERE parent || name = ?1",
[path.to_string_lossy().as_ref()], [path.to_string_lossy().as_ref()],
|r| r.get(0), |r| r.get(0),
) )

4
crates/quicksearch-core/src/db/mod.rs
View file

@ -17,8 +17,8 @@ pub mod schema;
pub use key::{process_key_hex, set_process_key}; pub use key::{process_key_hex, set_process_key};
pub use open::{ pub use open::{
index_needs_rebuild, open_existing, open_or_recreate, verify_process_key, index_needs_rebuild, key_mismatch_parts, open_existing, open_or_recreate, verify_process_key,
CURRENT_SCHEMA_VERSION, KEY_MISMATCH_PREFIX, KeyMismatch, CURRENT_SCHEMA_VERSION, KEY_MISMATCH_PREFIX,
}; };
/// Bumped whenever the index file is replaced rather than modified — a /// Bumped whenever the index file is replaced rather than modified — a

83
crates/quicksearch-core/src/db/open.rs
View file

@ -25,7 +25,7 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: ";
/// values go stale: `files.mime`, `files.type` and `content_state` are /// values go stale: `files.mime`, `files.type` and `content_state` are
/// computed at walk time and never re-derived for unchanged files, so a /// computed at walk time and never re-derived for unchanged files, so a
/// classification change needs the wipe to apply everywhere. /// classification change needs the wipe to apply everywhere.
pub const CURRENT_SCHEMA_VERSION: u32 = 7; pub const CURRENT_SCHEMA_VERSION: u32 = 8;
/// Open `db_path` and ensure the on-disk schema matches this build; if it /// Open `db_path` and ensure the on-disk schema matches this build; if it
/// doesn't (including a changed `tokenizer`), delete the file and recreate it /// doesn't (including a changed `tokenizer`), delete the file and recreate it
@ -241,6 +241,57 @@ fn is_notadb(e: &rusqlite::Error) -> bool {
) )
} }
/// Why a keyed open failed, as something the caller can branch on.
///
/// The three cases want three different things from a user — retype the
/// password, rebuild the index, supply a password at all — and only one of
/// them is "wrong password". They used to be distinguishable only by reading
/// the English in the message, which breaks the moment a database path
/// happens to contain that English.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum KeyMismatch {
/// A key was applied and the file did not accept it.
WrongPassword,
/// A key was applied but the file on disk is not encrypted at all —
/// protection was enabled and the rebuild that would encrypt it did not
/// finish.
NotEncrypted,
/// No key was applied and the file wants one.
PasswordRequired,
}
impl KeyMismatch {
/// The machine-readable token carried in the message, between
/// [`KEY_MISMATCH_PREFIX`] and the human detail.
fn token(self) -> &'static str {
match self {
KeyMismatch::WrongPassword => "wrong-password",
KeyMismatch::NotEncrypted => "not-encrypted",
KeyMismatch::PasswordRequired => "password-required",
}
}
fn from_token(token: &str) -> Option<KeyMismatch> {
match token {
"wrong-password" => Some(KeyMismatch::WrongPassword),
"not-encrypted" => Some(KeyMismatch::NotEncrypted),
"password-required" => Some(KeyMismatch::PasswordRequired),
_ => None,
}
}
}
/// Split a tagged mismatch message into its cause and the human detail.
///
/// `None` for any message that is not one — including a `KEY_MISMATCH_PREFIX`
/// message from an older build, which callers should treat as they always did.
pub fn key_mismatch_parts(message: &str) -> Option<(KeyMismatch, &str)> {
let rest = message.strip_prefix(KEY_MISMATCH_PREFIX)?;
let (token, detail) = rest.split_once(' ')?;
let token = token.strip_suffix(':')?;
Some((KeyMismatch::from_token(token)?, detail))
}
fn key_mismatch_message(db_path: &str, had_key: bool) -> String { fn key_mismatch_message(db_path: &str, had_key: bool) -> String {
// An unencrypted SQLite file still has its plaintext magic; sniffing it // An unencrypted SQLite file still has its plaintext magic; sniffing it
// distinguishes "wrong password" from "protection is enabled but the // distinguishes "wrong password" from "protection is enabled but the
@ -255,15 +306,31 @@ fn key_mismatch_message(db_path: &str, had_key: bool) -> String {
Some(&magic == b"SQLite format 3\0") Some(&magic == b"SQLite format 3\0")
}) })
.unwrap_or(false); .unwrap_or(false);
let detail = match (had_key, plaintext) { let (cause, detail) = match (had_key, plaintext) {
(true, true) => { (true, true) => (
KeyMismatch::NotEncrypted,
"password protection is enabled but the index is not encrypted; \ "password protection is enabled but the index is not encrypted; \
rebuild the index to encrypt it" rebuild the index to encrypt it",
} ),
(true, false) => "wrong password (or the file is not a QuickSearch index)", (true, false) => (
(false, _) => "the index is password-protected; a password is required", KeyMismatch::WrongPassword,
"wrong password (or the file is not a QuickSearch index)",
),
(false, _) => (
KeyMismatch::PasswordRequired,
"the index is password-protected; a password is required",
),
}; };
format!("{}index at {}: {}", KEY_MISMATCH_PREFIX, db_path, detail) // The token sits between the prefix and the detail so that every existing
// `starts_with(KEY_MISMATCH_PREFIX)` test still holds, while a caller that
// needs the cause can have it without reading prose.
format!(
"{}{}: index at {}: {}",
KEY_MISMATCH_PREFIX,
cause.token(),
db_path,
detail
)
} }
/// True iff the DB has a `schema_info` table whose `version` equals /// True iff the DB has a `schema_info` table whose `version` equals

70
crates/quicksearch-core/src/db/open_tests.rs
View file

@ -124,8 +124,8 @@ fn tokenizer_drift_wipes_db() {
let first_effective = { let first_effective = {
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
conn.execute( conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \ "INSERT INTO files (name, parent, size, mtime) \
VALUES ('x', '/x', '/', 0, 0)", VALUES ('x', '/', 0, 0)",
[], [],
) )
.unwrap(); .unwrap();
@ -164,8 +164,8 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() {
{ {
let conn = open_or_recreate(p.to_str().unwrap(), "unicode61").unwrap(); let conn = open_or_recreate(p.to_str().unwrap(), "unicode61").unwrap();
conn.execute( conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \ "INSERT INTO files (name, parent, size, mtime) \
VALUES ('note', '/note.txt', '/', 0, 0)", VALUES ('note', '/', 0, 0)",
[], [],
) )
.unwrap(); .unwrap();
@ -384,8 +384,8 @@ fn keyed_create_reopen_and_header_is_encrypted() {
{ {
let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&key)).unwrap(); let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&key)).unwrap();
conn.execute( conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \ "INSERT INTO files (name, parent, size, mtime) \
VALUES ('secret', '/secret.txt', '/', 0, 0)", VALUES ('secret', '/', 0, 0)",
[], [],
) )
.unwrap(); .unwrap();
@ -418,8 +418,8 @@ fn wrong_key_errors_without_wiping() {
let conn = let conn =
open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&test_key(0xa1))).unwrap(); open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&test_key(0xa1))).unwrap();
conn.execute( conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \ "INSERT INTO files (name, parent, size, mtime) \
VALUES ('x', '/x', '/', 0, 0)", VALUES ('x', '/', 0, 0)",
[], [],
) )
.unwrap(); .unwrap();
@ -532,15 +532,15 @@ fn open_existing_rw_allows_delete() {
{ {
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
conn.execute( conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \ "INSERT INTO files (name, parent, size, mtime) \
VALUES ('a', '/a', '/', 0, 0)", VALUES ('a', '/', 0, 0)",
[], [],
) )
.unwrap(); .unwrap();
} }
let conn = open_existing(p.to_str().unwrap(), true).unwrap(); let conn = open_existing(p.to_str().unwrap(), true).unwrap();
let removed = conn let removed = conn
.execute("DELETE FROM files WHERE path = '/a'", []) .execute("DELETE FROM files WHERE parent = '/' AND name = 'a'", [])
.unwrap(); .unwrap();
assert_eq!(removed, 1); assert_eq!(removed, 1);
drop(conn); drop(conn);
@ -571,8 +571,8 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() {
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
// A write, so the WAL and SHM exist to be checked. // A write, so the WAL and SHM exist to be checked.
conn.execute( conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \ "INSERT INTO files (name, parent, size, mtime) \
VALUES ('a', '/perm-a', '/', 0, 0)", VALUES ('a', '/', 0, 0)",
[], [],
) )
.unwrap(); .unwrap();
@ -598,3 +598,47 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() {
drop(conn); drop(conn);
std::fs::remove_file(&p).ok(); std::fs::remove_file(&p).ok();
} }
/// `maintain` must work on a *keyed* index, which is the one case the pragma
/// it reads does not answer with an integer.
///
/// SQLCipher intercepts `PRAGMA page_size` on a keyed connection, answers with
/// `cipher_page_size`, and hands that back as TEXT. Reading it straight into an
/// `i64` failed there and only there — so every unencrypted test passed while
/// every index with a password set silently skipped both its VACUUM and its
/// `PRAGMA optimize`. The assertion is simply that the call succeeds: it has to
/// get past all three pragma reads to return at all.
#[test]
fn maintain_reads_its_pragmas_on_a_keyed_index() {
let p = tmp_db_path();
let key = test_key(0xc3);
let dir = p.parent().unwrap().to_string_lossy().into_owned();
{
let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&key)).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)",
[],
)
.unwrap();
}
let conn = open_keyed_with_pragmas(p.to_str().unwrap(), true, Some(&key), PRAGMAS_MAINTENANCE)
.unwrap();
// A two-row index has no slack worth reclaiming, so `false` is the
// expected answer — what matters is that it is an answer and not an error.
assert_eq!(
crate::db::repo::maintain(&conn, &dir),
Ok(false),
"maintain must not fail on a keyed index"
);
// And the value itself has to be usable, not merely readable: a page size
// that parsed as 0 would size the free-space check at zero bytes and wave
// through a VACUUM that cannot fit.
assert!(
crate::db::repo::pragma_number(&conn, "page_size").unwrap() >= 512,
"a real page size, not a silent zero"
);
drop(conn);
std::fs::remove_file(&p).ok();
}

239
crates/quicksearch-core/src/db/repo.rs
View file

@ -62,7 +62,8 @@ fn set_state_clearing_failure(
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct NewFile<'a> { pub struct NewFile<'a> {
pub name: &'a str, pub name: &'a str,
pub path: &'a str, /// The containing directory, ending in the platform separator — see
/// [`crate::file_handling::split_db_path`], which produces the pair.
pub parent: &'a str, pub parent: &'a str,
pub size: u64, pub size: u64,
pub mtime: u64, pub mtime: u64,
@ -74,21 +75,29 @@ pub struct NewFile<'a> {
pub needs_content: bool, pub needs_content: bool,
} }
impl NewFile<'_> {
/// The file's path, for a log or error message. Not stored; see
/// [`super::schema::SCHEMA_CURRENT`].
fn path(&self) -> String {
format!("{}{}", self.parent, self.name)
}
}
/// Insert a new file row, returning its id. `content_state` comes from /// Insert a new file row, returning its id. `content_state` comes from
/// `needs_content`; there is no separate basic state, because the row /// `needs_content`; there is no separate basic state, because the row
/// existing *is* the basic-index state. `INSERT OR IGNORE`: a UNIQUE(path) /// existing *is* the basic-index state. `INSERT OR IGNORE`: a
/// collision returns `None` rather than aborting the batch. /// `UNIQUE(parent, name)` collision returns `None` rather than aborting the
/// batch.
pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> { pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> {
let rows = tx let rows = tx
.prepare_cached( .prepare_cached(
"INSERT OR IGNORE INTO files ( "INSERT OR IGNORE INTO files (
name, path, parent, size, mtime, mime, type, content_state, hash name, parent, size, mtime, mime, type, content_state, hash
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
) )
.and_then(|mut stmt| { .and_then(|mut stmt| {
stmt.execute(params![ stmt.execute(params![
f.name, f.name,
f.path,
f.parent, f.parent,
f.size as i64, f.size as i64,
f.mtime as i64, f.mtime as i64,
@ -98,7 +107,7 @@ pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>,
f.hash, f.hash,
]) ])
}) })
.map_err(|e| format!("insert file {}: {}", f.path, e))?; .map_err(|e| format!("insert file {}: {}", f.path(), e))?;
if rows == 0 { if rows == 0 {
return Ok(None); return Ok(None);
} }
@ -120,14 +129,14 @@ fn initial_content_state(f: &NewFile<'_>) -> i64 {
/// reset its content state from `f.needs_content`, clearing any extracted /// reset its content state from `f.needs_content`, clearing any extracted
/// content so the text-indexing pass re-processes it. Writes `size`, `mtime`, /// content so the text-indexing pass re-processes it. Writes `size`, `mtime`,
/// `hash`, `mime`, `type` and `content_state` — and only those; `name` and /// `hash`, `mime`, `type` and `content_state` — and only those; `name` and
/// `parent` are not refreshed here. /// `parent` are the key it matches on, so they cannot change here.
pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> { pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> {
let id: Option<i64> = tx let id: Option<i64> = tx
.prepare_cached( .prepare_cached(
"UPDATE files "UPDATE files
SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5, SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5,
content_state = ?6 content_state = ?6
WHERE path = ?7 WHERE parent = ?7 AND name = ?8
RETURNING id", RETURNING id",
) )
.and_then(|mut stmt| { .and_then(|mut stmt| {
@ -139,13 +148,14 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option
f.mime, f.mime,
f.ftype.bits() as i64, f.ftype.bits() as i64,
initial_content_state(f), initial_content_state(f),
f.path, f.parent,
f.name,
], ],
|r| r.get(0), |r| r.get(0),
) )
.optional() .optional()
}) })
.map_err(|e| format!("update file {}: {}", f.path, e))?; .map_err(|e| format!("update file {}: {}", f.path(), e))?;
let Some(id) = id else { let Some(id) = id else {
return Ok(None); return Ok(None);
}; };
@ -365,29 +375,38 @@ pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String>
} }
/// Delete a file row by path, keeping FTS in sync. Returns whether a row was /// Delete a file row by path, keeping FTS in sync. Returns whether a row was
/// removed. /// removed — including `false` for a string that cannot be a file's path at
/// all, which is not in the index by construction.
pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result<bool, String> { pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result<bool, String> {
let Some((parent, name)) = crate::file_handling::split_db_path(path) else {
return Ok(false);
};
let id: Option<i64> = tx let id: Option<i64> = tx
.prepare_cached("DELETE FROM files WHERE path = ?1 RETURNING id") .prepare_cached("DELETE FROM files WHERE parent = ?1 AND name = ?2 RETURNING id")
.and_then(|mut stmt| stmt.query_row(params![path], |r| r.get(0)).optional()) .and_then(|mut stmt| {
stmt.query_row(params![parent, name], |r| r.get(0))
.optional()
})
.map_err(|e| format!("delete file {}: {}", path, e))?; .map_err(|e| format!("delete file {}: {}", path, e))?;
let Some(id) = id else { return Ok(false) }; let Some(id) = id else { return Ok(false) };
remove_content_for_id(tx, id)?; remove_content_for_id(tx, id)?;
Ok(true) Ok(true)
} }
/// Delete every row whose path falls in the half-open range `[lo, hi)`, /// Delete every row whose parent falls in the half-open range `[lo, hi)`,
/// keeping the dependent tables in step. Returns how many `files` rows went. /// keeping the dependent tables in step. Returns how many `files` rows went.
/// ///
/// Five statements regardless of how many files the range holds, and the /// Four statements regardless of how many files the range holds, and the
/// range is an index seek on `UNIQUE(files.path)`. Build the bounds with /// range is an index seek on `UNIQUE(files.parent, files.name)`. Build the
/// [`crate::file_handling::ExtractCursor::for_root`], which is what makes /// bounds with [`crate::file_handling::ExtractCursor::for_root`], which is
/// them separator-correct. /// what makes them separator-correct — and note the range covers the root's
/// *own* files only because every stored parent ends in a separator (see
/// `dir_to_db_parent`).
pub fn delete_subtree(tx: &Transaction<'_>, lo: &str, hi: &str) -> Result<usize, String> { pub fn delete_subtree(tx: &Transaction<'_>, lo: &str, hi: &str) -> Result<usize, String> {
for (table, key) in DEPENDENT_TABLES { for (table, key) in DEPENDENT_TABLES {
let sql = format!( let sql = format!(
"DELETE FROM {} WHERE {} IN \ "DELETE FROM {} WHERE {} IN \
(SELECT id FROM files WHERE path >= ?1 AND path < ?2)", (SELECT id FROM files WHERE parent >= ?1 AND parent < ?2)",
table, key table, key
); );
exec(tx, &sql, params![lo, hi], || { exec(tx, &sql, params![lo, hi], || {
@ -396,13 +415,13 @@ pub fn delete_subtree(tx: &Transaction<'_>, lo: &str, hi: &str) -> Result<usize,
} }
exec( exec(
tx, tx,
"DELETE FROM files WHERE path >= ?1 AND path < ?2", "DELETE FROM files WHERE parent >= ?1 AND parent < ?2",
params![lo, hi], params![lo, hi],
|| format!("delete files under {}", lo), || format!("delete files under {}", lo),
) )
} }
/// Delete every row whose path falls in *none* of `ranges`. Returns how many /// Delete every row whose parent falls in *none* of `ranges`. Returns how many
/// `files` rows went. /// `files` rows went.
/// ///
/// A scan of `files` rather than a seek, reserved for the one transition that /// A scan of `files` rather than a seek, reserved for the one transition that
@ -423,7 +442,7 @@ pub fn delete_outside_ranges(
predicate.push_str(" AND "); predicate.push_str(" AND ");
} }
predicate.push_str(&format!( predicate.push_str(&format!(
"NOT (path >= ?{} AND path < ?{})", "NOT (parent >= ?{} AND parent < ?{})",
i * 2 + 1, i * 2 + 1,
i * 2 + 2 i * 2 + 2
)); ));
@ -485,8 +504,12 @@ pub fn delete_ids(tx: &Transaction<'_>, ids: &[i64]) -> Result<usize, String> {
Ok(removed) Ok(removed)
} }
/// Every indexed file directly inside `parent`, as `name -> mtime`. Served by /// Every indexed file directly inside `parent`, as `name -> mtime`.
/// `idx_files_parent`: one index range lookup. ///
/// `parent` must be in stored spelling — trailing separator and all; build it
/// with [`crate::file_handling::dir_to_db_parent`]. One `idx_files_parent`
/// range lookup for the names, then a row fetch each for the mtimes; see the
/// index's own comment for why that is the shape it is.
pub fn dir_rows( pub fn dir_rows(
conn: &Connection, conn: &Connection,
parent: &str, parent: &str,
@ -507,7 +530,9 @@ pub fn dir_rows(
Ok(out) Ok(out)
} }
/// A row the content pass has yet to extract: `(id, name, path, mime)`. /// A row the content pass has yet to extract: `(id, name, path, mime)`. The
/// path is reassembled here rather than stored — see
/// [`super::schema::SCHEMA_CURRENT`] — because the pass opens the file by it.
pub type PendingContentRow = (i64, String, String, Option<String>); pub type PendingContentRow = (i64, String, String, Option<String>);
/// One page of rows still awaiting content extraction under `cursor`'s range, /// One page of rows still awaiting content extraction under `cursor`'s range,
@ -526,11 +551,11 @@ pub fn pending_content_page(
.prepare_cached( .prepare_cached(
// `INDEXED BY` rather than a hint, because the planner gets this // `INDEXED BY` rather than a hint, because the planner gets this
// one wrong exactly when it costs most. Left to itself it takes // one wrong exactly when it costs most. Left to itself it takes
// `UNIQUE(path)` for the range and then sorts the survivors into a // `idx_files_parent` for the range and then sorts the survivors
// temp b-tree to satisfy `ORDER BY id` — which means every page // into a temp b-tree to satisfy `ORDER BY id` — which means every
// walks the whole root's range and fetches each row's heap entry // page walks the whole root's range and fetches each row's heap
// to test `content_state`. At `FEED_PAGE` rows per page that is // entry to test `content_state`. At `FEED_PAGE` rows per page that
// quadratic over a run. The partial index below is already // is quadratic over a run. The partial index below is already
// id-ordered, so it answers `id > ?` and the ORDER BY together and // id-ordered, so it answers `id > ?` and the ORDER BY together and
// holds only pending rows. Measured on 500k rows with everything // holds only pending rows. Measured on 500k rows with everything
// pending — the first index of a tree, i.e. the case that matters: // pending — the first index of a tree, i.e. the case that matters:
@ -539,9 +564,9 @@ pub fn pending_content_page(
// The planner only prefers it once pending rows are a small // The planner only prefers it once pending rows are a small
// minority, and never before ANALYZE has run at all, which is why // minority, and never before ANALYZE has run at all, which is why
// this cannot be left to statistics. // this cannot be left to statistics.
"SELECT id, name, path, mime FROM files INDEXED BY idx_files_content_pending "SELECT id, parent, name, mime FROM files INDEXED BY idx_files_content_pending
WHERE content_state = 0 AND size <= ?1 AND id > ?2 WHERE content_state = 0 AND size <= ?1 AND id > ?2
AND path >= ?3 AND path < ?4 AND parent >= ?3 AND parent < ?4
ORDER BY id ORDER BY id
LIMIT ?5", LIMIT ?5",
) )
@ -550,10 +575,13 @@ pub fn pending_content_page(
.query_map( .query_map(
params![max_size, cursor.last_id, cursor.lo, cursor.hi, limit], params![max_size, cursor.last_id, cursor.lo, cursor.hi, limit],
|row| { |row| {
let parent: String = row.get(1)?;
let name: String = row.get(2)?;
let path = format!("{}{}", parent, name);
Ok(( Ok((
row.get::<_, i64>(0)?, row.get::<_, i64>(0)?,
row.get::<_, String>(1)?, name,
row.get::<_, String>(2)?, path,
row.get::<_, Option<String>>(3)?, row.get::<_, Option<String>>(3)?,
)) ))
}, },
@ -565,10 +593,16 @@ pub fn pending_content_page(
/// A stored row as the scope reconciler sees it: enough to decide both /// A stored row as the scope reconciler sees it: enough to decide both
/// whether the path is still in scope and whether its content still is. /// whether the path is still in scope and whether its content still is.
///
/// `path` is `parent` and `name` joined, kept alongside them because the
/// reconciler tests it as a [`std::path::Path`] while the cursor resumes from
/// the key.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct ScopeRow { pub struct ScopeRow {
pub id: i64, pub id: i64,
pub path: String, pub path: String,
pub parent: String,
pub name: String,
pub size: u64, pub size: u64,
pub mime: Option<String>, pub mime: Option<String>,
pub content_state: i64, pub content_state: i64,
@ -590,7 +624,7 @@ pub struct RootCounts {
pub fts: i64, pub fts: i64,
} }
/// Count the rows in the half-open path range `[lo, hi)` and, in the same /// Count the rows in the half-open parent range `[lo, hi)` and, in the same
/// pass, how many of them have a full-text row. /// pass, how many of them have a full-text row.
/// ///
/// `content_state = STATE_DONE` *is* "has a `searchabletext` row": /// `content_state = STATE_DONE` *is* "has a `searchabletext` row":
@ -600,13 +634,13 @@ pub struct RootCounts {
/// contentless and keyed by `rowid`, so it has no path to range-scan on. /// contentless and keyed by `rowid`, so it has no path to range-scan on.
/// ///
/// One statement, but not a cheap one: `content_state` is not carried by the /// One statement, but not a cheap one: `content_state` is not carried by the
/// `UNIQUE(files.path)` index the range seeks on, so every row in the range is /// `idx_files_parent` index the range seeks on, so every row in the range is
/// fetched. Call it where a run has just read those rows anyway, not on a /// fetched. Call it where a run has just read those rows anyway, not on a
/// cadence. /// cadence.
pub fn count_root(conn: &Connection, lo: &str, hi: &str) -> Result<RootCounts, String> { pub fn count_root(conn: &Connection, lo: &str, hi: &str) -> Result<RootCounts, String> {
conn.prepare_cached( conn.prepare_cached(
"SELECT COUNT(*), COALESCE(SUM(content_state = ?3), 0) FROM files "SELECT COUNT(*), COALESCE(SUM(content_state = ?3), 0) FROM files
WHERE path >= ?1 AND path < ?2", WHERE parent >= ?1 AND parent < ?2",
) )
.and_then(|mut stmt| { .and_then(|mut stmt| {
stmt.query_row(params![lo, hi, STATE_DONE], |r| { stmt.query_row(params![lo, hi, STATE_DONE], |r| {
@ -619,37 +653,46 @@ pub fn count_root(conn: &Connection, lo: &str, hi: &str) -> Result<RootCounts, S
.map_err(|e| format!("count root {}: {}", lo, e)) .map_err(|e| format!("count root {}: {}", lo, e))
} }
/// One page of rows whose path is `> after` and `< hi`, in path order. /// One page of rows sorting after `(after_parent, after_name)` and inside the
/// parent range ending at `hi`, in `(parent, name)` order.
/// ///
/// Keyset on `path`: every page is an index walk with no sort step, and a row /// Keyset on the `idx_files_parent` key itself: every page is an index walk
/// is served at most once even though the caller is deleting behind the /// with no sort step, and a row is served at most once even though the caller
/// reader. Seed `after` with the range's `lo` bound, which is /// is deleting behind the reader. Seed the cursor with `(lo, "")` — no name is
/// `root + separator` and so can never equal a stored path. /// empty, so that lands exactly on the first row of the range.
///
/// The row-value comparison is what keeps it one seek; spelled out as
/// `parent > ? OR (parent = ? AND name > ?)` the planner is free to scan.
pub fn rows_in_range_page( pub fn rows_in_range_page(
conn: &Connection, conn: &Connection,
after: &str, after_parent: &str,
after_name: &str,
hi: &str, hi: &str,
limit: i64, limit: i64,
) -> Result<Vec<ScopeRow>, String> { ) -> Result<Vec<ScopeRow>, String> {
let mut stmt = conn let mut stmt = conn
.prepare_cached( .prepare_cached(
"SELECT id, path, size, mime, content_state FROM files "SELECT id, parent, name, size, mime, content_state FROM files
WHERE path > ?1 AND path < ?2 WHERE (parent, name) > (?1, ?2) AND parent < ?3
ORDER BY path ORDER BY parent, name
LIMIT ?3", LIMIT ?4",
) )
.map_err(|e| format!("prepare range page: {}", e))?; .map_err(|e| format!("prepare range page: {}", e))?;
let rows = stmt let rows = stmt
.query_map(params![after, hi, limit], |row| { .query_map(params![after_parent, after_name, hi, limit], |row| {
let parent: String = row.get(1)?;
let name: String = row.get(2)?;
Ok(ScopeRow { Ok(ScopeRow {
id: row.get(0)?, id: row.get(0)?,
path: row.get(1)?, path: format!("{}{}", parent, name),
size: row.get::<_, i64>(2)?.max(0) as u64, name,
mime: row.get(3)?, parent,
content_state: row.get(4)?, size: row.get::<_, i64>(3)?.max(0) as u64,
mime: row.get(4)?,
content_state: row.get(5)?,
}) })
}) })
.map_err(|e| format!("query range page after {}: {}", after, e))?; .map_err(|e| format!("query range page after {}: {}", after_parent, e))?;
rows.collect::<Result<Vec<_>, _>>() rows.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("read range page row: {}", e)) .map_err(|e| format!("read range page row: {}", e))
} }
@ -683,18 +726,26 @@ pub fn reset_content_pending(tx: &Transaction<'_>, file_id: i64) -> Result<(), S
/// files whose parent isn't the directory being read (a resolved symlink /// files whose parent isn't the directory being read (a resolved symlink
/// target), where [`dir_rows`] would not have them. /// target), where [`dir_rows`] would not have them.
pub fn mtime_for_path(conn: &Connection, path: &str) -> Result<Option<u64>, String> { pub fn mtime_for_path(conn: &Connection, path: &str) -> Result<Option<u64>, String> {
let Some((parent, name)) = crate::file_handling::split_db_path(path) else {
return Ok(None);
};
let mut stmt = conn let mut stmt = conn
.prepare_cached("SELECT mtime FROM files WHERE path = ?1") .prepare_cached("SELECT mtime FROM files WHERE parent = ?1 AND name = ?2")
.map_err(|e| format!("prepare mtime lookup for {}: {}", path, e))?; .map_err(|e| format!("prepare mtime lookup for {}: {}", path, e))?;
stmt.query_row(params![path], |r| r.get::<_, i64>(0)) stmt.query_row(params![parent, name], |r| r.get::<_, i64>(0))
.optional() .optional()
.map(|o| o.map(|m| m.max(0) as u64)) .map(|o| o.map(|m| m.max(0) as u64))
.map_err(|e| format!("mtime lookup for {}: {}", path, e)) .map_err(|e| format!("mtime lookup for {}: {}", path, e))
} }
/// Distinct `parent` values within the half-open path range `[lo, hi)`, /// Distinct `parent` values within the half-open range `[lo, hi)`, streamed to
/// streamed to `f` so nothing proportional to the tree is materialized. /// `f` so nothing proportional to the tree is materialized. `idx_files_parent`
/// `idx_files_parent` makes this an index-only scan. /// makes this an index-only scan.
///
/// The root's own directory is included: its stored parent is `root + SEP`,
/// which is exactly `lo`. It was not, back when the same bounds were applied
/// to a `path` column and the root's parent was spelled without the trailing
/// separator.
pub fn for_each_parent_in_range<F: FnMut(String)>( pub fn for_each_parent_in_range<F: FnMut(String)>(
conn: &Connection, conn: &Connection,
lo: &str, lo: &str,
@ -713,13 +764,17 @@ pub fn for_each_parent_in_range<F: FnMut(String)>(
Ok(()) Ok(())
} }
/// Paths of every file directly inside `parent`. /// Paths of every file directly inside `parent`, which must carry its trailing
/// separator — so the join below is a concatenation.
pub fn paths_in_dir(conn: &Connection, parent: &str) -> Result<Vec<String>, String> { pub fn paths_in_dir(conn: &Connection, parent: &str) -> Result<Vec<String>, String> {
let mut stmt = conn let mut stmt = conn
.prepare_cached("SELECT path FROM files WHERE parent = ?1") .prepare_cached("SELECT name FROM files WHERE parent = ?1")
.map_err(|e| format!("prepare paths in {}: {}", parent, e))?; .map_err(|e| format!("prepare paths in {}: {}", parent, e))?;
let rows = stmt let rows = stmt
.query_map(params![parent], |r| r.get::<_, String>(0)) .query_map(params![parent], |r| {
r.get::<_, String>(0)
.map(|name| format!("{}{}", parent, name))
})
.map_err(|e| format!("query paths in {}: {}", parent, e))?; .map_err(|e| format!("query paths in {}: {}", parent, e))?;
rows.collect::<Result<Vec<_>, _>>() rows.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("read path under {}: {}", parent, e)) .map_err(|e| format!("read path under {}: {}", parent, e))
@ -795,6 +850,33 @@ pub fn checkpoint_and_close(conn: Connection) {
drop(conn); drop(conn);
} }
/// Read a `PRAGMA` that reports a number.
///
/// Not simply `r.get::<i64>(0)`, because SQLCipher does not always answer with
/// one. On a **keyed** connection it intercepts `PRAGMA page_size`, answers
/// with `cipher_page_size` instead, and returns that as TEXT — so asking for an
/// integer fails with a type error. Unencrypted it is an INTEGER as usual,
/// which is why this only ever broke protected installs, and only in
/// [`maintain`]: every index with a password set skipped its VACUUM *and* its
/// `PRAGMA optimize` from the moment the free-space check was added.
///
/// A value that is neither is an error rather than a guess — the callers here
/// size a disk-space check with it.
pub(super) fn pragma_number(conn: &Connection, pragma: &str) -> Result<i64, String> {
use rusqlite::types::ValueRef;
conn.query_row(&format!("PRAGMA {}", pragma), [], |r| {
Ok(match r.get_ref(0)? {
ValueRef::Integer(n) => Some(n),
ValueRef::Text(t) => std::str::from_utf8(t)
.ok()
.and_then(|s| s.trim().parse().ok()),
_ => None,
})
})
.map_err(|e| format!("read {}: {}", pragma, e))?
.ok_or_else(|| format!("read {}: not a number", pragma))
}
/// Land the log, reclaim the file's slack, and refresh the query planner's /// Land the log, reclaim the file's slack, and refresh the query planner's
/// statistics. Returns whether it vacuumed. /// statistics. Returns whether it vacuumed.
/// ///
@ -822,14 +904,33 @@ pub fn maintain(conn: &Connection, db_dir: &str) -> Result<bool, String> {
crate::log_warn!("{}", e); crate::log_warn!("{}", e);
} }
let page_count: i64 = conn let page_count = pragma_number(conn, "page_count")?;
.query_row("PRAGMA page_count", [], |r| r.get(0)) let freelist = pragma_number(conn, "freelist_count")?;
.map_err(|e| format!("read page_count: {}", e))?;
let freelist: i64 = conn
.query_row("PRAGMA freelist_count", [], |r| r.get(0))
.map_err(|e| format!("read freelist_count: {}", e))?;
let vacuumed = freelist * 100 >= page_count * VACUUM_MIN_SLACK_PERCENT; let worth_it = freelist * 100 >= page_count * VACUUM_MIN_SLACK_PERCENT;
// The doc comment above puts VACUUM's peak transient need at roughly three
// times the index. Checking first turns "the volume filled up mid-rebuild"
// into a skipped compaction: a rollback is the *good* outcome there, and
// the bad one is that writes to the `-shm` mmap on a full filesystem come
// back as SIGBUS rather than as an error — see
// `indexing::pipeline::DISK_FLOOR`. Unknown free space is not a reason to
// skip.
let page_size = pragma_number(conn, "page_size")?;
let needed = (page_count.max(0) as u64).saturating_mul(page_size.max(0) as u64) * 3;
let room = match crate::platform::available_space(std::path::Path::new(db_dir)) {
Some(free) if free < needed => {
crate::log_warn!(
"skipping VACUUM: it needs about {} MiB free in {} and there is {} MiB",
needed / (1024 * 1024),
db_dir,
free / (1024 * 1024)
);
false
}
_ => true,
};
let vacuumed = worth_it && room;
if vacuumed { if vacuumed {
// `temp_store_directory` is a deprecated pragma that writes a global, // `temp_store_directory` is a deprecated pragma that writes a global,
// so it is set for the VACUUM and cleared straight after rather than // so it is set for the VACUUM and cleared straight after rather than

153
crates/quicksearch-core/src/db/repo_tests.rs
View file

@ -47,8 +47,7 @@ fn insert_update_delete_round_trip() {
&tx, &tx,
&NewFile { &NewFile {
name: "a.txt", name: "a.txt",
path: "/tmp/a.txt", parent: "/tmp/",
parent: "/tmp",
size: 42, size: 42,
mtime: 1_700_000_000, mtime: 1_700_000_000,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -101,8 +100,7 @@ fn insert_writes_content_state_from_needs_content() {
let tx = conn.transaction().unwrap(); let tx = conn.transaction().unwrap();
let mut row = NewFile { let mut row = NewFile {
name: "claimed.txt", name: "claimed.txt",
path: "/tmp/claimed.txt", parent: "/tmp/",
parent: "/tmp",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -112,7 +110,6 @@ fn insert_writes_content_state_from_needs_content() {
}; };
let claimed = insert_file(&tx, &row).unwrap().expect("unique path"); let claimed = insert_file(&tx, &row).unwrap().expect("unique path");
row.name = "unclaimed.mp4"; row.name = "unclaimed.mp4";
row.path = "/tmp/unclaimed.mp4";
row.mime = Some("video/mp4"); row.mime = Some("video/mp4");
row.needs_content = false; row.needs_content = false;
let unclaimed = insert_file(&tx, &row).unwrap().expect("unique path"); let unclaimed = insert_file(&tx, &row).unwrap().expect("unique path");
@ -139,8 +136,7 @@ fn update_writes_content_state_from_needs_content() {
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let mut row = NewFile { let mut row = NewFile {
name: "a.txt", name: "a.txt",
path: "/tmp/a.txt", parent: "/tmp/",
parent: "/tmp",
size: 10, size: 10,
mtime: 1, mtime: 1,
mime: None, mime: None,
@ -216,8 +212,7 @@ fn insert_file_twice_on_same_path_is_idempotent() {
let tx = conn.transaction().unwrap(); let tx = conn.transaction().unwrap();
let row = NewFile { let row = NewFile {
name: "dup.txt", name: "dup.txt",
path: "/tmp/dup.txt", parent: "/tmp/",
parent: "/tmp",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -234,8 +229,8 @@ fn insert_file_twice_on_same_path_is_idempotent() {
assert_eq!(count, 1); assert_eq!(count, 1);
let (id_read,): (i64,) = tx let (id_read,): (i64,) = tx
.query_row( .query_row(
"SELECT id FROM files WHERE path = ?1", "SELECT id FROM files WHERE parent = ?1 AND name = ?2",
params!["/tmp/dup.txt"], params!["/tmp/", "dup.txt"],
|r| Ok((r.get(0)?,)), |r| Ok((r.get(0)?,)),
) )
.unwrap(); .unwrap();
@ -250,13 +245,11 @@ fn delete_subtree_clears_every_dependent_table() {
let p = tmp_path(); let p = tmp_path();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let add = |tx: &Transaction<'_>, path: &str| -> i64 { let add = |tx: &Transaction<'_>, path: &str| -> i64 {
let name = path.rsplit('/').next().unwrap(); let (parent, name) = crate::file_handling::split_db_path(path).expect("a file's path");
let parent = &path[..path.rfind('/').unwrap()];
let id = insert_file( let id = insert_file(
tx, tx,
&NewFile { &NewFile {
name, name,
path,
parent, parent,
size: 1, size: 1,
mtime: 1, mtime: 1,
@ -306,7 +299,7 @@ fn delete_subtree_clears_every_dependent_table() {
let survivors: Vec<String> = { let survivors: Vec<String> = {
let mut stmt = conn let mut stmt = conn
.prepare("SELECT path FROM files ORDER BY path") .prepare("SELECT parent || name FROM files ORDER BY parent, name")
.unwrap(); .unwrap();
let v = stmt let v = stmt
.query_map([], |r| r.get::<_, String>(0)) .query_map([], |r| r.get::<_, String>(0))
@ -327,13 +320,12 @@ fn seeded(conn: &mut Connection, paths: &[&str]) -> std::collections::HashMap<St
let tx = conn.transaction().unwrap(); let tx = conn.transaction().unwrap();
let mut ids = std::collections::HashMap::new(); let mut ids = std::collections::HashMap::new();
for path in paths { for path in paths {
let name = path.rsplit('/').next().unwrap(); // The indexer's own split, so the separator stays with the parent.
let parent = &path[..path.rfind('/').unwrap()]; let (parent, name) = crate::file_handling::split_db_path(path).expect("a file's path");
let id = insert_file( let id = insert_file(
&tx, &tx,
&NewFile { &NewFile {
name, name,
path,
parent, parent,
size: 1, size: 1,
mtime: 1, mtime: 1,
@ -387,7 +379,7 @@ fn delete_outside_ranges_keeps_exactly_the_configured_roots() {
let survivors: Vec<String> = { let survivors: Vec<String> = {
let mut stmt = conn let mut stmt = conn
.prepare("SELECT path FROM files ORDER BY path") .prepare("SELECT parent || name FROM files ORDER BY parent, name")
.unwrap(); .unwrap();
let v = stmt let v = stmt
.query_map([], |r| r.get::<_, String>(0)) .query_map([], |r| r.get::<_, String>(0))
@ -607,40 +599,47 @@ fn rows_in_range_page_walks_the_range_once() {
let range = crate::file_handling::ExtractCursor::for_root("/t"); let range = crate::file_handling::ExtractCursor::for_root("/t");
let mut seen = Vec::new(); let mut seen = Vec::new();
let mut after = range.lo.clone(); // `(lo, "")`: no stored name is empty, so this sorts just below the
// range's first row.
let mut after = (range.lo.clone(), String::new());
loop { loop {
let page = rows_in_range_page(&conn, &after, &range.hi, 2).unwrap(); let page = rows_in_range_page(&conn, &after.0, &after.1, &range.hi, 2).unwrap();
let Some(last) = page.last() else { break }; let Some(last) = page.last() else { break };
after = last.path.clone(); after = (last.parent.clone(), last.name.clone());
seen.extend(page.into_iter().map(|r| r.path)); seen.extend(page.into_iter().map(|r| r.path));
} }
assert_eq!( assert_eq!(
seen, seen,
vec!["/t/a.txt", "/t/deep/b.txt", "/t/deep/deeper/c.txt"], vec!["/t/a.txt", "/t/deep/b.txt", "/t/deep/deeper/c.txt"],
"in path order, once each, and the prefix siblings are outside" "in (parent, name) order, once each, and the prefix siblings are outside"
); );
drop(conn); drop(conn);
std::fs::remove_file(&p).ok(); std::fs::remove_file(&p).ok();
} }
/// `idx_files_parent` carries `name` and `mtime` so `dir_rows` never touches /// `dir_rows` must *seek* on `idx_files_parent`, never scan the table.
/// the table heap; trimming it back to `(parent)` would silently reintroduce ///
/// a row fetch per entry. /// It is deliberately not index-only any more: the index stopped carrying
/// `mtime` when `UNIQUE(parent, name)` took over as the row key, so the plan
/// is a seek plus a row fetch per entry. See the index's own comment in
/// `schema.rs` for why that trade was taken — and note the thing this test
/// guards is the seek, which is what keeps the cost per *directory* rather
/// than per *tree*.
#[test] #[test]
fn dir_rows_is_served_entirely_from_the_index() { fn dir_rows_seeks_the_parent_index() {
let p = tmp_path(); let p = tmp_path();
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let plan: String = conn let plan: String = conn
.query_row( .query_row(
"EXPLAIN QUERY PLAN SELECT name, mtime FROM files WHERE parent = ?1", "EXPLAIN QUERY PLAN SELECT name, mtime FROM files WHERE parent = ?1",
params!["/some/dir"], params!["/some/dir/"],
|r| r.get(3), |r| r.get(3),
) )
.unwrap(); .unwrap();
assert!( assert!(
plan.contains("COVERING INDEX idx_files_parent"), plan.contains("SEARCH") && plan.contains("idx_files_parent"),
"dir_rows must be index-only, got: {}", "dir_rows must seek the parent index, got: {}",
plan plan
); );
drop(conn); drop(conn);
@ -655,7 +654,7 @@ fn the_subtree_range_is_an_index_seek_not_a_scan() {
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let plan: String = conn let plan: String = conn
.query_row( .query_row(
"EXPLAIN QUERY PLAN DELETE FROM files WHERE path >= ?1 AND path < ?2", "EXPLAIN QUERY PLAN DELETE FROM files WHERE parent >= ?1 AND parent < ?2",
params!["/tree/", "/tree0"], params!["/tree/", "/tree0"],
|r| r.get(3), |r| r.get(3),
) )
@ -669,6 +668,43 @@ fn the_subtree_range_is_an_index_seek_not_a_scan() {
std::fs::remove_file(&p).ok(); std::fs::remove_file(&p).ok();
} }
/// The keyset page must be one index walk: the row-value comparison against
/// `(parent, name)` is what lets SQLite seek straight to the cursor and read
/// forward, with no temp b-tree to satisfy the `ORDER BY`.
#[test]
fn the_reconcile_page_seeks_and_does_not_sort() {
let p = tmp_path();
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let mut stmt = conn
.prepare(
"EXPLAIN QUERY PLAN
SELECT id, parent, name, size, mime, content_state FROM files
WHERE (parent, name) > (?1, ?2) AND parent < ?3
ORDER BY parent, name
LIMIT ?4",
)
.unwrap();
let plan: Vec<String> = stmt
.query_map(params!["/tree/", "", "/tree0", 2], |r| r.get(3))
.unwrap()
.map(|r| r.unwrap())
.collect();
let plan = plan.join(" | ");
assert!(
plan.contains("SEARCH") && plan.contains("idx_files_parent"),
"keyset page must seek the parent index, got: {}",
plan
);
assert!(
!plan.contains("TEMP B-TREE"),
"the index order must satisfy the ORDER BY outright, got: {}",
plan
);
drop(stmt);
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test] #[test]
fn last_full_index_round_trip() { fn last_full_index_round_trip() {
let p = tmp_path(); let p = tmp_path();
@ -693,8 +729,7 @@ fn checkpoint_and_close_truncates_wal() {
&tx, &tx,
&NewFile { &NewFile {
name: "w.txt", name: "w.txt",
path: "/tmp/w.txt", parent: "/tmp/",
parent: "/tmp",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: None, mime: None,
@ -725,14 +760,12 @@ fn checkpoint_and_close_truncates_wal() {
fn seed_rows(conn: &mut Connection, range: std::ops::Range<usize>) { fn seed_rows(conn: &mut Connection, range: std::ops::Range<usize>) {
let tx = conn.transaction().unwrap(); let tx = conn.transaction().unwrap();
for i in range { for i in range {
let path = format!("/tmp/bulk/{}.txt", i);
let name = format!("{}.txt", i); let name = format!("{}.txt", i);
let id = insert_file( let id = insert_file(
&tx, &tx,
&NewFile { &NewFile {
name: &name, name: &name,
path: &path, parent: "/tmp/bulk/",
parent: "/tmp/bulk",
size: 1, size: 1,
mtime: 1, mtime: 1,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -815,14 +848,12 @@ fn a_busy_reader_defeats_the_autocheckpoint_but_not_a_forced_one() {
fn seed_bare(conn: &mut Connection, range: std::ops::Range<usize>) { fn seed_bare(conn: &mut Connection, range: std::ops::Range<usize>) {
let tx = conn.transaction().unwrap(); let tx = conn.transaction().unwrap();
for i in range { for i in range {
let path = format!("/tmp/bare/{}.txt", i);
let name = format!("{}.txt", i); let name = format!("{}.txt", i);
insert_file( insert_file(
&tx, &tx,
&NewFile { &NewFile {
name: &name, name: &name,
path: &path, parent: "/tmp/bare/",
parent: "/tmp/bare",
size: i as u64, size: i as u64,
mtime: 1, mtime: 1,
mime: None, mime: None,
@ -979,8 +1010,7 @@ fn set_content_failed_writes_failed_table() {
&tx, &tx,
&NewFile { &NewFile {
name: "oops.bin", name: "oops.bin",
path: "/tmp/oops.bin", parent: "/tmp/",
parent: "/tmp",
size: 0, size: 0,
mtime: 1, mtime: 1,
mime: None, mime: None,
@ -1019,13 +1049,11 @@ fn set_content_failed_writes_failed_table() {
/// Insert one row under `path`, born pending when `needs_content`. /// Insert one row under `path`, born pending when `needs_content`.
fn insert_at(tx: &Transaction<'_>, path: &str, needs_content: bool) -> i64 { fn insert_at(tx: &Transaction<'_>, path: &str, needs_content: bool) -> i64 {
let name = path.rsplit('/').next().unwrap(); let (parent, name) = crate::file_handling::split_db_path(path).expect("a file's path");
let parent = &path[..path.rfind('/').unwrap()];
insert_file( insert_file(
tx, tx,
&NewFile { &NewFile {
name, name,
path,
parent, parent,
size: 1, size: 1,
mtime: 1, mtime: 1,
@ -1176,3 +1204,40 @@ fn prune_root_stats_drops_every_figure_of_a_dropped_root() {
drop(conn); drop(conn);
std::fs::remove_file(&p).ok(); std::fs::remove_file(&p).ok();
} }
/// The vanished-directory sweep walks `for_each_parent_in_range`, and its
/// range has to include the root's *own* directory or the files sitting
/// directly in a root are never reconciled.
///
/// It does now only because every stored parent ends in a separator: the root's
/// parent is spelled `/tree/`, which is the range's `lo` exactly. With the bare
/// `/tree` an older schema stored, it sorted *below* `lo` and the sweep skipped
/// it — harmless then only because a readable root is always in `seen_dirs`.
#[test]
fn the_parent_scan_reaches_the_roots_own_directory() {
let p = tmp_path();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
seeded(
&mut conn,
&[
"/tree/top.txt", // directly in the root
"/tree/deep/b.txt", // a subdirectory
"/tree/deep/deeper/c.txt", // deeper still
"/tree2/outside.txt", // prefix sibling: outside
"/treeX/outside.txt", // and the LIKE-metacharacter neighbour
],
);
let range = crate::file_handling::ExtractCursor::for_root("/tree");
let mut seen = Vec::new();
for_each_parent_in_range(&conn, &range.lo, &range.hi, |parent| seen.push(parent)).unwrap();
seen.sort();
assert_eq!(
seen,
vec!["/tree/", "/tree/deep/", "/tree/deep/deeper/"],
"the root's own directory is in range, and the siblings are not"
);
drop(conn);
std::fs::remove_file(&p).ok();
}

46
crates/quicksearch-core/src/db/schema.rs
View file

@ -130,7 +130,11 @@ pub const PRAGMAS_READONLY: &str = "
/// ///
/// Two of these can exist per indexing root, so the cache size is multiplied /// Two of these can exist per indexing root, so the cache size is multiplied
/// by the root count. 1 MiB is sized for the walk's queries, which each read /// by the root count. 1 MiB is sized for the walk's queries, which each read
/// one range of `idx_files_parent` once and never revisit it. The feeder's /// one range of `idx_files_parent` once and never revisit it — plus, since
/// that index stopped carrying `mtime`, one table-row fetch per entry in the
/// range. Those land on a handful of pages while a directory's rows stay
/// rowid-adjacent; **this is the number to raise** if a tree churned across
/// many incremental runs ever scatters them far enough to matter. The feeder's
/// paging is the same shape, but its one-off `count_extract_scope` at pass /// paging is the same shape, but its one-off `count_extract_scope` at pass
/// start is not: that scans the root's whole path range fetching a row per /// start is not: that scans the root's whole path range fetching a row per
/// entry, so on a large root it is a cold read all the way through. It is /// entry, so on a large root it is a cold read all the way through. It is
@ -156,10 +160,20 @@ CREATE TABLE schema_info (
value TEXT NOT NULL value TEXT NOT NULL
); );
-- **There is no `path` column.** A file's path is `parent || name`, and
-- storing it a third time cost ~43% of the per-row footprint: over a
-- 400k-file corpus (avg path 77 bytes, parent 59, name 17) ~394 bytes/row
-- against the ~226 below, or ~170 MB per million files. It also widened the
-- row by a third, and three of the search cascade's passes scan every row —
-- on an encrypted index each extra page is an AES-CBC decrypt and an
-- HMAC-SHA512 verify. Reassembling a path is a `push_str` in the one place
-- that needs one (`file_handling::split_db_path` is the inverse).
--
-- `parent` always ends in the platform separator; see `dir_to_db_parent` for
-- why the whole design turns on that.
CREATE TABLE files ( CREATE TABLE files (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
name TEXT NOT NULL, name TEXT NOT NULL,
path TEXT NOT NULL UNIQUE,
parent TEXT NOT NULL, parent TEXT NOT NULL,
size INTEGER NOT NULL, size INTEGER NOT NULL,
mtime INTEGER NOT NULL, mtime INTEGER NOT NULL,
@ -169,15 +183,25 @@ CREATE TABLE files (
hash BLOB hash BLOB
); );
-- Covering, not just `(parent)`. The walk's row prefetcher issues -- The identity of a row, and the only index `parent` needs.
-- `SELECT name, mtime FROM files WHERE parent = ?` once per directory — the --
-- hottest read in a full run — and with the bare index that is an index probe -- It replaces both of what came before — a `UNIQUE(path)` and a covering
-- plus a table-row fetch per entry. Those fetches are cold by design: the walk -- `(parent, name, mtime)` — and dropping the second is the deliberate half.
-- reader deliberately runs on a 1 MiB page cache (see `PRAGMAS_WALK_READER`). -- The walk's row prefetcher issues `SELECT name, mtime FROM files WHERE
-- Carrying `name` and `mtime` in the index makes it an index-only scan. -- parent = ?` once per directory, the hottest read in a full run, and without
-- `parent` stays leading, so `SELECT DISTINCT parent` range scans and -- `mtime` in the index that is a table-row fetch per entry. It is affordable
-- `paths_in_dir` are unaffected. -- because the prefetcher is one thread ahead of four walk workers that each
CREATE INDEX idx_files_parent ON files(parent, name, mtime); -- spend a `stat` *and* a SHA-256 of the path per file (`crate::walk`), so it
-- has budget to spend; because a directory's rows are written in one batch and
-- so are rowid-adjacent; and because the row is now narrow enough that a
-- 1 MiB cache holds ~7,200 of them. `mtime` cannot simply be appended here —
-- `UNIQUE(parent, name, mtime)` would let the same file be inserted twice
-- under two mtimes.
--
-- If the prefetcher is ever measured falling behind, raise
-- `PRAGMAS_WALK_READER` — paid for out of the space this index no longer
-- occupies — rather than restoring the covering one.
CREATE UNIQUE INDEX idx_files_parent ON files(parent, name);
CREATE INDEX idx_files_mtime ON files(mtime); CREATE INDEX idx_files_mtime ON files(mtime);
CREATE INDEX idx_files_type ON files(type); CREATE INDEX idx_files_type ON files(type);
CREATE INDEX idx_files_mime ON files(mime); CREATE INDEX idx_files_mime ON files(mime);

237
crates/quicksearch-core/src/extract/office.rs
View file

@ -86,6 +86,34 @@ const ODF_SHEET: TextSpec = TextSpec {
separator: Some(' '), separator: Some(' '),
}; };
/// The text an `&entity;` or `&#1234;` reference stands for.
///
/// quick-xml 0.41 reports a reference as its own event instead of resolving
/// it inside the surrounding `Text`, so a reader that ignores this event
/// silently drops every `&amp;`, `&lt;` and `&#8217;` from the document —
/// no error, just missing characters in the index. Only the five predefined
/// entities and numeric references are resolvable without a DTD; anything
/// else is a document-defined entity we cannot expand, and is skipped.
fn entity_text(raw: &str) -> Option<String> {
if let Some(digits) = raw.strip_prefix('#') {
let code = match digits.strip_prefix(['x', 'X']) {
Some(hex) => u32::from_str_radix(hex, 16).ok()?,
None => digits.parse::<u32>().ok()?,
};
let c = char::from_u32(code)?;
// `char::from_u32` accepts far more than XML's character production
// does: every C0 control but tab, newline and carriage return is
// forbidden, and `&#0;` in particular would put a literal NUL into the
// indexed text and from there into an FTS5 column. `None` here reaches
// the callers as the same "unknown entity" error an unexpandable name
// gets — a `failed_files` row naming the file, which is the visible
// outcome this extractor prefers to a quietly mangled document.
let legal = !c.is_control() || matches!(c, '\t' | '\n' | '\r');
return legal.then(|| String::from(c));
}
quick_xml::escape::resolve_predefined_entity(raw).map(String::from)
}
/// Append the text `spec` selects out of `xml` to `out`. /// Append the text `spec` selects out of `xml` to `out`.
/// ///
/// `in_text` is a flag rather than a depth count, which means a closing /// `in_text` is a flag rather than a depth count, which means a closing
@ -93,7 +121,13 @@ const ODF_SHEET: TextSpec = TextSpec {
/// open. /// open.
fn collect_xml_text(xml: &str, spec: &TextSpec, out: &mut String) -> Result<(), Box<dyn Error>> { fn collect_xml_text(xml: &str, spec: &TextSpec, out: &mut String) -> Result<(), Box<dyn Error>> {
let mut reader = Reader::from_str(xml); let mut reader = Reader::from_str(xml);
reader.trim_text(true); // Deliberately no `trim_text`: it trims each *event*, and since 0.41 an
// entity reference splits the character data around it into separate
// events — so `Jack &amp; Jill` would come back as `Jack&Jill`, with the
// spaces trimmed off the ends of the two fragments. Nothing needs it
// either: whitespace between elements arrives while the `in_text`/`in_cell`
// flag is false and is ignored there, and whitespace *inside* a
// text-bearing element is content.
let mut buf = Vec::new(); let mut buf = Vec::new();
let mut in_text = false; let mut in_text = false;
@ -105,11 +139,23 @@ fn collect_xml_text(xml: &str, spec: &TextSpec, out: &mut String) -> Result<(),
} }
} }
Ok(Event::Text(e)) if in_text => { Ok(Event::Text(e)) if in_text => {
out.push_str(&e.unescape()?); out.push_str(&e.decode()?);
if let Some(sep) = spec.separator { if let Some(sep) = spec.separator {
out.push(sep); out.push(sep);
} }
} }
// An entity reference is its own event in 0.41; without this arm
// every `&amp;` in a document would vanish from the index.
Ok(Event::GeneralRef(e)) if in_text => {
let raw = e.decode()?;
// An entity nothing can expand is an error, as it was when
// `unescape` resolved these inline: dropping it would take
// characters out of the indexed text with nothing to show for
// it, and this reader has no DTD to define one with.
let text = entity_text(&raw)
.ok_or_else(|| format!("Error parsing XML: unknown entity &{};", raw))?;
out.push_str(&text);
}
Ok(Event::End(ref e)) => { Ok(Event::End(ref e)) => {
let name = e.name(); let name = e.name();
if spec.text.contains(&name.as_ref()) { if spec.text.contains(&name.as_ref()) {
@ -235,18 +281,46 @@ fn shared_strings<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Vec<String> {
return Vec::new(); return Vec::new();
}; };
let mut reader = Reader::from_str(&xml); let mut reader = Reader::from_str(&xml);
reader.trim_text(true); // Deliberately no `trim_text`: it trims each *event*, and since 0.41 an
// entity reference splits the character data around it into separate
// events — so `Jack &amp; Jill` would come back as `Jack&Jill`, with the
// spaces trimmed off the ends of the two fragments. Nothing needs it
// either: whitespace between elements arrives while the `in_text`/`in_cell`
// flag is false and is ignored there, and whitespace *inside* a
// text-bearing element is content.
let mut buf = Vec::new(); let mut buf = Vec::new();
let mut strings = Vec::new(); let mut strings = Vec::new();
let mut in_text = false; let mut in_text = false;
// One `<t>` is one shared string, but it is not one event: an entity
// reference inside it arrives separately and splits the character data
// around it. Accumulated here and pushed on the closing tag, or a cell
// containing `&amp;` would become three table entries and every later
// index would point at the wrong one.
let mut current = String::new();
loop { loop {
match reader.read_event_into(&mut buf) { match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) if e.name().as_ref() == b"t" => in_text = true, Ok(Event::Start(ref e)) if e.name().as_ref() == b"t" => {
Ok(Event::Text(e)) if in_text => match e.unescape() { in_text = true;
Ok(s) => strings.push(s.into_owned()), current.clear();
}
Ok(Event::Text(e)) if in_text => match e.decode() {
Ok(s) => current.push_str(&s),
Err(_) => return strings, Err(_) => return strings,
}, },
Ok(Event::End(ref e)) if e.name().as_ref() == b"t" => in_text = false, Ok(Event::GeneralRef(e)) if in_text => {
// Unlike the other two readers this one cannot fail — a
// missing table is not an error here — so an entity nothing
// can expand is simply left out.
if let Ok(raw) = e.decode() {
if let Some(text) = entity_text(&raw) {
current.push_str(&text);
}
}
}
Ok(Event::End(ref e)) if e.name().as_ref() == b"t" => {
in_text = false;
strings.push(std::mem::take(&mut current));
}
Ok(Event::Eof) | Err(_) => break, Ok(Event::Eof) | Err(_) => break,
_ => {} _ => {}
} }
@ -259,7 +333,13 @@ fn shared_strings<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Vec<String> {
/// rather than text of its own; every other type holds its value inline. /// rather than text of its own; every other type holds its value inline.
fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(), Box<dyn Error>> { fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(), Box<dyn Error>> {
let mut reader = Reader::from_str(xml); let mut reader = Reader::from_str(xml);
reader.trim_text(true); // Deliberately no `trim_text`: it trims each *event*, and since 0.41 an
// entity reference splits the character data around it into separate
// events — so `Jack &amp; Jill` would come back as `Jack&Jill`, with the
// spaces trimmed off the ends of the two fragments. Nothing needs it
// either: whitespace between elements arrives while the `in_text`/`in_cell`
// flag is false and is ignored there, and whitespace *inside* a
// text-bearing element is content.
let mut buf = Vec::new(); let mut buf = Vec::new();
let mut in_cell = false; let mut in_cell = false;
let mut cell_type = String::new(); let mut cell_type = String::new();
@ -287,19 +367,49 @@ fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(),
} }
} }
Ok(Event::Text(e)) if in_cell => { Ok(Event::Text(e)) if in_cell => {
let text = e.unescape()?; let text = e.decode()?;
if cell_type == "s" { if cell_type == "s" {
// A shared-string reference. An index past the end of the // A shared-string reference. An index past the end of the
// table is a corrupt workbook, not something to guess at. // table is a corrupt workbook, not something to guess at.
if let Some(s) = text.parse::<usize>().ok().and_then(|i| strings.get(i)) { //
// `trim` because this reader no longer sets `trim_text`
// (see the comment above): a generator that indents its
// XML hands `<v>` over as "\n 0\n", and an untrimmed
// parse would fail and drop the string with nothing to
// show for it. Whitespace around an integer index is not
// content, unlike whitespace inside a `<t>`.
if let Some(s) = text
.trim()
.parse::<usize>()
.ok()
.and_then(|i| strings.get(i))
{
out.push_str(s); out.push_str(s);
out.push(' '); out.push(' ');
} }
} else { } else if !text.trim().is_empty() {
// Whitespace-only fragments are the indentation *between*
// a cell's child elements, which reaches this arm now that
// the reader no longer sets `trim_text`. Skipped rather
// than pushed: `in_cell` is a flag, so it cannot tell an
// indent from a value, and a cell whose entire content is
// whitespace contributes nothing to a search index either
// way. The value itself is pushed whole — no `trim` — so a
// deliberate `xml:space="preserve"` inline string keeps
// its shape.
out.push_str(&text); out.push_str(&text);
out.push(' '); out.push(' ');
} }
} }
// See `entity_text`. Only inline values can carry one: a `t="s"`
// cell's text is an integer index, and an entity inside it would
// be a corrupt workbook rather than a character to recover.
Ok(Event::GeneralRef(e)) if in_cell && cell_type != "s" => {
let raw = e.decode()?;
let text = entity_text(&raw)
.ok_or_else(|| format!("Error parsing XML: unknown entity &{};", raw))?;
out.push_str(&text);
}
Ok(Event::End(ref e)) => { Ok(Event::End(ref e)) => {
let name = e.name(); let name = e.name();
if name.as_ref() == b"c" { if name.as_ref() == b"c" {
@ -376,6 +486,111 @@ mod tests {
use super::*; use super::*;
use std::io::Write; use std::io::Write;
/// Entity references must survive extraction.
///
/// quick-xml 0.41 reports `&amp;` as its own `GeneralRef` event instead of
/// resolving it into the surrounding text, so a reader that only handles
/// `Event::Text` loses the character with no error to show for it. This is
/// the test that makes that visible: it fails by producing "Blake Co"
/// rather than by failing to compile.
#[test]
fn entity_references_survive_extraction() {
let body = "<w:document><w:body><w:p><w:r>\
<w:t>Blake &amp; Co &lt;tags&gt; &#8217;24 &#x2019;25</w:t>\
</w:r></w:p></w:body></w:document>";
let path = container("docx-entities", "docx", &[("word/document.xml", body)]);
let out = OfficeExtractor.extract(&path).expect("extract");
assert!(
out.text.contains("Blake & Co"),
"predefined entity lost: {:?}",
out.text
);
assert!(
out.text.contains("<tags>"),
"angle-bracket entities lost: {:?}",
out.text
);
assert!(
out.text.contains('\u{2019}'),
"numeric entities lost: {:?}",
out.text
);
assert!(
!out.text.contains("&amp;") && !out.text.contains("&#"),
"entities left unresolved: {:?}",
out.text
);
}
/// The same, through the shared-string table an `.xlsx` cell indexes into
/// — a separate reader, and so a separate chance to drop the character.
#[test]
fn entity_references_survive_shared_strings() {
let shared = "<sst><si><t>Jack &amp; Jill</t></si></sst>";
let sheet = "<worksheet><sheetData><row>\
<c t=\"s\"><v>0</v></c></row></sheetData></worksheet>";
let path = container(
"xlsx-entities",
"xlsx",
&[
("xl/sharedStrings.xml", shared),
("xl/worksheets/sheet1.xml", sheet),
],
);
let out = OfficeExtractor.extract(&path).expect("extract");
assert!(
out.text.contains("Jack & Jill"),
"entity lost through the shared-string table: {:?}",
out.text
);
}
/// A shared-string reference must survive an indented `<v>`.
///
/// The reader deliberately does not set `trim_text` (an entity reference
/// splits the character data around it, and trimming each fragment would
/// eat the spaces at the split). A `t="s"` cell's `<v>` is an integer
/// index, though, so a generator that pretty-prints its sheet XML hands
/// this reader `"\n 0\n "` — and an untrimmed `parse::<usize>()`
/// fails, dropping the cell's text with no error and no `failed_files`
/// row. Whitespace-only fragments between a cell's children must not
/// reach the output either.
#[test]
fn an_indented_shared_string_reference_still_resolves() {
let shared = "<sst><si><t>Marmalade</t></si></sst>";
let sheet = "<worksheet>\n <sheetData>\n <row>\n \
<c t=\"s\">\n <v>\n 0\n </v>\n </c>\n \
<c t=\"n\">\n <v>17</v>\n </c>\n \
</row>\n </sheetData>\n</worksheet>";
let path = container(
"xlsx-indented",
"xlsx",
&[
("xl/sharedStrings.xml", shared),
("xl/worksheets/sheet1.xml", sheet),
],
);
let out = OfficeExtractor.extract(&path).expect("extract");
assert!(
out.text.contains("Marmalade"),
"the shared string was dropped by an indented index: {:?}",
out.text
);
assert!(
out.text.contains("17"),
"the inline value was dropped: {:?}",
out.text
);
// The indentation itself is not content: every run of whitespace in
// the output should be a separator this reader put there, never a
// line of the source XML's own layout.
assert!(
!out.text.contains("\n "),
"sheet indentation reached the indexed text: {:?}",
out.text
);
}
#[test] #[test]
fn supports_docx_and_friends() { fn supports_docx_and_friends() {
let e = OfficeExtractor; let e = OfficeExtractor;

37
crates/quicksearch-core/src/file_handling/batch.rs
View file

@ -104,7 +104,8 @@ pub fn process_batch_updates(
let updated = repo::update_file_basic(&tx, &rec.as_new_file()).map_err(|e| { let updated = repo::update_file_basic(&tx, &rec.as_new_file()).map_err(|e| {
format!( format!(
"Failed to update file record + clear stale content for {}: {}", "Failed to update file record + clear stale content for {}: {}",
rec.path, e rec.path(),
e
) )
})?; })?;
@ -115,7 +116,7 @@ pub fn process_batch_updates(
None => { None => {
crate::log_warn!( crate::log_warn!(
"no indexed row matched {} during update; inserting instead", "no indexed row matched {} during update; inserting instead",
rec.path rec.path()
); );
repo::insert_file(&tx, &rec.as_new_file()) repo::insert_file(&tx, &rec.as_new_file())
.map_err(|e| format!("Failed to insert file record: {}", e))? .map_err(|e| format!("Failed to insert file record: {}", e))?
@ -124,7 +125,7 @@ pub fn process_batch_updates(
}; };
if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) { if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
let zstd = body_or_skip!(bodies, i, rec.path); let zstd = body_or_skip!(bodies, i, rec.path());
repo::set_content_done(&tx, id, text, zstd)?; repo::set_content_done(&tx, id, text, zstd)?;
} }
} }
@ -149,7 +150,11 @@ pub fn process_batch_inserts(
return Ok(()); return Ok(());
} }
for batch in files_to_insert.chunks(config.processing.batch_size) { // `.max(1)`, as at every other use of this field: `chunks(0)` panics,
// and a panic here is on the indexing thread, before the arm that would
// publish `IndexingStatus::Error` — so a hand-edited `batch_size = 0`
// wedges indexing for the session while the UI still reads "Running".
for batch in files_to_insert.chunks(config.processing.batch_size.max(1)) {
if stop_flag.load(Ordering::Relaxed) { if stop_flag.load(Ordering::Relaxed) {
return Ok(()); return Ok(());
} }
@ -170,7 +175,7 @@ pub fn process_batch_inserts(
let id = repo::insert_file(&tx, &rec.as_new_file()) let id = repo::insert_file(&tx, &rec.as_new_file())
.map_err(|e| format!("Failed to insert file record: {}", e))?; .map_err(|e| format!("Failed to insert file record: {}", e))?;
if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) { if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
let zstd = body_or_skip!(bodies, i, rec.path); let zstd = body_or_skip!(bodies, i, rec.path());
repo::set_content_done(&tx, id, text, zstd)?; repo::set_content_done(&tx, id, text, zstd)?;
} }
} }
@ -236,13 +241,19 @@ pub fn cleanup_stale_index_entries(
/// Keyset cursor bounding everything stored beneath one directory. /// Keyset cursor bounding everything stored beneath one directory.
/// ///
/// `lo`/`hi` are the half-open path range `[dir + SEP, dir + (SEP + 1))`, so /// `lo`/`hi` are the half-open range `[dir + SEP, dir + (SEP + 1))` over
/// the pair is a pure index range on `UNIQUE(files.path)`. /// `files.parent`, so the pair is a pure index range on `idx_files_parent`.
/// ///
/// The separator must be the platform's own: `files.path` stores native /// It covers `dir`'s own files as well as its subdirectories' because every
/// separators, and the successor of `/` (`0x2F`) is `'0'` while the successor /// stored parent ends in a separator (see `dir_to_db_parent`): the files
/// of `\` (`0x5C`) is `']'` — the Unix pair on Windows yields /// directly in `dir` have parent `dir + SEP`, which is `lo` exactly. Without
/// `hi = "C:\Users\me0"`, which every stored path sorts *above*, silently /// that invariant `lo` would have to be the bare `dir`, and the range would
/// swallow siblings — `/a-b` sorts inside `["/a", "/a0")`.
///
/// The separator must be the platform's own: parents store native separators,
/// and the successor of `/` (`0x2F`) is `'0'` while the successor of `\`
/// (`0x5C`) is `']'` — the Unix pair on Windows yields
/// `hi = "C:\Users\me0"`, which every stored parent sorts *above*, silently
/// disabling content extraction and the vanished-directory sweep. /// disabling content extraction and the vanished-directory sweep.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct ExtractCursor { pub struct ExtractCursor {
@ -299,7 +310,7 @@ pub fn mark_oversize_pending_na(
) -> Result<(), String> { ) -> Result<(), String> {
conn.execute( conn.execute(
"UPDATE files SET content_state = 3 \ "UPDATE files SET content_state = 3 \
WHERE content_state = 0 AND size > ?1 AND path >= ?2 AND path < ?3", WHERE content_state = 0 AND size > ?1 AND parent >= ?2 AND parent < ?3",
rusqlite::params![max_text_file_size(config), cursor.lo, cursor.hi], rusqlite::params![max_text_file_size(config), cursor.lo, cursor.hi],
) )
.map_err(|e| format!("mark oversize files NA: {}", e))?; .map_err(|e| format!("mark oversize files NA: {}", e))?;
@ -322,7 +333,7 @@ pub fn count_extract_scope(
.query_row( .query_row(
"SELECT COALESCE(SUM(content_state = 0 AND size <= ?1), 0), \ "SELECT COALESCE(SUM(content_state = 0 AND size <= ?1), 0), \
COALESCE(SUM(content_state = 1), 0) \ COALESCE(SUM(content_state = 1), 0) \
FROM files WHERE path >= ?2 AND path < ?3", FROM files WHERE parent >= ?2 AND parent < ?3",
rusqlite::params![max_text_file_size(config), cursor.lo, cursor.hi], rusqlite::params![max_text_file_size(config), cursor.lo, cursor.hi],
|row| Ok((row.get(0)?, row.get(1)?)), |row| Ok((row.get(0)?, row.get(1)?)),
) )

9
crates/quicksearch-core/src/file_handling/mod.rs
View file

@ -18,8 +18,13 @@ pub use batch::{
Stored, Stored,
}; };
pub use counting::count_tree_entries_fast; pub use counting::count_tree_entries_fast;
pub use paths::{db_key_for_missing_path, filtered_dirs, filtered_walk, UnreadableDirs}; pub use paths::{
pub(crate) use paths::{normalize_root_string, path_to_db_string, warn_if_unrepresentable}; db_key_for_missing_path, filtered_dirs, filtered_walk, split_db_path, UnreadableDirs,
};
pub(crate) use paths::{
dir_to_db_parent, index_file_set, normalize_root_string, path_to_db_string,
warn_if_unrepresentable, INDEX_SIDECAR_SUFFIXES,
};
pub use records::{ pub use records::{
classify_by_mtime, classify_for_indexing, content_extractable, decide_content, classify_by_mtime, classify_for_indexing, content_extractable, decide_content,
extract_and_store, fts_finalize_after_text_indexing, hash_failure_counts, outcome_body, extract_and_store, fts_finalize_after_text_indexing, hash_failure_counts, outcome_body,

121
crates/quicksearch-core/src/file_handling/paths.rs
View file

@ -1,6 +1,7 @@
//! Path ↔ `files.path` string normalization and the filtered walkdir //! Path ↔ `files.parent`/`files.name` string normalization and the filtered
//! wrappers the reconcile passes use. //! walkdir wrappers the reconcile passes use.
use std::collections::HashSet;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::Mutex; use std::sync::Mutex;
@ -8,7 +9,8 @@ use walkdir::{DirEntry, WalkDir};
use crate::config::IgnoreSet; use crate::config::IgnoreSet;
/// Render a path as the string stored in `files.path`. /// Render a path in the spelling the index stores it in, which
/// [`split_db_path`] then splits into the `(parent, name)` key.
/// ///
/// `Path::canonicalize` on Windows hands back extended-length paths; the /// `Path::canonicalize` on Windows hands back extended-length paths; the
/// index stores plain ones. A UNC share canonicalizes to /// index stores plain ones. A UNC share canonicalizes to
@ -34,11 +36,55 @@ pub(crate) fn path_to_db_string(path: &Path) -> String {
} }
} }
/// What SQLite and we hang off the index's own filename.
///
/// `-wal`, `-shm` and `-journal` are SQLite's, spelled exactly as
/// `db::open` and the unlock screen already delete them; `.lock` is
/// [`crate::platform::IndexLock`]'s.
pub(crate) const INDEX_SIDECAR_SUFFIXES: [&str; 4] = ["-wal", "-shm", "-journal", ".lock"];
/// Every file belonging to the index at `db_path`, spelled the way the walk
/// spells the files it visits.
///
/// **Nothing may ever open one of these.** On POSIX, closing *any* descriptor
/// on an inode cancels every advisory lock the whole process holds on it, so a
/// walk worker that opens `index.sqlite-shm` to hash it destroys the DMS lock
/// SQLite took on that file — after which the next connection to attach, from
/// any process, truncates the wal-index to 3 bytes under our live mapping and
/// the next commit dies with SIGBUS. The same close cancels the main
/// database's own locks, which is SQLite's documented corruption hazard
/// (howtocorrupt.html §2.2). See [`crate::walk`], which prunes these before an
/// entry can become a candidate.
///
/// The *directory* is canonicalized rather than the files: `-wal`, `-shm` and
/// the lock come and go across a run, and `canonicalize` fails on a path that
/// is not there at the instant it is called. The directory outlives all of
/// them, so resolving it once and joining the names gives a stable answer that
/// does not depend on which sidecars happen to exist.
pub(crate) fn index_file_set(db_path: &Path) -> HashSet<PathBuf> {
let mut set = HashSet::new();
// An empty filename means `database_path` names a directory, not a file.
// Joining "" onto the parent would yield the directory itself and prune
// the entire tree below it.
let Some(name) = db_path.file_name().and_then(|s| s.to_str()) else {
return set;
};
let dir = db_path.parent().unwrap_or_else(|| Path::new("."));
let dir = PathBuf::from(path_to_db_string(
&dir.canonicalize().unwrap_or_else(|_| dir.to_path_buf()),
));
set.insert(dir.join(name));
for suffix in INDEX_SIDECAR_SUFFIXES {
set.insert(dir.join(format!("{}{}", name, suffix)));
}
set
}
/// Canonicalize a root string for storage/comparison. Multi-root strings /// Canonicalize a root string for storage/comparison. Multi-root strings
/// (newline-joined) fail canonicalize and pass through verbatim, which still /// (newline-joined) fail canonicalize and pass through verbatim, which still
/// compares consistently. /// compares consistently.
/// ///
/// This is the spelling `files.path` rows are prefixed with, so it is also the /// This is the spelling stored parents are prefixed with, so it is also the
/// form roots must be compared in: `~/docs` and `/home/me/docs` name one root /// form roots must be compared in: `~/docs` and `/home/me/docs` name one root
/// and must not read as a change. /// and must not read as a change.
pub(crate) fn normalize_root_string(indexing_path: &str) -> String { pub(crate) fn normalize_root_string(indexing_path: &str) -> String {
@ -49,11 +95,20 @@ pub(crate) fn normalize_root_string(indexing_path: &str) -> String {
} }
/// Warn and report `true` for a path that cannot round-trip through /// Warn and report `true` for a path that cannot round-trip through
/// `files.path`. /// the index.
/// ///
/// Everything downstream reopens the file by that TEXT column, and /// Everything downstream reopens the file by that TEXT column, and
/// [`path_to_db_string`] is lossy: a non-UTF-8 name would be stored as a /// [`path_to_db_string`] is lossy: a non-UTF-8 name would be stored as a path
/// path naming a file that does not exist. Such a file is skipped whole. /// naming a *different* file — U+FFFD is an ordinary filename character, so
/// the lossy spelling is somebody's real name. Such a file is skipped whole.
///
/// This is the check for callers that arrive with a single path and no
/// listing to prune: [`prepare_file_record_from_path`] on the incremental
/// route. The full walk screens far earlier, on the directory entry itself
/// (`crate::walk::read_directory`), because by the time a path exists a bad
/// *directory* component has already been joined into every path beneath it.
///
/// [`prepare_file_record_from_path`]: super::prepare_file_record_from_path
pub(crate) fn warn_if_unrepresentable(path: &Path) -> bool { pub(crate) fn warn_if_unrepresentable(path: &Path) -> bool {
if path.to_str().is_some() { if path.to_str().is_some() {
return false; return false;
@ -73,13 +128,21 @@ fn starts_with_drive_letter(s: &str) -> bool {
matches!((it.next(), it.next()), (Some(c), Some(':')) if c.is_ascii_alphabetic()) matches!((it.next(), it.next()), (Some(c), Some(':')) if c.is_ascii_alphabetic())
} }
/// The `files.path` key for a path that may no longer exist. /// The stored spelling of a path that may no longer exist.
/// ///
/// The insert side canonicalizes before storing, and plain `canonicalize` /// The insert side canonicalizes before storing, and plain `canonicalize`
/// fails on a path already gone — so this canonicalizes the deepest ancestor /// fails on a path already gone — so this canonicalizes the deepest ancestor
/// that still resolves and re-joins the missing tail. On Linux a root reached /// that still resolves and re-joins the missing tail. On Linux a root reached
/// through a symlinked parent (`/home` → `/mnt/home`) makes every removal a /// through a symlinked parent (`/home` → `/mnt/home`) makes every removal a
/// no-op without this. /// no-op without this.
///
/// **The caller must have screened `path` for representability.** This ends in
/// [`path_to_db_string`], which is lossy, and the answer is used to *delete* —
/// a row by path and then a whole subtree range beneath it. For a path that is
/// not valid UTF-8 the key returned names some other, real file, and deleting
/// by it takes that file's row and everything under it. There is nothing to
/// delete for such a path in any case: it could never have been indexed. See
/// [`crate::incremental::remove_paths`], which skips them.
pub fn db_key_for_missing_path(path: &Path) -> String { pub fn db_key_for_missing_path(path: &Path) -> String {
let mut tail: Vec<std::ffi::OsString> = Vec::new(); let mut tail: Vec<std::ffi::OsString> = Vec::new();
let mut cursor = path; let mut cursor = path;
@ -104,12 +167,42 @@ pub fn db_key_for_missing_path(path: &Path) -> String {
} }
} }
/// Parent directory of a path as a UTF-8 string, empty if root. /// Render a directory as the string stored in `files.parent`.
pub(super) fn parent_str(path: &str) -> String { ///
Path::new(path) /// **`files.parent` always ends in the platform separator**, and that is the
.parent() /// invariant the whole schema rests on:
.map(|p| p.to_string_lossy().into_owned()) ///
.unwrap_or_default() /// * A file's path is `parent` concatenated with `name` — no separator logic
/// at the join, and so no special case for `/` or `C:\`, whose children
/// would otherwise be spelled `//x` and `C:\\x`.
/// * A root's subtree is the single range `[root + SEP, root + succ(SEP))`,
/// because the root's *own* parent is `root + SEP` rather than `root`. With
/// a bare `root` there is no such range: strings between `root` and
/// `root + SEP` are siblings (`/a-b` sorts inside `["/a", "/a0")`), so it
/// would take two predicates and a sibling would sneak in through either.
///
/// See [`crate::db::schema::SCHEMA_CURRENT`] for what that buys.
pub(crate) fn dir_to_db_parent(dir: &Path) -> String {
let mut s = path_to_db_string(dir);
// The platform's own separator only. On Unix `\` is an ordinary filename
// character, so a directory genuinely named `weird\` must still get its
// `/` — testing both separators would leave that row unjoinable.
if !s.ends_with(std::path::MAIN_SEPARATOR) {
s.push(std::path::MAIN_SEPARATOR);
}
s
}
/// Split a stored path into the `(parent, name)` pair the index keys on.
///
/// The separator stays with the parent, so `parent` + `name` is the original
/// string back — see [`dir_to_db_parent`]. `None` for anything that cannot be
/// a file's path: a bare relative name with no separator at all, or a string
/// that ends in one and so names a directory rather than a file.
pub fn split_db_path(path: &str) -> Option<(&str, &str)> {
let cut = path.rfind(std::path::MAIN_SEPARATOR)?;
let (parent, name) = path.split_at(cut + 1);
(!name.is_empty()).then_some((parent, name))
} }
/// Paths a walk could not read, collected as it runs. /// Paths a walk could not read, collected as it runs.

33
crates/quicksearch-core/src/file_handling/records.rs
View file

@ -10,7 +10,6 @@ use std::time::UNIX_EPOCH;
use rusqlite::Connection; use rusqlite::Connection;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use super::paths::parent_str;
use super::*; use super::*;
use crate::config::Config; use crate::config::Config;
use crate::db::repo::{self, NewFile}; use crate::db::repo::{self, NewFile};
@ -109,7 +108,9 @@ pub fn fts_finalize_after_text_indexing(conn: &Connection) {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct OwnedNewFile { pub struct OwnedNewFile {
pub name: String, pub name: String,
pub path: String, /// The containing directory, ending in the platform separator. With
/// [`OwnedNewFile::name`] it is both the row's key and, concatenated, its
/// path — see [`super::paths::dir_to_db_parent`].
pub parent: String, pub parent: String,
pub size: u64, pub size: u64,
pub mtime: u64, pub mtime: u64,
@ -130,10 +131,15 @@ pub struct OwnedNewFile {
} }
impl OwnedNewFile { impl OwnedNewFile {
/// The file's path, rebuilt. Callers that only want it for a message
/// should say so — nothing stores this.
pub fn path(&self) -> String {
format!("{}{}", self.parent, self.name)
}
pub fn as_new_file(&self) -> NewFile<'_> { pub fn as_new_file(&self) -> NewFile<'_> {
NewFile { NewFile {
name: &self.name, name: &self.name,
path: &self.path,
parent: &self.parent, parent: &self.parent,
size: self.size, size: self.size,
mtime: self.mtime, mtime: self.mtime,
@ -165,12 +171,15 @@ pub fn hash_failure_counts() -> (u64, u64) {
/// already holds. The single implementation behind both full-run batches /// already holds. The single implementation behind both full-run batches
/// and incremental watcher updates. /// and incremental watcher updates.
/// ///
/// `path` must already be canonical and in `files.path` spelling (see /// `path` must already be canonical and in stored spelling (see
/// [`path_to_db_string`]), and must still name the file once parsed back into /// [`path_to_db_string`]), and must still name the file once parsed back into
/// a [`Path`] — this opens it by that string. A path that only survived /// a [`Path`] — this opens it by that string. A path that only survived
/// `to_string_lossy` does not qualify; callers holding the original [`Path`] /// `to_string_lossy` does not qualify: the lossy spelling of one name is the
/// screen it with [`warn_if_unrepresentable`] first. Callers holding an /// real name of another, so it would hash and index the wrong file. The walk
/// unresolved path want [`prepare_file_record_from_path`] instead. /// screens for that on the directory entry, before the path is even built
/// (`crate::walk::read_directory`); [`prepare_file_record_from_path`], which
/// is what callers holding an unresolved path want, screens with
/// [`warn_if_unrepresentable`].
/// ///
/// Returns `None` for anything that isn't a readable regular file, with a /// Returns `None` for anything that isn't a readable regular file, with a
/// warning when hashing fails. /// warning when hashing fails.
@ -213,10 +222,11 @@ pub fn prepare_file_record(
} }
}; };
let name = Path::new(path) // One split for both halves of the key, and the separator stays with the
.file_name() // parent so the two concatenate back into `path`. `None` here means the
.map(|n| n.to_string_lossy().into_owned())?; // caller handed us something that cannot be a file's path at all.
let parent = parent_str(path); let (parent, name) = split_db_path(path)?;
let (parent, name) = (parent.to_string(), name.to_string());
// Sniff from the bytes hashing already read; an empty head falls back to // Sniff from the bytes hashing already read; an empty head falls back to
// the extension. // the extension.
let mime = guess_mime_from_head(Path::new(path), &head); let mime = guess_mime_from_head(Path::new(path), &head);
@ -255,7 +265,6 @@ pub fn prepare_file_record(
Some(OwnedNewFile { Some(OwnedNewFile {
name, name,
path: path.to_string(),
parent, parent,
size, size,
mtime, mtime,

129
crates/quicksearch-core/src/file_handling/tests.rs
View file

@ -300,6 +300,41 @@ fn db_key_for_a_vanished_path_canonicalizes_what_remains() {
std::fs::remove_dir_all(&root).ok(); std::fs::remove_dir_all(&root).ok();
} }
/// The hazard every screen in the codebase exists for, stated once here.
///
/// `path_to_db_string` is many-to-one, and the collapsed spelling is not
/// garbage — it is a perfectly ordinary filename that a *different* file can
/// really have. So the stored key for an unrepresentable path is another
/// file's key, and any use of it (open, hash, delete by path, delete a subtree
/// range) lands on that file instead. `warn_if_unrepresentable` is how a
/// caller holding a single path avoids ever building one.
#[test]
fn the_stored_spelling_of_an_unrepresentable_path_is_another_files_key() {
let dir = Path::new("/docs");
let bad = dir.join(crate::testutil::unrepresentable_name("report", ".txt"));
let twin = dir.join(crate::testutil::lossy_twin("report", ".txt"));
assert!(warn_if_unrepresentable(&bad));
assert!(
!warn_if_unrepresentable(&twin),
"the twin is an ordinary name and must pass"
);
assert_eq!(
path_to_db_string(&bad),
path_to_db_string(&twin),
"two different files, one stored key"
);
// And the same one component deeper, which is why a bad *directory* has to
// be pruned rather than walked: the collision is inherited by everything
// beneath it.
assert_eq!(
path_to_db_string(&bad.join("child.txt")),
path_to_db_string(&twin.join("child.txt"))
);
assert!(warn_if_unrepresentable(&bad.join("child.txt")));
}
#[test] #[test]
fn db_key_for_an_entirely_missing_path_falls_back_to_the_raw_spelling() { fn db_key_for_an_entirely_missing_path_falls_back_to_the_raw_spelling() {
let nowhere = Path::new("relative-thing-that-does-not-exist.txt"); let nowhere = Path::new("relative-thing-that-does-not-exist.txt");
@ -396,3 +431,97 @@ fn hash_covers_size_and_head_only() {
std::fs::remove_dir_all(&root).ok(); std::fs::remove_dir_all(&root).ok();
} }
/// The invariant the whole schema rests on: `parent` and `name` concatenate
/// back into the path, with no separator logic at the join.
///
/// Checked against real paths built by `Path::join` rather than by string
/// formatting, so a platform whose separator is not `/` is tested too.
#[test]
fn a_split_path_concatenates_back_into_itself() {
let root = std::path::Path::new(if cfg!(windows) { r"C:\" } else { "/" });
let cases = [
root.join("a.txt"), // a file at the very root
root.join("home").join("me").join("x.md"), // the ordinary case
root.join("dir with spaces").join("y"),
root.join("weird.name").join("z.tar.gz"),
];
for path in cases {
let s = path_to_db_string(&path);
let (parent, name) = split_db_path(&s).expect("a file's path");
assert_eq!(
format!("{}{}", parent, name),
s,
"{:?} must round-trip through its (parent, name) key",
s
);
assert_eq!(
name,
path.file_name().unwrap().to_string_lossy(),
"the name half is the file name"
);
assert_eq!(
parent,
dir_to_db_parent(path.parent().unwrap()),
"the parent half is what `dir_to_db_parent` would store"
);
assert!(
parent.ends_with(MAIN_SEPARATOR),
"a stored parent always ends in a separator: {:?}",
parent
);
assert!(
!name.contains(MAIN_SEPARATOR),
"a stored name never contains one: {:?}",
name
);
}
}
/// A filesystem root already ends in a separator, so `dir_to_db_parent` must
/// not add a second one — otherwise every file directly at the root would be
/// stored under `//` (or `C:\\`) and never found again.
#[test]
fn a_root_directory_does_not_get_a_doubled_separator() {
let root = std::path::Path::new(if cfg!(windows) { r"C:\" } else { "/" });
let parent = dir_to_db_parent(root);
assert_eq!(parent, path_to_db_string(root));
assert!(!parent.ends_with(&format!("{}{}", MAIN_SEPARATOR, MAIN_SEPARATOR)));
// And the join still produces a path that names the same file.
assert_eq!(
format!("{}{}", parent, "a.txt"),
path_to_db_string(&root.join("a.txt"))
);
}
/// Strings that are not a file's path have no key, and must say so rather
/// than producing a half-formed one — every lookup helper reads `None` as
/// "not indexed".
#[test]
fn a_string_that_cannot_be_a_files_path_has_no_key() {
assert_eq!(split_db_path("bare-name.txt"), None, "no separator at all");
assert_eq!(
split_db_path(&format!("{}dir{}", MAIN_SEPARATOR, MAIN_SEPARATOR)),
None,
"trailing separator: names a directory, not a file"
);
assert_eq!(split_db_path(""), None);
}
/// On Unix a backslash is an ordinary filename character. Splitting on it
/// there would put half a name in the parent, and the row would be
/// unreachable by the path it was stored under.
#[test]
#[cfg(unix)]
fn a_backslash_is_just_a_character_on_unix() {
let (parent, name) = split_db_path(r"/home/me/back\slash.txt").expect("a file's path");
assert_eq!(parent, "/home/me/");
assert_eq!(name, r"back\slash.txt");
// Same on the directory side: a folder genuinely named `weird\` still
// gets its own separator appended.
assert_eq!(
dir_to_db_parent(std::path::Path::new(r"/tmp/weird\")),
r"/tmp/weird\/"
);
}

160
crates/quicksearch-core/src/incremental.rs
View file

@ -110,11 +110,17 @@ fn upsert_path(
// A moved-in tree surfaces as one directory event; walk it with // A moved-in tree surfaces as one directory event; walk it with
// the same filters as a full run. // the same filters as a full run.
// //
// A non-UTF-8 path is a whole subtree missing from the index, so it // A path the index cannot spell, alongside the ignore and hidden
// is an error (the caller schedules a full run) rather than a quiet // short-circuits above: a genuine "nothing indexable here", not a
// `Ok`. // failure.
//
// It reads as a whole subtree missing from the index, which it is —
// but reporting that as an error would set `needs_full_run`
// (`coordinator::inner::apply_pending`), and a full run screens the
// same subtree out for the same reason. The reindex could not fix it,
// so every write inside such a directory would buy another one.
let Some(root) = path.to_str() else { let Some(root) = path.to_str() else {
return Err(format!("directory path is not valid UTF-8: {:?}", path)); return Ok(Applied::Done);
}; };
// Streamed, not collected: `mv` of a large tree is one event, and // Streamed, not collected: `mv` of a large tree is one event, and
// materialising its entries first is a `DirEntry` per file resident // materialising its entries first is a `DirEntry` per file resident
@ -124,10 +130,18 @@ fn upsert_path(
// //
// `skip` rather than re-testing every entry: `upsert_file` on an // `skip` rather than re-testing every entry: `upsert_file` on an
// unchanged file is cheap but not free, and paying it again for // unchanged file is cheap but not free, and paying it again for
// everything already done would make a large tree quadratic in the // everything already done would cost a transaction and a hash per
// number of turns it takes. The walk order is deterministic for an // already-indexed file on every turn.
// unchanged tree; if the tree does change under us the count is only //
// an optimisation, and the next full run is what makes it exact. // Two honest limits on that. The walk itself is *not* skipped — the
// iterator still reads every directory it passes over (and on Windows
// stats every entry), so the traversal cost stays quadratic in the
// number of turns even though the writes do not. And the count is a
// position, not an identity: if the tree changes under us the entries
// shift, so `skip(n)` skips the wrong files and those get no row until
// the next full run. Both are acceptable for a moved-in tree, which is
// finite and static in the usual case; if either ever matters, resume
// by last-path rather than by count.
let mut done = budget.resume_from; let mut done = budget.resume_from;
for entry in filtered_walk( for entry in filtered_walk(
root, root,
@ -166,12 +180,12 @@ fn upsert_file(
let existing: Option<(i64, i64)> = tx let existing: Option<(i64, i64)> = tx
.query_row( .query_row(
"SELECT id, mtime FROM files WHERE path = ?1", "SELECT id, mtime FROM files WHERE parent = ?1 AND name = ?2",
rusqlite::params![rec.path], rusqlite::params![rec.parent, rec.name],
|r| Ok((r.get(0)?, r.get(1)?)), |r| Ok((r.get(0)?, r.get(1)?)),
) )
.optional() .optional()
.map_err(|e| format!("lookup {}: {}", rec.path, e))?; .map_err(|e| format!("lookup {}: {}", rec.path(), e))?;
let file_id = match existing { let file_id = match existing {
Some((_, mtime)) if mtime.max(0) as u64 == rec.mtime => return Ok(()), Some((_, mtime)) if mtime.max(0) as u64 == rec.mtime => return Ok(()),
@ -200,7 +214,7 @@ fn upsert_file(
extract_and_store( extract_and_store(
&tx, &tx,
file_id, file_id,
&rec.path, &rec.path(),
rec.mime.as_deref(), rec.mime.as_deref(),
registry, registry,
config, config,
@ -232,6 +246,14 @@ pub fn remove_paths(
.transaction() .transaction()
.map_err(|e| format!("begin incremental tx: {}", e))?; .map_err(|e| format!("begin incremental tx: {}", e))?;
for path in batch { for path in batch {
// A path the index cannot spell was never indexed, so there is
// nothing here to delete — and `db_key_for_missing_path` is lossy,
// so going ahead would key the row of whichever *different* file
// owns the lossy spelling and delete it, plus its whole subtree
// range below.
if path.to_str().is_none() {
continue;
}
// The insert side stores a canonicalized path, so the raw event // The insert side stores a canonicalized path, so the raw event
// spelling is not a usable key — but the file is already gone, so // spelling is not a usable key — but the file is already gone, so
// `canonicalize` cannot be called on it directly either. // `canonicalize` cannot be called on it directly either.
@ -327,7 +349,7 @@ mod tests {
fn row(&self, path: &str) -> Option<(i64, i64, i64)> { fn row(&self, path: &str) -> Option<(i64, i64, i64)> {
self.conn self.conn
.query_row( .query_row(
"SELECT id, mtime, content_state FROM files WHERE path = ?1", "SELECT id, mtime, content_state FROM files WHERE parent || name = ?1",
rusqlite::params![path], rusqlite::params![path],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
) )
@ -371,6 +393,48 @@ mod tests {
/// The collapse must not change what ends up deleted — only how much work /// The collapse must not change what ends up deleted — only how much work
/// it takes to get there. /// it takes to get there.
/// A resume point must not outlive the event it describes.
///
/// The coordinator prunes `resume_from` alongside the queues; this is the
/// half of that contract the module itself can state — resuming from a
/// count that belonged to some earlier walk skips real files, and they get
/// no row until the next full run.
#[test]
fn resuming_past_the_end_indexes_nothing_rather_than_the_wrong_files() {
let mut f = Fixture::new();
for i in 0..3 {
f.write(&format!("sub/f{i}.txt"), "body");
}
let sub = f.dir.join("sub");
// A count larger than the tree: every entry is skipped, and the result
// is an empty index rather than an arbitrary subset.
let outcome =
f.apply_resuming(&FsEvent::Create(sub.clone()), Duration::from_secs(3600), 99);
assert_eq!(outcome, Applied::Done);
assert_eq!(f.counts().0, 0);
// From zero — what a pruned resume point gives the next turn — the
// whole tree lands.
f.apply(&FsEvent::Create(sub));
assert_eq!(f.counts().0, 3);
}
/// `batch_size = 0` must not panic the writer.
///
/// `chunks(0)` panics, and this one runs on the indexing thread above the
/// arm that would publish `IndexingStatus::Error` — so before the clamp a
/// hand-edited zero wedged indexing for the session while the UI went on
/// reading "Running". Every sibling call site already had `.max(1)`.
#[test]
fn a_zero_batch_size_does_not_panic_the_writer() {
let mut f = Fixture::new();
f.config.processing.batch_size = 0;
f.write("a.txt", "body");
f.apply(&FsEvent::Create(f.dir.join("a.txt")));
assert_eq!(f.counts().0, 1, "the file should still be indexed");
}
/// A directory event is applied in slices, and a slice resumes where the /// A directory event is applied in slices, and a slice resumes where the
/// last one stopped instead of re-walking what it already did. /// last one stopped instead of re-walking what it already did.
/// ///
@ -635,6 +699,76 @@ mod tests {
assert!(f.row(&survivor).is_some()); assert!(f.row(&survivor).is_some());
} }
/// A `Remove` for a path the index cannot spell must delete nothing.
///
/// `db_key_for_missing_path` ends in `path_to_db_string`, which is lossy,
/// and the key it returns is used twice: to delete a row by path, and as
/// the low end of a range that deletes everything beneath it. For an
/// unrepresentable path that key names a *different*, real file — so the
/// event would take that file's row and its whole subtree, for a file that
/// was never indexed in the first place.
#[test]
fn removing_an_unrepresentable_path_spares_its_lossy_twin() {
let mut f = Fixture::new();
let twin = crate::testutil::lossy_twin("report", ".txt");
let kept = f.write(&twin, "the file that must survive");
f.apply(&FsEvent::Create(kept.clone()));
let canonical = f.canonical(&kept);
assert!(f.row(&canonical).is_some(), "seeded");
// Never written to disk: the event alone is enough, and a Remove is
// for a path that is already gone in any case.
let bad = f
.dir
.join(crate::testutil::unrepresentable_name("report", ".txt"));
f.apply(&FsEvent::Remove(bad));
assert!(
f.row(&canonical).is_some(),
"the real file's row was deleted by an event for a different file"
);
assert_eq!(f.counts().0, 1);
}
/// A directory event for a path the index cannot spell is a quiet `Ok`,
/// not an error.
///
/// `coordinator::inner::apply_pending` turns any `Err` here into
/// `needs_full_run`. A full run screens the same subtree out for the same
/// reason, so the reindex could not fix anything — it would just run
/// again on the next write into that directory, forever. Live on Windows,
/// where a `\\wsl.localhost\` or Samba tree can hold such a directory and
/// the root watch is recursive.
#[test]
fn a_directory_event_for_an_unrepresentable_path_is_not_an_error() {
let mut f = Fixture::new();
let bad = f.dir.join(crate::testutil::unrepresentable_name("dir", ""));
if std::fs::create_dir_all(&bad).is_err() {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
std::fs::write(bad.join("inside.txt"), "body").unwrap();
let outcome = apply_fs_event(
&mut f.conn,
&FsEvent::Create(bad),
&f.config,
&f.ignore,
&f.registry,
&Budget {
deadline: Instant::now() + Duration::from_secs(3600),
cancel: &AtomicBool::new(false),
resume_from: 0,
},
);
assert_eq!(
outcome,
Ok(Applied::Done),
"an unindexable subtree is not a failure"
);
assert_eq!(f.counts().0, 0, "and nothing under it is indexed");
}
#[test] #[test]
fn oversize_files_get_content_na() { fn oversize_files_get_content_na() {
let mut f = Fixture::new(); let mut f = Fixture::new();

45
crates/quicksearch-core/src/indexing/mod.rs
View file

@ -204,6 +204,20 @@ impl IndexingService {
} }
pub fn stop_indexing(&self) -> Result<(), String> { pub fn stop_indexing(&self) -> Result<(), String> {
self.stop_indexing_inner(true)
}
/// [`Self::stop_indexing`] for a caller that is about to delete the file.
///
/// Skips the checkpoint. `checkpoint_truncate` copies the whole write-ahead
/// log into the database — up to `maximum_wal_size`, 512 MiB by default —
/// and the next two statements delete both. It is pure cost, and it is
/// paid on the coordinator's thread while the user waits for a rebuild.
fn stop_indexing_for_delete(&self) -> Result<(), String> {
self.stop_indexing_inner(false)
}
fn stop_indexing_inner(&self, checkpoint: bool) -> Result<(), String> {
self.command_tx self.command_tx
.send(IndexingCommand::Stop) .send(IndexingCommand::Stop)
.map_err(|e| format!("Failed to send stop command: {}", e))?; .map_err(|e| format!("Failed to send stop command: {}", e))?;
@ -226,15 +240,27 @@ impl IndexingService {
// Flush the WAL and release the shared connection. WAL mode itself // Flush the WAL and release the shared connection. WAL mode itself
// stays on — it's the persistent journal mode for the index. // stays on — it's the persistent journal mode for the index.
if let Some(db_conn_arc) = crate::lock_ok(&self.db_connection).take() { if let Some(db_conn_arc) = crate::lock_ok(&self.db_connection).take() {
let conn = crate::lock_ok(&db_conn_arc); if checkpoint {
if let Err(e) = crate::db::repo::checkpoint_truncate(&conn) { let conn = crate::lock_ok(&db_conn_arc);
crate::log_warn!("{}", e); if let Err(e) = crate::db::repo::checkpoint_truncate(&conn) {
crate::log_warn!("{}", e);
}
} }
} }
Ok(()) Ok(())
} }
/// Publish a failure that happened *outside* a run, so the status bar
/// shows it rather than only the log ring.
///
/// A failed "Rebuild index" is the case this exists for: the user asked
/// for something, it did not happen, and a warning in a tab they are not
/// looking at is not telling them.
pub fn report_error(&self, message: String) {
*crate::lock_ok(&self.status) = IndexingStatus::Error(message);
}
pub fn get_status(&self) -> IndexingStatus { pub fn get_status(&self) -> IndexingStatus {
crate::lock_ok(&self.status).clone() crate::lock_ok(&self.status).clone()
} }
@ -279,7 +305,7 @@ impl IndexingService {
/// Stop indexing and delete the database file for a clean rebuild /// Stop indexing and delete the database file for a clean rebuild
pub fn delete_index_for_rebuild(&self, db_path: &str) -> Result<(), String> { pub fn delete_index_for_rebuild(&self, db_path: &str) -> Result<(), String> {
self.stop_indexing() self.stop_indexing_for_delete()
.map_err(|e| format!("Failed to stop indexing: {}", e))?; .map_err(|e| format!("Failed to stop indexing: {}", e))?;
// Cut short the optimize pass too: the file about to be deleted is // Cut short the optimize pass too: the file about to be deleted is
// the file it holds open. // the file it holds open.
@ -308,8 +334,15 @@ impl IndexingService {
// [`db::bump_index_epoch`]. // [`db::bump_index_epoch`].
db::bump_index_epoch(); db::bump_index_epoch();
if std::path::Path::new(db_path).exists() { let path = std::path::Path::new(db_path);
std::fs::remove_file(db_path) if path.exists() {
// Retried, as every other delete of this file is: on Windows it
// fails while any handle is open, and the readers are exactly the
// ones a user has just been using — the search worker holds its
// connection for `IDLE_RELEASE` after the last keystroke. The
// caller releases that one first; this covers a scan or a
// duplicates pass that has not noticed yet.
crate::platform::remove_file_retrying(path)
.map_err(|e| format!("Failed to delete database file: {}", e))?; .map_err(|e| format!("Failed to delete database file: {}", e))?;
} }
for suffix in ["-wal", "-shm", "-journal"] { for suffix in ["-wal", "-shm", "-journal"] {

97
crates/quicksearch-core/src/indexing/pipeline.rs
View file

@ -3,6 +3,7 @@
use rusqlite::Connection; use rusqlite::Connection;
use std::collections::HashSet; use std::collections::HashSet;
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::thread; use std::thread;
@ -74,6 +75,81 @@ fn wal_len(path: &str) -> u64 {
std::fs::metadata(path).map(|m| m.len()).unwrap_or(0) std::fs::metadata(path).map(|m| m.len()).unwrap_or(0)
} }
/// Free space at which a run gives up rather than keep writing.
///
/// Filling the volume the index sits on is not a clean failure. SQLite's
/// guard against a short wal-index only covers the moment that file is
/// *extended*; a later write goes through the `-shm` mmap, and a page fault
/// the filesystem cannot back is delivered as **SIGBUS**, which no `Result`
/// can catch. On a copy-on-write filesystem (btrfs, ZFS) even overwriting an
/// already-allocated page needs a new extent, so a full volume can take the
/// process down on a write to a page that has existed for hours. Stopping
/// with an error while there is still room is the only safe end.
const DISK_FLOOR: u64 = 128 * 1024 * 1024;
/// Share of the free space above [`DISK_FLOOR`] the log may occupy.
///
/// The log is what grows unboundedly between checkpoints, so its cap is the
/// figure that has to fit in what is left. A quarter leaves room for the
/// index's own growth, the FTS segments a merge writes beside it, and
/// whatever else on the machine wants the same volume.
const WAL_SHARE_OF_FREE: u64 = 4;
/// The configured checkpoint threshold, lowered to what the volume can
/// actually absorb.
///
/// `maximum_wal_size` is a stall-frequency knob chosen against a roomy disk;
/// on a nearly full one its 512 MiB default is more than everything left.
/// Checkpointing sooner costs some lock acquisitions and keeps the log inside
/// the space available — see [`DISK_FLOOR`] for why running out is not
/// survivable.
///
/// A configured `0` (forced checkpoints off) is bounded like any other value
/// rather than special-cased: the knob turns off a *performance* behaviour and
/// is not a licence to fill the disk. On a roomy volume the derived bound is
/// larger than any run's log, so `0` keeps its meaning without a second rule.
/// Unknown free space changes nothing.
fn wal_cap_for_volume(configured: u64, db_path: &Path) -> u64 {
let Some(free) = crate::platform::available_space(db_path) else {
return configured;
};
let effective = wal_cap_for_free(configured, free);
if effective != configured {
crate::log_info!(
"{} free where the index lives: forcing a WAL checkpoint every {} MiB \
instead of {}",
human_mib(free),
effective / (1024 * 1024),
match configured {
0 => "never".to_string(),
n => format!("{} MiB", n / (1024 * 1024)),
}
);
}
effective
}
/// The arithmetic of [`wal_cap_for_volume`], split from the syscall so it is
/// testable without a filesystem of a chosen size — the same split the rest of
/// the codebase makes for anything decidable without asking the kernel.
pub(super) fn wal_cap_for_free(configured: u64, free: u64) -> u64 {
let room = free.saturating_sub(DISK_FLOOR) / WAL_SHARE_OF_FREE;
// Never below the floor a configured value would be raised to: checkpoints
// more often than that cost more in lock acquisition than the log costs in
// space, and the in-run check is what actually stops a doomed run.
let capped = room.max(crate::config::MINIMUM_WAL_SIZE);
// `0` is "no cap", so it loses every `min` — hence the explicit arm.
if configured == 0 {
capped
} else {
configured.min(capped)
}
}
fn human_mib(bytes: u64) -> String {
format!("{} MiB", bytes / (1024 * 1024))
}
/// Flips an [`AtomicBool`] when dropped. Held by `run_indexing` so the /// Flips an [`AtomicBool`] when dropped. Held by `run_indexing` so the
/// per-root count subprocesses die on every exit path of a run. /// per-root count subprocesses die on every exit path of a run.
struct CancelOnDrop(Arc<AtomicBool>); struct CancelOnDrop(Arc<AtomicBool>);
@ -780,10 +856,11 @@ impl IndexingService {
// Log size at which to force a checkpoint; see [`wal_len`] for why // Log size at which to force a checkpoint; see [`wal_len`] for why
// SQLite's autocheckpoint cannot be left to do this. // SQLite's autocheckpoint cannot be left to do this.
let wal_path = format!("{}-wal", db_path); let wal_path = format!("{}-wal", db_path);
let wal_cap = match config.processing.maximum_wal_size { let configured_cap = match config.processing.maximum_wal_size {
0 => 0, 0 => 0,
n => n.max(crate::config::MINIMUM_WAL_SIZE), n => n.max(crate::config::MINIMUM_WAL_SIZE),
}; };
let wal_cap = wal_cap_for_volume(configured_cap, Path::new(db_path));
let mut checkpoint_at = wal_cap; let mut checkpoint_at = wal_cap;
// Walks first, one slice each, then a single extraction slice. // Walks first, one slice each, then a single extraction slice.
@ -852,6 +929,24 @@ impl IndexingService {
crate::log_warn!("{}", e); crate::log_warn!("{}", e);
} }
} }
// Only here, not every round: this is the moment the log is
// at its largest, and it costs one syscall per checkpoint
// rather than one per writer turn. The checkpoint above has
// just returned whatever it could, so what is left is the
// honest figure.
if let Some(free) = crate::platform::available_space(Path::new(db_path)) {
if free < DISK_FLOOR {
return Err(format!(
"Stopped: only {} free where the index lives ({}). \
Indexing needs room for its write-ahead log, and \
filling the disk can kill the process outright \
rather than fail cleanly. Free some space and run \
again — what is already indexed is kept.",
human_mib(free),
db_path
));
}
}
// Re-armed from what is on disk: a checkpoint that lost the // Re-armed from what is on disk: a checkpoint that lost the
// race then costs one attempt per further `wal_cap` of // race then costs one attempt per further `wal_cap` of
// growth, not a retry every round. // growth, not a retry every round.

66
crates/quicksearch-core/src/indexing/tests.rs
View file

@ -166,8 +166,11 @@ fn an_extracting_turn_lands_its_leftovers_one_slice_at_a_time() {
&tx, &tx,
&NewFile { &NewFile {
name: &format!("f{}.txt", i), name: &format!("f{}.txt", i),
path: &path.to_string_lossy(), // `dir_to_db_parent`, not `to_string_lossy`: a stored
parent: &tree.to_string_lossy(), // parent always ends in a separator, and a row spelled
// without one sorts below every `ExtractCursor` range that
// should contain it.
parent: &crate::file_handling::dir_to_db_parent(&tree),
size: 22, size: 22,
mtime: 1, mtime: 1,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -670,3 +673,62 @@ fn a_run_with_no_roots_is_complete_rather_than_unknown() {
assert_eq!(o.processed, 0); assert_eq!(o.processed, 0);
assert_eq!(o.total, Some(0)); assert_eq!(o.total, Some(0));
} }
/// A roomy volume leaves `maximum_wal_size` exactly as configured: this is a
/// safety valve, not a second tuning knob.
#[test]
fn a_roomy_volume_does_not_move_the_checkpoint_threshold() {
use super::pipeline::wal_cap_for_free;
let configured = 512 * 1024 * 1024;
assert_eq!(
wal_cap_for_free(configured, 500 * 1024 * 1024 * 1024),
configured
);
// Exactly enough: floor plus four times the log.
let just_enough = 128 * 1024 * 1024 + configured * 4;
assert_eq!(wal_cap_for_free(configured, just_enough), configured);
}
/// A tight volume checkpoints sooner, so the log cannot grow into the space
/// that is left. Running out is not a clean failure — the wal-index is reached
/// through an mmap, and a write the filesystem cannot back is a SIGBUS.
#[test]
fn a_tight_volume_lowers_the_checkpoint_threshold() {
use super::pipeline::wal_cap_for_free;
let configured = 512 * 1024 * 1024;
// 1 GiB free: 896 MiB above the floor, a quarter of which is 224 MiB.
let got = wal_cap_for_free(configured, 1024 * 1024 * 1024);
assert_eq!(got, 224 * 1024 * 1024);
assert!(got < configured);
}
/// `0` means "never force a checkpoint", which is a performance choice and not
/// a licence to fill the disk — so it is bounded by the volume like any other
/// value. On a roomy disk that bound is larger than any run's log, which is
/// how the setting keeps its meaning without a second rule to special-case it.
#[test]
fn disabled_checkpoints_are_still_bounded_by_the_volume() {
use super::pipeline::wal_cap_for_free;
let roomy = wal_cap_for_free(0, 500 * 1024 * 1024 * 1024);
assert!(
roomy > 100 * 1024 * 1024 * 1024,
"effectively never on a roomy disk, got {} bytes",
roomy
);
let tight = wal_cap_for_free(0, 1024 * 1024 * 1024);
assert_eq!(tight, 224 * 1024 * 1024, "a tight one checkpoints anyway");
}
/// Below the floor there is nothing sensible left to divide, and the run is
/// about to be stopped by the in-run check anyway — so the threshold bottoms
/// out at the same floor a configured value is raised to, never at zero.
#[test]
fn a_full_volume_bottoms_out_at_the_minimum_rather_than_zero() {
use super::pipeline::wal_cap_for_free;
for free in [0, 1024, 127 * 1024 * 1024] {
assert_eq!(
wal_cap_for_free(512 * 1024 * 1024, free),
crate::config::MINIMUM_WAL_SIZE
);
}
}

50
crates/quicksearch-core/src/live.rs
View file

@ -673,8 +673,16 @@ fn classify(
) { ) {
use notify::event::{ModifyKind, RenameMode}; use notify::event::{ModifyKind, RenameMode};
let key = |p: &PathBuf| p.to_string_lossy().into_owned(); // `to_str`, not `to_string_lossy`. Targets are paths out of the index, so
let is_target = |p: &PathBuf| targets.contains_key(&key(p)); // they are all representable, and a path that is *not* still has a lossy
// spelling — one that is a perfectly valid path for some other file. Keyed
// lossily, an event for a file we could never index would match a row we
// are displaying and fire a Gone or a Changed at it.
let target_key = |p: &PathBuf| {
p.to_str()
.filter(|k| targets.contains_key(*k))
.map(str::to_owned)
};
match event.kind { match event.kind {
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => {
@ -684,43 +692,55 @@ fn classify(
let (Some(from), Some(to)) = (event.paths.first(), event.paths.get(1)) else { let (Some(from), Some(to)) = (event.paths.first(), event.paths.get(1)) else {
return; return;
}; };
if is_target(from) { if let Some(from_key) = target_key(from) {
pending.insert(key(from), Op::Renamed(to.clone())); // A destination the index cannot spell cannot be recorded as a
} else if is_target(to) { // rename: the row would carry a path naming a different file,
// and the GUI opens rows by that path. The file did leave the
// searchable world, so `Gone` is the honest update — and the
// one the user would get anyway once the row was verified.
let op = match to.to_str() {
Some(_) => Op::Renamed(to.clone()),
None => Op::Gone,
};
pending.insert(from_key, op);
} else if let Some(to_key) = target_key(to) {
// The atomic-save shape: a temporary file renamed over a row // The atomic-save shape: a temporary file renamed over a row
// we are watching. The row did not move; its contents changed. // we are watching. The row did not move; its contents changed.
pending.insert(key(to), Op::Changed); pending.insert(to_key, Op::Changed);
} }
} }
EventKind::Modify(ModifyKind::Name(RenameMode::To)) => { EventKind::Modify(ModifyKind::Name(RenameMode::To)) => {
for path in &event.paths { for path in &event.paths {
if is_target(path) { if let Some(key) = target_key(path) {
pending.insert(key(path), Op::Changed); pending.insert(key, Op::Changed);
} else { } else if path.to_str().is_some() {
// Screened for the same reason: an orphan is paired with a
// lone `Gone` into a `Renamed` below, so an unrepresentable
// one would arrive at the same bad destination.
orphan_to.push(path.clone()); orphan_to.push(path.clone());
} }
} }
} }
EventKind::Modify(ModifyKind::Name(RenameMode::From)) => { EventKind::Modify(ModifyKind::Name(RenameMode::From)) => {
for path in &event.paths { for path in &event.paths {
if is_target(path) { if let Some(key) = target_key(path) {
// Provisional; a Both in this same window upgrades it. // Provisional; a Both in this same window upgrades it.
pending.entry(key(path)).or_insert(Op::Gone); pending.entry(key).or_insert(Op::Gone);
} }
} }
} }
EventKind::Remove(_) => { EventKind::Remove(_) => {
for path in &event.paths { for path in &event.paths {
if is_target(path) { if let Some(key) = target_key(path) {
pending.insert(key(path), Op::Gone); pending.insert(key, Op::Gone);
} }
} }
} }
EventKind::Create(_) | EventKind::Modify(_) => { EventKind::Create(_) | EventKind::Modify(_) => {
for path in &event.paths { for path in &event.paths {
if is_target(path) { if let Some(key) = target_key(path) {
// A Create at a watched path un-deletes the row. // A Create at a watched path un-deletes the row.
pending.insert(key(path), Op::Changed); pending.insert(key, Op::Changed);
} }
} }
} }

86
crates/quicksearch-core/src/live_tests.rs
View file

@ -170,6 +170,92 @@ fn an_ambiguous_windows_window_reports_gone_rather_than_guessing() {
assert_eq!(decided.get("/docs/b.txt"), Some(&Op::Gone)); assert_eq!(decided.get("/docs/b.txt"), Some(&Op::Gone));
} }
/// [`event`], for a path that cannot be spelled as a `&str`.
fn event_os(kind: EventKind, paths: &[PathBuf]) -> NotifyEvent {
NotifyEvent {
kind,
paths: paths.to_vec(),
attrs: Default::default(),
}
}
/// A path in the displayed set, and its unrepresentable neighbour that
/// `to_string_lossy` collapses onto exactly that spelling.
fn twin_pair() -> (String, PathBuf) {
let shown = format!("/docs/{}", crate::testutil::lossy_twin("report", ".txt"));
let bad = PathBuf::from("/docs").join(crate::testutil::unrepresentable_name("report", ".txt"));
assert_eq!(bad.to_string_lossy(), shown, "the two must collide");
(shown, bad)
}
/// A file the index could never hold still generates events, and its lossy
/// spelling is a displayed row's real path. Keyed lossily, every one of those
/// events lands on that row: a `Remove` of the file we cannot index would mark
/// a completely different file gone, on screen, while it sits there on disk.
#[test]
fn events_for_an_unrepresentable_path_never_touch_its_lossy_twin() {
let (shown, bad) = twin_pair();
let t = targets(&[&shown]);
let decided = window(
&t,
vec![
event_os(
EventKind::Remove(RemoveKind::File),
std::slice::from_ref(&bad),
),
event_os(
EventKind::Create(CreateKind::File),
std::slice::from_ref(&bad),
),
event_os(EventKind::Modify(ModifyKind::Any), &[bad]),
],
);
assert!(
decided.is_empty(),
"the displayed row must be untouched: {decided:?}"
);
}
/// Renamed *to* a name the index cannot spell, the row cannot keep a usable
/// path — `Op::Renamed` carries the destination, and the GUI opens rows by it,
/// so a lossy one would open some other file. It left the searchable world, so
/// the honest answer is `Gone`; dropping the event instead would leave a stale
/// row on screen until something else disturbed it.
#[test]
fn a_rename_to_an_unrepresentable_name_reports_gone() {
let (_, bad) = twin_pair();
let t = targets(&["/docs/a.txt"]);
let decided = window(
&t,
vec![event_os(
EventKind::Modify(ModifyKind::Name(RenameMode::Both)),
&[PathBuf::from("/docs/a.txt"), bad],
)],
);
assert_eq!(decided.get("/docs/a.txt"), Some(&Op::Gone));
}
/// The Windows shape of the same thing: the halves arrive separately, and a
/// lone `Gone` plus a lone arrival are paired into a rename. The arrival must
/// not be a path we cannot spell, or the pairing invents the same bad
/// destination the test above rejects.
#[test]
fn a_split_rename_is_not_paired_with_an_unrepresentable_arrival() {
let (_, bad) = twin_pair();
let t = targets(&["/docs/a.txt"]);
let decided = window(
&t,
vec![
event(
EventKind::Modify(ModifyKind::Name(RenameMode::From)),
&["/docs/a.txt"],
),
event_os(EventKind::Modify(ModifyKind::Name(RenameMode::To)), &[bad]),
],
);
assert_eq!(decided.get("/docs/a.txt"), Some(&Op::Gone));
}
/// A watched directory is full of files that are not on screen. None of them /// A watched directory is full of files that are not on screen. None of them
/// may produce an update — live results never add rows. /// may produce an update — live results never add rows.
#[test] #[test]

18
crates/quicksearch-core/src/log.rs
View file

@ -65,6 +65,24 @@ pub fn record(level: Level, message: String) {
Level::Warn => format!("Warning: {}", message), Level::Warn => format!("Warning: {}", message),
Level::Info => message, Level::Info => message,
}; };
// Almost every line here names a path, and a path is whatever someone
// called a file. An escape sequence in one reaches a terminal three ways:
// the stderr write below, a user running the GUI from a shell, and the
// Logs tab's Copy button, which puts the ring on the clipboard for pasting
// into a bug report. Diagnostics are not data — nothing downstream needs
// these bytes exactly — so they are scrubbed unconditionally.
//
// Line breaks are collapsed *first*, because `scrub_controls` counts them
// as controls and would leave `U+FFFD` where a space belongs. One record
// is one line — the ring renders it that way and `writeln!` adds the only
// newline there should be — so a message that arrives multi-line, such as
// a nested error's chain of causes, is flattened rather than boxed.
let text = if text.contains(['\n', '\r']) {
text.replace(['\n', '\r'], " ")
} else {
text
};
let text = crate::textenc::scrub_controls(&text).into_owned();
let _ = writeln!(std::io::stderr(), "{}", text); let _ = writeln!(std::io::stderr(), "{}", text);
lock().push(level, text); lock().push(level, text);
} }

327
crates/quicksearch-core/src/platform.rs
View file

@ -407,6 +407,15 @@ pub const WATCH_ROOTS_RECURSIVELY: bool = cfg!(windows);
/// other with `LIKE` would otherwise disagree with itself. `NOCASE` folds ASCII /// other with `LIKE` would otherwise disagree with itself. `NOCASE` folds ASCII
/// only, which matches what `LIKE` does — non-ASCII paths stay case-sensitive /// only, which matches what `LIKE` does — non-ASCII paths stay case-sensitive
/// on both sides, consistently. /// on both sides, consistently.
///
/// **No query interpolates this any more.** The folder filter was its last
/// caller, and it now needs one `LIKE` where it used to need `= COLLATE … OR
/// LIKE`. What is left is a *specification*: this constant and
/// `tests::collation_matches_like_case_folding` are where the rule is written
/// down, and the places that fold paths by hand —
/// [`PATHS_ARE_CASE_INSENSITIVE`], [`crate::config::IgnoreSet`],
/// [`crate::config::Config::is_index_file`] — are required to agree with it.
/// Deleting it would leave that agreement asserted nowhere.
pub const PATH_COLLATION: &str = if cfg!(windows) { "NOCASE" } else { "BINARY" }; pub const PATH_COLLATION: &str = if cfg!(windows) { "NOCASE" } else { "BINARY" };
/// Whether this platform's filesystem matches names without regard to case — /// Whether this platform's filesystem matches names without regard to case —
@ -573,6 +582,324 @@ pub fn open_regular_file(path: &Path) -> std::io::Result<std::fs::File> {
Ok(file) Ok(file)
} }
/// Why [`IndexLock::acquire`] did not hand back a lock.
#[derive(Debug)]
pub enum LockError {
/// Another process holds it. `pid` is whatever the holder recorded in the
/// file and is for the message only — it is never what decides.
Held { pid: Option<u32> },
/// The filesystem does not do locks. The caller must carry on regardless;
/// see [`IndexLock::acquire`].
Unsupported(String),
}
/// Proof that this process, and no other, owns the index at a given path.
///
/// Held for the life of the process: dropping it, or exiting, releases the
/// lock.
#[derive(Debug)]
pub struct IndexLock {
/// The lock lives on this open file description. Keeping the handle alive
/// is the whole mechanism, which is why the field is never read.
_file: std::fs::File,
path: PathBuf,
}
/// The lock this process is holding, if any.
///
/// A `static` because that is what the thing models: `flock` and `LockFileEx`
/// are owned by an open file description, one index is locked per process, and
/// what releases it is the process ending. Threading an `Arc<Mutex<_>>` from
/// `main` down through the unlock gate and into the settings handler would say
/// the same thing in three more places.
///
/// Never dropped — statics are not — and that costs nothing here: the kernel
/// releases the lock however the process goes away, which is the whole design.
static HELD_LOCK: std::sync::Mutex<Option<IndexLock>> = std::sync::Mutex::new(None);
impl IndexLock {
/// The lock file for the index at `db_path`.
///
/// Its own name, never the database or one of SQLite's sidecars: a
/// `flock` of ours on an inode SQLite also locks would be a second
/// locking protocol on one file, and on Unix our `close` of it would
/// cancel SQLite's locks — the very failure this whole change exists to
/// prevent.
pub fn path_for(db_path: &Path) -> PathBuf {
let name = db_path
.file_name()
.and_then(|s| s.to_str())
.unwrap_or("index.sqlite");
db_path.with_file_name(format!("{}.lock", name))
}
/// Take the index lock, or report who has it.
///
/// **The guard is the kernel's lock, never the file's existence.** `flock`
/// and `LockFileEx` are held by the open file description, so the kernel
/// drops them when the process goes away for *any* reason — a clean exit,
/// a panic, SIGKILL, the OOM killer, a SIGBUS, or the power going out. A
/// `.lock` file left behind by an unclean shutdown is therefore inert, and
/// the next start re-acquires it normally. Nothing here may ever branch on
/// the file being present; a stale-PID-file scheme would strand the user
/// behind a crash exactly when they most want to reopen the app.
///
/// [`LockError::Unsupported`] means the filesystem could not answer, not
/// that the lock is taken — NFS without `lockd`, some FUSE mounts, a few
/// network shares. Callers **must** start anyway: a convenience guard is
/// never a good enough reason to refuse to open.
pub fn acquire(db_path: &Path) -> Result<IndexLock, LockError> {
let path = IndexLock::path_for(db_path);
if let Some(dir) = path.parent() {
if !dir.as_os_str().is_empty() {
let _ = create_dir_private(dir);
}
}
let file = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&path)
{
Ok(f) => f,
// A read-only or unwritable index directory is not a second
// instance. Same rule as an unsupported filesystem: carry on.
Err(e) => return Err(LockError::Unsupported(format!("{}: {}", path.display(), e))),
};
lock_exclusive_nonblocking(&file).map_err(|e| match e {
LockAttempt::Held => LockError::Held {
pid: read_recorded_pid(&path),
},
LockAttempt::Unsupported(msg) => LockError::Unsupported(msg),
})?;
let lock = IndexLock {
_file: file,
path: path.clone(),
};
// Only after the lock is ours, and only so a second instance can name
// us in its message. Nothing reads this to make a decision.
lock.record_holder();
Ok(lock)
}
/// [`IndexLock::acquire`], keeping the lock in [`HELD_LOCK`] for the life
/// of the process. What a frontend calls at startup.
pub fn hold(db_path: &Path) -> Result<(), LockError> {
let lock = IndexLock::acquire(db_path)?;
*crate::lock_ok(&HELD_LOCK) = Some(lock);
Ok(())
}
/// Move the held lock onto the index at `db_path`, for a `database_path`
/// that changed while the app was running.
///
/// **The new lock is taken before the old one is let go.** A refusal then
/// leaves this process holding exactly what it held before, so the caller
/// can reject the change and stay in a state that is still true. Releasing
/// first would leave a window in which another instance could claim the
/// index we are about to keep using.
///
/// Naming the index already locked is [`LockError::Held`] and changes
/// nothing. [`LockError::Unsupported`] means the move *happened* — the old
/// lock is gone — and the new path simply cannot be locked; carry on, as at
/// startup. Holding the old path's lock in that case would be worse than
/// holding none: it guards a file this process no longer uses while barring
/// a second instance from it.
pub fn move_to(db_path: &Path) -> Result<(), LockError> {
let mut slot = crate::lock_ok(&HELD_LOCK);
// Short-circuit before acquiring: `flock` conflicts with itself across
// two open file descriptions even inside one process, so re-taking the
// path we already hold would report itself as `Held`.
if slot
.as_ref()
.is_some_and(|held| held.path == IndexLock::path_for(db_path))
{
return Ok(());
}
match IndexLock::acquire(db_path) {
Ok(lock) => {
*slot = Some(lock);
Ok(())
}
Err(LockError::Unsupported(why)) => {
*slot = None;
Err(LockError::Unsupported(why))
}
Err(e) => Err(e),
}
}
fn record_holder(&self) {
use std::io::Write;
// A fresh handle: writing through `_file` would move the shared file
// offset the lock has nothing to do with, and the failure here is
// cosmetic either way. That a *second* handle can write here at all is
// why the Windows lock byte sits at [`LOCK_BYTE_OFFSET`] rather than at
// offset 0 — its byte-range locks are mandatory and per-handle, so a
// lock over this byte would block our own write.
let written = std::fs::OpenOptions::new()
.write(true)
.truncate(true)
.open(&self.path)
.and_then(|mut f| write!(f, "{}", std::process::id()));
if let Err(e) = written {
crate::log_warn!(
"could not record the lock holder in {}: {}",
self.path.display(),
e
);
}
}
}
/// The PID a holder recorded, for a message. `None` whenever the file is
/// absent, empty or unparseable — all of which are ordinary.
fn read_recorded_pid(path: &Path) -> Option<u32> {
std::fs::read_to_string(path).ok()?.trim().parse().ok()
}
/// What one attempt at the lock found.
enum LockAttempt {
Held,
Unsupported(String),
}
/// Where the Windows lock byte lives, chosen to be nowhere near the PID the
/// lock file holds at offset 0. See [`lock_exclusive_nonblocking`]'s Windows
/// arm; the Unix arm has no equivalent, because `flock` locks the open file
/// description rather than a byte range.
#[cfg(windows)]
const LOCK_BYTE_OFFSET: u64 = 1 << 63;
#[cfg(unix)]
fn lock_exclusive_nonblocking(file: &std::fs::File) -> Result<(), LockAttempt> {
use std::os::unix::io::AsRawFd;
// `flock`, not `fcntl`: a POSIX record lock would be cancelled by any
// `close` this process makes on the same inode, which is the hazard
// documented in `file_handling::index_file_set`. A `flock` belongs to the
// open file description and is immune to it.
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
if rc == 0 {
return Ok(());
}
let err = std::io::Error::last_os_error();
match err.raw_os_error() {
Some(libc::EWOULDBLOCK) => Err(LockAttempt::Held),
_ => Err(LockAttempt::Unsupported(err.to_string())),
}
}
#[cfg(windows)]
fn lock_exclusive_nonblocking(file: &std::fs::File) -> Result<(), LockAttempt> {
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Foundation::ERROR_LOCK_VIOLATION;
use windows_sys::Win32::Storage::FileSystem::{
LockFileEx, LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY,
};
use windows_sys::Win32::System::IO::{OVERLAPPED, OVERLAPPED_0, OVERLAPPED_0_0};
// The locked byte sits at [`LOCK_BYTE_OFFSET`], far past anything this
// file will ever hold — **not** at offset 0, where the PID is written.
// Windows byte-range locks are *mandatory* and belong to the file object,
// not the process: a lock covering offset 0 makes our own
// `record_holder`, which writes through a second handle, fail with
// `ERROR_LOCK_VIOLATION`, and `read_recorded_pid` fail the same way — so
// the "already running" message would lose the PID on the one platform
// that has no terminal to print it to. Locking beyond end-of-file is
// explicitly legal and is the conventional way to use a file as a
// semaphore.
let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() };
overlapped.Anonymous = OVERLAPPED_0 {
Anonymous: OVERLAPPED_0_0 {
Offset: LOCK_BYTE_OFFSET as u32,
OffsetHigh: (LOCK_BYTE_OFFSET >> 32) as u32,
},
};
let ok = unsafe {
LockFileEx(
file.as_raw_handle() as _,
LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY,
0,
1,
0,
&mut overlapped,
)
};
if ok != 0 {
return Ok(());
}
let err = std::io::Error::last_os_error();
match err.raw_os_error() {
Some(code) if code == ERROR_LOCK_VIOLATION as i32 => Err(LockAttempt::Held),
_ => Err(LockAttempt::Unsupported(err.to_string())),
}
}
/// Bytes free to this user on the filesystem holding `path`, or `None` where
/// the platform will not say.
///
/// `None` is "unknown", never "zero": every caller treats it as no reason to
/// hold anything back.
///
/// The path need not exist — its nearest existing ancestor is what gets
/// asked, so this answers for a database that has not been created yet.
pub fn available_space(path: &Path) -> Option<u64> {
let mut probe = path;
loop {
if probe.exists() {
break;
}
probe = probe.parent()?;
}
available_space_of_existing(probe)
}
#[cfg(unix)]
fn available_space_of_existing(path: &Path) -> Option<u64> {
use std::ffi::CString;
use std::os::unix::ffi::OsStrExt;
let c_path = CString::new(path.as_os_str().as_bytes()).ok()?;
let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
if unsafe { libc::statvfs(c_path.as_ptr(), &mut stat) } != 0 {
return None;
}
// `f_bavail`, not `f_bfree`: the reserved blocks in the difference are
// root's, and this process is not root. `f_frsize` is the fragment size
// the block counts are in — `f_bsize` is the preferred I/O size and is
// the wrong multiplier.
let frsize = if stat.f_frsize > 0 {
stat.f_frsize
} else {
stat.f_bsize
};
(stat.f_bavail as u64).checked_mul(frsize as u64)
}
#[cfg(windows)]
fn available_space_of_existing(path: &Path) -> Option<u64> {
use std::os::windows::ffi::OsStrExt;
use windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW;
let wide: Vec<u16> = path
.as_os_str()
.encode_wide()
.chain(std::iter::once(0))
.collect();
let mut free_to_caller: u64 = 0;
let ok = unsafe {
GetDiskFreeSpaceExW(
wide.as_ptr(),
&mut free_to_caller,
std::ptr::null_mut(),
std::ptr::null_mut(),
)
};
(ok != 0).then_some(free_to_caller)
}
/// How long to keep retrying a delete that fails because something else holds /// How long to keep retrying a delete that fails because something else holds
/// the file open. /// the file open.
#[cfg(windows)] #[cfg(windows)]

239
crates/quicksearch-core/src/platform_tests.rs
View file

@ -1,5 +1,7 @@
use super::*; use super::*;
use std::time::{Duration, Instant};
#[test] #[test]
fn unc_spellings() { fn unc_spellings() {
assert!(is_unc_string(r"\\server\share")); assert!(is_unc_string(r"\\server\share"));
@ -242,3 +244,240 @@ fn sep_prefix() -> String {
"/".to_string() "/".to_string()
} }
} }
/// Two live instances cannot both hold the index.
#[test]
fn the_index_lock_is_exclusive_while_held() {
let db = crate::testutil::scratch_dir("lock-excl").join("index.sqlite");
let first = IndexLock::acquire(&db).expect("first acquire");
match IndexLock::acquire(&db) {
Err(LockError::Held { pid }) => {
// Recorded for the message only, but it should name us.
assert_eq!(pid, Some(std::process::id()));
}
Err(LockError::Unsupported(why)) => {
// A filesystem with no locks cannot answer; nothing to assert.
eprintln!("skipping: {}", why);
}
Ok(_) => panic!("the lock was handed out twice"),
}
drop(first);
// And it comes back once the holder lets go — but not necessarily in the
// same instant, which is why this retries instead of asserting outright.
//
// `flock` belongs to the *open file description*, and `fork` duplicates
// the descriptor table: between another thread's `fork` and its `exec`,
// the child shares every description this process has open, including the
// one we just released. `O_CLOEXEC` closes it at `exec` — verified, no
// descriptor survives into a spawned child — but until then the lock
// stays held. Several tests in this suite spawn processes (the sibling
// test below, and `file_handling::counting`'s `find`/`wc`), so under
// `cargo test` this window is reached often enough to be seen.
//
// It cannot reach the product: the lock is taken once at startup and held
// for the life of the process, never dropped and immediately retaken.
acquire_within(&db, Duration::from_secs(5));
}
/// [`IndexLock::acquire`], retried past the `fork`/`exec` window described in
/// [`the_index_lock_is_exclusive_while_held`].
fn acquire_within(db: &std::path::Path, budget: Duration) -> IndexLock {
let deadline = Instant::now() + budget;
loop {
match IndexLock::acquire(db) {
Ok(lock) => return lock,
Err(e) if Instant::now() >= deadline => panic!("never acquired: {:?}", e),
Err(_) => std::thread::sleep(Duration::from_millis(10)),
}
}
}
/// Env var naming the database whose lock [`lock_holder_child`] should take.
/// Absent in an ordinary run, which is what makes that test a no-op.
const LOCK_CHILD_DB: &str = "QS_LOCK_CHILD_DB";
/// **A crash must not lock the user out.**
///
/// Nothing unlinks the lock file, so it outlives an unclean exit. If startup
/// keyed on the file *existing*, one SIGKILL — or the SIGBUS this whole change
/// is about — would leave QuickSearch permanently unopenable. The guard is the
/// kernel's `flock`/`LockFileEx`, released when the holder's handle goes away
/// however it goes away, so the leftover file is inert.
///
/// Only a real killed process proves that, so this spawns one: nothing a
/// single process can do to itself reproduces "died without running a
/// destructor" while leaving a test alive to check the result.
#[test]
fn a_killed_holder_does_not_block_the_next_start() {
let db = crate::testutil::scratch_dir("lock-crash").join("index.sqlite");
let lock_path = IndexLock::path_for(&db);
// Probe first: on a filesystem without locks there is nothing to test.
match IndexLock::acquire(&db) {
Ok(lock) => drop(lock),
Err(LockError::Unsupported(why)) => {
eprintln!("skipping: {}", why);
return;
}
Err(LockError::Held { .. }) => panic!("a fresh path cannot be held"),
}
let exe = std::env::current_exe().expect("test binary path");
let mut child = std::process::Command::new(exe)
.args([
"--exact",
"platform::tests::lock_holder_child",
"--nocapture",
])
.env(LOCK_CHILD_DB, &db)
.spawn()
.expect("spawn the lock holder");
// Poll rather than read a pipe: a child that dies early then fails this
// test at the deadline instead of hanging it forever.
let deadline = Instant::now() + Duration::from_secs(30);
loop {
if matches!(IndexLock::acquire(&db), Err(LockError::Held { .. })) {
break;
}
if Instant::now() >= deadline {
let _ = child.kill();
panic!("the child never took the lock");
}
std::thread::sleep(Duration::from_millis(25));
}
// SIGKILL / TerminateProcess: no unwinding, no destructors, no cleanup —
// exactly what a SIGBUS leaves behind.
child.kill().expect("kill the holder");
child.wait().expect("reap the holder");
assert!(
lock_path.exists(),
"the crash should leave the lock file at {}",
lock_path.display()
);
// The point of the whole test: file present, holder dead, start succeeds.
// Retried for the reason `acquire_within` documents, not because a dead
// holder could still be holding anything.
acquire_within(&db, Duration::from_secs(5));
}
/// The child half of [`a_killed_holder_does_not_block_the_next_start`]: take
/// the lock, then wait to be killed. A no-op in an ordinary run.
#[test]
fn lock_holder_child() {
let Some(db) = std::env::var_os(LOCK_CHILD_DB) else {
return;
};
let db = std::path::PathBuf::from(db);
// The parent probes the lock to find out when we have it, so it may hold
// it for an instant just as we ask. Retry rather than lose the race.
let deadline = Instant::now() + Duration::from_secs(30);
let _lock = loop {
match IndexLock::acquire(&db) {
Ok(lock) => break lock,
Err(e) if Instant::now() >= deadline => panic!("child never acquired: {:?}", e),
Err(_) => std::thread::sleep(Duration::from_millis(10)),
}
};
// Killed long before this returns. The sleep is a backstop so a parent
// that dies first cannot strand this process.
std::thread::sleep(Duration::from_secs(120));
}
/// A `database_path` changed in Settings must carry the lock with it — and a
/// refused move must leave this process holding exactly what it held.
///
/// The whole point of taking the new lock before dropping the old: if the
/// destination is already somebody else's, the settings change is rejected and
/// the app goes on using the index it was using, still guarded. Releasing
/// first would open a window on the index we are about to keep writing to.
///
/// Uses the process-wide slot, so it is the one test that touches
/// [`HELD_LOCK`]; the paths are per-test scratch directories, so it does not
/// race the sibling tests that call [`IndexLock::acquire`] directly.
#[test]
fn the_held_lock_follows_the_database_path() {
let dir = crate::testutil::scratch_dir("lock-move");
let first = dir.join("first.sqlite");
let second = dir.join("second.sqlite");
// A filesystem with no locks cannot answer any of this.
match IndexLock::hold(&first) {
Ok(()) => {}
Err(LockError::Unsupported(why)) => {
eprintln!("skipping: {}", why);
return;
}
Err(LockError::Held { .. }) => panic!("a fresh path cannot be held"),
}
// Naming the index we already hold is a no-op, not a self-collision:
// `flock` conflicts with itself across two descriptions in one process.
IndexLock::move_to(&first).expect("re-holding the same path");
// Somebody else owns the destination, so the move is refused...
let rival = acquire_within(&second, Duration::from_secs(5));
assert!(
matches!(IndexLock::move_to(&second), Err(LockError::Held { .. })),
"a held destination must refuse the move"
);
// ...and the old path is still ours, which is what lets the caller reject
// the settings change and stay correct.
assert!(
matches!(IndexLock::acquire(&first), Err(LockError::Held { .. })),
"the original lock must survive a refused move"
);
// Once the destination frees up the move goes through, and the path we
// came from is released.
drop(rival);
let deadline = Instant::now() + Duration::from_secs(5);
while IndexLock::move_to(&second).is_err() && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(10));
}
assert!(
matches!(IndexLock::acquire(&second), Err(LockError::Held { .. })),
"the new path must be held after the move"
);
acquire_within(&first, Duration::from_secs(5));
}
/// `database_path` in hand, the lock is a sibling with its own name — never
/// the database or one of SQLite's sidecars, whose inodes we must not touch.
#[test]
fn the_lock_file_is_not_the_database_or_a_sidecar() {
let db = std::path::Path::new("/var/lib/qs/index.sqlite");
let lock = IndexLock::path_for(db);
assert_eq!(lock, std::path::Path::new("/var/lib/qs/index.sqlite.lock"));
for suffix in crate::file_handling::INDEX_SIDECAR_SUFFIXES {
if suffix == ".lock" {
continue;
}
assert_ne!(
lock,
std::path::PathBuf::from(format!("{}{}", db.display(), suffix))
);
}
assert_ne!(lock, db);
}
#[test]
fn available_space_answers_for_a_real_directory() {
let dir = crate::testutil::scratch_dir("space");
let free = available_space(&dir).expect("temp dir has a filesystem");
assert!(free > 0, "a writable scratch dir should have free space");
}
/// The database is asked about before it exists — the check runs at the start
/// of the first run, when nothing has created the file yet.
#[test]
fn available_space_walks_up_to_an_existing_ancestor() {
let missing = crate::testutil::scratch_dir("space-missing")
.join("not")
.join("created")
.join("index.sqlite");
assert!(!missing.exists());
assert!(available_space(&missing).is_some());
}

24
crates/quicksearch-core/src/query/split.rs
View file

@ -291,11 +291,14 @@ mod tests {
fn a_windows_drive_path_reaches_the_filter_intact() { fn a_windows_drive_path_reaches_the_filter_intact() {
let q = split_for_cascade(r"path:C:\Users\me\docs").unwrap(); let q = split_for_cascade(r"path:C:\Users\me\docs").unwrap();
assert_eq!(q.term, "", "the whole input is a filter"); assert_eq!(q.term, "", "the whole input is a filter");
assert!( // The folder filter binds one value: the subtree LIKE pattern built
matches!( // from the spelling the user typed. Asserted through the builder so
&q.filter_params[0], // this stays about the plumbing rather than about escaping rules.
Value::Text(t) if t == r"C:\Users\me\docs" assert_eq!(
), q.filter_params[0],
Value::Text(crate::query::translator::like_subtree_pattern(
r"C:\Users\me\docs"
)),
"{:?}", "{:?}",
q.filter_params q.filter_params
); );
@ -307,7 +310,9 @@ mod tests {
assert_eq!(q.term, ""); assert_eq!(q.term, "");
assert_eq!( assert_eq!(
q.filter_params[0], q.filter_params[0],
Value::Text("/home/me/My Documents".into()) Value::Text(crate::query::translator::like_subtree_pattern(
"/home/me/My Documents"
))
); );
} }
@ -439,9 +444,12 @@ mod tests {
let q = split_for_cascade("name:%*_").unwrap(); let q = split_for_cascade("name:%*_").unwrap();
assert!(matches!(&q.filter_params[0], Value::Text(t) if t == "%\\%%\\_%")); assert!(matches!(&q.filter_params[0], Value::Text(t) if t == "%\\%%\\_%"));
// path: values never glob. // path: values never glob — the star reaches the pattern as a literal.
let q = split_for_cascade("path:/da*ta").unwrap(); let q = split_for_cascade("path:/da*ta").unwrap();
assert!(matches!(&q.filter_params[0], Value::Text(t) if t == "/da*ta")); assert_eq!(
q.filter_params[0],
Value::Text(crate::query::translator::like_subtree_pattern("/da*ta"))
);
} }
#[test] #[test]

91
crates/quicksearch-core/src/query/translator.rs
View file

@ -146,18 +146,15 @@ pub fn build_filter(
// single root, so say it directly rather than by accident. // single root, so say it directly rather than by accident.
return Ok(frag("1=1", Vec::new())); return Ok(frag("1=1", Vec::new()));
} }
// The `=` half needs the collation spelled out: `LIKE` folds ASCII // One `LIKE`, where this used to need `parent = ? OR parent LIKE ?`
// case on its own, so without this the two halves of the same // with the collation spelled out to stop the two halves disagreeing
// filter disagree about `C:\Users` versus `c:\users`. // about `C:\Users` versus `c:\users`. Every stored parent now ends
// in a separator, so `dir + SEP + %` matches the folder's own files
// (`%` matching nothing) as well as its subdirectories', and the
// `=` half has nothing left to do.
Ok(frag( Ok(frag(
&format!( "f.parent LIKE ? ESCAPE '\\'",
"(f.parent = ? COLLATE {} OR f.parent LIKE ? ESCAPE '\\')", vec![Value::Text(like_subtree_pattern(&base))],
crate::platform::PATH_COLLATION
),
vec![
Value::Text(base.clone()),
Value::Text(like_subtree_pattern(&base)),
],
)) ))
} }
"name" | "filename" => { "name" | "filename" => {
@ -204,21 +201,23 @@ pub fn escape_like(s: &str) -> String {
out out
} }
/// Tidy a user-supplied folder value into the spelling `files.parent` stores. /// Tidy a user-supplied folder value: trim it, and drop any trailing
/// separator, of either flavour — that is how people naturally write a
/// directory, and either may show up on Windows.
/// ///
/// Trailing separators are how people naturally write directories, and either /// Empty out means "every folder", which is what a bare `/` or a blank value
/// separator may show up on Windows. A bare drive (`C:`) is *not* a path — the /// comes to; [`build_filter`] turns that into `1=1`.
/// stored parent is `C:\` — so the separator goes back on. ///
/// It used to special-case a bare drive (`C:` → `C:\`), because the filter's
/// `parent = ?` half had to match the stored spelling exactly. That half is
/// gone, and [`like_subtree_pattern`] puts the separator back itself, so both
/// spellings now produce the same pattern.
fn normalize_folder_value(value: &str) -> String { fn normalize_folder_value(value: &str) -> String {
let base = value.trim().trim_end_matches(['/', '\\']); value.trim().trim_end_matches(['/', '\\']).to_string()
if base.len() == 2 && base.ends_with(':') && base.starts_with(|c: char| c.is_ascii_alphabetic())
{
return format!("{}{}", base, std::path::MAIN_SEPARATOR);
}
base.to_string()
} }
/// A `LIKE ... ESCAPE '\'` pattern matching every path strictly beneath `dir`. /// A `LIKE ... ESCAPE '\'` pattern matching `dir`'s own files and everything
/// beneath it.
/// ///
/// The separator is escaped along with the base, because on Windows the /// The separator is escaped along with the base, because on Windows the
/// separator *is* the escape character — a hand-written `format!("{}/%", dir)` /// separator *is* the escape character — a hand-written `format!("{}/%", dir)`
@ -315,10 +314,15 @@ mod tests {
#[test] #[test]
fn path_filter_covers_the_folder_and_its_subtree() { fn path_filter_covers_the_folder_and_its_subtree() {
let f = frag("path", Op::Contains, "/home/me/docs"); let f = frag("path", Op::Contains, "/home/me/docs");
assert!(f.sql.contains("f.parent = ?")); // One `LIKE`, and one bound value: since every stored parent ends in a
assert!(f.sql.contains("f.parent LIKE ?")); // separator, `dir + SEP + %` reaches the folder's own files as well as
// The LIKE half must declare its escape character; without the clause // its subdirectories'. The `parent = ?` half this used to need — and
// a Windows separator would be eaten as an escape. // the explicit collation that went with it — is gone.
assert!(f.sql.contains("f.parent LIKE ?"), "{}", f.sql);
assert!(!f.sql.contains("f.parent = ?"), "{}", f.sql);
assert_eq!(f.params.len(), 1);
// The LIKE must declare its escape character; without the clause a
// Windows separator would be eaten as an escape.
assert!(f.sql.contains("ESCAPE '\\'"), "{}", f.sql); assert!(f.sql.contains("ESCAPE '\\'"), "{}", f.sql);
} }
@ -404,12 +408,14 @@ mod tests {
.unwrap(); .unwrap();
let base = format!("{}a{}b", root_prefix(), SEP); let base = format!("{}a{}b", root_prefix(), SEP);
// Stored parents always end in a separator, so each row here is spelled
// the way the indexer would spell it.
let rows = [ let rows = [
format!("{}{}sub", base, SEP), // inside format!("{}{}sub{}", base, SEP, SEP), // inside
format!("{}{}sub{}deep", base, SEP, SEP), // deeper format!("{}{}sub{}deep{}", base, SEP, SEP, SEP), // deeper
base.clone(), // the folder itself format!("{}{}", base, SEP), // the folder's own files
format!("{}a{}bc", root_prefix(), SEP), // prefix sibling: outside format!("{}a{}bc{}", root_prefix(), SEP, SEP), // prefix sibling: outside
format!("{}a", root_prefix()), // parent: outside format!("{}a{}", root_prefix(), SEP), // parent: outside
]; ];
for r in &rows { for r in &rows {
conn.execute("INSERT INTO files (parent) VALUES (?1)", [r]) conn.execute("INSERT INTO files (parent) VALUES (?1)", [r])
@ -423,7 +429,10 @@ mod tests {
|r| r.get(0), |r| r.get(0),
) )
.unwrap(); .unwrap();
assert_eq!(matched, 2, "only the two rows strictly beneath {}", base); // Three, not two: the trailing separator is what brings the folder's
// *own* files in, which is why the filter needs no second predicate.
// The prefix sibling and the parent stay out.
assert_eq!(matched, 3, "the subtree of {}, and only that", base);
} }
#[test] #[test]
@ -436,9 +445,9 @@ mod tests {
let base = format!("{}a_b", root_prefix()); let base = format!("{}a_b", root_prefix());
for r in [ for r in [
format!("{}{}inside", base, SEP), // real child format!("{}{}inside{}", base, SEP, SEP), // real child
format!("{}axb{}bait", root_prefix(), SEP), // `_` must not glob to `x` format!("{}axb{}bait{}", root_prefix(), SEP, SEP), // `_` must not glob to `x`
format!("{}100%_done{}x", root_prefix(), SEP), format!("{}100%_done{}x{}", root_prefix(), SEP, SEP),
] { ] {
conn.execute("INSERT INTO files (parent) VALUES (?1)", [&r]) conn.execute("INSERT INTO files (parent) VALUES (?1)", [&r])
.unwrap(); .unwrap();
@ -456,13 +465,17 @@ mod tests {
#[test] #[test]
fn folder_value_normalization() { fn folder_value_normalization() {
use std::path::MAIN_SEPARATOR as SEP;
// Trailing separators of either flavour are stripped. // Trailing separators of either flavour are stripped.
assert_eq!(normalize_folder_value("/home/me/"), "/home/me"); assert_eq!(normalize_folder_value("/home/me/"), "/home/me");
assert_eq!(normalize_folder_value(r"C:\Users\me\"), r"C:\Users\me"); assert_eq!(normalize_folder_value(r"C:\Users\me\"), r"C:\Users\me");
// A bare drive is not a path; the stored parent is `C:\`. // A bare drive and a rooted one now normalize alike: the pattern
assert_eq!(normalize_folder_value("C:"), format!("C:{}", SEP)); // builder puts the separator back either way.
assert_eq!(normalize_folder_value(r"C:\"), format!("C:{}", SEP)); assert_eq!(normalize_folder_value("C:"), "C:");
assert_eq!(normalize_folder_value(r"C:\"), "C:");
assert_eq!(
like_subtree_pattern(&normalize_folder_value("C:")),
like_subtree_pattern(&normalize_folder_value(r"C:\")),
);
// Empty means "everywhere". // Empty means "everywhere".
assert_eq!(normalize_folder_value("/"), ""); assert_eq!(normalize_folder_value("/"), "");
assert_eq!(normalize_folder_value(" "), ""); assert_eq!(normalize_folder_value(" "), "");

27
crates/quicksearch-core/src/scope.rs
View file

@ -5,8 +5,8 @@
//! produce. When the configuration changes, the two disagree — and almost //! produce. When the configuration changes, the two disagree — and almost
//! always in a way that can be *reconciled* rather than rebuilt: //! always in a way that can be *reconciled* rather than rebuilt:
//! //!
//! * A root was removed. Its rows are a contiguous `files.path` range, so //! * A root was removed. Its rows are a contiguous `files.parent` range, so
//! they go in five statements ([`crate::db::repo::delete_subtree`]). //! they go in four statements ([`crate::db::repo::delete_subtree`]).
//! * An ignore pattern was added, hidden files were switched off, symlinks //! * An ignore pattern was added, hidden files were switched off, symlinks
//! stopped being followed. The rows to drop are picked out by a predicate //! stopped being followed. The rows to drop are picked out by a predicate
//! no SQL range can express, so [`Scope::covers`] re-runs the walker's own //! no SQL range can express, so [`Scope::covers`] re-runs the walker's own
@ -51,7 +51,7 @@ use crate::indexing::ReconcileProgress;
/// through [`crate::db::InterruptGuard`]. /// through [`crate::db::InterruptGuard`].
pub const SLICE: Duration = Duration::from_millis(250); pub const SLICE: Duration = Duration::from_millis(250);
/// One configured root, with the `files.path` range it owns precomputed. /// One configured root, with the `files.parent` range it owns precomputed.
struct Root { struct Root {
path: PathBuf, path: PathBuf,
lo: String, lo: String,
@ -208,9 +208,9 @@ pub struct WorkCursor {
dropped_aliases: bool, dropped_aliases: bool,
/// Index into `scope.roots` of the range being scanned. /// Index into `scope.roots` of the range being scanned.
root_idx: usize, root_idx: usize,
/// Last path served by the scan — the keyset cursor. Empty means "start /// Last `(parent, name)` served by the scan — the keyset cursor. An empty
/// this root's range from its `lo` bound". /// parent means "start this root's range from its `lo` bound".
after: String, after: (String, String),
/// Set once the FTS automerge that follows a batch of deletions has run. /// Set once the FTS automerge that follows a batch of deletions has run.
finalized: bool, finalized: bool,
/// Rows deleted so far, for the log line when the work completes. /// Rows deleted so far, for the log line when the work completes.
@ -233,7 +233,7 @@ impl WorkCursor {
drop_idx: 0, drop_idx: 0,
dropped_aliases: false, dropped_aliases: false,
root_idx: 0, root_idx: 0,
after: String::new(), after: (String::new(), String::new()),
finalized: false, finalized: false,
deleted: 0, deleted: 0,
recontented: 0, recontented: 0,
@ -352,16 +352,19 @@ pub fn advance(
return Ok(()); return Ok(());
} }
let root = &cursor.scope.roots[cursor.root_idx]; let root = &cursor.scope.roots[cursor.root_idx];
if cursor.after.is_empty() { if cursor.after.0.is_empty() {
cursor.after = root.lo.clone(); // `(lo, "")` sorts below every row in the range: no stored name
// is empty, so the first page starts exactly at `lo`.
cursor.after = (root.lo.clone(), String::new());
} }
let rows = repo::rows_in_range_page(conn, &cursor.after, &root.hi, page)?; let rows =
repo::rows_in_range_page(conn, &cursor.after.0, &cursor.after.1, &root.hi, page)?;
let Some(last) = rows.last() else { let Some(last) = rows.last() else {
cursor.root_idx += 1; cursor.root_idx += 1;
cursor.after.clear(); cursor.after = (String::new(), String::new());
continue; continue;
}; };
cursor.after = last.path.clone(); cursor.after = (last.parent.clone(), last.name.clone());
cursor.examined += rows.len(); cursor.examined += rows.len();
let root = cursor.scope.roots[cursor.root_idx].path.clone(); let root = cursor.scope.roots[cursor.root_idx].path.clone();
let (deleted, recontented) = apply_page( let (deleted, recontented) = apply_page(

4
crates/quicksearch-core/src/scope_tests.rs
View file

@ -45,12 +45,12 @@ fn seed(conn: &mut Connection, paths: &[PathBuf]) {
let tx = conn.transaction().unwrap(); let tx = conn.transaction().unwrap();
for path in paths { for path in paths {
let path = path.to_string_lossy(); let path = path.to_string_lossy();
let (parent, name) = path.rsplit_once('/').unwrap(); // The indexer's own split, so the separator stays with the parent.
let (parent, name) = crate::file_handling::split_db_path(&path).expect("a file's path");
repo::insert_file( repo::insert_file(
&tx, &tx,
&repo::NewFile { &repo::NewFile {
name, name,
path: &path,
parent, parent,
size: 1, size: 1,
mtime: 1, mtime: 1,

9
crates/quicksearch-core/src/search/cascade.rs
View file

@ -277,11 +277,14 @@ fn rank_order(a: &SearchHit, b: &SearchHit) -> std::cmp::Ordering {
} }
/// The `files` columns every pass selects, in the order the passes index /// The `files` columns every pass selects, in the order the passes index
/// them: `0` id, `1` name, `2` path, `3` size, `4` mtime. Passes that also /// them: `0` id, `1` name, `2` parent, `3` size, `4` mtime. Passes that also
/// want the stored document text append `dt.text_zstd` as column `5`. A pass /// want the stored document text append `dt.text_zstd` as column `5`. A pass
/// spelling its own list in a different order would compile and then quietly /// spelling its own list in a different order would compile and then quietly
/// serve paths as names. /// serve parents as names.
const HIT_COLUMNS: &str = "f.id, f.name, f.path, f.size, f.mtime"; ///
/// There is no `path` column to select; [`Cx::scan_pass`] concatenates columns
/// 2 and 1 into one reused buffer and hands every classifier the result.
const HIT_COLUMNS: &str = "f.id, f.name, f.parent, f.size, f.mtime";
/// Columns 3 and 4. The clamp matters: `size` is `INTEGER` in SQLite and so /// Columns 3 and 4. The clamp matters: `size` is `INTEGER` in SQLite and so
/// signed; a corrupt row holding `-1` would otherwise become 18 exabytes on /// signed; a corrupt row holding `-1` would otherwise become 18 exabytes on

149
crates/quicksearch-core/src/search/cascade/passes.rs
View file

@ -62,41 +62,72 @@ impl<'a> Cx<'a> {
let mut deferred = Deferred::default(); let mut deferred = Deferred::default();
let mut scanned = 0usize; let mut scanned = 0usize;
let mut clock = FlushClock::new(); let mut clock = FlushClock::new();
// Set when the loop stops on the display limit rather than on the end
// of the candidate set — rows were left unexamined, so the result set
// is cut whatever the final flush happens to hold.
let mut cut_short = false;
// The reassembled path for the row in hand. One buffer for the whole
// scan, refilled per row: `files` stores `parent` and `name` and no
// longer a third column holding their concatenation, so this is where
// the concatenation happens. Reused rather than allocated because it
// runs for every *scanned* row — a full-table scan on three of the
// passes — while only the few that become hits keep an owned copy.
//
// A local, not a field on `self`: the classifiers take `&mut Self`, and
// a buffer borrowed out of `self` could not be handed to them.
let mut path = String::new();
while let Some(row) = rows.next().map_err(|e| e.to_string())? { while let Some(row) = rows.next().map_err(|e| e.to_string())? {
scanned += 1; scanned += 1;
if scanned.is_multiple_of(cancel_every) && self.cancelled() { if scanned.is_multiple_of(cancel_every) && self.cancelled() {
return Ok(false); return Ok(false);
} }
// The display limit is already full, and holding a row proves at
// least one more match exists than will be shown — so `limited` is
// exactly true here, and everything below is work whose result
// `flush_pass` would throw away. That work is not small: the
// full-text passes decompress the document, fold a copy of it, and
// cut a snippet, per row. `cascade::run` makes the same test
// between passes; without this one a single pass over a common
// term runs to the end of the candidate set.
if self.remaining() == 0 {
self.limited = true;
break;
}
let file_id: i64 = col(row, 0)?; let file_id: i64 = col(row, 0)?;
// Borrowed from the statement rather than `col::<String>`: this let borrowed = |idx| -> Result<&str, String> {
// runs for every *scanned* row — a full-table scan on the filename row.get_ref(idx)
// pass — while only the few that become hits need an owned copy. .map_err(|e| e.to_string())?
let path = row .as_str()
.get_ref(2) .map_err(|e| e.to_string())
.map_err(|e| e.to_string())? };
.as_str() path.clear();
.map_err(|e| e.to_string())?; // Parent first, and no separator between them: every stored parent
if self.skip(file_id, path) { // already ends in one. See `file_handling::dir_to_db_parent`.
path.push_str(borrowed(2)?);
path.push_str(borrowed(1)?);
if self.skip(file_id, &path) {
continue; continue;
} }
match classify(self, row, file_id, path)? { match classify(self, row, file_id, &path)? {
RowHit::Skip => {} RowHit::Skip => {}
RowHit::Emit(hit) => { RowHit::Emit(hit) => {
buf.push(hit); buf.push(hit);
overflowed |= self.enforce_cap(&mut buf); overflowed |= self.enforce_cap(&mut buf);
self.flush_if_due(&mut buf, &mut clock); self.flush_if_due(&mut buf, &mut clock);
// Stop once the display limit is full. Everything past
// this point is work `flush_pass` would throw away, and it
// is not small: the full-text passes decompress the
// document, fold a copy of it and cut a snippet, per row.
// `cascade::run` makes the same test between passes;
// without this one a single pass over a common term runs
// to the end of its candidate set.
//
// Tested *here* rather than at the top of the loop, where
// it is tempting: a scanned row is not a match. It may be
// one an earlier pass already emitted, or one `classify`
// rejects — the filename pass's SQL is deliberately a
// superset feeding both the name and the path tier — so
// stopping on one would end the pass over a row that was
// never going to be shown.
//
// The break is recorded rather than turned into `limited`
// on the spot. `flush_pass` decides that, because it knows
// how much it had to truncate; claiming it here would say
// a set was cut whenever it happened to end exactly at the
// limit. But leaving the scan *unfinished* silently is the
// opposite error — see the flush below.
if self.remaining() == 0 {
cut_short = true;
break;
}
} }
RowHit::Defer(hit) => { RowHit::Defer(hit) => {
deferred.hits.push(hit); deferred.hits.push(hit);
@ -114,6 +145,15 @@ impl<'a> Cx<'a> {
None => debug_assert!(deferred.hits.is_empty(), "deferred hits with no slot"), None => debug_assert!(deferred.hits.is_empty(), "deferred hits with no slot"),
} }
self.flush_pass(buf, overflowed); self.flush_pass(buf, overflowed);
// After the flush, so it cannot be undone by one that truncated
// nothing. A scan that broke on the limit left rows unexamined, and
// the two places that would otherwise notice both miss the boundary
// case: `flush_pass` compares `buf.len() > room`, which is `0 > 0`
// when a mid-scan flush landed exactly on the limit, and
// `cascade::run` tests `remaining()` at the top of the *next* pass,
// of which the last pass has none. Without this a cut result set can
// report itself complete.
self.limited |= cut_short;
Ok(true) Ok(true)
} }
@ -123,26 +163,57 @@ impl<'a> Cx<'a> {
let query = self.query; let query = self.query;
let pattern = &query.pattern; let pattern = &query.pattern;
let with_paths = path_tiers_enabled(pattern); let with_paths = path_tiers_enabled(pattern);
// A path always ends in its own name, so `path LIKE` is the
// superset that feeds both the name and the path tiers.
let sql = format!(
"SELECT {} FROM files f \
WHERE {} LIKE ? ESCAPE '\\'{}",
HIT_COLUMNS,
if with_paths { "f.path" } else { "f.name" },
query.filter_sql
);
// Wildcard patterns turn each star into an unescaped `%`; the // Wildcard patterns turn each star into an unescaped `%`; the
// substring wrap absorbs leading/trailing stars. User `%`/`_` // substring wrap absorbs leading/trailing stars. User `%`/`_`
// remain escaped literals either way. // remain escaped literals either way.
let like = pattern let like = format!(
.segments() "%{}%",
.iter() pattern
.map(|s| escape_like(s)) .segments()
.collect::<Vec<_>>() .iter()
.join("%"); .map(|s| escape_like(s))
let params = .collect::<Vec<_>>()
self.params_with_filters(vec![rusqlite::types::Value::Text(format!("%{}%", like))]); .join("%")
);
// There is no `path` column to `LIKE` against any more, and the
// prefilter has to stay a *superset* of what the classifier accepts or
// real hits vanish. Three cases:
//
// * Name tier only — unchanged, `name LIKE`.
// * Path tiers, and the pattern is a single segment containing no
// separator: `name LIKE ? OR parent LIKE ?` is then exactly
// equivalent to the old `path LIKE ?`. A match inside `parent || name`
// either sits wholly in one or straddles the boundary, and the
// boundary character is a separator the pattern does not contain. This
// is ordinary typing, so it is the case worth keeping cheap.
// * Anything else — a multi-segment wildcard whose `%` can span the
// boundary (`doc*q3` over `/x/docs/q3.txt`), or a term with a
// separator in it. No SQL predicate on one column covers those, so
// scan and let the classifier decide, exactly as passes C and E do.
let straddles = pattern.segments().len() > 1
|| pattern
.segments()
.iter()
.any(|s| s.contains(std::path::MAIN_SEPARATOR));
let (predicate, terms) = match (with_paths, straddles) {
(false, _) => (
"f.name LIKE ? ESCAPE '\\'".to_string(),
vec![rusqlite::types::Value::Text(like)],
),
(true, false) => (
"(f.name LIKE ? ESCAPE '\\' OR f.parent LIKE ? ESCAPE '\\')".to_string(),
vec![
rusqlite::types::Value::Text(like.clone()),
rusqlite::types::Value::Text(like),
],
),
(true, true) => ("1=1".to_string(), Vec::new()),
};
let sql = format!(
"SELECT {} FROM files f WHERE {}{}",
HIT_COLUMNS, predicate, query.filter_sql
);
let params = self.params_with_filters(terms);
self.scan_pass( self.scan_pass(
&sql, &sql,
params, params,

40
crates/quicksearch-core/src/search/duplicates.rs
View file

@ -57,16 +57,25 @@ pub fn find_duplicate_groups(
} }
} }
// `ORDER BY parent, name` rather than by path: it is the stored key, so
// the sort is free, and it groups a directory's copies together — which is
// what the tab shows anyway.
let mut member_stmt = conn let mut member_stmt = conn
.prepare("SELECT id, name, path, size, mtime FROM files WHERE hash = ?1 ORDER BY path") .prepare(
"SELECT id, name, parent, size, mtime FROM files \
WHERE hash = ?1 ORDER BY parent, name",
)
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
for group in &mut groups { for group in &mut groups {
let rows = member_stmt let rows = member_stmt
.query_map(params![group.hash], |r| { .query_map(params![group.hash], |r| {
let name: String = r.get(1)?;
let parent: String = r.get(2)?;
let path = format!("{}{}", parent, name);
Ok(( Ok((
r.get::<_, i64>(0)?, r.get::<_, i64>(0)?,
r.get::<_, String>(1)?, name,
r.get::<_, String>(2)?, path,
r.get::<_, i64>(3)?.max(0) as u64, r.get::<_, i64>(3)?.max(0) as u64,
r.get::<_, i64>(4)?, r.get::<_, i64>(4)?,
)) ))
@ -91,13 +100,12 @@ mod tests {
let p = crate::testutil::scratch_dir("dups").join("index.sqlite"); let p = crate::testutil::scratch_dir("dups").join("index.sqlite");
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let tx = conn.transaction().unwrap(); let tx = conn.transaction().unwrap();
let add = |name: &str, path: &str, size: u64, hash: Option<&[u8]>| { let add = |name: &str, size: u64, hash: Option<&[u8]>| {
insert_file( insert_file(
&tx, &tx,
&NewFile { &NewFile {
name, name,
path, parent: "/d/",
parent: "/d",
size, size,
mtime: 1_700_000_000, mtime: 1_700_000_000,
mime: None, mime: None,
@ -110,20 +118,20 @@ mod tests {
.expect("unique path"); .expect("unique path");
}; };
// Triple group of small files: redundant = 10 × 2 = 20. // Triple group of small files: redundant = 10 × 2 = 20.
add("a1.txt", "/d/a1.txt", 10, Some(b"AAA")); add("a1.txt", 10, Some(b"AAA"));
add("a2.txt", "/d/a2.txt", 10, Some(b"AAA")); add("a2.txt", 10, Some(b"AAA"));
add("a3.txt", "/d/a3.txt", 10, Some(b"AAA")); add("a3.txt", 10, Some(b"AAA"));
// Pair of large files: redundant = 100 × 1 = 100 — sorts first // Pair of large files: redundant = 100 × 1 = 100 — sorts first
// despite the smaller member count. // despite the smaller member count.
add("b1.txt", "/d/b1.txt", 100, Some(b"BBB")); add("b1.txt", 100, Some(b"BBB"));
add("b2.txt", "/d/b2.txt", 100, Some(b"BBB")); add("b2.txt", 100, Some(b"BBB"));
// Singletons and NULL hashes never appear. // Singletons and NULL hashes never appear.
add("c.txt", "/d/c.txt", 30, Some(b"CCC")); add("c.txt", 30, Some(b"CCC"));
add("n1.txt", "/d/n1.txt", 40, None); add("n1.txt", 40, None);
add("n2.txt", "/d/n2.txt", 40, None); add("n2.txt", 40, None);
// Zero-size files are trivially identical — excluded outright. // Zero-size files are trivially identical — excluded outright.
add("z1.txt", "/d/z1.txt", 0, Some(b"ZZZ")); add("z1.txt", 0, Some(b"ZZZ"));
add("z2.txt", "/d/z2.txt", 0, Some(b"ZZZ")); add("z2.txt", 0, Some(b"ZZZ"));
tx.commit().unwrap(); tx.commit().unwrap();
drop(conn); drop(conn);
p p

97
crates/quicksearch-core/src/search/mod.rs
View file

@ -187,11 +187,34 @@ struct SearchRequest {
/// its statement, tagged with the generation that owns it. /// its statement, tagged with the generation that owns it.
type InFlight = Arc<Mutex<Option<(u64, rusqlite::InterruptHandle)>>>; type InFlight = Arc<Mutex<Option<(u64, rusqlite::InterruptHandle)>>>;
/// One-shot channel the worker answers a release request on.
type ReleaseAck = Arc<Mutex<Option<mpsc::Sender<()>>>>;
/// How long [`SearchService::release_connection`] waits for the worker to
/// answer. Long enough for it to finish a statement and drop the handle,
/// short enough that a wedged worker cannot hold up a rebuild.
const RELEASE_WAIT: Duration = Duration::from_secs(2);
/// What the worker thread accepts. A search is the overwhelming majority;
/// the release exists because dropping the held connection is something only
/// the worker can do, and it is parked in a 30-second `recv_timeout` where a
/// flag would not reach it.
enum WorkerMsg {
Search(SearchRequest),
/// Drop the held connection now, so whoever asked can delete or replace
/// the index file. On Windows an open handle makes that fail outright;
/// everywhere else it keeps the deleted inode's blocks pinned.
ReleaseConnection,
}
pub struct SearchService { pub struct SearchService {
req_tx: mpsc::Sender<SearchRequest>, req_tx: mpsc::Sender<WorkerMsg>,
latest_gen: Arc<AtomicU64>, latest_gen: Arc<AtomicU64>,
in_flight: InFlight, in_flight: InFlight,
db_path: Arc<Mutex<PathBuf>>, db_path: Arc<Mutex<PathBuf>>,
/// Where the worker reports that it has let the connection go; see
/// [`SearchService::release_connection`].
release_ack: ReleaseAck,
handle: Option<JoinHandle<()>>, handle: Option<JoinHandle<()>>,
} }
@ -214,11 +237,12 @@ impl SearchService {
notify: Arc<dyn Fn() + Send + Sync>, notify: Arc<dyn Fn() + Send + Sync>,
idle_release: Duration, idle_release: Duration,
) -> (SearchService, mpsc::Receiver<SearchUpdate>) { ) -> (SearchService, mpsc::Receiver<SearchUpdate>) {
let (req_tx, req_rx) = mpsc::channel::<SearchRequest>(); let (req_tx, req_rx) = mpsc::channel::<WorkerMsg>();
let (update_tx, update_rx) = mpsc::channel::<SearchUpdate>(); let (update_tx, update_rx) = mpsc::channel::<SearchUpdate>();
let latest_gen = Arc::new(AtomicU64::new(0)); let latest_gen = Arc::new(AtomicU64::new(0));
let in_flight: InFlight = Arc::new(Mutex::new(None)); let in_flight: InFlight = Arc::new(Mutex::new(None));
let db_path = Arc::new(Mutex::new(db_path)); let db_path = Arc::new(Mutex::new(db_path));
let release_ack: ReleaseAck = Arc::new(Mutex::new(None));
let worker = Worker { let worker = Worker {
req_rx, req_rx,
@ -227,6 +251,7 @@ impl SearchService {
latest_gen: latest_gen.clone(), latest_gen: latest_gen.clone(),
in_flight: in_flight.clone(), in_flight: in_flight.clone(),
db_path: db_path.clone(), db_path: db_path.clone(),
release_ack: release_ack.clone(),
open: None, open: None,
idle_release, idle_release,
}; };
@ -241,6 +266,7 @@ impl SearchService {
latest_gen, latest_gen,
in_flight, in_flight,
db_path, db_path,
release_ack,
handle: Some(handle), handle: Some(handle),
}, },
update_rx, update_rx,
@ -254,14 +280,42 @@ impl SearchService {
// Interrupt before enqueueing: an idle worker can dequeue the new // Interrupt before enqueueing: an idle worker can dequeue the new
// request and be mid-statement within microseconds. // request and be mid-statement within microseconds.
self.interrupt_stale(); self.interrupt_stale();
let _ = self.req_tx.send(SearchRequest { let _ = self.req_tx.send(WorkerMsg::Search(SearchRequest {
generation, generation,
input: input.to_string(), input: input.to_string(),
options, options,
}); }));
generation generation
} }
/// Drop the connection the worker is holding, and wait briefly for it.
///
/// The worker keeps its connection for [`IDLE_RELEASE`] after the last
/// query so a typing session runs against a warm page cache. That is the
/// right default and the wrong thing to be holding when the index file is
/// about to be deleted: on Windows the delete fails outright, and a
/// "Rebuild index" that silently did not rebuild is worse than a slow one.
///
/// Cancelled first, and not merely as a courtesy. The worker coalesces its
/// queue, so a search enqueued just before this — a keystroke, then a
/// click on *Rebuild index* — is still current when the release is found,
/// and the worker would acknowledge the release and then **reopen** the
/// connection to serve it. Bumping the generation makes that request stale,
/// so it is dropped at the generation check instead. Results against a file
/// about to be deleted are worth nothing anyway.
///
/// Best-effort past that — it returns once the worker acknowledges or the
/// wait elapses, and the caller proceeds either way.
pub fn release_connection(&self) {
self.cancel();
let (ack_tx, ack_rx) = mpsc::channel();
*crate::lock_ok(&self.release_ack) = Some(ack_tx);
if self.req_tx.send(WorkerMsg::ReleaseConnection).is_err() {
return;
}
let _ = ack_rx.recv_timeout(RELEASE_WAIT);
}
/// Cancel without starting anything new. /// Cancel without starting anything new.
pub fn cancel(&self) { pub fn cancel(&self) {
self.latest_gen.fetch_add(1, Ordering::SeqCst); self.latest_gen.fetch_add(1, Ordering::SeqCst);
@ -320,12 +374,13 @@ pub fn classify_sql_err(error_msg: &str) -> String {
} }
struct Worker { struct Worker {
req_rx: mpsc::Receiver<SearchRequest>, req_rx: mpsc::Receiver<WorkerMsg>,
update_tx: mpsc::Sender<SearchUpdate>, update_tx: mpsc::Sender<SearchUpdate>,
notify: Arc<dyn Fn() + Send + Sync>, notify: Arc<dyn Fn() + Send + Sync>,
latest_gen: Arc<AtomicU64>, latest_gen: Arc<AtomicU64>,
in_flight: InFlight, in_flight: InFlight,
db_path: Arc<Mutex<PathBuf>>, db_path: Arc<Mutex<PathBuf>>,
release_ack: ReleaseAck,
/// The connection, and the index generation and path it was opened /// The connection, and the index generation and path it was opened
/// against. See [`Worker::take_connection`]. /// against. See [`Worker::take_connection`].
open: Option<OpenIndex>, open: Option<OpenIndex>,
@ -345,7 +400,11 @@ impl Worker {
fn run(mut self) { fn run(mut self) {
loop { loop {
let first = match self.req_rx.recv_timeout(self.idle_release) { let first = match self.req_rx.recv_timeout(self.idle_release) {
Ok(req) => req, Ok(WorkerMsg::ReleaseConnection) => {
self.release();
continue;
}
Ok(WorkerMsg::Search(req)) => req,
Err(mpsc::RecvTimeoutError::Timeout) => { Err(mpsc::RecvTimeoutError::Timeout) => {
// Dropping the connection frees `PRAGMAS_SEARCH`'s 32 MiB // Dropping the connection frees `PRAGMAS_SEARCH`'s 32 MiB
// page cache to glibc, which parks it in an arena rather // page cache to glibc, which parks it in an arena rather
@ -366,10 +425,19 @@ impl Worker {
Err(mpsc::RecvTimeoutError::Disconnected) => return, Err(mpsc::RecvTimeoutError::Disconnected) => return,
}; };
// A fast typist queues several requests; only the newest one // A fast typist queues several requests; only the newest one
// matters. // matters. A release queued behind them is *not* superseded — it
// is the one message that must not be coalesced away, so it is
// handled where it is found. Whether the search below still runs
// is then the generation check's business:
// `release_connection` cancels before it sends, so anything
// enqueued ahead of the release is stale and drops out rather than
// reopening the connection that was just let go of.
let mut req = first; let mut req = first;
while let Ok(newer) = self.req_rx.try_recv() { while let Ok(newer) = self.req_rx.try_recv() {
req = newer; match newer {
WorkerMsg::Search(newer) => req = newer,
WorkerMsg::ReleaseConnection => self.release(),
}
} }
if req.generation != self.latest_gen.load(Ordering::SeqCst) { if req.generation != self.latest_gen.load(Ordering::SeqCst) {
continue; continue;
@ -378,6 +446,19 @@ impl Worker {
} }
} }
/// Drop the held connection and tell whoever asked.
///
/// The same trim `run`'s idle path does, for the same reason: glibc parks
/// `PRAGMAS_SEARCH`'s page cache in an arena rather than returning it.
fn release(&mut self) {
if self.open.take().is_some() {
crate::platform::release_free_heap();
}
if let Some(ack) = crate::lock_ok(&self.release_ack).take() {
let _ = ack.send(());
}
}
/// Take the connection to run this request on, reopening if the one held /// Take the connection to run this request on, reopening if the one held
/// cannot be reused. Reuse is what keeps /// cannot be reused. Reuse is what keeps
/// [`crate::db::schema::PRAGMAS_SEARCH`]'s page cache warm across /// [`crate::db::schema::PRAGMAS_SEARCH`]'s page cache warm across

76
crates/quicksearch-core/src/testutil.rs
View file

@ -121,6 +121,56 @@ pub fn touch(path: &std::path::Path, body: &[u8]) {
std::fs::write(path, body).expect("write file"); std::fs::write(path, body).expect("write file");
} }
/// A filename that is legal on disk but cannot round-trip through the index,
/// spelled so that `to_string_lossy` yields exactly `{stem}\u{FFFD}{suffix}` —
/// which is itself a perfectly ordinary filename, and so a name a *different*
/// file can really have. That collision is what the screens in
/// `crate::walk::read_directory` and `crate::watcher` exist to prevent, and
/// pairing this with [`lossy_twin`] is how the tests reproduce it.
///
/// The two platforms fail in different ways and both are real:
///
/// * On Unix an `OsStr` is arbitrary bytes, so any invalid UTF-8 byte does it.
/// `0xFF` can never appear in well-formed UTF-8.
/// * On Windows a path is UTF-16 code units and NTFS does not check that they
/// are well-*formed*, so an unpaired surrogate is storable. Rust models this
/// with WTF-8, and `to_str()` returns `None` for precisely that case. Far
/// from theoretical: WSL's DrvFs encodes non-UTF-8 Linux names this way by
/// design, and Samba shares of Linux servers produce them from legacy
/// encodings.
///
/// Some filesystems (FAT, exFAT, some network redirectors) refuse the name —
/// tests that put one on disk must tolerate the creation failing rather than
/// asserting on it.
#[doc(hidden)]
pub fn unrepresentable_name(stem: &str, suffix: &str) -> std::ffi::OsString {
#[cfg(unix)]
{
use std::os::unix::ffi::OsStringExt;
let mut bytes = stem.as_bytes().to_vec();
bytes.push(0xFF);
bytes.extend_from_slice(suffix.as_bytes());
std::ffi::OsString::from_vec(bytes)
}
#[cfg(windows)]
{
use std::os::windows::ffi::OsStringExt;
let mut units: Vec<u16> = stem.encode_utf16().collect();
// A high surrogate with nothing after it to pair with.
units.push(0xD800);
units.extend(suffix.encode_utf16());
std::ffi::OsString::from_wide(&units)
}
}
/// The name [`unrepresentable_name`] collapses to under `to_string_lossy`, as
/// a name that is genuinely representable — so a test can put both on disk and
/// assert the real one survives what happens to the other.
#[doc(hidden)]
pub fn lossy_twin(stem: &str, suffix: &str) -> String {
format!("{}\u{FFFD}{}", stem, suffix)
}
/// Power-of-two bucket, so memory-map sizes group by what allocated them /// Power-of-two bucket, so memory-map sizes group by what allocated them
/// rather than by their exact size. Shared by the memory probes. /// rather than by their exact size. Shared by the memory probes.
#[doc(hidden)] #[doc(hidden)]
@ -162,6 +212,32 @@ mod tests {
assert_eq!(std::fs::read(&deep).unwrap(), b"hi"); assert_eq!(std::fs::read(&deep).unwrap(), b"hi");
} }
/// The premise every collision test rests on, pinned per platform: the
/// name really is unrepresentable, and its lossy image really is a name
/// another file could have. If this ever stops holding, those tests would
/// silently start asserting nothing.
#[test]
fn the_unrepresentable_name_collapses_onto_its_twin() {
let bad = unrepresentable_name("x", ".txt");
assert!(
bad.to_str().is_none(),
"the name must not be representable: {:?}",
bad
);
assert_eq!(
bad.to_string_lossy(),
lossy_twin("x", ".txt"),
"the two names must collide under to_string_lossy"
);
assert!(
std::path::Path::new(&lossy_twin("x", ".txt"))
.as_os_str()
.to_str()
.is_some(),
"the twin must itself be a perfectly ordinary name"
);
}
/// The sweep runs against a shared temp directory, so what it matches is /// The sweep runs against a shared temp directory, so what it matches is
/// the whole safety argument. `quicksearch-capture` is the one that would /// the whole safety argument. `quicksearch-capture` is the one that would
/// hurt: `packaging/capture.sh` puts a run's screenshots and screencasts /// hurt: `packaging/capture.sh` puts a run's screenshots and screencasts

25
crates/quicksearch-core/src/textenc.rs
View file

@ -145,6 +145,31 @@ pub fn decode_text(bytes: Vec<u8>, path: &Path) -> Result<String, String> {
} }
} }
/// Replace control characters with `U+FFFD`, borrowing when there are none.
///
/// Filenames may contain any byte but NUL and the separator, and extracted
/// text is whatever was in the file — so both can carry terminal escape
/// sequences. Printed raw they rewrite the line, retitle the window, or on a
/// terminal with OSC 52 enabled put text of the writer's choosing on the
/// user's clipboard. `ls` has scrubbed for this reason for decades.
///
/// Tab survives: it is a legitimate part of a filename and harmless. So does
/// everything above C1 — this is not a general sanitiser, and the point is to
/// stay byte-for-byte faithful wherever there is nothing dangerous to remove.
pub fn scrub_controls(s: &str) -> std::borrow::Cow<'_, str> {
fn dangerous(c: char) -> bool {
(c.is_control() && c != '\t') || ('\u{80}'..='\u{9f}').contains(&c)
}
if !s.chars().any(dangerous) {
return std::borrow::Cow::Borrowed(s);
}
std::borrow::Cow::Owned(
s.chars()
.map(|c| if dangerous(c) { '\u{fffd}' } else { c })
.collect(),
)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;

173
crates/quicksearch-core/src/walk.rs
View file

@ -26,8 +26,8 @@ use sha2::{Digest, Sha256};
use crate::config::{Config, IgnoreSet}; use crate::config::{Config, IgnoreSet};
use crate::extract::Registry; use crate::extract::Registry;
use crate::file_handling::{ use crate::file_handling::{
classify_by_mtime, classify_for_indexing, path_to_db_string, prepare_file_record, classify_by_mtime, classify_for_indexing, dir_to_db_parent, path_to_db_string,
warn_if_unrepresentable, DirRows, FileIndexAction, OwnedNewFile, UnreadableDirs, prepare_file_record, DirRows, FileIndexAction, OwnedNewFile, UnreadableDirs,
}; };
mod pool; mod pool;
@ -53,7 +53,8 @@ const NETWORK_THREADS: usize = 16;
/// One file the walk found, with everything the DB writer needs. /// One file the walk found, with everything the DB writer needs.
#[derive(Debug)] #[derive(Debug)]
pub struct WalkedFile { pub struct WalkedFile {
/// Canonical path, and the `files.path` key. /// Canonical path. The row it keys is `(parent, name)`; see
/// [`crate::file_handling::split_db_path`].
pub path: String, pub path: String,
pub action: FileIndexAction, pub action: FileIndexAction,
/// `None` when there is nothing to write: unchanged, or the record could /// `None` when there is nothing to write: unchanged, or the record could
@ -164,6 +165,13 @@ struct Ctx {
follow_symlinks: bool, follow_symlinks: bool,
include_hidden: bool, include_hidden: bool,
ignore: IgnoreSet, ignore: IgnoreSet,
/// The index's own files, which this walk must never so much as open.
/// See [`crate::file_handling::index_file_set`] for why opening one is
/// fatal rather than merely wasteful.
///
/// Precomputed rather than derived per entry: it is one canonicalize, and
/// the alternative is a syscall against every file in the tree.
index_files: HashSet<PathBuf>,
pruned: PruneCounts, pruned: PruneCounts,
config: Config, config: Config,
registry: Arc<Registry>, registry: Arc<Registry>,
@ -175,9 +183,16 @@ struct Ctx {
/// count is kept. Reset by [`reset_run_warnings`]. /// count is kept. Reset by [`reset_run_warnings`].
static UNREADABLE_WARNINGS: crate::log::Throttle = crate::log::Throttle::new(20); static UNREADABLE_WARNINGS: crate::log::Throttle = crate::log::Throttle::new(20);
/// Arm this module's per-run warning throttle. /// The same, for names that cannot round-trip through the index. Throttled
/// because a share can hold thousands of them: one legacy-encoded directory on
/// a Samba mount, or a `\\wsl.localhost\` tree, and every entry under it is a
/// separate occurrence.
static UNREPRESENTABLE_WARNINGS: crate::log::Throttle = crate::log::Throttle::new(20);
/// Arm this module's per-run warning throttles.
pub fn reset_run_warnings() { pub fn reset_run_warnings() {
UNREADABLE_WARNINGS.reset(); UNREADABLE_WARNINGS.reset();
UNREPRESENTABLE_WARNINGS.reset();
} }
/// Read one directory, apply the hidden/ignore rules, and split the result: /// Read one directory, apply the hidden/ignore rules, and split the result:
@ -204,7 +219,19 @@ fn read_directory(
if UNREADABLE_WARNINGS.allow() { if UNREADABLE_WARNINGS.allow() {
crate::log_warn!("cannot read {}: {}", dir.display(), e); crate::log_warn!("cannot read {}: {}", dir.display(), e);
} }
ctx.unreadable.record(dir.to_path_buf()); // "Gone" is not "could not look", and only the second one is a
// reason to distrust the walk. A directory deleted while the walk
// was in flight — a build tree, a browser cache — is a fact about
// the filesystem: its rows *should* fall to the stale sweep, and
// recording it here would both spare them and cost this root its
// stored walk count, which is what makes every later run pay for
// a second `find | wc` traversal of the whole tree.
//
// The same distinction the coordinator draws in `verb_for`:
// `NotFound` is unambiguous, every other errno is not.
if e.kind() != std::io::ErrorKind::NotFound {
ctx.unreadable.record(dir.to_path_buf());
}
return Vec::new(); return Vec::new();
} }
}; };
@ -231,13 +258,49 @@ fn read_directory(
}; };
let name = entry.file_name(); let name = entry.file_name();
let name = name.to_string_lossy(); // **The screen for names the index cannot spell, and the only one.**
//
// `files.name` and `files.parent` are TEXT, so a name that is not valid
// UTF-8 has no representation there; `to_string_lossy` would give one,
// but it is many-to-one, and every use of a path in this walk is a
// database *key*. A lossy name collides with the real name of a
// different file — U+FFFD is an ordinary filename character — and the
// collision is not a cosmetic one: the lossy parent makes the
// prefetcher hand this directory another directory's rows, and the diff
// below then reports all of them stale. Screening here, before any
// string is built, is what keeps that from being possible at all.
//
// On Unix this is any invalid byte sequence; on Windows it is an
// unpaired UTF-16 surrogate, which NTFS stores happily and WSL's DrvFs
// emits by design for non-UTF-8 Linux names.
//
// A directory is pruned whole and at its root: the join below would
// carry the bad component into every path beneath it, so nothing under
// it could be indexed either way, and stopping here costs one warning
// instead of one per descendant.
//
// Leaving the entry out of `present` is safe, despite the rule above
// that every `continue` must be a genuine "not indexable". No stored
// row can carry a name that is not valid UTF-8, so this entry has no
// row to protect; and a row whose name happens to *equal* the lossy
// spelling belongs to some other, representable entry, which this same
// listing yields separately and which marks itself present.
let Some(name) = name.to_str() else {
if UNREPRESENTABLE_WARNINGS.allow() {
crate::log_warn!(
"Skipping {:?} (name is not valid UTF-8, so it cannot be stored, hashed \
or text-indexed)",
entry.path()
);
}
continue;
};
// The closure runs only on Windows, where `entry.metadata()` is free — // The closure runs only on Windows, where `entry.metadata()` is free —
// the attributes came back with the directory read, and it reports the // the attributes came back with the directory read, and it reports the
// entry itself rather than a link target. // entry itself rather than a link target.
if !ctx.include_hidden { if !ctx.include_hidden {
if let Some(reason) = if let Some(reason) =
crate::platform::entry_hidden_reason(&name, || entry.metadata().ok()) crate::platform::entry_hidden_reason(name, || entry.metadata().ok())
{ {
match reason { match reason {
crate::platform::HiddenReason::DotPrefix => { crate::platform::HiddenReason::DotPrefix => {
@ -260,7 +323,7 @@ fn read_directory(
continue; continue;
} }
} }
if ctx.ignore.matches_component(&name) { if ctx.ignore.matches_component(name) {
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed); ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue; continue;
} }
@ -269,6 +332,20 @@ fn read_directory(
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed); ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue; continue;
} }
// The index's own database and sidecars. Not a user preference and
// not overridable, because hashing one is not a slow row, it is a
// process-wide cancellation of SQLite's locks on a file we are in the
// middle of writing — see `file_handling::index_file_set`.
//
// `continue` rather than a `WalkedFile::skipped`, deliberately: this
// leaves the name out of `present`, so any row an earlier run wrote
// for the index — before this pruning existed, or from a spell when
// `database_path` pointed elsewhere — falls to the stale sweep and is
// deleted. `skipped` would keep it forever.
if ctx.index_files.contains(&path) {
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue;
}
// `file_type` is the cached `d_type` from the directory read. // `file_type` is the cached `d_type` from the directory read.
match entry.file_type() { match entry.file_type() {
@ -291,7 +368,33 @@ fn read_directory(
// which full-path ignore patterns would never match and // which full-path ignore patterns would never match and
// `seen_dirs` could not dedup against an overlapping root. // `seen_dirs` could not dedup against an overlapping root.
if let Ok(target) = path.canonicalize() { if let Ok(target) = path.canonicalize() {
// The link's own name passed the screen above; the target
// is a different path and gets its own. Without this, a
// link to an unrepresentable path would be resolved to its
// *lossy* spelling — a path naming some other file
// entirely, which would then be walked or indexed in its
// place.
if target.to_str().is_none() {
if UNREPRESENTABLE_WARNINGS.allow() {
crate::log_warn!(
"Skipping {} (its target {:?} is not valid UTF-8, so it cannot \
be stored, hashed or text-indexed)",
path.display(),
target
);
}
continue;
}
let target = PathBuf::from(path_to_db_string(&target)); let target = PathBuf::from(path_to_db_string(&target));
// Again on the resolved target: the check above tested the
// link's own name, and a symlink pointing at the index
// would otherwise walk straight past it into an `open`.
// Harmless for a directory target — the set holds only
// files — which is why one check covers both arms.
if ctx.index_files.contains(&target) {
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue;
}
match fs::metadata(&target) { match fs::metadata(&target) {
Ok(m) if m.is_dir() => found.push(Found::Dir(target)), Ok(m) if m.is_dir() => found.push(Found::Dir(target)),
// The target's row belongs to its own directory, so // The target's row belongs to its own directory, so
@ -302,7 +405,7 @@ fn read_directory(
} }
} }
Ok(_) => { Ok(_) => {
present.insert(name.into_owned()); present.insert(name.to_string());
// `None` on Unix and on any reparse point: see // `None` on Unix and on any reparse point: see
// `entry_cached_metadata`. // `entry_cached_metadata`.
let cached = crate::platform::entry_cached_metadata(|| entry.metadata().ok()); let cached = crate::platform::entry_cached_metadata(|| entry.metadata().ok());
@ -310,7 +413,7 @@ fn read_directory(
} }
// Type unknown: mark it present so an existing row survives. // Type unknown: mark it present so an existing row survives.
Err(_) => { Err(_) => {
present.insert(name.into_owned()); present.insert(name.to_string());
} }
} }
} }
@ -318,7 +421,7 @@ fn read_directory(
if !unreadable_entry { if !unreadable_entry {
// Rebuild each stored path the way `prepare` does, by joining onto // Rebuild each stored path the way `prepare` does, by joining onto
// the canonical directory, so separators and roots match the // the canonical directory, so separators and roots match the
// `files.path` spelling exactly. // stored spelling exactly.
stale.extend( stale.extend(
rows.keys() rows.keys()
.filter(|name| !present.contains(name.as_str())) .filter(|name| !present.contains(name.as_str()))
@ -365,16 +468,23 @@ pub fn path_digest(path: &str) -> u128 {
/// see [`crate::platform::metadata_or_stat`]. /// see [`crate::platform::metadata_or_stat`].
fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile { fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
let PendingFile { path, cached } = file; let PendingFile { path, cached } = file;
// Every route here has already screened the path: a root is screened after
// canonicalizing (which can resolve onto a name the config string was
// not), `Job::Files` comes from a listing `read_directory` filtered, and
// `Job::Alias` from a target the symlink arm checked. That
// matters because `path_to_db_string` is lossy, and the string below is
// used as a database key *and* hashed into the run's duplicate-visit set —
// a lossy one would key another file's row and could consume its digest,
// silently dropping it from the index.
debug_assert!(
path.to_str().is_some(),
"an unrepresentable path reached prepare(): {:?}",
path
);
let db_path = path_to_db_string(&path); let db_path = path_to_db_string(&path);
let digest = path_digest(&db_path); let digest = path_digest(&db_path);
let aliased = matches!(known, Known::Exact(_)); let aliased = matches!(known, Known::Exact(_));
// A name that is not valid UTF-8 cannot be stored in `files.path`. Emitted
// as `Skip` because the caller reads a missing path as "deleted".
if warn_if_unrepresentable(&path) {
return WalkedFile::skipped(db_path, digest, aliased);
}
let Ok(meta) = crate::platform::metadata_or_stat(&path, cached) else { let Ok(meta) = crate::platform::metadata_or_stat(&path, cached) else {
// Seen but unreadable: a transient stat failure must not read as // Seen but unreadable: a transient stat failure must not read as
// "deleted". // "deleted".
@ -391,11 +501,15 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
let action = match known { let action = match known {
Known::InDir(rows) => { Known::InDir(rows) => {
// `to_str`, not `to_string_lossy`: this name is looked up in the
// directory's stored rows, and the lossy spelling of one file is a
// valid name for another. The screen in `read_directory` is what
// makes it always `Some`.
let name = path let name = path
.file_name() .file_name()
.map(|n| n.to_string_lossy().into_owned()) .and_then(|n| n.to_str())
.unwrap_or_default(); .unwrap_or_default();
classify_for_indexing(&name, mtime, rows) classify_for_indexing(name, mtime, rows)
} }
Known::Exact(stored) => classify_by_mtime(stored, mtime), Known::Exact(stored) => classify_by_mtime(stored, mtime),
}; };
@ -494,7 +608,7 @@ fn prefetcher(shared: &Shared, db_path: &str) {
while let Some(work) = shared.take_prefetch() { while let Some(work) = shared.take_prefetch() {
match work { match work {
PrefetchWork::Dir(dir) => { PrefetchWork::Dir(dir) => {
match crate::db::repo::dir_rows(&conn, &path_to_db_string(&dir)) { match crate::db::repo::dir_rows(&conn, &dir_to_db_parent(&dir)) {
Ok(rows) => shared.finish_prefetch(Job::Dir(dir, Arc::new(rows))), Ok(rows) => shared.finish_prefetch(Job::Dir(dir, Arc::new(rows))),
Err(e) => { Err(e) => {
crate::log_warn!("{}", e); crate::log_warn!("{}", e);
@ -551,7 +665,7 @@ impl ParallelWalk {
crate::lock_ok(&self.shared.queue) crate::lock_ok(&self.shared.queue)
.seen_dirs .seen_dirs
.iter() .iter()
.map(|d| path_to_db_string(d)) .map(|d| dir_to_db_parent(d))
.collect() .collect()
} }
@ -712,6 +826,22 @@ pub fn walk_indexable_files(
// //
// Roots themselves are never filtered — the user chose them. // Roots themselves are never filtered — the user chose them.
match fs::canonicalize(root) { match fs::canonicalize(root) {
// A root string is UTF-8 by construction — it came from the config
// — but `canonicalize` resolves symlinks, so what it resolves *to*
// need not be: `~/docs` can be a link to a directory whose real
// name the index cannot spell. Stored lossily, the root would be
// walked under a parent string that names some other directory
// entirely. Treated exactly like a root that would not resolve at
// all, which is what it amounts to: yields nothing, and is recorded
// so stale cleanup does not read that as "everything was deleted".
Ok(dir) if dir.to_str().is_none() => {
crate::log_warn!(
"cannot index root {}: it resolves to {:?}, whose name is not valid UTF-8",
root,
dir
);
unresolvable.push(PathBuf::from(root));
}
Ok(dir) => { Ok(dir) => {
let dir = PathBuf::from(path_to_db_string(&dir)); let dir = PathBuf::from(path_to_db_string(&dir));
if queue.seen_dirs.insert(dir.clone()) { if queue.seen_dirs.insert(dir.clone()) {
@ -738,6 +868,7 @@ pub fn walk_indexable_files(
follow_symlinks, follow_symlinks,
include_hidden, include_hidden,
ignore, ignore,
index_files: crate::file_handling::index_file_set(Path::new(db_path)),
pruned: PruneCounts::default(), pruned: PruneCounts::default(),
config, config,
registry, registry,

355
crates/quicksearch-core/src/walk/tests.rs
View file

@ -14,16 +14,13 @@ fn db_with(tag: &str, rows: &[(String, u64)]) -> PathBuf {
let p = crate::testutil::scratch_dir(tag).join("index.sqlite"); let p = crate::testutil::scratch_dir(tag).join("index.sqlite");
let conn = crate::db::open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let conn = crate::db::open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
for (path, mtime) in rows { for (path, mtime) in rows {
let as_path = Path::new(path); // Split the same way the indexer does, so the seeded parent carries
// its trailing separator and the prefetcher's `parent = ?` finds it.
let (parent, name) = crate::file_handling::split_db_path(path).expect("a file's path");
conn.execute( conn.execute(
"INSERT INTO files (name, path, parent, size, mtime, type, content_state) "INSERT INTO files (name, parent, size, mtime, type, content_state)
VALUES (?1, ?2, ?3, 0, ?4, 0, 3)", VALUES (?1, ?2, 0, ?3, 0, 3)",
rusqlite::params![ rusqlite::params![name, parent, *mtime as i64],
as_path.file_name().unwrap().to_string_lossy(),
path,
as_path.parent().unwrap().to_string_lossy(),
*mtime as i64,
],
) )
.unwrap(); .unwrap();
} }
@ -108,35 +105,237 @@ fn walks_a_nested_tree_exactly_once() {
fs::remove_dir_all(&root).ok(); fs::remove_dir_all(&root).ok();
} }
/// A name that is not valid UTF-8 must be skipped, not deleted. Unix only: /// Put `name` on disk under `dir`, or report that this filesystem refused it.
/// on Windows `OsString` comes from UTF-16 and the case cannot be built. ///
#[cfg(unix)] /// FAT, exFAT and some network redirectors reject the names
#[test] /// [`crate::testutil::unrepresentable_name`] builds. A test that cannot create
fn a_non_utf8_name_is_skipped_and_never_prepared() { /// one has nothing to assert and says so, rather than failing on the
use std::ffi::OsStr; /// filesystem's behalf.
use std::os::unix::ffi::OsStrExt; fn try_touch(path: &Path) -> bool {
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, b"x").is_ok() && path.symlink_metadata().is_ok()
}
fn try_mkdir(path: &Path) -> bool {
fs::create_dir_all(path).is_ok() && path.symlink_metadata().is_ok()
}
/// A name the index cannot spell is dropped at the directory entry, before any
/// string is built from it — so it is neither prepared nor reported.
///
/// Both platforms, and the Windows half is not the exotic one: `OsString`
/// there is WTF-8 over UTF-16, and `OsString::from_wide(&[0xD800])` builds an
/// unpaired surrogate that NTFS stores happily. (An older comment here claimed
/// the case could not be built on Windows. It can, which is exactly why WSL
/// and Samba trees hit it.)
#[test]
fn a_non_utf8_name_is_dropped_at_the_entry() {
let root = tmp_tree("nonutf8"); let root = tmp_tree("nonutf8");
touch(&root.join("plain.txt")); touch(&root.join("plain.txt"));
// 0xFF only survives `to_string_lossy` as U+FFFD. let bad = root.join(crate::testutil::unrepresentable_name("DRH257", "~X.MP4"));
let bad = root.join(OsStr::from_bytes(b"DRH257\xff~X.MP4")); if !try_touch(&bad) {
touch(&bad); eprintln!("skipped: this filesystem will not store an unrepresentable name");
assert!(bad.symlink_metadata().is_ok(), "the file really is on disk"); return;
}
let files = walk(&root, &empty_db("nonutf8")); let files = walk(&root, &empty_db("nonutf8"));
// Both are yielded, so neither reads as deleted... // Not yielded at all — a `WalkedFile` for it would carry the lossy path as
assert_eq!(files.len(), 2, "the bad name is still reported as seen"); // its key and its duplicate-visit digest.
// ...but only the representable one is prepared for insertion. assert_eq!(names(&files), vec!["plain.txt"]);
let prepared: Vec<&WalkedFile> = files.iter().filter(|f| f.record.is_some()).collect();
assert_eq!(prepared.len(), 1);
assert!(prepared[0].path.ends_with("plain.txt"));
let skipped = files.iter().find(|f| f.record.is_none()).unwrap();
assert!(matches!(skipped.action, FileIndexAction::Skip));
assert!( assert!(
skipped.path.contains('\u{FFFD}'), files.iter().all(|f| !f.path.contains('\u{FFFD}')),
"stored spelling is the lossy one" "no lossy spelling may reach the writer: {:?}",
files.iter().map(|f| &f.path).collect::<Vec<_>>()
);
fs::remove_dir_all(&root).ok();
}
/// The safety argument for dropping the entry, pinned: a `continue` in
/// `read_directory` leaves the name out of `present`, and everything left out
/// of `present` is deleted.
///
/// It is safe because the dropped entry has no row of its own — an
/// unrepresentable name cannot be stored — and because the row that *is*
/// named its lossy spelling belongs to a different file, which the same
/// listing yields separately and which marks itself present. This test is what
/// says that second half still happens.
#[test]
fn dropping_a_bad_entry_deletes_nothing() {
let root = tmp_tree("nonutf8-stale");
let twin = crate::testutil::lossy_twin("DRH257", "~X.MP4");
touch(&root.join("plain.txt"));
touch(&root.join(&twin));
let bad = root.join(crate::testutil::unrepresentable_name("DRH257", "~X.MP4"));
if !try_touch(&bad) {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
// Seed both real files as already indexed, so anything the walk reports
// stale is a row it wants deleted.
let db = db_with(
"nonutf8-stale",
&[
(path_to_db_string(&root.join("plain.txt")), 0),
(path_to_db_string(&root.join(&twin)), 0),
],
);
let stale = stale_only(walk_indexable_files(
&[root.to_string_lossy().into_owned()],
false,
false,
IgnoreSet::compile(&[]).unwrap(),
db.to_str().unwrap(),
Config::default(),
Arc::new(Registry::default_set()),
Arc::new(AtomicBool::new(false)),
4,
));
assert!(
stale.is_empty(),
"both files are on disk; nothing may be deleted: {:?}",
stale
);
fs::remove_dir_all(&root).ok();
}
/// A root spelled in UTF-8 can still *resolve* onto a name that is not:
/// canonicalizing follows symlinks, so `~/docs` may be a link to a directory
/// the index cannot spell.
///
/// Stored lossily, that root's parent string names some other directory — so
/// the walk would attribute a completely unrelated tree to it, or read nothing
/// and report the root's real rows as deleted. It has to be recorded as
/// unreadable instead, which is what keeps stale cleanup off it.
#[cfg(unix)]
#[test]
fn a_root_that_resolves_onto_an_unrepresentable_name_is_not_walked() {
let base = tmp_tree("nonutf8-root");
let real = base.join(crate::testutil::unrepresentable_name("target", ""));
if !try_mkdir(&real) {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
touch(&real.join("inside.txt"));
// The root the user configures is an ordinary, spellable string.
let link = base.join("docs");
std::os::unix::fs::symlink(&real, &link).unwrap();
let root = link.to_str().expect("the configured root is spellable");
let mut walk = walk_indexable_files(
&[root.to_string()],
false,
false,
IgnoreSet::compile(&[]).unwrap(),
empty_db("nonutf8-root").to_str().unwrap(),
Config::default(),
Arc::new(Registry::default_set()),
Arc::new(AtomicBool::new(false)),
4,
);
let events: Vec<WalkEvent> = walk.by_ref().collect();
assert!(
events.is_empty(),
"nothing under an unspellable root may be walked: {:?}",
events
);
assert!(
!walk.unreadable().is_empty(),
"the root must read as unreadable, or its rows fall to stale cleanup"
);
std::fs::remove_dir_all(&base).ok();
}
/// The collision that made this worth fixing: a file whose *lossy* spelling is
/// another file's real name must not stand in for it.
///
/// Before the screen moved to the directory entry, the bad name was still
/// turned into a `WalkedFile` carrying the lossy path — and the pipeline hashes
/// that path into `seen_paths` before it looks for a record, so whichever of
/// the two the walk happened to reach second was dropped from the index.
#[test]
fn a_bad_name_cannot_stand_in_for_its_lossy_twin() {
let root = tmp_tree("nonutf8-twin");
let twin = crate::testutil::lossy_twin("x", ".txt");
touch(&root.join(&twin));
let bad = root.join(crate::testutil::unrepresentable_name("x", ".txt"));
if !try_touch(&bad) {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
let files = walk(&root, &empty_db("nonutf8-twin"));
let prepared: Vec<&WalkedFile> = files.iter().filter(|f| f.record.is_some()).collect();
assert_eq!(
prepared.len(),
1,
"exactly the representable file is prepared"
);
assert_eq!(
prepared[0].path,
path_to_db_string(&root.join(&twin)),
"and it is the real one, under its own name"
);
fs::remove_dir_all(&root).ok();
}
/// A directory the index cannot spell is pruned at its root: every path
/// beneath it would carry the bad component, so none of them is indexable
/// either way.
///
/// The reason this is the *directory* case and not just N file cases: the
/// walk's row prefetcher keys on the directory's lossy parent string, so a bad
/// directory sitting beside a real one with the colliding name was handed the
/// real one's rows — and then reported every one of them stale.
#[test]
fn a_bad_directory_is_pruned_with_its_whole_subtree() {
let root = tmp_tree("nonutf8-dir");
let twin = crate::testutil::lossy_twin("dir", "");
touch(&root.join(&twin).join("a.txt"));
let bad = root.join(crate::testutil::unrepresentable_name("dir", ""));
if !try_mkdir(&bad) {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
for n in 0..5 {
touch(&bad.join(format!("b{}.txt", n)));
}
touch(&bad.join("deep/c.txt"));
let db = db_with(
"nonutf8-dir",
&[(path_to_db_string(&root.join(&twin).join("a.txt")), 0)],
);
let stale = stale_only(walk_indexable_files(
&[root.to_string_lossy().into_owned()],
false,
false,
IgnoreSet::compile(&[]).unwrap(),
db.to_str().unwrap(),
Config::default(),
Arc::new(Registry::default_set()),
Arc::new(AtomicBool::new(false)),
4,
));
assert!(
stale.is_empty(),
"the real directory's row must survive its bad-named sibling: {:?}",
stale
);
let files = walk(&root, &db);
assert_eq!(
names(&files),
vec!["a.txt"],
"nothing under the bad directory is walked"
); );
fs::remove_dir_all(&root).ok(); fs::remove_dir_all(&root).ok();
@ -804,3 +1003,97 @@ fn local_temp_dir_is_not_detected_as_network() {
); );
fs::remove_dir_all(&root).ok(); fs::remove_dir_all(&root).ok();
} }
/// **The index must never be walked into.**
///
/// Hashing a file means opening it, and on Unix closing a descriptor on an
/// inode cancels every advisory lock the whole process holds on it. Doing that
/// to `index.sqlite-shm` destroys the DMS lock SQLite took, after which an
/// attaching connection truncates the wal-index under our live mapping and the
/// next commit dies with SIGBUS — which is the crash this pruning exists to
/// prevent. The same close cancels the database's own locks, SQLite's
/// documented corruption hazard.
///
/// `include_hidden` is on because that is what exposes the default layout:
/// the index lives under `~/.local/share`, which a default walk skips for
/// being dot-named — so the hazard is real but latent until a user turns
/// hidden files on.
#[test]
fn the_index_and_its_sidecars_are_never_walked() {
let root = tmp_tree("walk-self-index");
touch(&root.join("ordinary.txt"));
// The index inside the tree being walked, as it is by default: the
// default root is the home directory and the default database sits
// beneath it.
let db = root.join("data").join("index.sqlite");
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
// A write, so the WAL and SHM exist to be walked over.
conn.execute(
"INSERT INTO files (name, parent, size, mtime, type, content_state)
VALUES ('a', '/', 0, 0, 0, 3)",
[],
)
.unwrap();
let found = names(&walk_with(&root, &db, false, true));
assert!(
found.contains(&"ordinary.txt".to_string()),
"the walk should still report ordinary files: {:?}",
found
);
for name in ["index.sqlite", "index.sqlite-wal", "index.sqlite-shm"] {
assert!(
!found.contains(&name.to_string()),
"{} was walked; found {:?}",
name,
found
);
}
drop(conn);
}
/// Pruning it is not enough — a row written before this existed has to go, or
/// the live watcher and duplicate verification keep opening the file from a
/// result list forever.
///
/// This is why the walk `continue`s past the index rather than emitting a
/// `WalkedFile::skipped`: a skip keeps the row, and only leaving the name out
/// of the directory's `present` set hands it to the stale sweep.
#[test]
fn a_stored_row_for_the_index_falls_to_the_stale_sweep() {
let root = crate::testutil::scratch_dir_canonical("walk-self-index-stale");
// The database the walk uses has to be the one inside the walked tree,
// or there is nothing self-referential to sweep.
let db = root.join("data").join("index.sqlite");
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
// The row an older build would have written for the database itself,
// inserted exactly the way `db_with` inserts one.
let stored = path_to_db_string(&db);
let (parent, name) = crate::file_handling::split_db_path(&stored).expect("a file's path");
conn.execute(
"INSERT INTO files (name, parent, size, mtime, type, content_state)
VALUES (?1, ?2, 0, 0, 0, 3)",
rusqlite::params![name, parent],
)
.unwrap();
drop(conn);
let stale = stale_only(walk_indexable_files(
&[root.to_string_lossy().into_owned()],
false,
true,
IgnoreSet::compile(&[]).unwrap(),
db.to_str().unwrap(),
Config::default(),
Arc::new(Registry::default_set()),
Arc::new(AtomicBool::new(false)),
4,
));
assert!(
stale.contains(&stored),
"the index's own row should be swept; got {:?}",
stale
);
}

94
crates/quicksearch-core/src/watcher.rs
View file

@ -30,7 +30,7 @@
//! root looks live while going silently stale. A single directory the kernel //! root looks live while going silently stale. A single directory the kernel
//! refuses is logged and skipped instead (see [`add_watch`]). //! refuses is logged and skipped instead (see [`add_watch`]).
use std::collections::{HashMap, HashSet}; use std::collections::{BTreeSet, HashMap};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{mpsc, Arc, Mutex}; use std::sync::{mpsc, Arc, Mutex};
@ -97,6 +97,11 @@ pub enum WatchError {
KernelLimit { KernelLimit {
registered: usize, registered: usize,
}, },
/// The kernel's event queue overflowed and events were dropped. Unlike
/// the two above this says nothing about the watcher's *capacity* — it
/// keeps working — only that the index is now out of step with the disk
/// by an unknown amount, so a full run is owed.
Overflowed,
Other(String), Other(String),
} }
@ -122,6 +127,11 @@ impl std::fmt::Display for WatchError {
"" ""
} }
), ),
WatchError::Overflowed => write!(
f,
"the system event queue overflowed and changes were missed; \
reindexing to catch up"
),
WatchError::Other(msg) => write!(f, "{}", msg), WatchError::Other(msg) => write!(f, "{}", msg),
} }
} }
@ -156,8 +166,9 @@ pub struct WatcherConfig {
/// How often the tick loop inspects the throttle map. Short ticks mean /// How often the tick loop inspects the throttle map. Short ticks mean
/// low latency for first-in-a-burst; long ticks lower CPU at idle. /// low latency for first-in-a-burst; long ticks lower CPU at idle.
pub tick_interval: Duration, pub tick_interval: Duration,
/// Maximum directories processed per tick. Caps the time spent in a /// Maximum directories *selected* per tick. A ceiling on the work one
/// single flush pass so long backlogs don't monopolize the thread. /// pass takes on; [`FLUSH_BUDGET`] is what bounds how long it may spend
/// on them, and is the real limit.
pub max_dirs_per_tick: usize, pub max_dirs_per_tick: usize,
/// When to garbage-collect stale throttle entries (idle > window * N). /// When to garbage-collect stale throttle entries (idle > window * N).
pub prune_max_age_multiplier: u32, pub prune_max_age_multiplier: u32,
@ -172,7 +183,7 @@ impl Default for WatcherConfig {
pending_max_defer: Duration::from_secs(30), pending_max_defer: Duration::from_secs(30),
throttle_window: Duration::from_secs(30), throttle_window: Duration::from_secs(30),
tick_interval: Duration::from_millis(500), tick_interval: Duration::from_millis(500),
max_dirs_per_tick: 64, max_dirs_per_tick: 512,
prune_max_age_multiplier: 10, prune_max_age_multiplier: 10,
max_watched_dirs: DEFAULT_MAX_WATCHED_DIRS, max_watched_dirs: DEFAULT_MAX_WATCHED_DIRS,
} }
@ -187,10 +198,23 @@ impl Default for WatcherConfig {
/// never blocks behind a large subtree registration. /// never blocks behind a large subtree registration.
struct WatchRegistry { struct WatchRegistry {
raw: RecommendedWatcher, raw: RecommendedWatcher,
dirs: HashSet<PathBuf>, /// Ordered, not hashed, so [`WatchRegistry::remove_tree`] can take the
/// subtree as a range instead of scanning every watched directory. The
/// set reaches `max_watched_dirs` (128k by default) and `rm -rf` deletes
/// bottom-up, so every directory in a deleted tree hits that path.
dirs: BTreeSet<PathBuf>,
cap: usize, cap: usize,
} }
/// How long one flush pass may spend handing events to the sink.
///
/// Paired with `max_dirs_per_tick`: the count decides how many directories a
/// pass takes on, this decides when it stops regardless. A backlog then
/// drains at whatever the machine can actually do rather than at a fixed
/// directories-per-second, while one enormous directory still cannot hold the
/// tick loop.
const FLUSH_BUDGET: Duration = Duration::from_millis(50);
/// The mode every `watch()` call uses on this platform. See /// The mode every `watch()` call uses on this platform. See
/// [`crate::platform::WATCH_ROOTS_RECURSIVELY`] for why it differs. /// [`crate::platform::WATCH_ROOTS_RECURSIVELY`] for why it differs.
const WATCH_MODE: RecursiveMode = if crate::platform::WATCH_ROOTS_RECURSIVELY { const WATCH_MODE: RecursiveMode = if crate::platform::WATCH_ROOTS_RECURSIVELY {
@ -243,10 +267,15 @@ impl WatchRegistry {
if !self.dirs.contains(dir) { if !self.dirs.contains(dir) {
return 0; return 0;
} }
// A range from `dir`, stopping at the first entry that is no longer
// beneath it: descendants sort immediately after their ancestor, so
// this visits the subtree and one entry more, rather than the whole
// set. `starts_with` is still the test — it compares whole components,
// where a raw string prefix would take `/a/bc` for a child of `/a/b`.
let doomed: Vec<PathBuf> = self let doomed: Vec<PathBuf> = self
.dirs .dirs
.iter() .range(dir.to_path_buf()..)
.filter(|d| d.starts_with(dir)) .take_while(|d| d.starts_with(dir))
.cloned() .cloned()
.collect(); .collect();
for d in &doomed { for d in &doomed {
@ -311,6 +340,21 @@ impl Watcher {
let raw = RecommendedWatcher::new( let raw = RecommendedWatcher::new(
move |res: notify::Result<NotifyEvent>| match res { move |res: notify::Result<NotifyEvent>| match res {
Ok(ev) => { Ok(ev) => {
// An overflow of the kernel's own event queue arrives here
// — on the *Ok* arm, as `EventKind::Other` with the rescan
// flag and no paths at all — so the error arm below never
// sees it and the per-path loop downstream iterates zero
// times. Left alone it is silent data loss: an arbitrary
// set of creates, modifies and removes never reaches the
// index while the watcher goes on reporting itself
// healthy. It is reported rather than repaired here
// because the events are simply gone; only a full run can
// find out what happened.
if ev.need_rescan() {
let mut slot = crate::lock_ok(&degraded_cb);
slot.get_or_insert(WatchError::Overflowed);
return;
}
// A closed receiver just means the watcher was stopped; ignore. // A closed receiver just means the watcher was stopped; ignore.
let _ = tx.send(ev); let _ = tx.send(ev);
} }
@ -330,7 +374,7 @@ impl Watcher {
let registry = Arc::new(Mutex::new(WatchRegistry { let registry = Arc::new(Mutex::new(WatchRegistry {
raw, raw,
dirs: HashSet::new(), dirs: BTreeSet::new(),
cap: config.max_watched_dirs, cap: config.max_watched_dirs,
})); }));
@ -416,6 +460,19 @@ impl Watcher {
crate::lock_ok(&self.degraded).is_some() crate::lock_ok(&self.degraded).is_some()
} }
/// Forget the recorded reason, so a later one can take its place.
///
/// For [`WatchError::Overflowed`] only, and the distinction is the whole
/// point of the method: the other two reasons are *standing* — the watch
/// budget does not come back — while an overflow is a one-shot "you missed
/// some" from a watcher that is still delivering. Left in place it would
/// re-trigger on every coordinator tick and, worse, mask a real
/// [`WatchError::KernelLimit`] arriving afterwards, because the callback
/// records with `get_or_insert`.
pub fn clear_degraded(&self) {
*crate::lock_ok(&self.degraded) = None;
}
/// Signal the background thread to stop and wait for it to join. Safe to /// Signal the background thread to stop and wait for it to join. Safe to
/// call multiple times. /// call multiple times.
pub fn stop(&mut self) { pub fn stop(&mut self) {
@ -594,6 +651,16 @@ fn unwatch_tree(ctx: &LoopCtx, path: &Path) {
/// on which subtrees exist. Under a recursive root watch it is the *only* /// on which subtrees exist. Under a recursive root watch it is the *only*
/// thing keeping `node_modules` churn out of the throttle map. /// thing keeping `node_modules` churn out of the throttle map.
fn is_event_interesting(ctx: &LoopCtx, path: &Path) -> bool { fn is_event_interesting(ctx: &LoopCtx, path: &Path) -> bool {
// A path the index cannot spell, screened here because this is the one
// gate every `FsEvent` passes through. Such a file is never indexed, so
// there is no row for a Create to update and none for a Remove to delete —
// but the incremental side keys on `path_to_db_string`, which is lossy, so
// letting the event through means acting on whichever *different* file
// happens to own the lossy spelling. The event carries no information and
// every use of it is a mistake.
if path.to_str().is_none() {
return false;
}
if ctx.filters.ignore.matches_path(path) { if ctx.filters.ignore.matches_path(path) {
return false; return false;
} }
@ -696,7 +763,18 @@ fn flush_ready(
} }
} }
} }
// The count above bounds how many directories are *selected*; this bounds
// how long draining them may take, which is the thing that actually
// matters. Directory queues differ by orders of magnitude, so a fixed
// count is either too small after a large delete — at 64 per 500 ms tick
// the drain rate is 128 directories a second regardless of backlog, and a
// 20k-directory unpack takes minutes to reach the index — or too large for
// one deep directory.
let deadline = now + FLUSH_BUDGET;
for dir in ready { for dir in ready {
if Instant::now() >= deadline {
break;
}
if let Some(entry) = throttle.get_mut(&dir) { if let Some(entry) = throttle.get_mut(&dir) {
let drained: Vec<(PathBuf, QueuedOp)> = entry.queue.drain().collect(); let drained: Vec<(PathBuf, QueuedOp)> = entry.queue.drain().collect();
entry.immediate = false; entry.immediate = false;

69
crates/quicksearch-core/src/watcher_tests.rs
View file

@ -189,6 +189,75 @@ fn e2e_create_modify_remove_surfaces() {
std::fs::remove_dir_all(&dir).ok(); std::fs::remove_dir_all(&dir).ok();
} }
/// A file the index cannot spell must produce no event at all.
///
/// It is never indexed, so no row exists for a `Create` to update or a
/// `Remove` to delete — but the incremental side keys on `path_to_db_string`,
/// which is lossy, so an event that got through would be applied to whichever
/// *different* file owns the lossy spelling. Screened in
/// [`is_event_interesting`], the one gate every `FsEvent` passes through.
///
/// A real file is created alongside, so a run where the watcher simply saw
/// nothing cannot pass by accident.
#[test]
fn events_for_an_unrepresentable_name_never_surface() {
let dir = tmp_dir("e2e-unrepresentable");
let bad = dir.join(crate::testutil::unrepresentable_name("report", ".txt"));
if std::fs::write(&bad, "hi").is_err() {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
std::fs::remove_dir_all(&dir).ok();
return;
}
std::fs::remove_file(&bad).unwrap();
let (sink, got) = sink_to_vec();
let mut w = Watcher::start(
std::iter::once(&dir),
default_filters(),
fast_config(),
sink,
)
.unwrap();
std::fs::write(&bad, "hi").unwrap();
std::thread::sleep(Duration::from_millis(150));
std::fs::remove_file(&bad).unwrap();
// The control: whatever the backend does for the bad name, it certainly
// reports this one, so an empty event list means the watcher was working.
let good = dir.join("ordinary.txt");
std::fs::write(&good, "hi").unwrap();
std::thread::sleep(Duration::from_millis(250));
w.stop();
let events = got.lock().unwrap().clone();
assert!(
events.iter().any(|e| matches!(
e,
FsEvent::Create(p) | FsEvent::Modify(p) if p == &good
)),
"the control file produced no event, so this test proves nothing: {:?}",
events
);
let leaked: Vec<&FsEvent> = events
.iter()
.filter(|e| {
let paths: Vec<&PathBuf> = match e {
FsEvent::Create(p) | FsEvent::Modify(p) | FsEvent::Remove(p) => vec![p],
FsEvent::Rename { from, to } => vec![from, to],
};
paths.iter().any(|p| p.to_str().is_none())
})
.collect();
assert!(
leaked.is_empty(),
"an unrepresentable path reached the sink: {:?}",
leaked
);
std::fs::remove_dir_all(&dir).ok();
}
/// Descriptors are not spent on directories the indexer would discard. /// Descriptors are not spent on directories the indexer would discard.
#[test] #[test]
fn ignored_and_hidden_dirs_are_not_registered() { fn ignored_and_hidden_dirs_are_not_registered() {

182
crates/quicksearch-core/tests/cascade.rs
View file

@ -31,15 +31,18 @@ impl Seeder {
} }
/// Insert a file; `text: Some(..)` also content-indexes it. /// Insert a file; `text: Some(..)` also content-indexes it.
///
/// `dir` is spelled without a trailing separator, as a caller naturally
/// would; the stored parent always carries one, because that is what makes
/// the path `parent || name` (see `file_handling::dir_to_db_parent`).
fn add(&mut self, name: &str, dir: &str, mtime: u64, text: Option<&str>) -> i64 { fn add(&mut self, name: &str, dir: &str, mtime: u64, text: Option<&str>) -> i64 {
let path = format!("{}/{}", dir, name); let parent = format!("{}/", dir);
let tx = self.conn.transaction().unwrap(); let tx = self.conn.transaction().unwrap();
let id = insert_file( let id = insert_file(
&tx, &tx,
&NewFile { &NewFile {
name, name,
path: &path, parent: &parent,
parent: dir,
size: 42, size: 42,
mtime, mtime,
mime: Some("text/plain"), mime: Some("text/plain"),
@ -561,6 +564,45 @@ fn limit_truncates_and_flags() {
std::fs::remove_file(&p).ok(); std::fs::remove_file(&p).ok();
} }
/// Stopping at the limit must not cost a result that belongs in it.
///
/// The scan breaks out as soon as the display limit is full, which is what
/// keeps a common term from decompressing its whole candidate set. The break
/// sits after classification for a reason: a *scanned* row is not a match —
/// the filename pass's SQL is a superset feeding two rank tiers — so breaking
/// on one would end the pass early over a row that was never going to be
/// shown, and the hits below it would be lost.
#[test]
fn stopping_at_the_limit_keeps_the_best_ranked_hits() {
let p = tmp_db("limit-break");
let mut s = Seeder::new(&p, true);
// Exact-name matches rank above substring matches, and are seeded last so
// that a pass which stopped too early would miss them.
for i in 0..20 {
s.add(&format!("zz-match-{:02}.txt", i), "/d", 1, None);
}
s.add("match", "/d", 1, None);
let conn = s.done();
let options = SearchOptions {
limit: 2,
..SearchOptions::default()
};
let (hits, outcome) = run_collect(&conn, "match", &options);
assert_eq!(hits.len(), 2);
assert!(
outcome.limited,
"21 matches under a limit of 2 is a cut set"
);
assert_eq!(
hits[0].name, "match",
"the exact match must survive the break"
);
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test] #[test]
fn session_ignores_hide_hits_before_the_cap() { fn session_ignores_hide_hits_before_the_cap() {
let p = tmp_db("ignores"); let p = tmp_db("ignores");
@ -1318,12 +1360,60 @@ fn the_connection_is_released_once_searching_stops() {
std::fs::remove_file(&p).ok(); std::fs::remove_file(&p).ok();
} }
/// An explicit release must actually leave the file closed — including when a
/// search was queued a moment before it.
///
/// The worker coalesces its request queue, so a release found behind a search
/// is handled where it sits and the search then runs. Without a cancel it
/// would run *after* the acknowledgement, reopening the index while the caller
/// — `Backend::rebuild_index` — is on its way to delete it. `release_connection`
/// therefore bumps the generation first, which is what this asserts: the
/// request made before the release is stale afterwards, so the worker drops it
/// instead of serving it.
///
/// The generation is checked rather than the descriptor because the race is
/// won or lost in microseconds; an fd assertion here would pass with the bug
/// in place most of the time and fail the rest, which is worse than no test.
#[test]
fn a_release_supersedes_a_search_queued_before_it() {
let p = tmp_db("relcancel");
let mut s = Seeder::new(&p, true);
s.add("held.txt", "/d", 1, Some("shared body"));
drop(s.done());
let (service, updates) = SearchService::new_with_idle_release(
p.clone(),
Arc::new(|| {}),
std::time::Duration::from_secs(60),
);
let queued = service.search("shared", SearchOptions::default());
service.release_connection();
let after = service.search("shared", SearchOptions::default());
assert!(
after > queued + 1,
"the release must advance the generation past {}, got {}",
queued,
after
);
// The service still works: cancelling is how the release is made safe, not
// a way of shutting the worker down.
assert_eq!(
search_names(&service, &updates, "shared").unwrap(),
vec!["held.txt"]
);
service.shutdown();
std::fs::remove_file(&p).ok();
}
/// The point of the whole change: a pass hands hits over *while* it scans, so /// The point of the whole change: a pass hands hits over *while* it scans, so
/// the UI has something to show long before the scan ends. /// the UI has something to show long before the scan ends.
/// ///
/// Proven by ordering rather than by batch count — `flush_pass` has always /// Proven by ordering rather than by batch count — `flush_pass` has always
/// chunked its output, so counting sink calls proves nothing. Pass A scans in /// chunked its output, so counting sink calls proves nothing. Pass A scans in
/// `files.path` order, so seeding a *worse* match at an early path and a /// stored `(parent, name)` order, so seeding a *worse* match early and a
/// *better* one at a late path separates the two designs: emitting at the end /// *better* one at a late path separates the two designs: emitting at the end
/// sorts them and leads with rank 1, while streaming hands over the rank-3 hit /// sorts them and leads with rank 1, while streaming hands over the rank-3 hit
/// before the scan has even reached the rank-1 one. /// before the scan has even reached the rank-1 one.
@ -1568,3 +1658,87 @@ fn a_fuzzy_mark_is_not_truncated_to_a_leading_part_of_the_term() {
&snip.window[a..b] &snip.window[a..b]
); );
} }
/// A path-tier match that straddles the directory/name boundary must still be
/// found.
///
/// `files` stores `parent` and `name` and no concatenation of the two, so the
/// filename pass cannot prefilter with a single `path LIKE`. For a one-piece
/// term it uses `name LIKE ? OR parent LIKE ?`, which is exactly equivalent —
/// a term with no separator in it cannot span the boundary. A multi-segment
/// wildcard *can*: `doc*q3` matches `/x/docs/q3.txt` with `doc` in the parent
/// and `q3` in the name, and neither half of that OR would fire. Those fall
/// back to scanning and letting the classifier decide, and this is the fixture
/// that fails if that fallback is ever "optimised" away.
#[test]
fn a_wildcard_spanning_the_directory_boundary_is_still_found() {
let p = tmp_db("pathstraddle");
let mut s = Seeder::new(&p, true);
let straddling = s.add("q3.txt", "/x/docs", 1, None);
// Same two pieces, both inside the name: found either way, so it proves
// the query ran rather than that the fallback was reached.
let in_name = s.add("doc-q3.txt", "/other", 2, None);
let _miss = s.add("q3.txt", "/x/plans", 3, None);
let conn = s.done();
let (hits, _) = run_collect(&conn, "doc*q3", &SearchOptions::default());
let mut ids: Vec<i64> = hits.iter().map(|h| h.file_id).collect();
ids.sort();
let mut want = vec![straddling, in_name];
want.sort();
assert_eq!(ids, want, "the straddling path match must not be dropped");
// And it is a path-tier hit carrying the whole path as its snippet.
let hit = hits.iter().find(|h| h.file_id == straddling).unwrap();
assert_eq!(hit.stage, 9);
assert_eq!(hit.path, "/x/docs/q3.txt", "parent and name rejoined");
drop(conn);
std::fs::remove_file(&p).ok();
}
/// The one-piece prefilter's other half: a term that is a whole directory
/// component matches through `parent LIKE`, with nothing in the name.
#[test]
fn a_term_matching_only_the_directory_is_found_through_the_parent() {
let p = tmp_db("pathparent");
let mut s = Seeder::new(&p, true);
let dir_hit = s.add("z.bin", "/srv/invoices", 1, None);
let _miss = s.add("z.bin", "/srv/other", 2, None);
let conn = s.done();
let (hits, _) = run_collect(&conn, "invoices", &SearchOptions::default());
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].file_id, dir_hit);
assert_eq!(hits[0].stage, 9);
drop(conn);
std::fs::remove_file(&p).ok();
}
/// `folder:` must reach the folder's own files, not just its subdirectories'.
///
/// The filter is a single `parent LIKE 'dir/%'`, which covers both only
/// because every stored parent ends in a separator — the folder's own files
/// have parent `dir/`, and `%` matches nothing. It used to need a second
/// `parent = ?` term to catch them.
#[test]
fn the_folder_filter_covers_the_folder_itself_and_its_subtree() {
let p = tmp_db("folderself");
let mut s = Seeder::new(&p, true);
let own = s.add("top.txt", "/srv/data", 1, None);
let nested = s.add("deep.txt", "/srv/data/2024", 2, None);
let _sibling = s.add("other.txt", "/srv/data-archive", 3, None);
let _outside = s.add("far.txt", "/srv", 4, None);
let conn = s.done();
let (hits, _) = run_collect(&conn, "txt folder:/srv/data", &SearchOptions::default());
let mut ids: Vec<i64> = hits.iter().map(|h| h.file_id).collect();
ids.sort();
let mut want = vec![own, nested];
want.sort();
assert_eq!(ids, want, "the folder's own files count as inside it");
drop(conn);
std::fs::remove_file(&p).ok();
}

6
crates/quicksearch-core/tests/encrypted.rs
View file

@ -91,12 +91,12 @@ fn encrypted_index_lifecycle() {
{ {
let conn = db::open_existing(&db_path.to_string_lossy(), true).unwrap(); let conn = db::open_existing(&db_path.to_string_lossy(), true).unwrap();
conn.execute_batch( conn.execute_batch(
"INSERT INTO files (name, path, parent, size, mtime, type, content_state) "INSERT INTO files (name, parent, size, mtime, type, content_state)
WITH RECURSIVE n(i) AS ( WITH RECURSIVE n(i) AS (
SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 20000 SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 20000
) )
SELECT 'p' || i, '/pad/' || i, '/pad', 0, 0, 0, 3 FROM n; SELECT 'p' || i, '/pad/', 0, 0, 0, 3 FROM n;
DELETE FROM files WHERE parent = '/pad';", DELETE FROM files WHERE parent = '/pad/';",
) )
.unwrap(); .unwrap();
drop(conn); drop(conn);

20
crates/quicksearch-core/tests/full_index.rs
View file

@ -47,7 +47,7 @@ fn index_once(root: &Path, db: &Path, config: &Config) {
fn rows(db: &Path) -> Vec<(String, i64, i64)> { fn rows(db: &Path) -> Vec<(String, i64, i64)> {
let conn = rusqlite::Connection::open(db).unwrap(); let conn = rusqlite::Connection::open(db).unwrap();
let mut stmt = conn let mut stmt = conn
.prepare("SELECT path, mtime, content_state FROM files ORDER BY path") .prepare("SELECT parent || name, mtime, content_state FROM files ORDER BY parent, name")
.unwrap(); .unwrap();
let out = stmt let out = stmt
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))) .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
@ -756,11 +756,11 @@ fn content_rows(db: &Path) -> Vec<ContentRow> {
let conn = rusqlite::Connection::open(db).unwrap(); let conn = rusqlite::Connection::open(db).unwrap();
let mut stmt = conn let mut stmt = conn
.prepare( .prepare(
"SELECT f.path, f.content_state, ff.reason, LENGTH(d.text_zstd) "SELECT f.parent || f.name, f.content_state, ff.reason, LENGTH(d.text_zstd)
FROM files f FROM files f
LEFT JOIN documents_text d ON d.file_id = f.id LEFT JOIN documents_text d ON d.file_id = f.id
LEFT JOIN failed_files ff ON ff.file_id = f.id LEFT JOIN failed_files ff ON ff.file_id = f.id
ORDER BY f.path", ORDER BY f.parent, f.name",
) )
.unwrap(); .unwrap();
let out = stmt let out = stmt
@ -778,7 +778,7 @@ fn stored_text(db: &Path, suffix: &str) -> Option<String> {
.query_row( .query_row(
"SELECT d.text_zstd FROM documents_text d "SELECT d.text_zstd FROM documents_text d
JOIN files f ON f.id = d.file_id JOIN files f ON f.id = d.file_id
WHERE f.path LIKE '%' || ?1", WHERE f.parent || f.name LIKE '%' || ?1",
[suffix], [suffix],
|r| r.get(0), |r| r.get(0),
) )
@ -900,7 +900,7 @@ fn undecodable_small_files_are_reported_as_failures_not_silently_skipped() {
.query_row( .query_row(
"SELECT f.content_state, ff.reason FROM files f \ "SELECT f.content_state, ff.reason FROM files f \
LEFT JOIN failed_files ff ON ff.file_id = f.id \ LEFT JOIN failed_files ff ON ff.file_id = f.id \
WHERE f.path LIKE '%bad.txt'", WHERE f.parent || f.name LIKE '%bad.txt'",
[], [],
|r| Ok((r.get(0)?, r.get(1)?)), |r| Ok((r.get(0)?, r.get(1)?)),
) )
@ -943,7 +943,7 @@ fn an_unreadable_legacy_office_file_fails_with_a_reason() {
.query_row( .query_row(
"SELECT f.content_state, ff.reason FROM files f \ "SELECT f.content_state, ff.reason FROM files f \
LEFT JOIN failed_files ff ON ff.file_id = f.id \ LEFT JOIN failed_files ff ON ff.file_id = f.id \
WHERE f.path LIKE '%broken.doc'", WHERE f.parent || f.name LIKE '%broken.doc'",
[], [],
|r| Ok((r.get(0)?, r.get(1)?)), |r| Ok((r.get(0)?, r.get(1)?)),
) )
@ -981,7 +981,7 @@ fn extensionless_text_files_are_indexed() {
let conn = rusqlite::Connection::open(&db).unwrap(); let conn = rusqlite::Connection::open(&db).unwrap();
let state_of = |name: &str| -> i64 { let state_of = |name: &str| -> i64 {
conn.query_row( conn.query_row(
"SELECT content_state FROM files WHERE path LIKE '%' || ?1", "SELECT content_state FROM files WHERE parent || name LIKE '%' || ?1",
[name], [name],
|r| r.get(0), |r| r.get(0),
) )
@ -1162,7 +1162,7 @@ fn an_empty_file_is_done_with_no_snippet_sidecar() {
let (state, sidecars): (i64, i64) = conn let (state, sidecars): (i64, i64) = conn
.query_row( .query_row(
"SELECT f.content_state, (SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id) "SELECT f.content_state, (SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id)
FROM files f WHERE f.path LIKE '%empty.txt'", FROM files f WHERE f.parent || f.name LIKE '%empty.txt'",
[], [],
|r| Ok((r.get(0)?, r.get(1)?)), |r| Ok((r.get(0)?, r.get(1)?)),
) )
@ -1188,7 +1188,7 @@ fn the_content_extension_filter_still_excludes_small_text_files() {
let conn = rusqlite::Connection::open(&db).unwrap(); let conn = rusqlite::Connection::open(&db).unwrap();
let states: Vec<(String, i64)> = conn let states: Vec<(String, i64)> = conn
.prepare("SELECT path, content_state FROM files ORDER BY path") .prepare("SELECT parent || name, content_state FROM files ORDER BY parent, name")
.unwrap() .unwrap()
.query_map([], |r| Ok((r.get(0)?, r.get(1)?))) .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))
.unwrap() .unwrap()
@ -1933,7 +1933,7 @@ fn high_byte_binaries_are_listed_but_not_text_extracted() {
"SELECT f.content_state, "SELECT f.content_state,
(SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id), (SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id),
(SELECT COUNT(*) FROM failed_files x WHERE x.file_id = f.id) (SELECT COUNT(*) FROM failed_files x WHERE x.file_id = f.id)
FROM files f WHERE f.path LIKE '%' || ?1", FROM files f WHERE f.parent || f.name LIKE '%' || ?1",
[suffix], [suffix],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
) )

12
crates/quicksearch-core/tests/reconcile.rs
View file

@ -62,14 +62,22 @@ fn conn(db: &Path) -> rusqlite::Connection {
rusqlite::Connection::open(db).unwrap() rusqlite::Connection::open(db).unwrap()
} }
/// Every indexed path, in **path** order.
///
/// Sorted here rather than in SQL: the index's own order is `(parent, name)`,
/// which groups by directory and is not the same sequence — `/r/keep.txt`
/// sorts *before* `/r/sub/k2.txt` there and after it by path. These assertions
/// are about which rows survived, so they read best in the order a person
/// would list them.
fn paths(db: &Path) -> Vec<String> { fn paths(db: &Path) -> Vec<String> {
let c = conn(db); let c = conn(db);
let mut stmt = c.prepare("SELECT path FROM files ORDER BY path").unwrap(); let mut stmt = c.prepare("SELECT parent || name FROM files").unwrap();
let out = stmt let mut out: Vec<String> = stmt
.query_map([], |r| r.get::<_, String>(0)) .query_map([], |r| r.get::<_, String>(0))
.unwrap() .unwrap()
.map(|r| r.unwrap()) .map(|r| r.unwrap())
.collect(); .collect();
out.sort();
out out
} }

5
crates/quicksearch-core/tests/search_perf.rs
View file

@ -114,13 +114,12 @@ fn seed(path: &std::path::Path) {
let w1 = WORDS[(rng.next() as usize) % WORDS.len()]; let w1 = WORDS[(rng.next() as usize) % WORDS.len()];
let w2 = WORDS[(rng.next() as usize) % WORDS.len()]; let w2 = WORDS[(rng.next() as usize) % WORDS.len()];
let name = format!("{}-{}-{:07}.txt", w1, w2, i); let name = format!("{}-{}-{:07}.txt", w1, w2, i);
let dir = format!("/seed/{:03}", i % 500); // Stored parents always end in a separator.
let full = format!("{}/{}", dir, name); let dir = format!("/seed/{:03}/", i % 500);
let id = insert_file( let id = insert_file(
&tx, &tx,
&NewFile { &NewFile {
name: &name, name: &name,
path: &full,
parent: &dir, parent: &dir,
size: 4096, size: 4096,
mtime: 1_700_000_000 + i as u64, mtime: 1_700_000_000 + i as u64,

45
crates/quicksearch-gui/src/app.rs
View file

@ -11,6 +11,7 @@ use quicksearch_core::db;
use quicksearch_core::indexing::{ use quicksearch_core::indexing::{
overall_progress, ConfigChange, IndexingStatus, PrepStep, RootPhase, RootProgress, overall_progress, ConfigChange, IndexingStatus, PrepStep, RootPhase, RootProgress,
}; };
use quicksearch_core::platform::{IndexLock, LockError};
use quicksearch_core::search::SearchOptions; use quicksearch_core::search::SearchOptions;
use quicksearch_core::security::{derive_key, generate_salt, salt_to_hex, IndexKey}; use quicksearch_core::security::{derive_key, generate_salt, salt_to_hex, IndexKey};
use quicksearch_core::watcher::WatchError; use quicksearch_core::watcher::WatchError;
@ -322,10 +323,45 @@ impl QuickSearchApp {
.watch_cap_warned_roots .watch_cap_warned_roots
.retain(|root| new.paths.indexing_paths.contains(root)); .retain(|root| new.paths.indexing_paths.contains(root));
let actions = diff_actions(&self.cfg, &new); let actions = diff_actions(&self.cfg, &new);
// The instance lock follows the database path, and it has to move
// before the change is written: a `database_path` that another
// instance already holds must never reach the config file, or the next
// launch reads it and refuses to start at all. `move_to` takes the new
// lock before dropping the old, so this rejection leaves us holding
// what we already had.
if actions.search_db_changed {
match IndexLock::move_to(&new.resolved_database_path()) {
Ok(()) => {}
Err(LockError::Held { pid }) => {
let who = match pid {
Some(pid) => format!(" (process {})", pid),
None => String::new(),
};
self.config_error = Some(format!(
"Not applied: another QuickSearch{} is using that index.",
who
));
return false;
}
// The move happened and the new path simply cannot be locked.
// Same rule as at startup: a convenience guard is never a good
// enough reason to refuse.
Err(LockError::Unsupported(why)) => {
quicksearch_core::log_warn!("cannot lock the index ({}); continuing", why);
}
}
}
// A config that could not be written must not take effect either: it // A config that could not be written must not take effect either: it
// would apply to this process, revert on restart, and show nothing // would apply to this process, revert on restart, and show nothing
// unsaved in between. // unsaved in between.
if let Err(e) = new.save() { if let Err(e) = new.save() {
// Put the lock back on the path the config still names, or this
// process would go on using the old index while guarding the new
// one — leaving the index it is actually writing open to a second
// instance.
if actions.search_db_changed {
let _ = IndexLock::move_to(&self.cfg.resolved_database_path());
}
self.config_error = Some(e); self.config_error = Some(e);
return false; return false;
} }
@ -350,7 +386,7 @@ impl QuickSearchApp {
self.backend.coordinator.apply_config(new.clone()); self.backend.coordinator.apply_config(new.clone());
if actions.requires_rebuild { if actions.requires_rebuild {
if self.backend.coordinator.state().mode == IndexMode::Auto { if self.backend.coordinator.state().mode == IndexMode::Auto {
self.backend.coordinator.rebuild_index(); self.backend.rebuild_index();
} else { } else {
let changes = self let changes = self
.backend .backend
@ -440,7 +476,9 @@ impl QuickSearchApp {
if let Some(rx) = &self.backend.dup_job { if let Some(rx) = &self.backend.dup_job {
use std::sync::mpsc::TryRecvError; use std::sync::mpsc::TryRecvError;
let done = match rx.try_recv() { let done = match rx.try_recv() {
Ok(Ok(groups)) => Some(DupState::Loaded(groups)), Ok(Ok(groups)) => Some(DupState::Loaded(crate::duplicates_tab::LoadedGroups::new(
groups,
))),
Ok(Err(e)) => Some(DupState::Error(e)), Ok(Err(e)) => Some(DupState::Error(e)),
Err(TryRecvError::Empty) => None, Err(TryRecvError::Empty) => None,
Err(TryRecvError::Disconnected) => { Err(TryRecvError::Disconnected) => {
@ -692,7 +730,8 @@ impl eframe::App for QuickSearchApp {
if let Some(paths) = actions.verify { if let Some(paths) = actions.verify {
let paths: Vec<std::path::PathBuf> = let paths: Vec<std::path::PathBuf> =
paths.into_iter().map(std::path::PathBuf::from).collect(); paths.into_iter().map(std::path::PathBuf::from).collect();
self.backend.start_verify(paths.clone(), ctx.clone()); self.backend
.start_verify(paths.clone(), &self.cfg, ctx.clone());
self.verify = Some(VerifyModal::new(paths)); self.verify = Some(VerifyModal::new(paths));
} }
} }

18
crates/quicksearch-gui/src/app/modals.rs
View file

@ -50,7 +50,7 @@ impl QuickSearchApp {
)); ));
ui.horizontal(|ui| { ui.horizontal(|ui| {
if ui.button("Rebuild now").clicked() { if ui.button("Rebuild now").clicked() {
self.backend.coordinator.rebuild_index(); self.backend.rebuild_index();
close = true; close = true;
} }
if ui.button("Later").clicked() { if ui.button("Later").clicked() {
@ -97,7 +97,7 @@ impl QuickSearchApp {
} }
if stale_index_window(ctx, self.key_source) { if stale_index_window(ctx, self.key_source) {
self.stale_index_prompt = false; self.stale_index_prompt = false;
self.backend.coordinator.rebuild_index(); self.backend.rebuild_index();
self.dups.state = DupState::NotLoaded; self.dups.state = DupState::NotLoaded;
} }
} }
@ -158,6 +158,18 @@ impl QuickSearchApp {
group_thousands(*registered as u64), group_thousands(*registered as u64),
)); ));
} }
// Not reachable through this modal today: an overflow leaves
// the watcher running and schedules a full run instead of
// disabling live updates, so it never becomes a
// `WatcherStatus::Disabled`. Spelled out rather than folded
// into a catch-all so that a future variant is a compile
// error here, which is how this arm came to be written.
WatchError::Overflowed => {
ui.label(
"The system dropped some change notifications, so the index \
is being rebuilt to catch up.",
);
}
WatchError::Other(msg) => { WatchError::Other(msg) => {
ui.label(format!("Live updates are unavailable: {}", msg)); ui.label(format!("Live updates are unavailable: {}", msg));
} }
@ -227,7 +239,7 @@ impl QuickSearchApp {
// resurrect what was just deleted, nor the next launch // resurrect what was just deleted, nor the next launch
// undo the stop. // undo the stop.
self.set_index_mode(false); self.set_index_mode(false);
self.backend.coordinator.clear_index(); self.backend.clear_index();
self.dups.state = DupState::NotLoaded; self.dups.state = DupState::NotLoaded;
return true; return true;
} }

2
crates/quicksearch-gui/src/app/security.rs
View file

@ -377,7 +377,7 @@ impl QuickSearchApp {
} }
} }
db::set_process_key(new_key); db::set_process_key(new_key);
self.backend.coordinator.rebuild_index(); self.backend.rebuild_index();
self.dups.state = DupState::NotLoaded; self.dups.state = DupState::NotLoaded;
} }
} }

39
crates/quicksearch-gui/src/backend.rs
View file

@ -51,6 +51,31 @@ pub struct Backend {
} }
impl Backend { impl Backend {
/// Ask the coordinator to rebuild, after letting go of everything that
/// holds the index file open.
///
/// The search worker keeps its connection for half a minute after the last
/// keystroke so a typing session runs against a warm cache — which is
/// exactly the wrong thing to be holding when the file is about to be
/// deleted. Without this the delete fails on Windows and the rebuild
/// silently becomes an ordinary run against the old index; after a
/// password change that leaves the config claiming protection the file on
/// disk does not have.
pub fn rebuild_index(&self) {
if let Some(search) = &self.search {
search.release_connection();
}
self.coordinator.rebuild_index();
}
/// [`Backend::rebuild_index`]'s reasoning, for the delete-only path.
pub fn clear_index(&self) {
if let Some(search) = &self.search {
search.release_connection();
}
self.coordinator.clear_index();
}
pub fn start(config: &Config, ctx: egui::Context) -> Result<Backend, String> { pub fn start(config: &Config, ctx: egui::Context) -> Result<Backend, String> {
// eframe is reactive: a run the coordinator schedules on its own // eframe is reactive: a run the coordinator schedules on its own
// would sit unseen behind a settled window until the pointer moved. // would sit unseen behind a settled window until the pointer moved.
@ -89,10 +114,17 @@ impl Backend {
pub fn watch_live( pub fn watch_live(
&self, &self,
query: &str, query: &str,
targets: Vec<quicksearch_core::live::Target>, mut targets: Vec<quicksearch_core::live::Target>,
config: &Config, config: &Config,
) { ) {
let Some(live) = &self.live else { return }; let Some(live) = &self.live else { return };
// The watcher re-reads a row's file to re-cut its snippet, and the one
// file it must never open is the index it is reading the row from:
// closing a descriptor on it cancels SQLite's locks process-wide. The
// walk no longer writes such rows, but one from an older build lives
// until the stale sweep reaches its directory, and it can be on screen
// before then.
targets.retain(|t| !config.is_index_file(std::path::Path::new(&t.path)));
if targets.is_empty() { if targets.is_empty() {
live.clear(); live.clear();
} else { } else {
@ -133,10 +165,13 @@ impl Backend {
/// Read a duplicate group through on a worker thread, comparing every /// Read a duplicate group through on a worker thread, comparing every
/// member against the first byte for byte. Replaces any run already going. /// member against the first byte for byte. Replaces any run already going.
pub fn start_verify(&mut self, paths: Vec<PathBuf>, ctx: egui::Context) { pub fn start_verify(&mut self, mut paths: Vec<PathBuf>, config: &Config, ctx: egui::Context) {
if let Some(job) = &self.verify_job { if let Some(job) = &self.verify_job {
job.cancel(); job.cancel();
} }
// Byte-for-byte comparison opens every member. See `watch_live` for
// why the index must not be one of them.
paths.retain(|p| !config.is_index_file(p));
let (tx, rx) = mpsc::channel(); let (tx, rx) = mpsc::channel();
let cancel = Arc::new(AtomicBool::new(false)); let cancel = Arc::new(AtomicBool::new(false));
let worker_cancel = cancel.clone(); let worker_cancel = cancel.clone();

133
crates/quicksearch-gui/src/cli.rs
View file

@ -10,6 +10,7 @@ use quicksearch_core::db;
use quicksearch_core::query::split::split_for_cascade; use quicksearch_core::query::split::split_for_cascade;
use quicksearch_core::search::{cascade, SearchHit, SearchOptions}; use quicksearch_core::search::{cascade, SearchHit, SearchOptions};
use quicksearch_core::security::{derive_key, IndexKey}; use quicksearch_core::security::{derive_key, IndexKey};
use quicksearch_core::textenc::scrub_controls;
use zeroize::Zeroizing; use zeroize::Zeroizing;
use crate::format::{fmt_mtime, human_size}; use crate::format::{fmt_mtime, human_size};
@ -127,15 +128,22 @@ pub(crate) fn resolve_key(
if let Some(hex) = keychain_hex { if let Some(hex) = keychain_hex {
match IndexKey::from_hex(&hex).map_err(|e| format!("keychain entry: {}", e)) { match IndexKey::from_hex(&hex).map_err(|e| format!("keychain entry: {}", e)) {
Ok(key) => { Ok(key) => match try_key(key) {
match try_key(key) { Ok(()) => return Ok(()),
Ok(()) => return Ok(()), Err(e) => match mismatch_cause(&e) {
Err(e) if e.starts_with(db::KEY_MISMATCH_PREFIX) => { Some((db::KeyMismatch::WrongPassword, _)) => {
eprintln!("warning: the key remembered in the OS keychain no longer opens this index"); eprintln!(
"warning: the key remembered in the OS keychain no longer \
opens this index"
);
} }
Err(e) => return Err(e), // Not a stale keychain entry at all: the index is not
} // encrypted, or wants a key this one is not. Say which,
} // then fall through to the other sources as before.
Some((_, detail)) => eprintln!("warning: {}", detail),
None => return Err(e),
},
},
Err(e) => eprintln!("warning: {}", e), Err(e) => eprintln!("warning: {}", e),
} }
} }
@ -145,11 +153,17 @@ pub(crate) fn resolve_key(
drop(password); drop(password);
return match try_key(key) { return match try_key(key) {
Ok(()) => Ok(()), Ok(()) => Ok(()),
Err(e) if e.starts_with(db::KEY_MISMATCH_PREFIX) => Err(format!( Err(e) => match mismatch_cause(&e) {
"{} does not match this index's password", Some((db::KeyMismatch::WrongPassword, _)) => Err(format!(
PASSWORD_ENV "{} does not match this index's password",
)), PASSWORD_ENV
Err(e) => Err(e), )),
// Blaming the environment variable for "the index is not
// encrypted" sends the user to change the one thing that is
// not wrong.
Some((_, detail)) => Err(detail),
None => Err(e),
},
}; };
} }
@ -168,15 +182,31 @@ pub(crate) fn resolve_key(
drop(password); drop(password);
match try_key(key) { match try_key(key) {
Ok(()) => return Ok(()), Ok(()) => return Ok(()),
Err(e) if e.starts_with(db::KEY_MISMATCH_PREFIX) => { Err(e) => match mismatch_cause(&e) {
eprintln!("Wrong password."); // Only a wrong password is worth another attempt. The other
} // two causes are facts about the index — it is not encrypted,
Err(e) => return Err(e), // or it wants a password this build did not apply — and
// retrying makes the user type a *correct* password twice more
// before being told something that was never about their
// typing. The same reasoning as the unlock screen's.
Some((db::KeyMismatch::WrongPassword, _)) => eprintln!("Wrong password."),
Some((_, detail)) => return Err(detail),
None => return Err(e),
},
} }
} }
Err("wrong password (3 attempts)".to_string()) Err("wrong password (3 attempts)".to_string())
} }
/// [`db::key_mismatch_parts`] with the detail copied out.
///
/// Owned rather than borrowed so a caller can hand the original error back on
/// the `None` arm: `key_mismatch_parts` borrows from the message, and a `match`
/// on its result would hold that borrow across a `return Err(e)`.
fn mismatch_cause(error: &str) -> Option<(db::KeyMismatch, String)> {
db::key_mismatch_parts(error).map(|(cause, detail)| (cause, detail.to_string()))
}
/// Wire [`resolve_key`] to the real terminal, environment, keychain and /// Wire [`resolve_key`] to the real terminal, environment, keychain and
/// database, installing the verified key as the process key. /// database, installing the verified key as the process key.
fn resolve_key_for_terminal(security: &SecurityConfig, db_path: &str) -> Result<(), String> { fn resolve_key_for_terminal(security: &SecurityConfig, db_path: &str) -> Result<(), String> {
@ -269,7 +299,21 @@ fn run_query(query: &str, fuzzy: bool, limit: Option<usize>, long: bool) -> i32
match outcome { match outcome {
Ok(Some(outcome)) => { Ok(Some(outcome)) => {
let color = long && std::io::stdout().is_terminal() && enable_vt(); let tty = std::io::stdout().is_terminal();
let color = long && tty && enable_vt();
// Scrubbed only for a terminal, the way `ls` does it and the way
// the colour above already decides: a filename may contain escape
// sequences, and printed raw they rewrite the line, retitle the
// window, or reach the clipboard on a terminal with OSC 52 on.
// Piped output stays byte-exact, because the next program in the
// pipe wants the real name and interprets nothing.
let show = |s: &str| -> String {
if tty {
scrub_controls(s).into_owned()
} else {
s.to_string()
}
};
for hit in &hits { for hit in &hits {
if long { if long {
println!( println!(
@ -277,13 +321,13 @@ fn run_query(query: &str, fuzzy: bool, limit: Option<usize>, long: bool) -> i32
hit.rank, hit.rank,
human_size(hit.size), human_size(hit.size),
fmt_mtime(hit.mtime), fmt_mtime(hit.mtime),
hit.path show(&hit.path)
); );
if let Some(snip) = &hit.snippet { if let Some(snip) = &hit.snippet {
println!(" {}", render_snippet(snip, color)); println!(" {}", render_snippet(snip, color, tty));
} }
} else { } else {
println!("{}", hit.path); println!("{}", show(&hit.path));
} }
} }
if outcome.limited { if outcome.limited {
@ -334,28 +378,58 @@ fn enable_vt() -> bool {
} }
/// One-line snippet with matches emphasized (ANSI bold on TTYs). /// One-line snippet with matches emphasized (ANSI bold on TTYs).
fn render_snippet(snip: &quicksearch_core::snippet::Snippet, color: bool) -> String { fn render_snippet(snip: &quicksearch_core::snippet::Snippet, color: bool, tty: bool) -> String {
// The window is document text, so it can hold anything the file did —
// including the escape sequences this function is about to add its own
// emphasis codes around. Each piece is scrubbed as it goes in, before the
// codes, so the only escapes in the result are the ones put there here.
let mut out = String::new(); let mut out = String::new();
let push = |out: &mut String, piece: &str| {
// Line breaks are flattened *before* the scrub, not after the whole
// string is assembled. A snippet is one line by contract for both
// destinations, and `scrub_controls` turns `\n` into `U+FFFD` — so a
// trailing `replace(['\n', '\r'], " ")` would find nothing left to
// collapse on a terminal and print `<60>` where the space belongs.
let flat = flatten_lines(piece);
if tty {
out.push_str(&scrub_controls(&flat));
} else {
out.push_str(&flat);
}
};
if snip.truncated_start { if snip.truncated_start {
out.push('…'); out.push('…');
} }
let mut cursor = 0; let mut cursor = 0;
for &(start, end) in &snip.ranges { for &(start, end) in &snip.ranges {
out.push_str(&snip.window[cursor..start]); push(&mut out, &snip.window[cursor..start]);
if color { if color {
out.push_str("\x1b[1m"); out.push_str("\x1b[1m");
out.push_str(&snip.window[start..end]); push(&mut out, &snip.window[start..end]);
out.push_str("\x1b[0m"); out.push_str("\x1b[0m");
} else { } else {
out.push_str(&snip.window[start..end]); push(&mut out, &snip.window[start..end]);
} }
cursor = end; cursor = end;
} }
out.push_str(&snip.window[cursor..]); push(&mut out, &snip.window[cursor..]);
if snip.truncated_end { if snip.truncated_end {
out.push('…'); out.push('…');
} }
out.replace(['\n', '\r'], " ") out
}
/// `piece` with line breaks flattened to spaces, borrowed when it has none.
///
/// A `Cow` for the same reason [`scrub_controls`] is one: this runs per
/// fragment of every snippet of every hit, and the overwhelmingly common case
/// has nothing to change.
fn flatten_lines(piece: &str) -> std::borrow::Cow<'_, str> {
if piece.contains(['\n', '\r']) {
std::borrow::Cow::Owned(piece.replace(['\n', '\r'], " "))
} else {
std::borrow::Cow::Borrowed(piece)
}
} }
#[cfg(test)] #[cfg(test)]
@ -372,7 +446,10 @@ mod tests {
} }
fn mismatch() -> Result<(), String> { fn mismatch() -> Result<(), String> {
Err(format!("{}wrong password", db::KEY_MISMATCH_PREFIX)) Err(format!(
"{}wrong-password: index at /x: wrong password",
db::KEY_MISMATCH_PREFIX
))
} }
fn argv(args: &[&str]) -> Vec<String> { fn argv(args: &[&str]) -> Vec<String> {

55
crates/quicksearch-gui/src/duplicates_tab.rs
View file

@ -9,10 +9,45 @@ use crate::ui_util::hint;
pub enum DupState { pub enum DupState {
NotLoaded, NotLoaded,
Loading, Loading,
Loaded(Vec<DuplicateGroup>), Loaded(LoadedGroups),
Error(String), Error(String),
} }
/// The scan's result, with each group's header line already built.
///
/// The titles are four formatted numbers each and the list runs to 500, so
/// building them in the render loop meant ~2000 allocations *per frame* — and
/// this list overflows by definition, which keeps `more_below_hint`'s 20 Hz
/// repaint running for as long as the tab is open. They depend only on the
/// data, so they are built once, here, where the data arrives.
pub struct LoadedGroups {
pub groups: Vec<DuplicateGroup>,
titles: Vec<String>,
}
impl LoadedGroups {
pub fn new(groups: Vec<DuplicateGroup>) -> LoadedGroups {
let titles = groups
.iter()
.map(|group| {
let name = group
.members
.first()
.map(|m| m.1.as_str())
.unwrap_or("(unknown)");
format!(
"{} × {}: {} reclaimable ({} total)",
group_thousands(group.count as u64),
name,
human_size(group.redundant_size.max(0) as u64),
human_size(group.total_size.max(0) as u64),
)
})
.collect();
LoadedGroups { groups, titles }
}
}
pub struct DuplicatesTab { pub struct DuplicatesTab {
pub state: DupState, pub state: DupState,
} }
@ -73,7 +108,8 @@ impl DuplicatesTab {
DupState::Error(e) => { DupState::Error(e) => {
ui.colored_label(ui.visuals().error_fg_color, e); ui.colored_label(ui.visuals().error_fg_color, e);
} }
DupState::Loaded(groups) => { DupState::Loaded(loaded) => {
let groups = &loaded.groups;
if groups.is_empty() { if groups.is_empty() {
ui.label("No duplicate files found."); ui.label("No duplicate files found.");
return actions; return actions;
@ -85,18 +121,9 @@ impl DuplicatesTab {
.auto_shrink([false; 2]) .auto_shrink([false; 2])
.show(ui, |ui| { .show(ui, |ui| {
for (i, group) in groups.iter().enumerate() { for (i, group) in groups.iter().enumerate() {
let name = group // Built once when the scan landed; see
.members // `LoadedGroups`.
.first() let title = loaded.titles[i].as_str();
.map(|m| m.1.as_str())
.unwrap_or("(unknown)");
let title = format!(
"{} × {}: {} reclaimable ({} total)",
group_thousands(group.count as u64),
name,
human_size(group.redundant_size.max(0) as u64),
human_size(group.total_size.max(0) as u64),
);
let header = let header =
egui::CollapsingHeader::new(title) egui::CollapsingHeader::new(title)
.id_salt(i) .id_salt(i)

4
crates/quicksearch-gui/src/duplicates_tab/tests.rs
View file

@ -23,7 +23,7 @@ fn group(paths: &[&str]) -> DuplicateGroup {
fn loaded(paths: &[&str]) -> DuplicatesTab { fn loaded(paths: &[&str]) -> DuplicatesTab {
DuplicatesTab { DuplicatesTab {
state: DupState::Loaded(vec![group(paths)]), state: DupState::Loaded(LoadedGroups::new(vec![group(paths)])),
} }
} }
@ -177,7 +177,7 @@ fn a_member_row_offers_the_verification_too() {
fn an_empty_result_says_so_rather_than_showing_an_empty_list() { fn an_empty_result_says_so_rather_than_showing_an_empty_list() {
let ctx = crate::test_ui::ctx(); let ctx = crate::test_ui::ctx();
let mut tab = DuplicatesTab { let mut tab = DuplicatesTab {
state: DupState::Loaded(Vec::new()), state: DupState::Loaded(LoadedGroups::new(Vec::new())),
}; };
let painted = painted_text(&frame(&ctx, &mut tab, false, Vec::new()).0); let painted = painted_text(&frame(&ctx, &mut tab, false, Vec::new()).0);
assert!( assert!(

16
crates/quicksearch-gui/src/format.rs
View file

@ -43,10 +43,14 @@ pub fn fmt_ago(unix_secs: u64) -> String {
/// A configured interval as a phrase to drop after "every": `90 min`, /// A configured interval as a phrase to drop after "every": `90 min`,
/// `24 h`, `3 days`. /// `24 h`, `3 days`.
pub fn fmt_interval(minutes: u64) -> String { pub fn fmt_interval(minutes: u64) -> String {
if minutes == 0 { // The same clamp the scheduler applies. Zero is only reachable from a
// The scheduler treats 0 as always-due. // hand-edited config — the spinner's floor is 5 — and it used to read
return "run".to_string(); // "run", from a time when the scheduler took it as always-due. It no
} // longer does (`coordinator::inner::periodic_due` raises it to one
// minute, because "reindex continuously" is not a setting anyone wants
// and manual mode is how you say "never"), so saying "run" here would
// describe a behaviour the app does not have.
let minutes = minutes.max(1);
if minutes < 60 { if minutes < 60 {
return format!("{} min", minutes); return format!("{} min", minutes);
} }
@ -170,7 +174,9 @@ mod tests {
#[test] #[test]
fn intervals() { fn intervals() {
assert_eq!(fmt_interval(0), "run"); // Clamped, exactly as `periodic_due` clamps it: the two must not
// describe different behaviour.
assert_eq!(fmt_interval(0), "1 min");
assert_eq!(fmt_interval(1), "1 min"); assert_eq!(fmt_interval(1), "1 min");
assert_eq!(fmt_interval(59), "59 min"); assert_eq!(fmt_interval(59), "59 min");
assert_eq!(fmt_interval(60), "1 h", "the shipped default"); assert_eq!(fmt_interval(60), "1 h", "the shipped default");

40
crates/quicksearch-gui/src/main.rs
View file

@ -37,6 +37,7 @@ mod unlock;
mod version; mod version;
use quicksearch_core::config::Config; use quicksearch_core::config::Config;
use quicksearch_core::platform::{IndexLock, LockError};
/// The window icon, shown in the titlebar, taskbar and alt-tab switcher. /// The window icon, shown in the titlebar, taskbar and alt-tab switcher.
/// ///
@ -82,6 +83,45 @@ fn main() {
}; };
let initial_query = seed_query(); let initial_query = seed_query();
// After the CLI early-exit above, deliberately: `quicksearch <query>` only
// reads, and must keep working from a terminal while the window is open.
// Two *windows* on one index are the problem — two indexers writing, and,
// once either has cancelled the other's SQLite locks, an attach that
// truncates the wal-index under a live mapping.
//
// Held for the life of the process in `platform`'s own slot, so the
// settings handler can move it when `database_path` changes; the kernel
// releases it on exit, however that exit happens.
match IndexLock::hold(&config.resolved_database_path()) {
Ok(()) => {}
Err(LockError::Held { pid }) => {
let who = match pid {
Some(pid) => format!(" (process {})", pid),
None => String::new(),
};
let msg = format!(
"QuickSearch is already running{}.\n\nOnly one window can use \
the index at a time. Switch to the running window, or close \
it and try again.",
who
);
eprintln!("{}", msg);
// The app is normally launched from a desktop icon or a hotkey,
// where nothing is watching stderr.
rfd::MessageDialog::new()
.set_level(rfd::MessageLevel::Info)
.set_title("QuickSearch")
.set_description(&msg)
.show();
std::process::exit(1);
}
// Not "the lock is taken" — the filesystem could not answer. A
// convenience guard is never a good enough reason to refuse to open.
Err(LockError::Unsupported(why)) => {
eprintln!("warning: cannot lock the index ({}); starting anyway", why);
}
}
// With protection on, try the keychain before the window opens; a // With protection on, try the keychain before the window opens; a
// verified key means no prompt at all. `None` starts locked, and no // verified key means no prompt at all. `None` starts locked, and no
// index is touched until unlocked. // index is touched until unlocked.

30
crates/quicksearch-gui/src/unlock.rs
View file

@ -211,19 +211,23 @@ impl UnlockScreen {
match result { match result {
Ok(key) => return self.unlocked(ctx, key), Ok(key) => return self.unlocked(ctx, key),
Err(e) => { Err(e) => {
// A tagged mismatch has three distinct causes and // A tagged mismatch has three distinct causes and the
// `key_mismatch_message` already told them apart. Only one // opener already told them apart. Only one is a wrong
// of them is a wrong password; the others are "the config // password; the others are "the config says protected but
// says protected but the index on disk is not" (a crash // the index on disk is not" (a crash between saving the
// between saving the config and rebuilding) and "this // config and rebuilding) and "this index wants a password
// index wants a password at all". Collapsing them all into // at all". Collapsing them all into "Wrong password."
// "Wrong password." tells a user with the right password // tells a user with the right password that it is wrong,
// that it is wrong, and the only button on this screen // and the only button on this screen deletes their index
// deletes their index and turns protection off. The detail // and turns protection off.
// names the database path and nothing secret. //
self.error = Some(match e.strip_prefix(db::KEY_MISMATCH_PREFIX) { // Matched on the cause rather than on the wording: the
Some(_) if e.contains("wrong password") => "Wrong password.".to_string(), // message carries the database path, so a path that
Some(detail) => detail.to_string(), // happens to contain the phrase would otherwise decide
// this. The detail names the path and nothing secret.
self.error = Some(match db::key_mismatch_parts(&e) {
Some((db::KeyMismatch::WrongPassword, _)) => "Wrong password.".to_string(),
Some((_, detail)) => detail.to_string(),
None => e, None => e,
}); });
// The field was cleared on submit; put the caret back // The field was cleared on submit; put the caret back

1
vendor/pdf-extract/.cargo-ok vendored
View file

@ -1 +0,0 @@
{"v":1}