From 2e2936b249916156037f34919cc92158b0ac4a75 Mon Sep 17 00:00:00 2001 From: Jeremy Karst Date: Thu, 20 Aug 2026 02:34:08 -0400 Subject: [PATCH] Storage, safety, and correctness hardening --- Cargo.lock | 340 +++++------------ Cargo.toml | 2 +- README.md | 38 +- config_example.toml | 12 + crates/quicksearch-core/Cargo.toml | 6 +- crates/quicksearch-core/examples/walkprobe.rs | 2 +- crates/quicksearch-core/src/cli.rs | 54 +-- crates/quicksearch-core/src/config/diff.rs | 2 +- crates/quicksearch-core/src/config/mod.rs | 227 +++++++++-- crates/quicksearch-core/src/config/tests.rs | 52 +++ crates/quicksearch-core/src/content.rs | 5 +- crates/quicksearch-core/src/coordinator.rs | 2 + .../quicksearch-core/src/coordinator/inner.rs | 181 ++++++++- .../quicksearch-core/src/coordinator/tests.rs | 2 +- crates/quicksearch-core/src/db/mod.rs | 4 +- crates/quicksearch-core/src/db/open.rs | 83 +++- crates/quicksearch-core/src/db/open_tests.rs | 70 +++- crates/quicksearch-core/src/db/repo.rs | 239 ++++++++---- crates/quicksearch-core/src/db/repo_tests.rs | 153 +++++--- crates/quicksearch-core/src/db/schema.rs | 46 ++- crates/quicksearch-core/src/extract/office.rs | 237 +++++++++++- .../src/file_handling/batch.rs | 37 +- .../quicksearch-core/src/file_handling/mod.rs | 9 +- .../src/file_handling/paths.rs | 121 +++++- .../src/file_handling/records.rs | 33 +- .../src/file_handling/tests.rs | 129 +++++++ crates/quicksearch-core/src/incremental.rs | 160 +++++++- crates/quicksearch-core/src/indexing/mod.rs | 45 ++- .../quicksearch-core/src/indexing/pipeline.rs | 97 ++++- crates/quicksearch-core/src/indexing/tests.rs | 66 +++- crates/quicksearch-core/src/live.rs | 50 ++- crates/quicksearch-core/src/live_tests.rs | 86 +++++ crates/quicksearch-core/src/log.rs | 18 + crates/quicksearch-core/src/platform.rs | 327 ++++++++++++++++ crates/quicksearch-core/src/platform_tests.rs | 239 ++++++++++++ crates/quicksearch-core/src/query/split.rs | 24 +- .../quicksearch-core/src/query/translator.rs | 91 +++-- crates/quicksearch-core/src/scope.rs | 27 +- crates/quicksearch-core/src/scope_tests.rs | 4 +- crates/quicksearch-core/src/search/cascade.rs | 9 +- .../src/search/cascade/passes.rs | 149 ++++++-- .../quicksearch-core/src/search/duplicates.rs | 40 +- crates/quicksearch-core/src/search/mod.rs | 97 ++++- crates/quicksearch-core/src/testutil.rs | 76 ++++ crates/quicksearch-core/src/textenc.rs | 25 ++ crates/quicksearch-core/src/walk.rs | 173 +++++++-- crates/quicksearch-core/src/walk/tests.rs | 355 ++++++++++++++++-- crates/quicksearch-core/src/watcher.rs | 94 ++++- crates/quicksearch-core/src/watcher_tests.rs | 69 ++++ crates/quicksearch-core/tests/cascade.rs | 182 ++++++++- crates/quicksearch-core/tests/encrypted.rs | 6 +- crates/quicksearch-core/tests/full_index.rs | 20 +- crates/quicksearch-core/tests/reconcile.rs | 12 +- crates/quicksearch-core/tests/search_perf.rs | 5 +- crates/quicksearch-gui/src/app.rs | 45 ++- crates/quicksearch-gui/src/app/modals.rs | 18 +- crates/quicksearch-gui/src/app/security.rs | 2 +- crates/quicksearch-gui/src/backend.rs | 39 +- crates/quicksearch-gui/src/cli.rs | 133 +++++-- crates/quicksearch-gui/src/duplicates_tab.rs | 55 ++- .../src/duplicates_tab/tests.rs | 4 +- crates/quicksearch-gui/src/format.rs | 16 +- crates/quicksearch-gui/src/main.rs | 40 ++ crates/quicksearch-gui/src/unlock.rs | 30 +- vendor/pdf-extract/.cargo-ok | 1 - 65 files changed, 4177 insertions(+), 838 deletions(-) delete mode 100644 vendor/pdf-extract/.cargo-ok diff --git a/Cargo.lock b/Cargo.lock index db2027a..9fee860 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,7 +86,7 @@ dependencies = [ "android-properties", "bitflags 2.13.1", "cc", - "jni 0.22.4", + "jni", "libc", "log", "ndk", @@ -501,9 +501,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.6.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "calloop" @@ -577,12 +577,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cesu8" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" - [[package]] name = "cfb" version = "0.7.3" @@ -745,16 +739,6 @@ dependencies = [ "libc", ] -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -768,7 +752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" dependencies = [ "bitflags 1.3.2", - "core-foundation 0.9.4", + "core-foundation", "core-graphics-types", "foreign-types 0.5.0", "libc", @@ -781,7 +765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" dependencies = [ "bitflags 1.3.2", - "core-foundation 0.9.4", + "core-foundation", "libc", ] @@ -814,9 +798,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] @@ -1261,11 +1245,10 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -1429,15 +1412,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-lite" @@ -1454,26 +1437,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 3.0.3", ] [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-io", @@ -1481,7 +1464,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -1764,21 +1746,23 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", + "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1787,109 +1771,81 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "1.5.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", + "icu_locale_core", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] [[package]] -name = "icu_provider_macros" -version = "1.5.0" +name = "idna" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", + "idna_adapter", + "smallvec", + "utf8_iter", ] [[package]] -name = "idna" -version = "1.0.0" +name = "idna_adapter" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", - "smallvec", - "utf8_iter", ] [[package]] @@ -1980,22 +1936,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" -[[package]] -name = "jni" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" -dependencies = [ - "cesu8", - "cfg-if", - "combine", - "jni-sys 0.3.0", - "log", - "thiserror 1.0.61", - "walkdir", - "windows-sys 0.45.0", -] - [[package]] name = "jni" version = "0.22.4" @@ -2191,9 +2131,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.7.3" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "litrs" @@ -2964,12 +2904,6 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "piper" version = "0.2.3" @@ -3045,6 +2979,15 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -3087,15 +3030,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" -[[package]] -name = "quick-xml" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" -dependencies = [ - "memchr", -] - [[package]] name = "quick-xml" version = "0.41.0" @@ -3107,7 +3041,7 @@ dependencies = [ [[package]] name = "quicksearch-core" -version = "1.1.0" +version = "1.1.2" dependencies = [ "argon2", "cfb", @@ -3124,7 +3058,7 @@ dependencies = [ "mime_guess", "notify", "pdf-extract", - "quick-xml 0.31.0", + "quick-xml", "regex", "rtf-parser", "rusqlite", @@ -3140,7 +3074,7 @@ dependencies = [ [[package]] name = "quicksearch-gui" -version = "1.1.0" +version = "1.1.2" dependencies = [ "ashpd", "chrono", @@ -3164,9 +3098,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -3864,9 +3798,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -4058,9 +3992,9 @@ checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "url" -version = "2.5.1" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -4074,12 +4008,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -4305,7 +4233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" dependencies = [ "proc-macro2", - "quick-xml 0.41.0", + "quick-xml", "quote", ] @@ -4343,15 +4271,15 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.0.6" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f1243ef785213e3a32fa0396093424a3a6ea566f9948497e5a2309261a4c97" +checksum = "ef62a3d5f7b2411119a11b6f62570dbff91d7105e011a20fb83fbf8f5761c40f" dependencies = [ - "core-foundation 0.10.1", - "jni 0.21.1", + "jni", "log", "ndk-context", "objc2 0.6.4", + "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", "url", "web-sys", @@ -4550,15 +4478,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -4604,21 +4523,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -4667,12 +4571,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4691,12 +4589,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -4715,12 +4607,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -4751,12 +4637,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -4775,12 +4655,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -4799,12 +4673,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -4823,12 +4691,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -4862,7 +4724,7 @@ dependencies = [ "calloop 0.13.0", "cfg_aliases", "concurrent-queue", - "core-foundation 0.9.4", + "core-foundation", "core-graphics", "cursor-icon", "dpi", @@ -4922,17 +4784,11 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x11-dl" @@ -5009,11 +4865,10 @@ checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" [[package]] name = "yoke" -version = "0.7.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -5021,9 +4876,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -5115,18 +4970,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -5155,10 +5010,21 @@ dependencies = [ ] [[package]] -name = "zerovec" -version = "0.10.2" +name = "zerotrie" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -5167,13 +5033,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" +checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 3.0.3", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9b2677c..d7f3350 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ exclude = ["vendor/pdf-extract"] pdf-extract = { path = "vendor/pdf-extract" } [workspace.package] -version = "1.1.1" +version = "1.1.2" edition = "2021" license = "GPL-3.0-or-later" authors = ["Jeremy "] diff --git a/README.md b/README.md index 2c2d043..dfaad84 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,16 @@ inside that folder. ### GUI -`quicksearch` with no query arguments opens the app: +`quicksearch` with no query arguments opens the app. **One window at a +time**: a second launch reports that QuickSearch is already running and +exits, because two processes indexing one database corrupt it. Terminal +search (`quicksearch `) only reads and keeps working while the +window is open. The guard is a kernel lock on `.lock`, held +by the running process — so a crash or a power cut releases it, and the +leftover file never locks you out. The lock follows `database_path`: point +Settings at a different index and it moves with you, and if that index +belongs to another instance the change is refused rather than written, so a +path you cannot open can never end up in the config file. - **Search**: results appear as you type; every keystroke cancels the previous search. One checkbox enables the two fuzzy passes, and once a @@ -491,7 +500,14 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. `wal_checkpoint(TRUNCATE)` every `processing.maximum_wal_size` bytes of log, because SQLite's own autocheckpoint can only reset the log at an instant no reader holds it — and a run keeps a reader per root querying - throughout, so left alone the log grows for the whole run. `files` holds + throughout, so left alone the log grows for the whole run. That threshold + is lowered to fit the volume when free space is short, and a run stops + with an error rather than fill the disk: SQLite reaches the wal-index + through an mmap, and a page fault the filesystem cannot back arrives as + **SIGBUS**, which no `Result` can catch. The index's own files are, for + the same reason, never walked into — hashing one means opening it, and on + POSIX closing any descriptor on an inode cancels every advisory lock the + process holds on it, SQLite's documented corruption hazard. `files` holds metadata (name, path, size, mtime, hash, MIME/type bitmask, content state); `searchabletext` is a *contentless* FTS5 table over one column, the document body (postings only, configurable tokenizer, trigram by @@ -572,7 +588,7 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. tokenizing can do; the scheduling shares it fairly and keeps the walk first, it does not raise it. Every run ends — whether it completed or was stopped — with an optimize pass on its own connection: - checkpoint, VACUUM if the file has at least 10% slack to reclaim, `PRAGMA + checkpoint, VACUUM if the file has at least 20% slack to reclaim, `PRAGMA optimize`, checkpoint again. Progress streams through a polled `IndexingStatus`, which reads `Optimizing` for the duration of that pass — and `Preparing` for everything a run does before its first file is walked: @@ -674,13 +690,15 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. to that by construction, because a buffer allocated and freed inside one loop iteration never moves RSS. `DocDecoder` must therefore never fall back to a per-row allocating decode, and there is a trap waiting there: - `zstd::encode_all`, which the indexer writes with, is *stream*-based and so - records no content size in the frame header, meaning - `get_frame_content_size` returns `None` for every row this ever sees. The - "cannot happen" branch is the only branch. Sizing the buffer from the header - and handing the `None` case to `zstd::decode_all` looks obviously right and - costs ~2.4 MiB per document, because `decode_all` builds a streaming decoder - per call — 27 of the 30 GiB a fuzzy search moved. Growing this buffer and + the indexer writes through `DocEncoder`, i.e. `zstd::bulk::Compressor` and + so `ZSTD_compress2`, which is handed the whole document at once and records + its length in the frame header. `get_frame_content_size` therefore returns + `Some` for every row this writer produced, the reservation is normally + exact, and the growth loop below it is the fallback for a frame written by + a *stream* encoder rather than the only branch. What must not come back is + handing the `None` case to `zstd::decode_all`, which looks obviously right + and costs ~2.4 MiB per document because it builds a streaming decoder per + call — 27 of the 30 GiB a fuzzy search moved. Growing this buffer and keeping it is what makes decoding a row allocate nothing at all. Measured over the same 77k-file index, per query: `cascade` 582 → 14 MiB, `function` 6.0 GiB → 29 MiB, `--fuzzy cascade` diff --git a/config_example.toml b/config_example.toml index 41efad9..b40d516 100644 --- a/config_example.toml +++ b/config_example.toml @@ -74,6 +74,11 @@ content_extensions = [] # The Windows defaults add: "$RECYCLE.BIN", "System Volume Information", # "pagefile.sys", "hiberfil.sys", "swapfile.sys", "Thumbs.db", # "desktop.ini". +# +# The index itself (database_path and its -wal/-shm/-journal/.lock +# siblings) is always skipped and needs no pattern here — not as a +# convenience but because merely opening one of those files to hash it +# would cancel SQLite's locks on it for the whole process. ignore_patterns = [".git", "node_modules", "*.tmp", ".venv", "venv"] # Worth adding by hand if you index a whole Windows drive rather than just @@ -124,6 +129,13 @@ fts_update_batch_size = 1000 # unattended the log grows for the whole run and can end up larger than the # index. Set to 0 to disable forced checkpoints; any other value below # 16777216 is raised to it. +# +# This is a stall-frequency knob, not a safety one: when the volume holding +# the index is short on space the indexer checkpoints sooner than asked — +# including when this is 0 — and stops the run with an error rather than +# fill the disk. Running out is not a clean failure; SQLite reaches its +# wal-index through a memory map, and a write the filesystem cannot back +# kills the process with SIGBUS instead of returning an error. maximum_wal_size = 536870912 # FTS5 tokenizer: 'trigram' (substring matching, the default; gets # remove_diacritics 1 appended), 'unicode61', 'porter', or a full FTS5 diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index 69d0947..5b21d31 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -28,7 +28,7 @@ walkdir = "2.5.0" # containers (docx/xlsx/pptx/odt/ods/odp), whose entries are always deflate or # stored, so zstd/bzip2/aes-crypto are all dead weight here. zip = { version = "0.6", default-features = false, features = ["deflate"] } -quick-xml = "0.31" +quick-xml = "0.41" # OLE2 compound-file reader, for the pre-2007 binary Office formats # (.doc/.xls/.ppt) whose text lives in named streams rather than a zip. Already # in the lockfile transitively via infer, so naming it directly compiles @@ -101,6 +101,10 @@ windows-sys = { version = "0.52", features = [ # the directory-count path passes null for it. "Win32_Security", "Win32_Storage_FileSystem", + # OVERLAPPED, which `LockFileEx` takes for the index's instance lock. + # `LockFileEx` and `GetDiskFreeSpaceExW` themselves are in + # Win32_Storage_FileSystem above. + "Win32_System_IO", "Win32_System_Threading", "Win32_System_WindowsProgramming", ] } diff --git a/crates/quicksearch-core/examples/walkprobe.rs b/crates/quicksearch-core/examples/walkprobe.rs index d85541c..84ff5b9 100644 --- a/crates/quicksearch-core/examples/walkprobe.rs +++ b/crates/quicksearch-core/examples/walkprobe.rs @@ -80,7 +80,7 @@ fn serial(root: &str, config: &Config, existing: &DirRows) -> (usize, usize) { for entry in filtered_walk(root, false, false, &ignore, &UnreadableDirs::default()) { seen += 1; // Same rule as the real walk: a name that is not valid UTF-8 cannot be - // stored in `files.path` and reopened by it, so it is skipped before + // stored in the index and reopened by its path, so it is skipped before // anything tries to hash it. Counted as seen, never prepared. let Some(path) = entry.path().to_str().map(str::to_owned) else { continue; diff --git a/crates/quicksearch-core/src/cli.rs b/crates/quicksearch-core/src/cli.rs index 77c420e..17e5ed1 100644 --- a/crates/quicksearch-core/src/cli.rs +++ b/crates/quicksearch-core/src/cli.rs @@ -103,17 +103,23 @@ impl SizeReport { /// failure reason comes from `failed_files`, the one place it is written. pub fn status_for_path(db_path: &str, path: &str) -> Result { let conn = open_existing(db_path, false)?; - let row: Option<(i64, Option)> = conn - .query_row( - "SELECT f.content_state, ff.reason \ + let split = crate::file_handling::split_db_path(path); + let row: Option<(i64, Option)> = match split { + // Not a path a row could ever have been stored under, so it is not + // indexed — the same answer as a miss. + None => None, + Some((parent, name)) => conn + .query_row( + "SELECT f.content_state, ff.reason \ FROM files f \ LEFT JOIN failed_files ff ON ff.file_id = f.id \ - WHERE f.path = ?1", - params![path], - |r| Ok((r.get(0)?, r.get(1)?)), - ) - .optional() - .map_err(|e| format!("status_for_path({}): {}", path, e))?; + WHERE f.parent = ?1 AND f.name = ?2", + params![parent, name], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional() + .map_err(|e| format!("status_for_path({}): {}", path, e))?, + }; Ok(match row { None => FileStatus { path: path.to_string(), @@ -138,7 +144,7 @@ pub fn list_failed(db_path: &str, limit: Option) -> Result None => String::new(), }; let sql = format!( - "SELECT ff.file_id, f.path, ff.reason, ff.ts \ + "SELECT ff.file_id, f.parent, f.name, ff.reason, ff.ts \ FROM failed_files ff \ JOIN files f ON f.id = ff.file_id \ ORDER BY ff.ts DESC{}", @@ -149,11 +155,13 @@ pub fn list_failed(db_path: &str, limit: Option) -> Result .map_err(|e| format!("list_failed prepare: {}", e))?; let rows = stmt .query_map([], |r| { + let parent: String = r.get(1)?; + let name: String = r.get(2)?; Ok(FailedEntry { file_id: r.get(0)?, - path: r.get(1)?, - reason: r.get(2)?, - ts: r.get(3)?, + path: format!("{}{}", parent, name), + reason: r.get(3)?, + ts: r.get(4)?, }) }) .map_err(|e| format!("list_failed query: {}", e))?; @@ -252,8 +260,7 @@ mod tests { &tx, &NewFile { name: "a.txt", - path: "/tmp/a.txt", - parent: "/tmp", + parent: "/tmp/", size: 1, mtime: 1, mime: Some("text/plain"), @@ -269,8 +276,7 @@ mod tests { &tx, &NewFile { name: "b.bin", - path: "/tmp/b.bin", - parent: "/tmp", + parent: "/tmp/", size: 1, mtime: 1, mime: None, @@ -343,8 +349,7 @@ mod tests { &tx, &NewFile { name: "c.txt", - path: "/tmp/c.txt", - parent: "/tmp", + parent: "/tmp/", size: 1, mtime: 1, mime: Some("text/plain"), @@ -361,8 +366,7 @@ mod tests { &tx, &NewFile { name: "d.bin", - path: "/tmp/d.bin", - parent: "/tmp", + parent: "/tmp/", size: 1, mtime: 1, mime: None, @@ -422,8 +426,7 @@ mod tests { &tx, &NewFile { name: "big.txt", - path: "/tmp/big.txt", - parent: "/tmp", + parent: "/tmp/", size: 1, mtime: 1, mime: Some("text/plain"), @@ -477,13 +480,12 @@ mod tests { { let mut conn = open_or_recreate(dbp, "unicode61").unwrap(); let tx = conn.transaction().unwrap(); - for (name, path) in [("a.txt", "/tmp/a.txt"), ("b.txt", "/tmp/b.txt")] { + for name in ["a.txt", "b.txt"] { insert_file( &tx, &NewFile { name, - path, - parent: "/tmp", + parent: "/tmp/", size: 1, mtime: 1, mime: Some("text/plain"), diff --git a/crates/quicksearch-core/src/config/diff.rs b/crates/quicksearch-core/src/config/diff.rs index e495cac..2e20fc6 100644 --- a/crates/quicksearch-core/src/config/diff.rs +++ b/crates/quicksearch-core/src/config/diff.rs @@ -15,7 +15,7 @@ use super::*; /// app was closed. See [`crate::scope`] for the pass that applies it. #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct IndexWork { - /// Roots that are no longer configured, in `files.path` spelling. Every + /// Roots that are no longer configured, in stored spelling. Every /// row beneath one is deleted; no filesystem access is involved, so a /// root whose folder is gone is handled the same as one that still /// exists. diff --git a/crates/quicksearch-core/src/config/mod.rs b/crates/quicksearch-core/src/config/mod.rs index 8184a6e..8ae4b80 100644 --- a/crates/quicksearch-core/src/config/mod.rs +++ b/crates/quicksearch-core/src/config/mod.rs @@ -567,13 +567,37 @@ impl Config { // open again. `rename` is atomic on both platforms, and the `sync_all` // before it means the bytes are on the disk before the name points at // them. - let tmp = path.with_extension("toml.tmp"); - write_private(&tmp, content.as_bytes()) - .map_err(|e| format!("Failed to write config file {}: {}", tmp.display(), e))?; + let tmp = write_private_temp(&path, content.as_bytes()).map_err(|e| { + format!( + "Failed to write config file beside {}: {}", + path.display(), + e + ) + })?; fs::rename(&tmp, &path).map_err(|e| { let _ = fs::remove_file(&tmp); format!("Failed to replace config file {}: {}", path.display(), e) })?; + // The rename is atomic but not durable: it is a directory entry, and + // the directory has its own dirty state. Without this a power cut just + // after can leave *neither* name, which is the salt loss the whole + // dance exists to prevent. Best-effort, and on Windows it is a no-op + // every time rather than only on an exotic filesystem: opening a + // directory as a file needs `FILE_FLAG_BACKUP_SEMANTICS`, which + // `fs::File::open` does not pass, so this fails and is skipped. NTFS + // journals the rename itself, which is the guarantee this is reaching + // for; on Unix it has to be asked for. + if let Some(dir) = path.parent() { + if let Ok(handle) = fs::File::open(dir) { + let _ = handle.sync_all(); + } + // Leftovers from a save that died between `create_new` and the + // rename. The old fixed `config.toml.tmp` overwrote itself, so + // there was never more than one; a unique name per attempt is what + // makes the write safe (see `write_private_temp`) and what makes + // them accumulate, so they are swept here instead. + sweep_stale_temps(dir, &path); + } Ok(()) } @@ -598,8 +622,72 @@ impl Config { } } + /// Whether `path` is the index itself — the database, one of SQLite's + /// `-wal`/`-shm`/`-journal` sidecars, or the instance lock. + /// + /// For the paths that arrive from a `files` row rather than from a walk: + /// once the walk stops indexing the index + /// ([`crate::file_handling::index_file_set`], which explains why opening + /// one is fatal), such rows are swept away, but a row written by an older + /// build survives until the sweep reaches it and can still be opened from + /// a result list in the meantime. + /// + /// The name is compared first and the directory only on a hit, so the + /// overwhelmingly common miss costs one string comparison rather than the + /// `canonicalize` the directory check needs. + /// + /// Names are folded where the filesystem folds them + /// ([`crate::platform::PATHS_ARE_CASE_INSENSITIVE`]): a stored row spelling + /// the database `Index.sqlite` names the same file as a config spelling it + /// `index.sqlite`, and opening it has the same consequence. ASCII only, + /// matching `PATH_COLLATION` and [`IgnoreSet`]. + pub fn is_index_file(&self, path: &Path) -> bool { + let db = self.resolved_database_path(); + let (Some(db_name), Some(name)) = ( + db.file_name().and_then(|s| s.to_str()), + path.file_name().and_then(|s| s.to_str()), + ) else { + return false; + }; + let same_name = |a: &str, b: &str| { + if crate::platform::PATHS_ARE_CASE_INSENSITIVE { + a.eq_ignore_ascii_case(b) + } else { + a == b + } + }; + // `str::get`, not `name[..cut]`: the cut is `db_name`'s *byte* length + // and the two names are unrelated strings, so it can land inside a + // multi-byte character — `€xyz` against a two-byte `db_name` is six + // bytes either way — and indexing there panics. A non-boundary is + // simply not a match. + let name_matches = same_name(name, db_name) + || crate::file_handling::INDEX_SIDECAR_SUFFIXES + .iter() + .any(|s| { + let cut = db_name.len(); + name.len() == cut + s.len() + && name.get(..cut).is_some_and(|head| same_name(head, db_name)) + && name.get(cut..).is_some_and(|tail| same_name(tail, s)) + }); + if !name_matches { + return false; + } + let same_dir = |a: &Path, b: &Path| { + a == b + || a.canonicalize().unwrap_or_else(|_| a.to_path_buf()) + == b.canonicalize().unwrap_or_else(|_| b.to_path_buf()) + }; + match (path.parent(), db.parent()) { + (Some(a), Some(b)) => same_dir(a, b), + // Both at a filesystem root, or neither: the name match stands. + (None, None) => true, + _ => false, + } + } + /// `resolved_indexing_paths` canonicalized and spelled the way - /// `files.path` prefixes them. + /// stored parents are prefixed with them. /// /// The form roots must be compared in: `~/docs`, `docs` in a portable /// config and `/home/me/docs` are one root under three spellings, and a @@ -631,30 +719,121 @@ impl Config { } } -/// Write `bytes` to `path`, owner-readable only, and flush them to the disk -/// before returning. +/// Write `bytes` to a fresh temporary file beside `target`, owner-only, with +/// its contents flushed to the disk, and return its path for the caller to +/// rename into place. /// -/// `O_NOFOLLOW` on Unix: the config directory is not always somewhere only -/// this user can write — a portable install can sit in a shared or removable -/// directory — and a symlink left at the config's name would otherwise -/// redirect this write onto whatever it points at. -fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> { +/// `create_new` and a unique name, not a fixed one: `O_NOFOLLOW` refuses a +/// symlink but says nothing about a *regular* file or a hardlink that is +/// already sitting at the name we were going to use. The config directory is +/// not always somewhere only this user can write — a portable install can sit +/// in a shared or removable directory — and an attacker who pre-creates the +/// temp file as a hardlink to a file they can read would otherwise be handed +/// `[security].salt`. `mode(0o600)` only applies to a file this call creates, +/// which is the same reason. +fn write_private_temp(target: &Path, bytes: &[u8]) -> std::io::Result { use std::io::Write; - let mut opts = fs::OpenOptions::new(); - opts.write(true).create(true).truncate(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - opts.custom_flags(libc::O_NOFOLLOW); - opts.mode(0o600); + let dir = target.parent().unwrap_or_else(|| Path::new(".")); + let stem = target + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| "config.toml".to_string()); + + // A handful of attempts, then give up rather than spin: if something is + // racing us for every name we pick, failing the save is the honest + // outcome — the caller treats a failed save as fatal to the change. + let mut last_err = None; + for _ in 0..8 { + let tmp = dir.join(format!("{}.{}.tmp", stem, unique_suffix())); + let mut opts = fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.custom_flags(libc::O_NOFOLLOW); + opts.mode(0o600); + } + match opts.open(&tmp) { + Ok(mut f) => { + let wrote = f.write_all(bytes).and_then(|()| f.sync_all()); + return match wrote { + Ok(()) => Ok(tmp), + Err(e) => { + let _ = fs::remove_file(&tmp); + Err(e) + } + }; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => last_err = Some(e), + Err(e) => return Err(e), + } } - let mut f = opts.open(path)?; - f.write_all(bytes)?; - // The rename that follows is atomic with respect to the *directory*, not - // to the file's contents: without this, a crash can leave the new name - // pointing at a block of zeroes. - f.sync_all() + Err(last_err.unwrap_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not create a temporary config file", + ) + })) +} + +/// How old an abandoned temp file must be before [`sweep_stale_temps`] takes +/// it. A save is a serialize, a write and a rename — milliseconds — so an hour +/// is far past any doubt, while still short enough that leftovers do not +/// accumulate across a run of crashes. +const TEMP_SWEEP_AGE: std::time::Duration = std::time::Duration::from_secs(60 * 60); + +/// Delete abandoned `.*.tmp` files beside the config. +/// +/// [`write_private_temp`] must use a *unique* name — a fixed one can be +/// pre-created by someone else as a hardlink — and a unique name is one that +/// nothing later overwrites, so a process that dies between `create_new` and +/// the rename leaves its temp file behind for good. The old fixed +/// `config.toml.tmp` was reused by the next save and so never accumulated; +/// this is what replaces that property. +/// +/// Age is the discriminator, not the recorded PID: a temp file created seconds +/// ago may belong to a save running *right now* in another process, and +/// deleting that would destroy the very write this whole dance protects. +/// Entirely best-effort — a directory that cannot be listed is not a reason to +/// fail a save that already succeeded. +fn sweep_stale_temps(dir: &Path, target: &Path) { + let Some(stem) = target.file_name().and_then(|n| n.to_str()) else { + return; + }; + // The trailing dot matters: without it the config file itself would be a + // prefix match. + let prefix = format!("{}.", stem); + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { continue }; + if !name.starts_with(&prefix) || !name.ends_with(".tmp") { + continue; + } + let abandoned = entry + .metadata() + .and_then(|m| m.modified()) + .is_ok_and(|t| t.elapsed().is_ok_and(|age| age >= TEMP_SWEEP_AGE)); + if abandoned { + let _ = fs::remove_file(entry.path()); + } + } +} + +/// A short, non-guessable-enough suffix for the temp name. +/// +/// This does not need to be unpredictable to an attacker — `create_new` is +/// what makes the write safe — only unlikely to collide with a leftover from +/// an interrupted save, so the process id and a clock reading are plenty. +fn unique_suffix() -> String { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos()) + .unwrap_or(0); + format!("{}-{}", std::process::id(), nanos) } /// Reserved `content_extensions` entry standing for "files with no diff --git a/crates/quicksearch-core/src/config/tests.rs b/crates/quicksearch-core/src/config/tests.rs index fcdc4cc..d3b0806 100644 --- a/crates/quicksearch-core/src/config/tests.rs +++ b/crates/quicksearch-core/src/config/tests.rs @@ -1043,3 +1043,55 @@ fn the_search_table_ships_without_size_or_modified() { assert_eq!(older.search.display_limit, 500); assert!(older.search.live_results, "live results default to on"); } + +/// The guard the live watcher and duplicate verification consult before +/// opening a path that came from a `files` row. See +/// [`crate::file_handling::index_file_set`] for what opening one costs. +#[test] +fn the_index_and_every_sidecar_are_recognised() { + let mut c = Config::default(); + c.paths.database_path = "/var/lib/qs/index.sqlite".to_string(); + + for name in [ + "index.sqlite", + "index.sqlite-wal", + "index.sqlite-shm", + "index.sqlite-journal", + "index.sqlite.lock", + ] { + let p = PathBuf::from(format!("/var/lib/qs/{}", name)); + assert!(c.is_index_file(&p), "{} should be recognised", name); + } + + // A near miss on the name, and the right name in the wrong directory. + for p in [ + "/var/lib/qs/index.sqlite-walrus", + "/var/lib/qs/notes-wal", + "/var/lib/qs/index.sqlite2", + "/var/lib/other/index.sqlite-wal", + "/var/lib/qs/sub/index.sqlite", + ] { + assert!( + !c.is_index_file(Path::new(p)), + "{} is not part of the index", + p + ); + } +} + +/// `database_path` is resolved the same way everywhere else resolves it, so a +/// `~` or a relative spelling has to compare equal to the absolute file. +#[test] +fn a_tilde_database_path_still_matches_the_real_file() { + let Some(home) = crate::platform::home_dir() else { + return; + }; + let mut c = Config::default(); + c.paths.database_path = "~/.local/share/quicksearch/index.sqlite".to_string(); + + let absolute = PathBuf::from(home).join(".local/share/quicksearch/index.sqlite"); + assert!(c.is_index_file(&absolute)); + let wal = absolute.with_file_name("index.sqlite-wal"); + assert!(c.is_index_file(&wal)); + assert!(!c.is_index_file(&absolute.with_file_name("other.sqlite"))); +} diff --git a/crates/quicksearch-core/src/content.rs b/crates/quicksearch-core/src/content.rs index 7cf908a..cfdf4b0 100644 --- a/crates/quicksearch-core/src/content.rs +++ b/crates/quicksearch-core/src/content.rs @@ -396,8 +396,7 @@ mod tests { &tx, &NewFile { name: f.file_name().unwrap().to_str().unwrap(), - path: f.to_str().unwrap(), - parent: d.to_str().unwrap(), + parent: &crate::file_handling::dir_to_db_parent(&d), size: std::fs::metadata(&f).unwrap().len(), mtime: 1, mime: Some("text/plain"), @@ -494,7 +493,7 @@ mod tests { .lock() .unwrap() .query_row( - "SELECT content_state FROM files WHERE path = ?1", + "SELECT content_state FROM files WHERE parent || name = ?1", rusqlite::params![p.to_str().unwrap()], |r| r.get(0), ) diff --git a/crates/quicksearch-core/src/coordinator.rs b/crates/quicksearch-core/src/coordinator.rs index 59cfb24..07d1716 100644 --- a/crates/quicksearch-core/src/coordinator.rs +++ b/crates/quicksearch-core/src/coordinator.rs @@ -234,6 +234,8 @@ impl IndexCoordinator { last_event_at: None, pending_since: None, needs_full_run: false, + run_retry_at: None, + run_retry_delay: inner::RUN_RETRY_BASE, pending_work: None, reconcile_done: None, reconcile_cut_short: false, diff --git a/crates/quicksearch-core/src/coordinator/inner.rs b/crates/quicksearch-core/src/coordinator/inner.rs index 8a37e9f..fa8b0d9 100644 --- a/crates/quicksearch-core/src/coordinator/inner.rs +++ b/crates/quicksearch-core/src/coordinator/inner.rs @@ -3,6 +3,32 @@ use super::*; +/// First wait after a full run is refused or fails to start. +pub(super) const RUN_RETRY_BASE: Duration = Duration::from_secs(30); + +/// Ceiling for that wait. A misconfiguration that is never fixed then costs +/// one attempt every few minutes instead of one per second. +const RUN_RETRY_MAX: Duration = Duration::from_secs(300); + +/// Individually-logged occurrences of each refusal before it goes quiet. The +/// message is identical every time, and the log ring is the only record of +/// what *else* happened; these are reset the moment a run starts. +static NO_ROOTS: crate::log::Throttle = crate::log::Throttle::new(3); +static NESTED_ROOTS: crate::log::Throttle = crate::log::Throttle::new(3); +static START_FAILURES: crate::log::Throttle = crate::log::Throttle::new(3); + +/// Why a full run is being started, which is what decides how a refusal is +/// reported. See [`Inner::refuse`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum RunTrigger { + /// Something the user did: *Index now*, *Rebuild index*, or a settings + /// change wide enough to need a walk. A refusal owes them an answer. + Requested, + /// The periodic timer, or repair the coordinator scheduled for itself. A + /// refusal here is worth one log line and then quiet. + Scheduled, +} + pub(super) struct Inner { pub(super) config: Config, pub(super) indexing: Arc, @@ -40,6 +66,18 @@ pub(super) struct Inner { /// defer application past `pending_max_defer`. pub(super) pending_since: Option, pub(super) needs_full_run: bool, + /// Earliest time another full run may be *attempted*, when the last one + /// was refused or failed. Without it a config with no roots, or with + /// nested roots, or an index on a full disk, retries every tick forever: + /// two stderr writes and two ring pushes a second, which evicts the whole + /// 5000-line log ring in under an hour and takes the diagnostics for the + /// actual problem with it — and in the failed-run case re-walks the tree + /// each time. + pub(super) run_retry_at: Option, + /// How long the next refusal waits, doubling to [`RUN_RETRY_MAX`] so a + /// permanent misconfiguration costs almost nothing, and reset once a run + /// actually starts. + pub(super) run_retry_delay: Duration, /// Reconciliation owed to a config change, part-applied across ticks. /// Unlike `needs_full_run`, this is acted on in manual mode too. pub(super) pending_work: Option, @@ -95,7 +133,7 @@ impl Inner { CoordCmd::SetMode(IndexMode::ManualStopped) => self.enter_manual_stopped(), // ManualRunning isn't directly settable; ReindexNow is the verb. CoordCmd::SetMode(IndexMode::ManualRunning) | CoordCmd::ReindexNow => { - self.start_full_run(); + self.start_full_run(RunTrigger::Requested); if self.mode != IndexMode::Auto { self.mode = IndexMode::ManualRunning; } @@ -139,10 +177,18 @@ impl Inner { self.pending_work = None; self.reconcile_done = None; if let Err(e) = self.indexing.delete_index_for_rebuild(&db) { + // Not a warning to step over: a run started now would + // reopen the *old* index and present itself as the rebuild + // the user asked for. Worse after a password change, where + // the config already claims a protection the surviving + // file does not have. Report it and leave the index alone. crate::log_warn!("coordinator: rebuild: {}", e); + self.indexing + .report_error(format!("could not rebuild the index: {}", e)); + return; } self.clear_root_counts(); - self.start_full_run(); + self.start_full_run(RunTrigger::Requested); if self.mode != IndexMode::Auto { self.mode = IndexMode::ManualRunning; } @@ -174,7 +220,7 @@ impl Inner { // delivers anything else, so nothing downstream checks, and // a file renamed *out* of every root would otherwise be // written into the index at its new home. Roots in the same - // spelling `files.path` uses — the caller's paths are. + // spelling the index uses — the caller's paths are. let prefixes: Vec = self .config .normalized_indexing_paths() @@ -224,6 +270,12 @@ impl Inner { if self.saw_running { self.saw_running = false; self.was_busy = true; + // A run that errored never stamps `last_full_index`, so + // `periodic_due` stays true and the next tick starts another one — + // a fresh whole-tree walk per second against, say, a full disk. + if matches!(status, IndexingStatus::Error(_)) { + self.defer_runs(); + } self.refresh_last_full_index(); // Eager re-read: the run just changed the number on screen. self.files_at = None; @@ -263,8 +315,9 @@ impl Inner { worked = true; } - if self.needs_full_run || self.periodic_due() { - self.start_full_run(); + let deferred = self.run_retry_at.is_some_and(|at| Instant::now() < at); + if (self.needs_full_run || self.periodic_due()) && !deferred { + self.start_full_run(RunTrigger::Scheduled); worked = true; } @@ -408,7 +461,7 @@ impl Inner { // Widening adds files only a walk can produce; mirrors `ReindexNow`, // including the manual-mode round trip back to stopped. if cursor.reindex() { - self.start_full_run(); + self.start_full_run(RunTrigger::Requested); if self.mode != IndexMode::Auto { self.mode = IndexMode::ManualRunning; } @@ -555,6 +608,9 @@ impl Inner { Err(e) => { crate::log_warn!("coordinator: targeted update unavailable: {}", e); self.targeted.clear(); + // As in `start_full_run`: a resume point outlives its event + // here too, and would misapply to the next one for that path. + self.resume_from.retain(|p, _| self.pending.contains_key(p)); return; } }; @@ -633,10 +689,15 @@ impl Inner { } fn periodic_due(&self) -> bool { + // `.max(1)`: at zero every tick is "due", so a full run would start + // the moment the previous one finished, forever. There is no setting + // that means "reindex continuously" — manual mode is how you say + // "never", so zero is a typo rather than an intent. let interval_secs = self .config .indexing .reindex_interval_minutes + .max(1) .saturating_mul(60); let last = crate::lock_ok(&self.shared).last_full_index; match last { @@ -651,7 +712,20 @@ impl Inner { } } - fn start_full_run(&mut self) { + fn start_full_run(&mut self, trigger: RunTrigger) { + // A run already in progress is not a refusal: whoever called this + // wanted a run and there is one. Returning here rather than falling + // through to `start_indexing`'s "already running" error keeps an + // *Index now* pressed mid-run from backing off future runs and from + // replacing the progress the user is watching with an error. It also + // covers `Optimizing`, which `start_indexing` does not reject even + // though it holds a write transaction over the whole file. + if !matches!( + self.indexing.get_status(), + IndexingStatus::Idle | IndexingStatus::Error(_) + ) { + return; + } let roots: Vec = self .config .resolved_indexing_paths() @@ -659,20 +733,23 @@ impl Inner { .map(|p| p.to_string_lossy().into_owned()) .collect(); if roots.is_empty() { - crate::log_warn!("coordinator: no indexing roots configured"); + self.refuse(trigger, &NO_ROOTS, "no indexing roots are configured"); return; } // Backstop for hand-edited configs; the GUI rejects nested roots // itself. let nested = crate::config::nested_roots(&roots); if !nested.is_empty() { - for (child, parent) in &nested { - crate::log_warn!( - "coordinator: refusing to index: root {} is nested under {}", - child, - parent - ); - } + let detail = nested + .iter() + .map(|(child, parent)| format!("{} is nested under {}", child, parent)) + .collect::>() + .join("; "); + self.refuse( + trigger, + &NESTED_ROOTS, + &format!("refusing to index nested roots: {}", detail), + ); return; } // The full run owns the DB (and may wipe/rebuild the file). @@ -683,6 +760,12 @@ impl Inner { // unreadable directory (`unreadable.covers`) or of an aliased symlink // target (`aliased_paths`), and those rows would leak until a rebuild. self.pending.retain(|_, ev| is_removal(ev)); + // Resume points describe events that no longer exist. Left behind, the + // next `Create` for that path would `skip` the first n entries of a + // walk that has nothing to do with the one the count came from, and + // those files would have no row until a later full run. + self.resume_from + .retain(|p, _| self.pending.contains_key(p) || self.targeted.contains_key(p)); if self.pending.is_empty() { self.last_event_at = None; self.pending_since = None; @@ -691,15 +774,58 @@ impl Inner { .indexing .start_indexing(roots, self.db_path(), self.config.clone()) { - crate::log_warn!("coordinator: start indexing: {}", e); + self.refuse( + trigger, + &START_FAILURES, + &format!("could not start indexing: {}", e), + ); return; } + // A run started, so whatever was wrong is not wrong any more. + self.run_retry_at = None; + self.run_retry_delay = RUN_RETRY_BASE; + NO_ROOTS.reset(); + NESTED_ROOTS.reset(); + START_FAILURES.reset(); // `start_indexing` claims Running before returning, so there is no // window in which this thread believes the service idle and writes to // a database the run is about to reopen. self.saw_running = true; } + /// Refuse a full run: back off, and say so at a volume the trigger earns. + /// + /// The throttles exist for the *scheduled* retry, which repeats the same + /// sentence every few minutes for as long as the configuration stays + /// broken and would otherwise evict the log ring. They must not silence a + /// run somebody asked for: after three refusals a fourth press of *Index + /// now* would log nothing, show nothing and change nothing, which reads as + /// a button that does not work. So a requested run resets the throttle — + /// it is a fresh occurrence in the user's eyes — and publishes the reason + /// where they are already looking, the same way a failed rebuild does. + fn refuse(&mut self, trigger: RunTrigger, throttle: &crate::log::Throttle, reason: &str) { + self.defer_runs(); + if trigger == RunTrigger::Requested { + throttle.reset(); + self.indexing.report_error(reason.to_string()); + } + if throttle.allow() { + crate::log_warn!("coordinator: {}", reason); + } + } + + /// Hold off further full runs for a while, doubling the wait each time. + /// + /// Called wherever a run is refused or fails to start. `needs_full_run` is + /// cleared with it: the two refusal paths sit above the point that clears + /// it, so without this the flag stays set and `tick` retries immediately, + /// every second, for as long as the configuration stays broken. + fn defer_runs(&mut self) { + self.needs_full_run = false; + self.run_retry_at = Some(Instant::now() + self.run_retry_delay); + self.run_retry_delay = (self.run_retry_delay * 2).min(RUN_RETRY_MAX); + } + fn enter_auto(&mut self) { self.mode = IndexMode::Auto; self.config.indexing.auto_index = true; @@ -811,6 +937,29 @@ impl Inner { let Some(mut reason) = w.degraded_reason() else { return; }; + // An overflow is not a capacity problem: the watcher is still good, + // it just cannot tell us what it missed. Schedule the run that finds + // out, rather than disabling live updates for the session over + // something a busy minute can cause. + // + // Not a restart. `inotify` and `ReadDirectoryChangesW` both keep + // delivering after their queue overflows — the rescan flag means "you + // missed some", not "this watch is broken" — so tearing down and + // re-registering would re-`stat` every one of `max_watched_dirs` + // (128k) directories, generate events of its own, and do it again the + // next time a busy tree overflowed the queue, which is exactly when + // the machine can least afford it. + // + // Consuming the reason is what makes this a one-shot: the slot is + // never otherwise cleared, so a standing `Overflowed` would re-arm + // `needs_full_run` on every tick and hide a `KernelLimit` recorded + // afterwards. + if matches!(reason, WatchError::Overflowed) { + w.clear_degraded(); + crate::log_warn!("watcher: {}", reason); + self.needs_full_run = true; + return; + } // The async notify callback can't know the count; fill it in here. if let WatchError::KernelLimit { registered } = &mut reason { if *registered == 0 { diff --git a/crates/quicksearch-core/src/coordinator/tests.rs b/crates/quicksearch-core/src/coordinator/tests.rs index 2879cd8..d911cf4 100644 --- a/crates/quicksearch-core/src/coordinator/tests.rs +++ b/crates/quicksearch-core/src/coordinator/tests.rs @@ -512,7 +512,7 @@ impl Fixture { fn stored_mtime(&self, path: &std::path::Path) -> Option { let conn = db::open_existing(&self.db.to_string_lossy(), false).ok()?; conn.query_row( - "SELECT mtime FROM files WHERE path = ?1", + "SELECT mtime FROM files WHERE parent || name = ?1", [path.to_string_lossy().as_ref()], |r| r.get(0), ) diff --git a/crates/quicksearch-core/src/db/mod.rs b/crates/quicksearch-core/src/db/mod.rs index 1082657..1fa9a09 100644 --- a/crates/quicksearch-core/src/db/mod.rs +++ b/crates/quicksearch-core/src/db/mod.rs @@ -17,8 +17,8 @@ pub mod schema; pub use key::{process_key_hex, set_process_key}; pub use open::{ - index_needs_rebuild, open_existing, open_or_recreate, verify_process_key, - CURRENT_SCHEMA_VERSION, KEY_MISMATCH_PREFIX, + index_needs_rebuild, key_mismatch_parts, open_existing, open_or_recreate, verify_process_key, + KeyMismatch, CURRENT_SCHEMA_VERSION, KEY_MISMATCH_PREFIX, }; /// Bumped whenever the index file is replaced rather than modified — a diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs index 311d93d..a4b81d5 100644 --- a/crates/quicksearch-core/src/db/open.rs +++ b/crates/quicksearch-core/src/db/open.rs @@ -25,7 +25,7 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: "; /// values go stale: `files.mime`, `files.type` and `content_state` are /// computed at walk time and never re-derived for unchanged files, so a /// classification change needs the wipe to apply everywhere. -pub const CURRENT_SCHEMA_VERSION: u32 = 7; +pub const CURRENT_SCHEMA_VERSION: u32 = 8; /// Open `db_path` and ensure the on-disk schema matches this build; if it /// doesn't (including a changed `tokenizer`), delete the file and recreate it @@ -241,6 +241,57 @@ fn is_notadb(e: &rusqlite::Error) -> bool { ) } +/// Why a keyed open failed, as something the caller can branch on. +/// +/// The three cases want three different things from a user — retype the +/// password, rebuild the index, supply a password at all — and only one of +/// them is "wrong password". They used to be distinguishable only by reading +/// the English in the message, which breaks the moment a database path +/// happens to contain that English. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyMismatch { + /// A key was applied and the file did not accept it. + WrongPassword, + /// A key was applied but the file on disk is not encrypted at all — + /// protection was enabled and the rebuild that would encrypt it did not + /// finish. + NotEncrypted, + /// No key was applied and the file wants one. + PasswordRequired, +} + +impl KeyMismatch { + /// The machine-readable token carried in the message, between + /// [`KEY_MISMATCH_PREFIX`] and the human detail. + fn token(self) -> &'static str { + match self { + KeyMismatch::WrongPassword => "wrong-password", + KeyMismatch::NotEncrypted => "not-encrypted", + KeyMismatch::PasswordRequired => "password-required", + } + } + + fn from_token(token: &str) -> Option { + match token { + "wrong-password" => Some(KeyMismatch::WrongPassword), + "not-encrypted" => Some(KeyMismatch::NotEncrypted), + "password-required" => Some(KeyMismatch::PasswordRequired), + _ => None, + } + } +} + +/// Split a tagged mismatch message into its cause and the human detail. +/// +/// `None` for any message that is not one — including a `KEY_MISMATCH_PREFIX` +/// message from an older build, which callers should treat as they always did. +pub fn key_mismatch_parts(message: &str) -> Option<(KeyMismatch, &str)> { + let rest = message.strip_prefix(KEY_MISMATCH_PREFIX)?; + let (token, detail) = rest.split_once(' ')?; + let token = token.strip_suffix(':')?; + Some((KeyMismatch::from_token(token)?, detail)) +} + fn key_mismatch_message(db_path: &str, had_key: bool) -> String { // An unencrypted SQLite file still has its plaintext magic; sniffing it // distinguishes "wrong password" from "protection is enabled but the @@ -255,15 +306,31 @@ fn key_mismatch_message(db_path: &str, had_key: bool) -> String { Some(&magic == b"SQLite format 3\0") }) .unwrap_or(false); - let detail = match (had_key, plaintext) { - (true, true) => { + let (cause, detail) = match (had_key, plaintext) { + (true, true) => ( + KeyMismatch::NotEncrypted, "password protection is enabled but the index is not encrypted; \ - rebuild the index to encrypt it" - } - (true, false) => "wrong password (or the file is not a QuickSearch index)", - (false, _) => "the index is password-protected; a password is required", + rebuild the index to encrypt it", + ), + (true, false) => ( + KeyMismatch::WrongPassword, + "wrong password (or the file is not a QuickSearch index)", + ), + (false, _) => ( + KeyMismatch::PasswordRequired, + "the index is password-protected; a password is required", + ), }; - format!("{}index at {}: {}", KEY_MISMATCH_PREFIX, db_path, detail) + // The token sits between the prefix and the detail so that every existing + // `starts_with(KEY_MISMATCH_PREFIX)` test still holds, while a caller that + // needs the cause can have it without reading prose. + format!( + "{}{}: index at {}: {}", + KEY_MISMATCH_PREFIX, + cause.token(), + db_path, + detail + ) } /// True iff the DB has a `schema_info` table whose `version` equals diff --git a/crates/quicksearch-core/src/db/open_tests.rs b/crates/quicksearch-core/src/db/open_tests.rs index ab6081b..6e11a67 100644 --- a/crates/quicksearch-core/src/db/open_tests.rs +++ b/crates/quicksearch-core/src/db/open_tests.rs @@ -124,8 +124,8 @@ fn tokenizer_drift_wipes_db() { let first_effective = { let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); conn.execute( - "INSERT INTO files (name, path, parent, size, mtime) \ - VALUES ('x', '/x', '/', 0, 0)", + "INSERT INTO files (name, parent, size, mtime) \ + VALUES ('x', '/', 0, 0)", [], ) .unwrap(); @@ -164,8 +164,8 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() { { let conn = open_or_recreate(p.to_str().unwrap(), "unicode61").unwrap(); conn.execute( - "INSERT INTO files (name, path, parent, size, mtime) \ - VALUES ('note', '/note.txt', '/', 0, 0)", + "INSERT INTO files (name, parent, size, mtime) \ + VALUES ('note', '/', 0, 0)", [], ) .unwrap(); @@ -384,8 +384,8 @@ fn keyed_create_reopen_and_header_is_encrypted() { { let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&key)).unwrap(); conn.execute( - "INSERT INTO files (name, path, parent, size, mtime) \ - VALUES ('secret', '/secret.txt', '/', 0, 0)", + "INSERT INTO files (name, parent, size, mtime) \ + VALUES ('secret', '/', 0, 0)", [], ) .unwrap(); @@ -418,8 +418,8 @@ fn wrong_key_errors_without_wiping() { let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&test_key(0xa1))).unwrap(); conn.execute( - "INSERT INTO files (name, path, parent, size, mtime) \ - VALUES ('x', '/x', '/', 0, 0)", + "INSERT INTO files (name, parent, size, mtime) \ + VALUES ('x', '/', 0, 0)", [], ) .unwrap(); @@ -532,15 +532,15 @@ fn open_existing_rw_allows_delete() { { let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); conn.execute( - "INSERT INTO files (name, path, parent, size, mtime) \ - VALUES ('a', '/a', '/', 0, 0)", + "INSERT INTO files (name, parent, size, mtime) \ + VALUES ('a', '/', 0, 0)", [], ) .unwrap(); } let conn = open_existing(p.to_str().unwrap(), true).unwrap(); let removed = conn - .execute("DELETE FROM files WHERE path = '/a'", []) + .execute("DELETE FROM files WHERE parent = '/' AND name = 'a'", []) .unwrap(); assert_eq!(removed, 1); drop(conn); @@ -571,8 +571,8 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() { let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); // A write, so the WAL and SHM exist to be checked. conn.execute( - "INSERT INTO files (name, path, parent, size, mtime) \ - VALUES ('a', '/perm-a', '/', 0, 0)", + "INSERT INTO files (name, parent, size, mtime) \ + VALUES ('a', '/', 0, 0)", [], ) .unwrap(); @@ -598,3 +598,47 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() { drop(conn); std::fs::remove_file(&p).ok(); } + +/// `maintain` must work on a *keyed* index, which is the one case the pragma +/// it reads does not answer with an integer. +/// +/// SQLCipher intercepts `PRAGMA page_size` on a keyed connection, answers with +/// `cipher_page_size`, and hands that back as TEXT. Reading it straight into an +/// `i64` failed there and only there — so every unencrypted test passed while +/// every index with a password set silently skipped both its VACUUM and its +/// `PRAGMA optimize`. The assertion is simply that the call succeeds: it has to +/// get past all three pragma reads to return at all. +#[test] +fn maintain_reads_its_pragmas_on_a_keyed_index() { + let p = tmp_db_path(); + let key = test_key(0xc3); + let dir = p.parent().unwrap().to_string_lossy().into_owned(); + { + let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&key)).unwrap(); + conn.execute( + "INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)", + [], + ) + .unwrap(); + } + let conn = open_keyed_with_pragmas(p.to_str().unwrap(), true, Some(&key), PRAGMAS_MAINTENANCE) + .unwrap(); + // A two-row index has no slack worth reclaiming, so `false` is the + // expected answer — what matters is that it is an answer and not an error. + assert_eq!( + crate::db::repo::maintain(&conn, &dir), + Ok(false), + "maintain must not fail on a keyed index" + ); + + // And the value itself has to be usable, not merely readable: a page size + // that parsed as 0 would size the free-space check at zero bytes and wave + // through a VACUUM that cannot fit. + assert!( + crate::db::repo::pragma_number(&conn, "page_size").unwrap() >= 512, + "a real page size, not a silent zero" + ); + + drop(conn); + std::fs::remove_file(&p).ok(); +} diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs index 95c11a7..fc766fe 100644 --- a/crates/quicksearch-core/src/db/repo.rs +++ b/crates/quicksearch-core/src/db/repo.rs @@ -62,7 +62,8 @@ fn set_state_clearing_failure( #[derive(Debug, Clone)] pub struct NewFile<'a> { pub name: &'a str, - pub path: &'a str, + /// The containing directory, ending in the platform separator — see + /// [`crate::file_handling::split_db_path`], which produces the pair. pub parent: &'a str, pub size: u64, pub mtime: u64, @@ -74,21 +75,29 @@ pub struct NewFile<'a> { pub needs_content: bool, } +impl NewFile<'_> { + /// The file's path, for a log or error message. Not stored; see + /// [`super::schema::SCHEMA_CURRENT`]. + fn path(&self) -> String { + format!("{}{}", self.parent, self.name) + } +} + /// Insert a new file row, returning its id. `content_state` comes from /// `needs_content`; there is no separate basic state, because the row -/// existing *is* the basic-index state. `INSERT OR IGNORE`: a UNIQUE(path) -/// collision returns `None` rather than aborting the batch. +/// existing *is* the basic-index state. `INSERT OR IGNORE`: a +/// `UNIQUE(parent, name)` collision returns `None` rather than aborting the +/// batch. pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, String> { let rows = tx .prepare_cached( "INSERT OR IGNORE INTO files ( - name, path, parent, size, mtime, mime, type, content_state, hash - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + name, parent, size, mtime, mime, type, content_state, hash + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", ) .and_then(|mut stmt| { stmt.execute(params![ f.name, - f.path, f.parent, f.size as i64, f.mtime as i64, @@ -98,7 +107,7 @@ pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, f.hash, ]) }) - .map_err(|e| format!("insert file {}: {}", f.path, e))?; + .map_err(|e| format!("insert file {}: {}", f.path(), e))?; if rows == 0 { return Ok(None); } @@ -120,14 +129,14 @@ fn initial_content_state(f: &NewFile<'_>) -> i64 { /// reset its content state from `f.needs_content`, clearing any extracted /// content so the text-indexing pass re-processes it. Writes `size`, `mtime`, /// `hash`, `mime`, `type` and `content_state` — and only those; `name` and -/// `parent` are not refreshed here. +/// `parent` are the key it matches on, so they cannot change here. pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, String> { let id: Option = tx .prepare_cached( "UPDATE files SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5, content_state = ?6 - WHERE path = ?7 + WHERE parent = ?7 AND name = ?8 RETURNING id", ) .and_then(|mut stmt| { @@ -139,13 +148,14 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result