From e1e4c19ef8683357ff40ecc547fe070164df601d Mon Sep 17 00:00:00 2001 From: = <=> Date: Tue, 21 Apr 2026 23:00:47 -0400 Subject: [PATCH] Added pagination to search results, handling of more file types, and several structural changes working towards baloo integration. --- Cargo.lock | 626 +++++- Cargo.toml | 25 +- config_example.toml | 50 +- crates/quicksearch-core/Cargo.toml | 25 + crates/quicksearch-core/src/cli.rs | 283 +++ .../quicksearch-core/src}/config.rs | 185 +- crates/quicksearch-core/src/db/migrate.rs | 281 +++ crates/quicksearch-core/src/db/mod.rs | 13 + crates/quicksearch-core/src/db/repo.rs | 495 +++++ crates/quicksearch-core/src/db/schema.rs | 84 + .../src}/document_extraction.rs | 728 +++--- crates/quicksearch-core/src/extract/audio.rs | 96 + crates/quicksearch-core/src/extract/image.rs | 96 + crates/quicksearch-core/src/extract/mod.rs | 149 ++ crates/quicksearch-core/src/extract/office.rs | 69 + crates/quicksearch-core/src/extract/pdf.rs | 77 + .../quicksearch-core/src/extract/plaintext.rs | 65 + .../quicksearch-core/src}/file_handling.rs | 276 +-- .../quicksearch-core/src}/indexing.rs | 1972 +++++++++-------- crates/quicksearch-core/src/lib.rs | 12 + crates/quicksearch-core/src/mime.rs | 238 ++ crates/quicksearch-core/src/query/ast.rs | 58 + crates/quicksearch-core/src/query/lexer.rs | 214 ++ crates/quicksearch-core/src/query/mod.rs | 28 + crates/quicksearch-core/src/query/parser.rs | 284 +++ .../quicksearch-core/src/query/translator.rs | 654 ++++++ crates/quicksearch-core/src/search_sql.rs | 328 +++ crates/quicksearch-core/src/shutdown.rs | 27 + crates/quicksearch-core/src/watcher.rs | 496 +++++ crates/quicksearch-gui/Cargo.toml | 15 + .../quicksearch-gui/assets}/app.js | 464 ++-- .../quicksearch-gui/assets}/index.html | 42 +- .../quicksearch-gui/assets}/styles.css | 598 ++--- .../quicksearch-gui/src}/frontend.rs | 751 ++++--- crates/quicksearch-gui/src/main.rs | 64 + crates/quicksearch-gui/src/search.rs | 512 +++++ index.html | 170 -- run.bat | 2 +- run.sh | 4 +- src/main.rs | 71 - src/search.rs | 404 ---- 41 files changed, 7750 insertions(+), 3281 deletions(-) create mode 100644 crates/quicksearch-core/Cargo.toml create mode 100644 crates/quicksearch-core/src/cli.rs rename {src => crates/quicksearch-core/src}/config.rs (75%) create mode 100644 crates/quicksearch-core/src/db/migrate.rs create mode 100644 crates/quicksearch-core/src/db/mod.rs create mode 100644 crates/quicksearch-core/src/db/repo.rs create mode 100644 crates/quicksearch-core/src/db/schema.rs rename {src => crates/quicksearch-core/src}/document_extraction.rs (97%) create mode 100644 crates/quicksearch-core/src/extract/audio.rs create mode 100644 crates/quicksearch-core/src/extract/image.rs create mode 100644 crates/quicksearch-core/src/extract/mod.rs create mode 100644 crates/quicksearch-core/src/extract/office.rs create mode 100644 crates/quicksearch-core/src/extract/pdf.rs create mode 100644 crates/quicksearch-core/src/extract/plaintext.rs rename {src => crates/quicksearch-core/src}/file_handling.rs (77%) rename {src => crates/quicksearch-core/src}/indexing.rs (80%) create mode 100644 crates/quicksearch-core/src/lib.rs create mode 100644 crates/quicksearch-core/src/mime.rs create mode 100644 crates/quicksearch-core/src/query/ast.rs create mode 100644 crates/quicksearch-core/src/query/lexer.rs create mode 100644 crates/quicksearch-core/src/query/mod.rs create mode 100644 crates/quicksearch-core/src/query/parser.rs create mode 100644 crates/quicksearch-core/src/query/translator.rs create mode 100644 crates/quicksearch-core/src/search_sql.rs create mode 100644 crates/quicksearch-core/src/shutdown.rs create mode 100644 crates/quicksearch-core/src/watcher.rs create mode 100644 crates/quicksearch-gui/Cargo.toml rename {assets => crates/quicksearch-gui/assets}/app.js (97%) rename {assets => crates/quicksearch-gui/assets}/index.html (97%) rename {assets => crates/quicksearch-gui/assets}/styles.css (94%) rename {src => crates/quicksearch-gui/src}/frontend.rs (83%) create mode 100644 crates/quicksearch-gui/src/main.rs create mode 100644 crates/quicksearch-gui/src/search.rs delete mode 100644 index.html delete mode 100644 src/main.rs delete mode 100644 src/search.rs diff --git a/Cargo.lock b/Cargo.lock index 0365510..0d7c745 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adobe-cmap-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3" +dependencies = [ + "pom", +] + [[package]] name = "aes" version = "0.8.4" @@ -56,10 +65,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] -name = "anyhow" -version = "1.0.86" +name = "android_system_properties" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] [[package]] name = "ashpd" @@ -490,6 +502,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -681,19 +704,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -705,9 +715,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -722,46 +732,12 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" -[[package]] -name = "crossterm" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" -dependencies = [ - "bitflags 1.3.2", - "crossterm_winapi", - "libc", - "mio", - "parking_lot", - "signal-hook", - "signal-hook-mio", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - [[package]] name = "crunchy" version = "0.2.4" @@ -863,6 +839,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + [[package]] name = "deranged" version = "0.5.3" @@ -1004,7 +986,7 @@ dependencies = [ "futures-util", "generational-box", "global-hotkey", - "infer", + "infer 0.11.0", "muda", "objc", "objc_id", @@ -1086,7 +1068,7 @@ dependencies = [ "dioxus-core", "dioxus-html-internal-macro", "enumset", - "euclid", + "euclid 0.22.10", "futures-channel", "generational-box", "keyboard-types", @@ -1222,15 +1204,6 @@ dependencies = [ "syn 2.0.66", ] -[[package]] -name = "dpc-pariter" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a6f60e0061d01135c7e9b77486a7f5a8e63ba14acbdbca73e42007cfd8a1c91" -dependencies = [ - "pariter", -] - [[package]] name = "dtoa" version = "1.0.9" @@ -1252,6 +1225,21 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "endi" version = "1.1.0" @@ -1316,6 +1304,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "euclid" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad" +dependencies = [ + "num-traits", +] + [[package]] name = "euclid" version = "0.22.10" @@ -1384,6 +1381,17 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -1442,6 +1450,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futf" version = "0.1.5" @@ -2015,6 +2032,30 @@ dependencies = [ "itoa 1.0.11", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core 0.62.1", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -2180,6 +2221,35 @@ dependencies = [ "cfb", ] +[[package]] +name = "infer" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb33622da908807a06f9513c19b3c1ad50fab3e4137d82a78107d502075aa199" +dependencies = [ + "cfb", +] + +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.1.4" @@ -2309,6 +2379,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kamadak-exif" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4fc70d0ab7e5b6bafa30216a6b48705ea964cdfc29c050f2412295eba58077" +dependencies = [ + "mutate_once", +] + [[package]] name = "keyboard-types" version = "0.7.0" @@ -2320,6 +2399,26 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "krates" version = "0.16.10" @@ -2358,6 +2457,18 @@ version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +[[package]] +name = "libredox" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +dependencies = [ + "bitflags 2.5.0", + "libc", + "plain", + "redox_syscall 0.7.4", +] + [[package]] name = "libsqlite3-sys" version = "0.28.0" @@ -2388,6 +2499,12 @@ dependencies = [ "x11", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -2410,6 +2527,32 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lofty" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8436c21042327836a52493b8cb2e1bb8a8f44c723a838d8c8b01f0bb657ea2" +dependencies = [ + "byteorder", + "data-encoding", + "flate2", + "lofty_attr", + "log", + "ogg_pager", + "paste", +] + +[[package]] +name = "lofty_attr" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e2dcdba64a01c3ce936b02401ab17b52db3c72720cf484c9df3387ca7517d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "log" version = "0.4.21" @@ -2422,6 +2565,43 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86" +[[package]] +name = "lopdf" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e775e4ee264e8a87d50a9efef7b67b4aa988cf94e75630859875fc347e6c872b" +dependencies = [ + "chrono", + "encoding_rs", + "flate2", + "itoa 1.0.11", + "linked-hash-map", + "log", + "md5", + "nom", + "rayon", + "time", + "weezl", +] + +[[package]] +name = "lopdf" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5c8ecfc6c72051981c0459f75ccc585e7ff67c70829560cda8e647882a9abff" +dependencies = [ + "encoding_rs", + "flate2", + "indexmap 2.2.6", + "itoa 1.0.11", + "log", + "md-5", + "nom", + "rangemap", + "time", + "weezl", +] + [[package]] name = "lru" version = "0.12.3" @@ -2466,6 +2646,16 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "md5" version = "0.7.0" @@ -2487,6 +2677,28 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.3" @@ -2527,6 +2739,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mutate_once" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" + [[package]] name = "ndk" version = "0.7.0" @@ -2592,6 +2810,35 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.5.0", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "walkdir", + "windows-sys 0.48.0", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2696,6 +2943,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ogg_pager" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87b0bef808533c5890ab77279538212efdbbbd9aa4ef1ccdfcfbf77a42f7e6fa" +dependencies = [ + "byteorder", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -2752,17 +3008,6 @@ dependencies = [ "system-deps", ] -[[package]] -name = "pariter" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a62b9e7b5f270c0acc92a2040f8028bb643f959f9c068f11a7864f327e3d9" -dependencies = [ - "crossbeam", - "crossbeam-channel", - "num_cpus", -] - [[package]] name = "parking" version = "2.2.0" @@ -2787,7 +3032,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.1", "smallvec", "windows-targets 0.52.5", ] @@ -2803,6 +3048,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pbkdf2" version = "0.11.0" @@ -2815,6 +3066,21 @@ dependencies = [ "sha2", ] +[[package]] +name = "pdf-extract" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb3a5387b94b9053c1e69d8abfd4dd6dae7afda65a5c5279bc1f42ab39df575" +dependencies = [ + "adobe-cmap-parser", + "encoding_rs", + "euclid 0.20.14", + "lopdf 0.34.0", + "postscript", + "type1-encoding-parser", + "unicode-normalization", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2972,6 +3238,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "png" version = "0.17.13" @@ -3006,6 +3278,18 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22686f4785f02a4fcc856d3b3bb19bf6c8160d103f7a99cc258bddd0251dc7f2" +[[package]] +name = "pom" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" + +[[package]] +name = "postscript" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" + [[package]] name = "powerfmt" version = "0.2.0" @@ -3103,24 +3387,36 @@ dependencies = [ ] [[package]] -name = "quicksearch" +name = "quicksearch-core" version = "0.1.0" dependencies = [ "ctrlc", - "dioxus", - "dioxus-desktop", - "dpc-pariter", + "infer 0.15.0", + "kamadak-exif", + "lofty", + "lopdf 0.32.0", + "mime_guess", + "notify", + "pdf-extract", "quick-xml", "rusqlite", "serde", "sha2", - "tokio", "toml", - "tqdm", "walkdir", "zip", ] +[[package]] +name = "quicksearch-gui" +version = "0.1.0" +dependencies = [ + "dioxus", + "dioxus-desktop", + "quicksearch-core", + "tokio", +] + [[package]] name = "quote" version = "1.0.36" @@ -3211,6 +3507,12 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + [[package]] name = "raw-window-handle" version = "0.5.2" @@ -3223,6 +3525,26 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.1" @@ -3232,6 +3554,15 @@ dependencies = [ "bitflags 2.5.0", ] +[[package]] +name = "redox_syscall" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "regex" version = "1.10.5" @@ -3566,17 +3897,6 @@ dependencies = [ "signal-hook-registry", ] -[[package]] -name = "signal-hook-mio" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" -dependencies = [ - "libc", - "mio", - "signal-hook", -] - [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -3893,6 +4213,7 @@ dependencies = [ "powerfmt", "serde", "time-core", + "time-macros", ] [[package]] @@ -3901,6 +4222,16 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -3911,6 +4242,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "to_method" version = "1.1.0" @@ -3990,17 +4336,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tqdm" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2d2932240205a99b65f15d9861992c95fbb8c9fb280b3a1f17a92db6dc611f" -dependencies = [ - "anyhow", - "crossterm", - "once_cell", -] - [[package]] name = "tracing" version = "0.1.40" @@ -4058,6 +4393,15 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "type1-encoding-parser" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa10c302f5a53b7ad27fd42a3996e23d096ba39b5b8dd6d9e683a05b01bee749" +dependencies = [ + "pom", +] + [[package]] name = "typenum" version = "1.17.0" @@ -4075,12 +4419,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -4367,6 +4726,12 @@ dependencies = [ "windows-core 0.52.0", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi" version = "0.3.9" @@ -4437,10 +4802,23 @@ version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" dependencies = [ - "windows-result", + "windows-result 0.1.2", "windows-targets 0.52.5", ] +[[package]] +name = "windows-core" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6844ee5416b285084d3d3fffd743b925a6c9385455f64f6d4fa3031c4c2749a9" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link", + "windows-result 0.4.0", + "windows-strings", +] + [[package]] name = "windows-implement" version = "0.52.0" @@ -4463,6 +4841,17 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "windows-interface" version = "0.52.0" @@ -4485,6 +4874,17 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "windows-link" version = "0.2.0" @@ -4500,6 +4900,24 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "windows-result" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.45.0" diff --git a/Cargo.toml b/Cargo.toml index 6bea883..6535d4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,19 +1,10 @@ -[package] -name = "quicksearch" +[workspace] +resolver = "2" +members = [ + "crates/quicksearch-core", + "crates/quicksearch-gui", +] + +[workspace.package] version = "0.1.0" edition = "2021" - -[dependencies] -dioxus = { version = "0.5.1", features = ["desktop"] } -dioxus-desktop = "0.5.1" -dpc-pariter = "0.5.1" -rusqlite = { version = "0.31.0", features = ["bundled"] } -sha2 = "0.10.8" -tqdm = "0.7.0" -walkdir = "2.5.0" -zip = "0.6" -quick-xml = "0.31" -tokio = { version = "1.0", features = ["time", "signal"] } -serde = { version = "1.0", features = ["derive"] } -toml = "0.8" -ctrlc = "3.4" diff --git a/config_example.toml b/config_example.toml index 0d8ffc6..67debd1 100644 --- a/config_example.toml +++ b/config_example.toml @@ -1,25 +1,27 @@ -[paths] -default_indexing_path = "C:\\" -database_path = "QuickSearch.db" - -[processing] -# Amount of data in bytes read from start/end of files used to calculate hash -hash_length = 8192 -# Maximum text content to index per file (bytes) -maximum_text_size = 262144 -# Maximum file size to process for text extraction (bytes) -maximum_text_file_size = 2097152 -# Number of files to process in each batch (directory walk / inserts / text extraction batches) -batch_size = 200 -# Files per transaction for incremental UPDATE files + DELETE from searchabletext (FTS); larger = fewer commits, more RAM per chunk -fts_update_batch_size = 1000 -# If true, run a fast shell-backed tree count before Phase 1 (enables % progress; Linux uses GNU find -printf '\n' | wc -l when available). -# If false, Phase 1 shows file counts without a percentage. -precount_files_for_progress = false -# If true, follow symbolic links during directory walks (indexing only; shell precount unchanged). -follow_symlinks = false -# If true, hidden files and directories will be indexed. -include_hidden = false -# FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61') -# Look here for more information https://www.sqlite.org/fts5.html#tokenizers +[paths] +# One or more directory roots to index. Walked in order; duplicate and +# nested roots are de-duplicated automatically. +indexing_paths = ["C:\\"] +database_path = "QuickSearch.db" + +[processing] +# Amount of data in bytes read from start/end of files used to calculate hash +hash_length = 8192 +# Maximum text content to index per file (bytes) +maximum_text_size = 262144 +# Maximum file size to process for text extraction (bytes) +maximum_text_file_size = 2097152 +# Number of files to process in each batch (directory walk / inserts / text extraction batches) +batch_size = 200 +# Files per transaction for incremental UPDATE files + DELETE from searchabletext (FTS); larger = fewer commits, more RAM per chunk +fts_update_batch_size = 1000 +# If true, run a fast shell-backed tree count before Phase 1 (enables % progress; Linux uses GNU find -printf '\n' | wc -l when available). +# If false, Phase 1 shows file counts without a percentage. +precount_files_for_progress = false +# If true, follow symbolic links during directory walks (indexing only; shell precount unchanged). +follow_symlinks = false +# If true, hidden files and directories will be indexed. +include_hidden = false +# FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61') +# Look here for more information https://www.sqlite.org/fts5.html#tokenizers tokenize = "trigram" \ No newline at end of file diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml new file mode 100644 index 0000000..d327f70 --- /dev/null +++ b/crates/quicksearch-core/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "quicksearch-core" +version.workspace = true +edition.workspace = true + +[lib] +name = "quicksearch_core" +path = "src/lib.rs" + +[dependencies] +rusqlite = { version = "0.31.0", features = ["bundled"] } +sha2 = "0.10.8" +walkdir = "2.5.0" +zip = "0.6" +quick-xml = "0.31" +serde = { version = "1.0", features = ["derive"] } +toml = "0.8" +mime_guess = "2.0" +infer = "0.15" +pdf-extract = "0.7" +lopdf = "0.32" +lofty = "0.19" +kamadak-exif = "0.5" +notify = "6.1" +ctrlc = "3.4" diff --git a/crates/quicksearch-core/src/cli.rs b/crates/quicksearch-core/src/cli.rs new file mode 100644 index 0000000..bb3f1f0 --- /dev/null +++ b/crates/quicksearch-core/src/cli.rs @@ -0,0 +1,283 @@ +//! Programmatic read-only query helpers. +//! +//! Pure functions that open a DB, run a query, and return structured data. +//! No stdout, no CLI framing — callers (GUI, future CLI binaries, Set B +//! `balooctl`) format the result as they see fit. Mutating operations live +//! on [`crate::indexing::IndexingService`] since they require a running +//! worker thread. + +use rusqlite::{params, OptionalExtension}; + +use crate::db::open_and_migrate; +use crate::db::repo::{STATE_DONE, STATE_FAILED, STATE_NA, STATE_PENDING}; + +/// Per-file indexing status, mirroring Baloo's multi-state reporting. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IndexState { + NotIndexed, + Pending, + Done, + Failed, + NotApplicable, +} + +impl From for IndexState { + fn from(v: i64) -> Self { + match v { + x if x == STATE_PENDING => IndexState::Pending, + x if x == STATE_DONE => IndexState::Done, + x if x == STATE_FAILED => IndexState::Failed, + x if x == STATE_NA => IndexState::NotApplicable, + _ => IndexState::Pending, + } + } +} + +/// Indexing status for a single file. `basic` is the metadata row state +/// (indexed or not); `content` is the extractor state. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileStatus { + pub path: String, + pub basic: IndexState, + pub content: IndexState, + pub failure_reason: Option, +} + +/// Per-file entry returned by [`list_failed`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FailedEntry { + pub file_id: i64, + pub path: String, + pub reason: Option, + pub ts: i64, +} + +/// Storage footprint report. "Partitions" correspond to SQL tables for our +/// SQLite layout (Baloo's LMDB has named sub-DBs; our equivalent is per-table +/// row/size counts). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SizeReport { + pub file_size_bytes: u64, + pub files_row_count: i64, + pub properties_row_count: i64, + pub failed_files_row_count: i64, + pub searchabletext_row_count: i64, +} + +/// Query the per-file indexing status. Returns `FileStatus` with +/// `basic == NotIndexed` if the path isn't in the database. +pub fn status_for_path(db_path: &str, path: &str) -> Result { + let conn = open_and_migrate(db_path, "trigram")?; + let row: Option<(i64, i64, Option)> = conn + .query_row( + "SELECT basic_state, content_state, failure_msg FROM files WHERE path = ?1", + params![path], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .optional() + .map_err(|e| format!("status_for_path({}): {}", path, e))?; + Ok(match row { + None => FileStatus { + path: path.to_string(), + basic: IndexState::NotIndexed, + content: IndexState::NotIndexed, + failure_reason: None, + }, + Some((basic, content, reason)) => FileStatus { + path: path.to_string(), + basic: IndexState::from(basic), + content: IndexState::from(content), + failure_reason: reason, + }, + }) +} + +/// Return every file that failed content extraction, newest first. +pub fn list_failed(db_path: &str, limit: Option) -> Result, String> { + let conn = open_and_migrate(db_path, "trigram")?; + let limit_sql = match limit { + Some(n) => format!(" LIMIT {}", n), + None => String::new(), + }; + let sql = format!( + "SELECT ff.file_id, f.path, ff.reason, ff.ts \ + FROM failed_files ff \ + JOIN files f ON f.id = ff.file_id \ + ORDER BY ff.ts DESC{}", + limit_sql + ); + let mut stmt = conn + .prepare(&sql) + .map_err(|e| format!("list_failed prepare: {}", e))?; + let rows = stmt + .query_map([], |r| { + Ok(FailedEntry { + file_id: r.get(0)?, + path: r.get(1)?, + reason: r.get(2)?, + ts: r.get(3)?, + }) + }) + .map_err(|e| format!("list_failed query: {}", e))?; + rows.collect::, _>>() + .map_err(|e| format!("list_failed row: {}", e)) +} + +/// Return a rough size breakdown of the database on disk and by table. +pub fn index_size_breakdown(db_path: &str) -> Result { + let file_size_bytes = std::fs::metadata(db_path) + .map(|m| m.len()) + .unwrap_or(0); + let conn = open_and_migrate(db_path, "trigram")?; + let count = |table: &str| -> Result { + conn.query_row(&format!("SELECT COUNT(*) FROM {}", table), [], |r| r.get(0)) + .map_err(|e| format!("count {}: {}", table, e)) + }; + Ok(SizeReport { + file_size_bytes, + files_row_count: count("files")?, + properties_row_count: count("properties")?, + failed_files_row_count: count("failed_files")?, + searchabletext_row_count: count("searchabletext")?, + }) +} + +/// Remove a single file from the index. Returns whether a row was deleted. +/// Keeps FTS/documents/properties in sync via the repo helpers. +pub fn clear_path(db_path: &str, path: &str) -> Result { + let mut conn = open_and_migrate(db_path, "trigram")?; + let tx = conn + .transaction() + .map_err(|e| format!("clear_path begin tx: {}", e))?; + let removed = crate::db::repo::delete_file_by_path(&tx, path)?; + tx.commit() + .map_err(|e| format!("clear_path commit: {}", e))?; + Ok(removed) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::repo::{insert_file, set_content_done, set_content_failed, NewFile}; + use crate::mime::FileType; + + fn tmp_path() -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "qs-cli-test-{}-{}.sqlite", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + p + } + + fn seed_fixture(db_path: &str) -> (i64, i64) { + let mut conn = open_and_migrate(db_path, "trigram").unwrap(); + let (a, b) = { + let tx = conn.transaction().unwrap(); + let a = insert_file( + &tx, + &NewFile { + name: "a.txt", + path: "/tmp/a.txt", + parent: "/tmp", + size: 1, + mtime: 1, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + }, + ) + .unwrap() + .expect("unique path"); + set_content_done(&tx, a, "a.txt", "hello", &[]).unwrap(); + let b = insert_file( + &tx, + &NewFile { + name: "b.bin", + path: "/tmp/b.bin", + parent: "/tmp", + size: 1, + mtime: 1, + inode: None, + device_id: None, + mime: None, + ftype: FileType::EMPTY, + hash: None, + }, + ) + .unwrap() + .expect("unique path"); + set_content_failed(&tx, b, "bad extract").unwrap(); + tx.commit().unwrap(); + (a, b) + }; + drop(conn); + (a, b) + } + + #[test] + fn status_for_path_returns_states() { + let p = tmp_path(); + let (_a, _b) = seed_fixture(p.to_str().unwrap()); + + let st_a = status_for_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap(); + assert_eq!(st_a.basic, IndexState::Done); + assert_eq!(st_a.content, IndexState::Done); + + let st_b = status_for_path(p.to_str().unwrap(), "/tmp/b.bin").unwrap(); + assert_eq!(st_b.basic, IndexState::Done); + assert_eq!(st_b.content, IndexState::Failed); + assert_eq!(st_b.failure_reason.as_deref(), Some("bad extract")); + + let st_missing = status_for_path(p.to_str().unwrap(), "/tmp/never.txt").unwrap(); + assert_eq!(st_missing.basic, IndexState::NotIndexed); + + std::fs::remove_file(&p).ok(); + } + + #[test] + fn list_failed_returns_failed_rows() { + let p = tmp_path(); + let (_a, b) = seed_fixture(p.to_str().unwrap()); + + let failed = list_failed(p.to_str().unwrap(), None).unwrap(); + assert_eq!(failed.len(), 1); + assert_eq!(failed[0].file_id, b); + assert_eq!(failed[0].path, "/tmp/b.bin"); + assert_eq!(failed[0].reason.as_deref(), Some("bad extract")); + + std::fs::remove_file(&p).ok(); + } + + #[test] + fn index_size_breakdown_counts_rows() { + let p = tmp_path(); + let _ = seed_fixture(p.to_str().unwrap()); + + let r = index_size_breakdown(p.to_str().unwrap()).unwrap(); + assert!(r.file_size_bytes > 0); + assert_eq!(r.files_row_count, 2); + assert_eq!(r.failed_files_row_count, 1); + + std::fs::remove_file(&p).ok(); + } + + #[test] + fn clear_path_removes_file_and_returns_true() { + let p = tmp_path(); + let _ = seed_fixture(p.to_str().unwrap()); + + assert!(clear_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap()); + let st = status_for_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap(); + assert_eq!(st.basic, IndexState::NotIndexed); + assert!(!clear_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap()); + + std::fs::remove_file(&p).ok(); + } +} diff --git a/src/config.rs b/crates/quicksearch-core/src/config.rs similarity index 75% rename from src/config.rs rename to crates/quicksearch-core/src/config.rs index caaed75..e39acd6 100644 --- a/src/config.rs +++ b/crates/quicksearch-core/src/config.rs @@ -1,86 +1,99 @@ -use serde::{Deserialize, Serialize}; -use std::fs; -use std::path::Path; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct Config { - pub paths: PathConfig, - pub processing: ProcessingConfig, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct PathConfig { - pub default_indexing_path: String, - pub database_path: String, -} - -fn default_fts_update_batch_size() -> usize { - 1000 -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct ProcessingConfig { - pub hash_length: usize, - pub maximum_text_size: usize, - pub maximum_text_file_size: u64, - pub batch_size: usize, - #[serde(default = "default_fts_update_batch_size")] - pub fts_update_batch_size: usize, - pub tokenize: String, - #[serde(default)] - pub precount_files_for_progress: bool, - #[serde(default)] - pub follow_symlinks: bool, - #[serde(default)] - pub include_hidden: bool, -} - -impl Default for Config { - fn default() -> Self { - Config { - paths: PathConfig { - default_indexing_path: "C:\\".to_string(), - database_path: "QuickSearch.db".to_string(), - }, - processing: ProcessingConfig { - hash_length: 1024 * 8, - maximum_text_size: 1024 * 256, - maximum_text_file_size: 1024 * 1024 * 2, - batch_size: 200, - fts_update_batch_size: 1000, - tokenize: "trigram".to_string(), - precount_files_for_progress: false, - follow_symlinks: false, - include_hidden: false, - }, - } - } -} - -impl Config { - pub fn load() -> Result { - let config_path = "config.toml"; - - if Path::new(config_path).exists() { - let content = fs::read_to_string(config_path) - .map_err(|e| format!("Failed to read config file: {}", e))?; - - toml::from_str(&content) - .map_err(|e| format!("Failed to parse config file: {}", e)) - } else { - let default_config = Config::default(); - default_config.save()?; - Ok(default_config) - } - } - - pub fn save(&self) -> Result<(), String> { - let content = toml::to_string_pretty(self) - .map_err(|e| format!("Failed to serialize config: {}", e))?; - - fs::write("config.toml", content) - .map_err(|e| format!("Failed to write config file: {}", e))?; - - Ok(()) - } -} +use serde::{Deserialize, Serialize}; +use std::fs; +use std::path::Path; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Config { + pub paths: PathConfig, + pub processing: ProcessingConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct PathConfig { + /// One or more directory roots to index. Indexing walks each root + /// independently; duplicates and nested roots are de-duplicated by the + /// indexer at run time. Must contain at least one entry. + pub indexing_paths: Vec, + pub database_path: String, +} + +fn default_fts_update_batch_size() -> usize { + 1000 +} + +/// Platform-sensible default for the first indexing root when no config +/// exists. `$HOME` on Unix, `%USERPROFILE%` on Windows; falls back to the +/// current directory. +fn default_home_path() -> String { + if let Some(home) = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) { + return home.to_string_lossy().into_owned(); + } + ".".to_string() +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ProcessingConfig { + pub hash_length: usize, + pub maximum_text_size: usize, + pub maximum_text_file_size: u64, + pub batch_size: usize, + #[serde(default = "default_fts_update_batch_size")] + pub fts_update_batch_size: usize, + pub tokenize: String, + #[serde(default)] + pub precount_files_for_progress: bool, + #[serde(default)] + pub follow_symlinks: bool, + #[serde(default)] + pub include_hidden: bool, +} + +impl Default for Config { + fn default() -> Self { + Config { + paths: PathConfig { + indexing_paths: vec![default_home_path()], + database_path: "QuickSearch.db".to_string(), + }, + processing: ProcessingConfig { + hash_length: 1024 * 8, + maximum_text_size: 1024 * 256, + maximum_text_file_size: 1024 * 1024 * 2, + batch_size: 200, + fts_update_batch_size: 1000, + tokenize: "trigram".to_string(), + precount_files_for_progress: false, + follow_symlinks: false, + include_hidden: false, + }, + } + } +} + +impl Config { + pub fn load() -> Result { + let config_path = "config.toml"; + + if Path::new(config_path).exists() { + let content = fs::read_to_string(config_path) + .map_err(|e| format!("Failed to read config file: {}", e))?; + + toml::from_str(&content) + .map_err(|e| format!("Failed to parse config file: {}", e)) + } else { + let default_config = Config::default(); + default_config.save()?; + Ok(default_config) + } + } + + pub fn save(&self) -> Result<(), String> { + let content = toml::to_string_pretty(self) + .map_err(|e| format!("Failed to serialize config: {}", e))?; + + fs::write("config.toml", content) + .map_err(|e| format!("Failed to write config file: {}", e))?; + + Ok(()) + } +} diff --git a/crates/quicksearch-core/src/db/migrate.rs b/crates/quicksearch-core/src/db/migrate.rs new file mode 100644 index 0000000..bcbe68b --- /dev/null +++ b/crates/quicksearch-core/src/db/migrate.rs @@ -0,0 +1,281 @@ +//! Schema version detection and upgrade. +//! +//! Callers should always enter the DB via [`open_and_migrate`]. It applies +//! pragmas, detects the on-disk schema version, and upgrades or recreates as +//! required. +//! +//! Current policy: Set A introduces schema v1 and is the first versioned +//! release. Any pre-A database (has the legacy `files(name, path, size, +//! moddate, hash)` shape and no `schema_info` table) is wiped and rebuilt — +//! the user will re-index. A prominent log line is printed so the behavior is +//! not silent. Future migrations should prefer ALTER TABLE. + +use std::path::Path; + +use rusqlite::{params, Connection, OptionalExtension}; + +use super::schema::{fts_create_sql, PRAGMAS_FAST, SCHEMA_CURRENT}; + +pub const CURRENT_SCHEMA_VERSION: u32 = 2; + +/// Open the database at `db_path`, apply pragmas, and ensure the schema is at +/// [`CURRENT_SCHEMA_VERSION`]. Recreates the DB if a pre-versioned layout is +/// detected. +/// +/// `tokenizer` is used when (re)creating the FTS5 virtual table. It has no +/// effect on an already-current DB. +pub fn open_and_migrate(db_path: &str, tokenizer: &str) -> Result { + let path_for_rebuild = Path::new(db_path).to_path_buf(); + let mut conn = Connection::open(db_path) + .map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?; + + conn.execute_batch(PRAGMAS_FAST) + .map_err(|e| format!("Failed to apply pragmas: {}", e))?; + + let version = read_schema_version(&conn)?; + match version { + Some(v) if v == CURRENT_SCHEMA_VERSION => {} + Some(v) if v > CURRENT_SCHEMA_VERSION => { + return Err(format!( + "Database schema version {} is newer than this build ({}). \ + Use a newer QuickSearch or move the database aside.", + v, CURRENT_SCHEMA_VERSION + )); + } + Some(v) => { + // Older schema version. Set A's upgrade policy: wipe and rebuild. + // When we start adding ALTER-based migrations this match arm + // will gain a proper stepwise runner. + eprintln!( + "QuickSearch: database at {} is schema v{}; rebuilding to v{}. \ + Existing rows will be re-scanned.", + db_path, v, CURRENT_SCHEMA_VERSION + ); + conn = wipe_and_reopen(conn, &path_for_rebuild)?; + apply_current_schema(&conn, tokenizer)?; + } + None => { + // No schema_info row. Either an empty DB (good — just create) or a + // legacy pre-A layout (detected by presence of the old `files` + // table). Legacy layouts are wiped. + if has_legacy_layout(&conn)? { + eprintln!( + "QuickSearch: legacy database detected at {}; rebuilding index with schema v{}. \ + Existing file rows will be re-scanned.", + db_path, CURRENT_SCHEMA_VERSION + ); + conn = wipe_and_reopen(conn, &path_for_rebuild)?; + } + apply_current_schema(&conn, tokenizer)?; + } + } + + Ok(conn) +} + +fn wipe_and_reopen( + conn: Connection, + path_for_rebuild: &std::path::Path, +) -> Result { + drop(conn); + std::fs::remove_file(path_for_rebuild) + .map_err(|e| format!("Failed to remove old database: {}", e))?; + // Remove WAL/SHM/journal sidecars defensively even though journal_mode=OFF. + for suffix in ["-wal", "-shm", "-journal"] { + let sidecar = path_for_rebuild.with_file_name(format!( + "{}{}", + path_for_rebuild + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or(""), + suffix + )); + let _ = std::fs::remove_file(sidecar); + } + let conn = Connection::open(path_for_rebuild) + .map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?; + conn.execute_batch(PRAGMAS_FAST) + .map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?; + Ok(conn) +} + +fn read_schema_version(conn: &Connection) -> Result, String> { + let has_info: bool = conn + .query_row( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_info'", + [], + |_| Ok(true), + ) + .optional() + .map_err(|e| format!("sqlite_master schema_info: {}", e))? + .unwrap_or(false); + if !has_info { + return Ok(None); + } + let v: Option = conn + .query_row( + "SELECT value FROM schema_info WHERE key = 'version'", + [], + |r| r.get(0), + ) + .optional() + .map_err(|e| format!("read schema_info.version: {}", e))?; + match v { + Some(s) => s + .parse::() + .map(Some) + .map_err(|e| format!("invalid schema_info.version {:?}: {}", s, e)), + None => Ok(None), + } +} + +fn has_legacy_layout(conn: &Connection) -> Result { + // Old layout has a `files` table without an `id INTEGER PRIMARY KEY`. + let has_files: bool = conn + .query_row( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='files'", + [], + |_| Ok(true), + ) + .optional() + .map_err(|e| format!("sqlite_master files: {}", e))? + .unwrap_or(false); + if !has_files { + return Ok(false); + } + // Check whether the columns match the legacy shape. + let mut stmt = conn + .prepare("PRAGMA table_info(files)") + .map_err(|e| format!("pragma table_info: {}", e))?; + let has_id = stmt + .query_map([], |row| row.get::<_, String>(1)) + .map_err(|e| format!("table_info query: {}", e))? + .filter_map(|r| r.ok()) + .any(|name| name == "id"); + Ok(!has_id) +} + +fn apply_current_schema(conn: &Connection, tokenizer: &str) -> Result<(), String> { + conn.execute_batch(SCHEMA_CURRENT) + .map_err(|e| format!("Failed to create current schema tables: {}", e))?; + let fts = fts_create_sql(tokenizer); + conn.execute_batch(&fts) + .map_err(|e| format!("Failed to create searchabletext: {}", e))?; + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + conn.execute( + "INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)", + params![CURRENT_SCHEMA_VERSION.to_string(), now.to_string(), tokenizer], + ) + .map_err(|e| format!("Failed to seed schema_info: {}", e))?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp_db_path() -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "quicksearch-test-{}-{}.sqlite", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + p + } + + #[test] + fn fresh_db_gets_current_version() { + let p = tmp_db_path(); + let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let v: String = conn + .query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0)) + .unwrap(); + assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string()); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn reopen_is_idempotent() { + let p = tmp_db_path(); + { + let _ = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + } + let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let v: String = conn + .query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0)) + .unwrap(); + assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string()); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn older_versioned_db_is_wiped_and_recreated() { + // Simulate a DB that was created at a previous schema version. + let p = tmp_db_path(); + { + let conn = Connection::open(&p).unwrap(); + conn.execute("CREATE TABLE schema_info (key TEXT PRIMARY KEY, value TEXT NOT NULL)", []).unwrap(); + conn.execute( + "INSERT INTO schema_info(key,value) VALUES('version','1')", + [], + ) + .unwrap(); + conn.execute("CREATE TABLE files (id INTEGER PRIMARY KEY, name TEXT)", []).unwrap(); + conn.execute("INSERT INTO files(name) VALUES('a.txt')", []).unwrap(); + } + let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let v: String = conn + .query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0)) + .unwrap(); + assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string()); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 0, "old rows should be wiped"); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn legacy_db_is_wiped_and_recreated() { + let p = tmp_db_path(); + { + // Simulate a pre-A database. + let conn = Connection::open(&p).unwrap(); + conn.execute( + "CREATE TABLE files (name TEXT, path TEXT, size INTEGER, moddate INTEGER, hash BLOB)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO files VALUES ('a.txt', '/tmp/a.txt', 1, 2, X'00')", + [], + ) + .unwrap(); + } + let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + // Old row should be gone. + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 0); + // New columns should exist. + let _ = conn + .query_row("SELECT basic_state, content_state, type, mime FROM files LIMIT 0", [], |_| Ok(())) + .or_else(|e| if matches!(e, rusqlite::Error::QueryReturnedNoRows) { Ok(()) } else { Err(e) }) + .unwrap(); + drop(conn); + std::fs::remove_file(&p).ok(); + } +} diff --git a/crates/quicksearch-core/src/db/mod.rs b/crates/quicksearch-core/src/db/mod.rs new file mode 100644 index 0000000..34f35aa --- /dev/null +++ b/crates/quicksearch-core/src/db/mod.rs @@ -0,0 +1,13 @@ +//! SQLite schema, migrations, and row-level repository helpers. +//! +//! The only "live" schema is `CURRENT_SCHEMA_VERSION` (see [`schema`]). Older +//! databases are detected in [`migrate::open_and_migrate`] and recreated from +//! scratch — Set A of the QuickSearch → Baloo work is the first schema bump +//! and carries no rows we'd want to preserve. Subsequent migrations should +//! prefer `ALTER TABLE` and versioned steps. + +pub mod migrate; +pub mod repo; +pub mod schema; + +pub use migrate::{open_and_migrate, CURRENT_SCHEMA_VERSION}; diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs new file mode 100644 index 0000000..a643627 --- /dev/null +++ b/crates/quicksearch-core/src/db/repo.rs @@ -0,0 +1,495 @@ +//! Row-level write helpers that keep the FTS5 contentless table in sync with +//! `files`/`documents`/`properties`. +//! +//! FTS5 contentless tables store postings only. Updating them requires the +//! old row values to compute which terms to remove. These helpers centralize +//! that bookkeeping so callers never have to remember the order of operations. +//! +//! States (mirrors `basic_state` / `content_state` columns): +//! +//! | value | meaning | +//! |------:|---------| +//! | 0 | pending | +//! | 1 | done | +//! | 2 | failed | +//! | 3 | not applicable (content only) | + +use rusqlite::{params, Connection, OptionalExtension, Transaction}; + +use crate::mime::FileType; + +pub const STATE_PENDING: i64 = 0; +pub const STATE_DONE: i64 = 1; +pub const STATE_FAILED: i64 = 2; +pub const STATE_NA: i64 = 3; + +/// Everything needed to insert a fresh file row. +#[derive(Debug, Clone)] +pub struct NewFile<'a> { + pub name: &'a str, + pub path: &'a str, + pub parent: &'a str, + pub size: u64, + pub mtime: u64, + pub inode: Option, + pub device_id: Option, + pub mime: Option<&'a str>, + pub ftype: FileType, + pub hash: Option<&'a [u8]>, +} + +/// Insert a new file row, returning its id. `basic_state` is set to DONE +/// (row existing *is* the basic-index state); `content_state` is PENDING +/// unless the MIME maps to a type we won't extract text from, in which case +/// the caller can later set it to NA. +/// +/// Uses `INSERT OR IGNORE` so a UNIQUE(path) collision (which indicates the +/// caller fed the same path twice in one run) becomes a silent no-op +/// returning `None` rather than aborting the whole batch. The walker is +/// expected to dedupe visits upstream; this is a defense-in-depth backstop. +pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, String> { + let rows = tx + .execute( + "INSERT OR IGNORE INTO files ( + name, path, parent, size, mtime, inode, device_id, + mime, type, basic_state, content_state, hash + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + params![ + f.name, + f.path, + f.parent, + f.size as i64, + f.mtime as i64, + f.inode.map(|x| x as i64), + f.device_id.map(|x| x as i64), + f.mime, + f.ftype.bits() as i64, + STATE_DONE, + STATE_PENDING, + f.hash, + ], + ) + .map_err(|e| format!("insert file {}: {}", f.path, e))?; + if rows == 0 { + // Existing row with the same path (e.g. duplicate visit within the run). + return Ok(None); + } + Ok(Some(tx.last_insert_rowid())) +} + +/// Update a file's metadata in place (same path, changed size/mtime/hash). +/// Clears any extracted content so the text-indexing pass re-processes it. +pub fn update_file_basic( + tx: &Transaction<'_>, + path: &str, + size: u64, + mtime: u64, + hash: Option<&[u8]>, + mime: Option<&str>, + ftype: FileType, +) -> Result, String> { + let id: Option = tx + .query_row( + "SELECT id FROM files WHERE path = ?1", + params![path], + |r| r.get(0), + ) + .optional() + .map_err(|e| format!("lookup file id {}: {}", path, e))?; + let Some(id) = id else { + return Ok(None); + }; + tx.execute( + "UPDATE files + SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5, + content_state = ?6, failure_msg = NULL + WHERE id = ?7", + params![ + size as i64, + mtime as i64, + hash, + mime, + ftype.bits() as i64, + STATE_PENDING, + id, + ], + ) + .map_err(|e| format!("update file {}: {}", path, e))?; + // Any prior extracted content is stale — remove it along with its FTS row. + remove_content_for_id(tx, id)?; + Ok(Some(id)) +} + +/// Mark a file's content indexing as complete and write the extracted text + +/// properties atomically. FTS5 stores the canonical text for this file; +/// `properties` are stored both as a structured side-table (for exact +/// retrieval) and concatenated into the FTS `properties` column (for MATCH). +pub fn set_content_done( + tx: &Transaction<'_>, + file_id: i64, + name: &str, + text: &str, + properties: &[(String, String)], +) -> Result<(), String> { + // Clear any previous extraction (in case of re-run). + remove_content_for_id(tx, file_id)?; + + for (k, v) in properties { + tx.execute( + "INSERT INTO properties(file_id, key, value) VALUES (?1, ?2, ?3)", + params![file_id, k, v], + ) + .map_err(|e| format!("insert property {}={}: {}", k, v, e))?; + } + let props_blob = encode_properties_for_fts(properties); + tx.execute( + "INSERT INTO searchabletext(rowid, name, text, properties) VALUES (?1, ?2, ?3, ?4)", + params![file_id, name, text, props_blob], + ) + .map_err(|e| format!("insert FTS row {}: {}", file_id, e))?; + + tx.execute( + "UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2", + params![STATE_DONE, file_id], + ) + .map_err(|e| format!("update content_state DONE {}: {}", file_id, e))?; + // Clear any prior failed-file record. + tx.execute("DELETE FROM failed_files WHERE file_id = ?1", params![file_id]) + .map_err(|e| format!("clear failed_files {}: {}", file_id, e))?; + Ok(()) +} + +/// Mark a file's content extraction as failed. Keeps the basic row in place. +pub fn set_content_failed( + tx: &Transaction<'_>, + file_id: i64, + reason: &str, +) -> Result<(), String> { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) as i64; + tx.execute( + "UPDATE files SET content_state = ?1, failure_msg = ?2 WHERE id = ?3", + params![STATE_FAILED, reason, file_id], + ) + .map_err(|e| format!("update content_state FAILED {}: {}", file_id, e))?; + tx.execute( + "INSERT OR REPLACE INTO failed_files(file_id, reason, ts) VALUES (?1, ?2, ?3)", + params![file_id, reason, now], + ) + .map_err(|e| format!("insert failed_files {}: {}", file_id, e))?; + Ok(()) +} + +/// Mark content extraction as not applicable (e.g. binary format we don't +/// support). The file row still contributes to filename search. +pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> { + tx.execute( + "UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2", + params![STATE_NA, file_id], + ) + .map_err(|e| format!("update content_state NA {}: {}", file_id, e))?; + tx.execute("DELETE FROM failed_files WHERE file_id = ?1", params![file_id]) + .map_err(|e| format!("clear failed_files {}: {}", file_id, e))?; + Ok(()) +} + +/// Delete a file row by path, keeping FTS in sync. Returns whether a row was +/// removed. +pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result { + let id: Option = tx + .query_row( + "SELECT id FROM files WHERE path = ?1", + params![path], + |r| r.get(0), + ) + .optional() + .map_err(|e| format!("lookup {} for delete: {}", path, e))?; + let Some(id) = id else { return Ok(false) }; + remove_content_for_id(tx, id)?; + tx.execute("DELETE FROM files WHERE id = ?1", params![id]) + .map_err(|e| format!("delete file {}: {}", path, e))?; + Ok(true) +} + +/// Remove the FTS row and any `properties` rows for a given file id. Does +/// not touch the `files` row itself. Idempotent — a missing FTS row is fine. +pub fn remove_content_for_id(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> { + // Regular FTS5 supports a plain DELETE by rowid; no need to supply the + // old column values the way a contentless table would require. + tx.execute( + "DELETE FROM searchabletext WHERE rowid = ?1", + params![file_id], + ) + .map_err(|e| format!("FTS delete row {}: {}", file_id, e))?; + tx.execute("DELETE FROM properties WHERE file_id = ?1", params![file_id]) + .map_err(|e| format!("delete properties {}: {}", file_id, e))?; + Ok(()) +} + +/// Serialize properties for the FTS `properties` column. `key:value` pairs +/// separated by spaces so `MATCH 'properties:artist:beatles'` works. +fn encode_properties_for_fts(props: &[(String, String)]) -> String { + let mut buf = String::new(); + for (i, (k, v)) in props.iter().enumerate() { + if i > 0 { + buf.push(' '); + } + buf.push_str(k); + buf.push(':'); + buf.push_str(v); + } + buf +} + +/// Flush and close a connection, restoring durable PRAGMAs. Call on clean +/// shutdown so the next open sees a consistent DB. +pub fn checkpoint_and_close(conn: Connection) { + let _ = conn.execute_batch( + "PRAGMA journal_mode = DELETE; \ + PRAGMA synchronous = FULL; \ + PRAGMA wal_checkpoint(FULL);", + ); + drop(conn); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::open_and_migrate; + + fn tmp_path() -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "quicksearch-repo-{}-{}.sqlite", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + p + } + + #[test] + fn insert_update_delete_round_trip() { + let p = tmp_path(); + let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + { + let tx = conn.transaction().unwrap(); + let id = insert_file( + &tx, + &NewFile { + name: "a.txt", + path: "/tmp/a.txt", + parent: "/tmp", + size: 42, + mtime: 1_700_000_000, + inode: Some(7), + device_id: Some(64768), + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: Some(&[1, 2, 3]), + }, + ) + .unwrap() + .expect("unique path"); + set_content_done( + &tx, + id, + "a.txt", + "hello world", + &[("title".to_string(), "hi".to_string())], + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Text is findable via FTS. + let hit: i64 = conn + .query_row( + "SELECT rowid FROM searchabletext WHERE searchabletext MATCH 'hello'", + [], + |r| r.get(0), + ) + .unwrap(); + assert!(hit > 0); + + // Delete cleans up. + { + let tx = conn.transaction().unwrap(); + assert!(delete_file_by_path(&tx, "/tmp/a.txt").unwrap()); + tx.commit().unwrap(); + } + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 0); + let fts_count: i64 = conn + .query_row("SELECT COUNT(*) FROM searchabletext", [], |r| r.get(0)) + .unwrap(); + assert_eq!(fts_count, 0); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn update_resets_content_state() { + let p = tmp_path(); + let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let id = { + let tx = conn.transaction().unwrap(); + let id = insert_file( + &tx, + &NewFile { + name: "a.txt", + path: "/tmp/a.txt", + parent: "/tmp", + size: 10, + mtime: 1, + inode: None, + device_id: None, + mime: None, + ftype: FileType::EMPTY, + hash: None, + }, + ) + .unwrap() + .expect("unique path"); + set_content_done(&tx, id, "a.txt", "old text", &[]).unwrap(); + tx.commit().unwrap(); + id + }; + + { + let tx = conn.transaction().unwrap(); + let got = update_file_basic( + &tx, + "/tmp/a.txt", + 20, + 2, + None, + Some("text/plain"), + FileType::TEXT, + ) + .unwrap(); + assert_eq!(got, Some(id)); + tx.commit().unwrap(); + } + + let (state, content): (i64, i64) = conn + .query_row( + "SELECT basic_state, content_state FROM files WHERE id = ?1", + params![id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(state, STATE_DONE); + assert_eq!(content, STATE_PENDING); + + // FTS row for the stale content should be gone. + let fts_hits: i64 = conn + .query_row( + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'old'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(fts_hits, 0); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn insert_file_twice_on_same_path_is_idempotent() { + // Defense-in-depth: when the walker visits a canonical path twice + // (overlapping roots, symlink resolution quirks), the second INSERT + // must be a silent no-op, not a run-ending error. + let p = tmp_path(); + let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let tx = conn.transaction().unwrap(); + let row = NewFile { + name: "dup.txt", + path: "/tmp/dup.txt", + parent: "/tmp", + size: 1, + mtime: 1, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + }; + let id1 = insert_file(&tx, &row).unwrap().expect("first insert"); + let id2 = insert_file(&tx, &row).unwrap(); + assert!(id2.is_none(), "second insert of same path must return None"); + // Only one row exists. + let count: i64 = tx + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 1); + let (id_read,): (i64,) = tx + .query_row( + "SELECT id FROM files WHERE path = ?1", + params!["/tmp/dup.txt"], + |r| Ok((r.get(0)?,)), + ) + .unwrap(); + assert_eq!(id_read, id1); + tx.commit().unwrap(); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn set_content_failed_writes_failed_table() { + let p = tmp_path(); + let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let id = { + let tx = conn.transaction().unwrap(); + let id = insert_file( + &tx, + &NewFile { + name: "oops.bin", + path: "/tmp/oops.bin", + parent: "/tmp", + size: 0, + mtime: 1, + inode: None, + device_id: None, + mime: None, + ftype: FileType::EMPTY, + hash: None, + }, + ) + .unwrap() + .expect("unique path"); + set_content_failed(&tx, id, "bad parse").unwrap(); + tx.commit().unwrap(); + id + }; + + let reason: String = conn + .query_row( + "SELECT reason FROM failed_files WHERE file_id = ?1", + params![id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(reason, "bad parse"); + let content_state: i64 = conn + .query_row( + "SELECT content_state FROM files WHERE id = ?1", + params![id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(content_state, STATE_FAILED); + + drop(conn); + std::fs::remove_file(&p).ok(); + } +} diff --git a/crates/quicksearch-core/src/db/schema.rs b/crates/quicksearch-core/src/db/schema.rs new file mode 100644 index 0000000..df8e134 --- /dev/null +++ b/crates/quicksearch-core/src/db/schema.rs @@ -0,0 +1,84 @@ +//! SQL strings for the current schema. Versioned; [`migrate`](super::migrate) +//! drives the upgrade path. + +/// Pragmas applied on every connection open. Tuned for write throughput during +/// indexing; a clean shutdown re-enables journal_mode/synchronous via +/// [`super::repo::checkpoint_and_close`]. +pub const PRAGMAS_FAST: &str = " + PRAGMA journal_mode = OFF; + PRAGMA synchronous = 0; + PRAGMA cache_size = 10000; + PRAGMA temp_store = MEMORY; + PRAGMA foreign_keys = ON; +"; + +/// The full current schema. Applied by [`migrate::open_and_migrate`] when +/// the DB is fresh or has been wiped during upgrade. +/// +/// FTS5 is a *regular* (non-contentless) virtual table so `snippet()` and +/// `highlight()` can read the stored text. This also means there is no +/// separate `documents` table — FTS5 *is* the text store. +pub const SCHEMA_CURRENT: &str = r#" +CREATE TABLE schema_info ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE files ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + path TEXT NOT NULL UNIQUE, + parent TEXT NOT NULL, + size INTEGER NOT NULL, + mtime INTEGER NOT NULL, + inode INTEGER, + device_id INTEGER, + mime TEXT, + type INTEGER NOT NULL DEFAULT 0, + basic_state INTEGER NOT NULL DEFAULT 0, -- 0=pending 1=done 2=failed + content_state INTEGER NOT NULL DEFAULT 0, -- 0=pending 1=done 2=failed 3=n/a + failure_msg TEXT, + hash BLOB +); + +CREATE INDEX idx_files_parent ON files(parent); +CREATE INDEX idx_files_mtime ON files(mtime); +CREATE INDEX idx_files_type ON files(type); +CREATE INDEX idx_files_mime ON files(mime); +CREATE INDEX idx_files_hash ON files(hash); +CREATE INDEX idx_files_content_pending ON files(id) WHERE content_state = 0; + +CREATE TABLE properties ( + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (file_id, key) +); + +CREATE TABLE failed_files ( + file_id INTEGER PRIMARY KEY REFERENCES files(id) ON DELETE CASCADE, + reason TEXT, + ts INTEGER NOT NULL +); + +CREATE TABLE config_validation ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +"#; + +/// FTS5 virtual table DDL. Separate because the tokenizer is config-driven. +/// +/// Regular (not contentless, not external-content) FTS5: the table stores +/// its own text, which enables `snippet()`/`highlight()` and makes row-level +/// INSERT/UPDATE/DELETE work with normal SQL semantics. `rowid` is supplied +/// by the caller and must equal `files.id`. +pub fn fts_create_sql(tokenizer: &str) -> String { + format!( + "CREATE VIRTUAL TABLE searchabletext USING fts5(\ + name, text, properties, \ + tokenize='{}'\ + );", + tokenizer.replace('\'', "''") + ) +} diff --git a/src/document_extraction.rs b/crates/quicksearch-core/src/document_extraction.rs similarity index 97% rename from src/document_extraction.rs rename to crates/quicksearch-core/src/document_extraction.rs index 02cd6d2..926f3dd 100644 --- a/src/document_extraction.rs +++ b/crates/quicksearch-core/src/document_extraction.rs @@ -1,364 +1,364 @@ -use std::ffi::OsString; -use std::fs::File; -use std::io::{Read, BufReader}; - -use zip::ZipArchive; -use quick_xml::Reader; -use quick_xml::events::Event; - -/// Extract text from DOCX files by parsing the word/document.xml -pub fn extract_text_from_docx(file_path: &OsString) -> Result> { - let file = File::open(file_path)?; - let mut archive = ZipArchive::new(BufReader::new(file))?; - - let mut document_xml = archive.by_name("word/document.xml")?; - let mut content = String::new(); - document_xml.read_to_string(&mut content)?; - - let mut reader = Reader::from_str(&content); - reader.trim_text(true); - - let mut text_content = String::new(); - let mut buf = Vec::new(); - let mut in_text = false; - - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(ref e)) => { - if e.name().as_ref() == b"w:t" { - in_text = true; - } - } - Ok(Event::Text(e)) if in_text => { - text_content.push_str(&e.unescape()?.into_owned()); - } - Ok(Event::End(ref e)) => { - if e.name().as_ref() == b"w:t" { - in_text = false; - } else if e.name().as_ref() == b"w:p" { - text_content.push('\n'); - } - } - Ok(Event::Eof) => break, - Err(e) => return Err(format!("Error parsing XML: {}", e).into()), - _ => {} - } - buf.clear(); - } - - Ok(text_content) -} - -/// Extract text from XLSX files by parsing worksheet XML files -pub fn extract_text_from_xlsx(file_path: &OsString) -> Result> { - let file = File::open(file_path)?; - let mut archive = ZipArchive::new(BufReader::new(file))?; - - let mut text_content = String::new(); - - // First, read shared strings if they exist - let mut shared_strings = Vec::new(); - if let Ok(mut shared_strings_xml) = archive.by_name("xl/sharedStrings.xml") { - let mut content = String::new(); - shared_strings_xml.read_to_string(&mut content)?; - - let mut reader = Reader::from_str(&content); - reader.trim_text(true); - let mut buf = Vec::new(); - let mut in_text = false; - - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(ref e)) => { - if e.name().as_ref() == b"t" { - in_text = true; - } - } - Ok(Event::Text(e)) if in_text => { - shared_strings.push(e.unescape()?.into_owned()); - } - Ok(Event::End(ref e)) => { - if e.name().as_ref() == b"t" { - in_text = false; - } - } - Ok(Event::Eof) => break, - _ => {} - } - buf.clear(); - } - } - - // Read worksheets - for i in 0..archive.len() { - let file_name = archive.by_index(i)?.name().to_string(); - if file_name.starts_with("xl/worksheets/sheet") && file_name.ends_with(".xml") { - let mut sheet_xml = archive.by_index(i)?; - let mut content = String::new(); - sheet_xml.read_to_string(&mut content)?; - - let mut reader = Reader::from_str(&content); - reader.trim_text(true); - let mut buf = Vec::new(); - let mut in_cell = false; - let mut cell_type = String::new(); - - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(ref e)) => { - if e.name().as_ref() == b"c" { - in_cell = true; - cell_type.clear(); - for attr in e.attributes() { - let attr = attr?; - if attr.key.as_ref() == b"t" { - cell_type = String::from_utf8_lossy(&attr.value).to_string(); - } - } - } else if e.name().as_ref() == b"v" && in_cell { - // Value element - } - } - Ok(Event::Text(e)) if in_cell => { - let text = e.unescape()?.into_owned(); - if cell_type == "s" { - // Shared string reference - if let Ok(index) = text.parse::() { - if index < shared_strings.len() { - text_content.push_str(&shared_strings[index]); - text_content.push(' '); - } - } - } else { - text_content.push_str(&text); - text_content.push(' '); - } - } - Ok(Event::End(ref e)) => { - if e.name().as_ref() == b"c" { - in_cell = false; - } else if e.name().as_ref() == b"row" { - text_content.push('\n'); - } - } - Ok(Event::Eof) => break, - _ => {} - } - buf.clear(); - } - } - } - - Ok(text_content) -} - -/// Extract text from PPTX files by parsing slide XML files -pub fn extract_text_from_pptx(file_path: &OsString) -> Result> { - let file = File::open(file_path)?; - let mut archive = ZipArchive::new(BufReader::new(file))?; - - let mut text_content = String::new(); - - // Read all slide files - for i in 0..archive.len() { - let file_name = archive.by_index(i)?.name().to_string(); - if file_name.starts_with("ppt/slides/slide") && file_name.ends_with(".xml") { - let mut slide_xml = archive.by_index(i)?; - let mut content = String::new(); - slide_xml.read_to_string(&mut content)?; - - let mut reader = Reader::from_str(&content); - reader.trim_text(true); - let mut buf = Vec::new(); - let mut in_text = false; - - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(ref e)) => { - if e.name().as_ref() == b"a:t" { - in_text = true; - } - } - Ok(Event::Text(e)) if in_text => { - text_content.push_str(&e.unescape()?.into_owned()); - } - Ok(Event::End(ref e)) => { - if e.name().as_ref() == b"a:t" { - in_text = false; - } else if e.name().as_ref() == b"a:p" { - text_content.push('\n'); - } - } - Ok(Event::Eof) => break, - _ => {} - } - buf.clear(); - } - text_content.push_str("\n--- New Slide ---\n"); - } - } - - Ok(text_content) -} - -/// Extract text from ODT files (OpenDocument Text) -pub fn extract_text_from_odt(file_path: &OsString) -> Result> { - let file = File::open(file_path)?; - let mut archive = ZipArchive::new(BufReader::new(file))?; - - let mut content_xml = archive.by_name("content.xml")?; - let mut content = String::new(); - content_xml.read_to_string(&mut content)?; - - let mut reader = Reader::from_str(&content); - reader.trim_text(true); - - let mut text_content = String::new(); - let mut buf = Vec::new(); - let mut in_text = false; - - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(ref e)) => { - let name = e.name(); - if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" || name.as_ref() == b"text:span" { - in_text = true; - } - } - Ok(Event::Text(e)) if in_text => { - text_content.push_str(&e.unescape()?.into_owned()); - } - Ok(Event::End(ref e)) => { - let name = e.name(); - if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" { - text_content.push('\n'); - in_text = false; - } else if name.as_ref() == b"text:span" { - in_text = false; - } - } - Ok(Event::Eof) => break, - _ => {} - } - buf.clear(); - } - - Ok(text_content) -} - -/// Extract text from ODP files (OpenDocument Presentation) -pub fn extract_text_from_odp(file_path: &OsString) -> Result> { - let file = File::open(file_path)?; - let mut archive = ZipArchive::new(BufReader::new(file))?; - - let mut content_xml = archive.by_name("content.xml")?; - let mut content = String::new(); - content_xml.read_to_string(&mut content)?; - - let mut reader = Reader::from_str(&content); - reader.trim_text(true); - - let mut text_content = String::new(); - let mut buf = Vec::new(); - let mut in_text = false; - - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(ref e)) => { - let name = e.name(); - if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" || name.as_ref() == b"text:span" { - in_text = true; - } - } - Ok(Event::Text(e)) if in_text => { - text_content.push_str(&e.unescape()?.into_owned()); - } - Ok(Event::End(ref e)) => { - let name = e.name(); - if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" { - text_content.push('\n'); - in_text = false; - } else if name.as_ref() == b"text:span" { - in_text = false; - } - } - Ok(Event::Eof) => break, - _ => {} - } - buf.clear(); - } - - Ok(text_content) -} - -/// Extract text from ODS files (OpenDocument Spreadsheet) -pub fn extract_text_from_ods(file_path: &OsString) -> Result> { - let file = File::open(file_path)?; - let mut archive = ZipArchive::new(BufReader::new(file))?; - - let mut content_xml = archive.by_name("content.xml")?; - let mut content = String::new(); - content_xml.read_to_string(&mut content)?; - - let mut reader = Reader::from_str(&content); - reader.trim_text(true); - - let mut text_content = String::new(); - let mut buf = Vec::new(); - let mut in_text = false; - - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(ref e)) => { - let name = e.name(); - if name.as_ref() == b"text:p" || name.as_ref() == b"text:span" { - in_text = true; - } - } - Ok(Event::Text(e)) if in_text => { - text_content.push_str(&e.unescape()?.into_owned()); - text_content.push(' '); - } - Ok(Event::End(ref e)) => { - let name = e.name(); - if name.as_ref() == b"text:p" { - text_content.push('\n'); - in_text = false; - } else if name.as_ref() == b"text:span" { - in_text = false; - } - } - Ok(Event::Eof) => break, - _ => {} - } - buf.clear(); - } - - Ok(text_content) -} - -/// Extract text from various document formats -pub fn extract_document_text(file_path: &OsString, extension: &str) -> Result> { - match extension { - "docx" => extract_text_from_docx(file_path), - "doc" => { - // DOC format is binary and complex to parse without external tools - // For now, return an empty result - Ok(String::new()) - } - "xlsx" => extract_text_from_xlsx(file_path), - "xls" => { - // XLS format is binary and complex to parse without external tools - Ok(String::new()) - } - "pptx" => extract_text_from_pptx(file_path), - "ppt" => { - // PPT format is binary and complex to parse without external tools - Ok(String::new()) - } - "odt" => extract_text_from_odt(file_path), - "odp" => extract_text_from_odp(file_path), - "ods" => extract_text_from_ods(file_path), - _ => Ok(String::new()) - } -} +use std::ffi::OsString; +use std::fs::File; +use std::io::{Read, BufReader}; + +use zip::ZipArchive; +use quick_xml::Reader; +use quick_xml::events::Event; + +/// Extract text from DOCX files by parsing the word/document.xml +pub fn extract_text_from_docx(file_path: &OsString) -> Result> { + let file = File::open(file_path)?; + let mut archive = ZipArchive::new(BufReader::new(file))?; + + let mut document_xml = archive.by_name("word/document.xml")?; + let mut content = String::new(); + document_xml.read_to_string(&mut content)?; + + let mut reader = Reader::from_str(&content); + reader.trim_text(true); + + let mut text_content = String::new(); + let mut buf = Vec::new(); + let mut in_text = false; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) => { + if e.name().as_ref() == b"w:t" { + in_text = true; + } + } + Ok(Event::Text(e)) if in_text => { + text_content.push_str(&e.unescape()?.into_owned()); + } + Ok(Event::End(ref e)) => { + if e.name().as_ref() == b"w:t" { + in_text = false; + } else if e.name().as_ref() == b"w:p" { + text_content.push('\n'); + } + } + Ok(Event::Eof) => break, + Err(e) => return Err(format!("Error parsing XML: {}", e).into()), + _ => {} + } + buf.clear(); + } + + Ok(text_content) +} + +/// Extract text from XLSX files by parsing worksheet XML files +pub fn extract_text_from_xlsx(file_path: &OsString) -> Result> { + let file = File::open(file_path)?; + let mut archive = ZipArchive::new(BufReader::new(file))?; + + let mut text_content = String::new(); + + // First, read shared strings if they exist + let mut shared_strings = Vec::new(); + if let Ok(mut shared_strings_xml) = archive.by_name("xl/sharedStrings.xml") { + let mut content = String::new(); + shared_strings_xml.read_to_string(&mut content)?; + + let mut reader = Reader::from_str(&content); + reader.trim_text(true); + let mut buf = Vec::new(); + let mut in_text = false; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) => { + if e.name().as_ref() == b"t" { + in_text = true; + } + } + Ok(Event::Text(e)) if in_text => { + shared_strings.push(e.unescape()?.into_owned()); + } + Ok(Event::End(ref e)) => { + if e.name().as_ref() == b"t" { + in_text = false; + } + } + Ok(Event::Eof) => break, + _ => {} + } + buf.clear(); + } + } + + // Read worksheets + for i in 0..archive.len() { + let file_name = archive.by_index(i)?.name().to_string(); + if file_name.starts_with("xl/worksheets/sheet") && file_name.ends_with(".xml") { + let mut sheet_xml = archive.by_index(i)?; + let mut content = String::new(); + sheet_xml.read_to_string(&mut content)?; + + let mut reader = Reader::from_str(&content); + reader.trim_text(true); + let mut buf = Vec::new(); + let mut in_cell = false; + let mut cell_type = String::new(); + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) => { + if e.name().as_ref() == b"c" { + in_cell = true; + cell_type.clear(); + for attr in e.attributes() { + let attr = attr?; + if attr.key.as_ref() == b"t" { + cell_type = String::from_utf8_lossy(&attr.value).to_string(); + } + } + } else if e.name().as_ref() == b"v" && in_cell { + // Value element + } + } + Ok(Event::Text(e)) if in_cell => { + let text = e.unescape()?.into_owned(); + if cell_type == "s" { + // Shared string reference + if let Ok(index) = text.parse::() { + if index < shared_strings.len() { + text_content.push_str(&shared_strings[index]); + text_content.push(' '); + } + } + } else { + text_content.push_str(&text); + text_content.push(' '); + } + } + Ok(Event::End(ref e)) => { + if e.name().as_ref() == b"c" { + in_cell = false; + } else if e.name().as_ref() == b"row" { + text_content.push('\n'); + } + } + Ok(Event::Eof) => break, + _ => {} + } + buf.clear(); + } + } + } + + Ok(text_content) +} + +/// Extract text from PPTX files by parsing slide XML files +pub fn extract_text_from_pptx(file_path: &OsString) -> Result> { + let file = File::open(file_path)?; + let mut archive = ZipArchive::new(BufReader::new(file))?; + + let mut text_content = String::new(); + + // Read all slide files + for i in 0..archive.len() { + let file_name = archive.by_index(i)?.name().to_string(); + if file_name.starts_with("ppt/slides/slide") && file_name.ends_with(".xml") { + let mut slide_xml = archive.by_index(i)?; + let mut content = String::new(); + slide_xml.read_to_string(&mut content)?; + + let mut reader = Reader::from_str(&content); + reader.trim_text(true); + let mut buf = Vec::new(); + let mut in_text = false; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) => { + if e.name().as_ref() == b"a:t" { + in_text = true; + } + } + Ok(Event::Text(e)) if in_text => { + text_content.push_str(&e.unescape()?.into_owned()); + } + Ok(Event::End(ref e)) => { + if e.name().as_ref() == b"a:t" { + in_text = false; + } else if e.name().as_ref() == b"a:p" { + text_content.push('\n'); + } + } + Ok(Event::Eof) => break, + _ => {} + } + buf.clear(); + } + text_content.push_str("\n--- New Slide ---\n"); + } + } + + Ok(text_content) +} + +/// Extract text from ODT files (OpenDocument Text) +pub fn extract_text_from_odt(file_path: &OsString) -> Result> { + let file = File::open(file_path)?; + let mut archive = ZipArchive::new(BufReader::new(file))?; + + let mut content_xml = archive.by_name("content.xml")?; + let mut content = String::new(); + content_xml.read_to_string(&mut content)?; + + let mut reader = Reader::from_str(&content); + reader.trim_text(true); + + let mut text_content = String::new(); + let mut buf = Vec::new(); + let mut in_text = false; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) => { + let name = e.name(); + if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" || name.as_ref() == b"text:span" { + in_text = true; + } + } + Ok(Event::Text(e)) if in_text => { + text_content.push_str(&e.unescape()?.into_owned()); + } + Ok(Event::End(ref e)) => { + let name = e.name(); + if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" { + text_content.push('\n'); + in_text = false; + } else if name.as_ref() == b"text:span" { + in_text = false; + } + } + Ok(Event::Eof) => break, + _ => {} + } + buf.clear(); + } + + Ok(text_content) +} + +/// Extract text from ODP files (OpenDocument Presentation) +pub fn extract_text_from_odp(file_path: &OsString) -> Result> { + let file = File::open(file_path)?; + let mut archive = ZipArchive::new(BufReader::new(file))?; + + let mut content_xml = archive.by_name("content.xml")?; + let mut content = String::new(); + content_xml.read_to_string(&mut content)?; + + let mut reader = Reader::from_str(&content); + reader.trim_text(true); + + let mut text_content = String::new(); + let mut buf = Vec::new(); + let mut in_text = false; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) => { + let name = e.name(); + if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" || name.as_ref() == b"text:span" { + in_text = true; + } + } + Ok(Event::Text(e)) if in_text => { + text_content.push_str(&e.unescape()?.into_owned()); + } + Ok(Event::End(ref e)) => { + let name = e.name(); + if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" { + text_content.push('\n'); + in_text = false; + } else if name.as_ref() == b"text:span" { + in_text = false; + } + } + Ok(Event::Eof) => break, + _ => {} + } + buf.clear(); + } + + Ok(text_content) +} + +/// Extract text from ODS files (OpenDocument Spreadsheet) +pub fn extract_text_from_ods(file_path: &OsString) -> Result> { + let file = File::open(file_path)?; + let mut archive = ZipArchive::new(BufReader::new(file))?; + + let mut content_xml = archive.by_name("content.xml")?; + let mut content = String::new(); + content_xml.read_to_string(&mut content)?; + + let mut reader = Reader::from_str(&content); + reader.trim_text(true); + + let mut text_content = String::new(); + let mut buf = Vec::new(); + let mut in_text = false; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) => { + let name = e.name(); + if name.as_ref() == b"text:p" || name.as_ref() == b"text:span" { + in_text = true; + } + } + Ok(Event::Text(e)) if in_text => { + text_content.push_str(&e.unescape()?.into_owned()); + text_content.push(' '); + } + Ok(Event::End(ref e)) => { + let name = e.name(); + if name.as_ref() == b"text:p" { + text_content.push('\n'); + in_text = false; + } else if name.as_ref() == b"text:span" { + in_text = false; + } + } + Ok(Event::Eof) => break, + _ => {} + } + buf.clear(); + } + + Ok(text_content) +} + +/// Extract text from various document formats +pub fn extract_document_text(file_path: &OsString, extension: &str) -> Result> { + match extension { + "docx" => extract_text_from_docx(file_path), + "doc" => { + // DOC format is binary and complex to parse without external tools + // For now, return an empty result + Ok(String::new()) + } + "xlsx" => extract_text_from_xlsx(file_path), + "xls" => { + // XLS format is binary and complex to parse without external tools + Ok(String::new()) + } + "pptx" => extract_text_from_pptx(file_path), + "ppt" => { + // PPT format is binary and complex to parse without external tools + Ok(String::new()) + } + "odt" => extract_text_from_odt(file_path), + "odp" => extract_text_from_odp(file_path), + "ods" => extract_text_from_ods(file_path), + _ => Ok(String::new()) + } +} diff --git a/crates/quicksearch-core/src/extract/audio.rs b/crates/quicksearch-core/src/extract/audio.rs new file mode 100644 index 0000000..933934f --- /dev/null +++ b/crates/quicksearch-core/src/extract/audio.rs @@ -0,0 +1,96 @@ +//! Audio tag extraction via [`lofty`]. Pulls title/artist/album/genre/year/ +//! track/duration into properties; concatenates tag values into `text` so +//! full-text search works across them. + +use std::path::Path; + +use lofty::{ + file::{AudioFile, TaggedFileExt}, + probe::Probe, + tag::{Accessor, ItemKey}, +}; + +use super::{ExtractError, ExtractedContent, Extractor}; + +pub struct AudioExtractor; + +impl Extractor for AudioExtractor { + fn supports(&self, mime: &str) -> bool { + mime.starts_with("audio/") + } + + fn extract(&self, path: &Path) -> Result { + let tagged = Probe::open(path) + .map_err(|e| format!("lofty probe {}: {}", path.display(), e))? + .read() + .map_err(|e| format!("lofty read {}: {}", path.display(), e))?; + + let mut out = ExtractedContent::default(); + + // Duration in seconds. + let duration_secs = tagged.properties().duration().as_secs(); + if duration_secs > 0 { + out.properties + .insert("duration".to_string(), duration_secs.to_string()); + } + + if let Some(tag) = tagged.primary_tag().or_else(|| tagged.first_tag()) { + for (key, item_key) in [ + ("title", ItemKey::TrackTitle), + ("artist", ItemKey::TrackArtist), + ("album", ItemKey::AlbumTitle), + ("genre", ItemKey::Genre), + ("year", ItemKey::Year), + ("track", ItemKey::TrackNumber), + ("comment", ItemKey::Comment), + ] { + if let Some(v) = tag.get_string(&item_key) { + if !v.is_empty() { + out.properties.insert(key.to_string(), v.to_string()); + } + } + } + // Accessor shortcuts for common fields if the ItemKey lookup missed. + if !out.properties.contains_key("title") { + if let Some(t) = tag.title() { + out.properties.insert("title".to_string(), t.to_string()); + } + } + if !out.properties.contains_key("artist") { + if let Some(a) = tag.artist() { + out.properties.insert("artist".to_string(), a.to_string()); + } + } + if !out.properties.contains_key("album") { + if let Some(a) = tag.album() { + out.properties.insert("album".to_string(), a.to_string()); + } + } + } + + // Join the searchable tag values into one blob so FTS hits on any of them. + let mut pieces: Vec<&str> = Vec::new(); + for k in ["title", "artist", "album", "genre", "comment"] { + if let Some(v) = out.properties.get(k) { + pieces.push(v.as_str()); + } + } + out.text = pieces.join(" "); + + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supports_audio_mimes() { + let e = AudioExtractor; + assert!(e.supports("audio/mpeg")); + assert!(e.supports("audio/flac")); + assert!(!e.supports("video/mp4")); + assert!(!e.supports("image/png")); + } +} diff --git a/crates/quicksearch-core/src/extract/image.rs b/crates/quicksearch-core/src/extract/image.rs new file mode 100644 index 0000000..9b0920d --- /dev/null +++ b/crates/quicksearch-core/src/extract/image.rs @@ -0,0 +1,96 @@ +//! Image metadata extraction via [`kamadak_exif`]. Reads EXIF tags (camera +//! make/model, date, GPS, dimensions) into properties. `text` is left empty +//! — this extractor does not OCR. + +use std::fs::File; +use std::io::BufReader; +use std::path::Path; + +use exif::{In, Reader, Tag, Value}; + +use super::{ExtractError, ExtractedContent, Extractor}; + +pub struct ImageExtractor; + +impl Extractor for ImageExtractor { + fn supports(&self, mime: &str) -> bool { + mime.starts_with("image/") + } + + fn extract(&self, path: &Path) -> Result { + let file = File::open(path) + .map_err(|e| format!("image open {}: {}", path.display(), e))?; + let mut bufreader = BufReader::new(&file); + let mut out = ExtractedContent::default(); + + // Not every image has EXIF (e.g. PNGs usually don't). Treat "no EXIF" + // as a successful no-op rather than a failure. + let exif = match Reader::new().read_from_container(&mut bufreader) { + Ok(exif) => exif, + Err(_) => return Ok(out), + }; + + for (key, tag) in [ + ("make", Tag::Make), + ("model", Tag::Model), + ("date_taken", Tag::DateTimeOriginal), + ("software", Tag::Software), + ("orientation", Tag::Orientation), + ("width", Tag::PixelXDimension), + ("height", Tag::PixelYDimension), + ("iso", Tag::PhotographicSensitivity), + ("f_number", Tag::FNumber), + ("exposure", Tag::ExposureTime), + ("focal_length", Tag::FocalLength), + ("gps_latitude", Tag::GPSLatitude), + ("gps_longitude", Tag::GPSLongitude), + ] { + if let Some(field) = exif.get_field(tag, In::PRIMARY) { + let s = field_to_string(&field.value); + if !s.is_empty() { + out.properties.insert(key.to_string(), s); + } + } + } + + Ok(out) + } +} + +fn field_to_string(value: &Value) -> String { + match value { + Value::Ascii(bytes_vec) => { + let mut parts = Vec::new(); + for bytes in bytes_vec { + let s: String = bytes + .iter() + .take_while(|&&b| b != 0) + .map(|&b| b as char) + .collect(); + if !s.is_empty() { + parts.push(s); + } + } + parts.join(" ") + } + other => { + // The EXIF crate's Display formatter handles integer/rational + // types. Use it for any value we didn't handle explicitly. + format!("{}", other.display_as(exif::Tag::Copyright)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supports_image_mimes() { + let e = ImageExtractor; + assert!(e.supports("image/jpeg")); + assert!(e.supports("image/png")); + assert!(!e.supports("video/mp4")); + assert!(!e.supports("text/plain")); + } +} diff --git a/crates/quicksearch-core/src/extract/mod.rs b/crates/quicksearch-core/src/extract/mod.rs new file mode 100644 index 0000000..6c089d9 --- /dev/null +++ b/crates/quicksearch-core/src/extract/mod.rs @@ -0,0 +1,149 @@ +//! Content extractors: text plus structured properties (title, artist, EXIF, …). +//! +//! An [`Extractor`] decides whether it can handle a given MIME type and, if +//! so, produces [`ExtractedContent`] for the file. The [`Registry`] picks the +//! first registered extractor that accepts the MIME and runs it. Callers can +//! also fall back to an extension-based match for files with no detected MIME. + +use std::collections::HashMap; +use std::path::Path; + +pub mod audio; +pub mod image; +pub mod office; +pub mod pdf; +pub mod plaintext; + +/// Result of a successful extraction. `text` feeds the FTS5 `text` column; +/// `properties` feeds both the `properties` FTS5 column (as `key:value` +/// tokens) and the structured `properties` table for later retrieval. +/// +/// Extractors may return an empty `text` when the file has no narrative +/// content (e.g. an image where only EXIF matters). Filename search still +/// works in that case. +#[derive(Debug, Default, Clone)] +pub struct ExtractedContent { + pub text: String, + pub properties: HashMap, +} + +impl ExtractedContent { + pub fn with_text(text: impl Into) -> Self { + Self { + text: text.into(), + properties: HashMap::new(), + } + } + + pub fn with_property(mut self, key: impl Into, value: impl Into) -> Self { + self.properties.insert(key.into(), value.into()); + self + } + + /// Convert properties into the `Vec<(String, String)>` shape expected by + /// [`crate::db::repo::set_content_done`]. Keys are sorted for determinism + /// in tests and snapshots. + pub fn properties_sorted(&self) -> Vec<(String, String)> { + let mut v: Vec<(String, String)> = self + .properties + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + v.sort_by(|a, b| a.0.cmp(&b.0)); + v + } +} + +/// Boxed error type for extractor failures. A string reason is stored on the +/// file row (see [`crate::db::repo::set_content_failed`]), so extractors +/// should surface human-readable messages. +pub type ExtractError = String; + +/// A pluggable content extractor. Stateless; implementors should not hold +/// file handles across calls. +pub trait Extractor: Send + Sync { + /// Whether this extractor can handle the given MIME type. `mime` is + /// normalized to lowercase before dispatch. + fn supports(&self, mime: &str) -> bool; + + /// Read the file at `path` and return its extracted content. Return an + /// [`ExtractError`] to mark the file's content state as failed (so it + /// won't be retried every run). + fn extract(&self, path: &Path) -> Result; +} + +/// An ordered dispatch table of extractors. The first extractor whose +/// [`supports`](Extractor::supports) returns true for the MIME is used. +pub struct Registry { + extractors: Vec>, +} + +impl Registry { + pub fn new() -> Self { + Self { extractors: Vec::new() } + } + + pub fn with(mut self, e: impl Extractor + 'static) -> Self { + self.extractors.push(Box::new(e)); + self + } + + /// Look up a handler for `mime` and run it against `path`. Returns + /// `Ok(None)` if no extractor claims the MIME — the caller should then + /// decide whether the file is "not applicable" (text state NA). + pub fn extract( + &self, + path: &Path, + mime: &str, + ) -> Result, ExtractError> { + let lower = mime.to_ascii_lowercase(); + for e in &self.extractors { + if e.supports(&lower) { + return e.extract(path).map(Some); + } + } + Ok(None) + } + + /// The default set wired up for Set A: plaintext, office docs, PDF, + /// audio tags, image EXIF. + pub fn default_set() -> Self { + Self::new() + .with(plaintext::PlaintextExtractor) + .with(office::OfficeExtractor) + .with(pdf::PdfExtractor) + .with(audio::AudioExtractor) + .with(image::ImageExtractor) + } +} + +impl Default for Registry { + fn default() -> Self { + Self::default_set() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_registry_returns_none() { + let r = Registry::new(); + let out = r + .extract(Path::new("/tmp/x"), "text/plain") + .expect("no error"); + assert!(out.is_none()); + } + + #[test] + fn properties_sorted_is_deterministic() { + let c = ExtractedContent::with_text("hi") + .with_property("b", "2") + .with_property("a", "1"); + assert_eq!( + c.properties_sorted(), + vec![("a".to_string(), "1".to_string()), ("b".to_string(), "2".to_string())] + ); + } +} diff --git a/crates/quicksearch-core/src/extract/office.rs b/crates/quicksearch-core/src/extract/office.rs new file mode 100644 index 0000000..b2e009e --- /dev/null +++ b/crates/quicksearch-core/src/extract/office.rs @@ -0,0 +1,69 @@ +//! Office document extraction: DOCX, XLSX, PPTX, ODT, ODP, ODS. +//! +//! Delegates to [`crate::document_extraction`] — a single entry point based +//! on file extension rather than MIME. We translate the MIME to the +//! extension expected by that module. + +use std::ffi::OsString; +use std::path::Path; + +use crate::document_extraction::extract_document_text; + +use super::{ExtractError, ExtractedContent, Extractor}; + +pub struct OfficeExtractor; + +fn mime_to_ext(mime: &str) -> Option<&'static str> { + match mime { + "application/msword" => Some("doc"), + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => Some("docx"), + "application/vnd.ms-excel" => Some("xls"), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => Some("xlsx"), + "application/vnd.ms-powerpoint" => Some("ppt"), + "application/vnd.openxmlformats-officedocument.presentationml.presentation" => Some("pptx"), + "application/vnd.oasis.opendocument.text" => Some("odt"), + "application/vnd.oasis.opendocument.spreadsheet" => Some("ods"), + "application/vnd.oasis.opendocument.presentation" => Some("odp"), + _ => None, + } +} + +impl Extractor for OfficeExtractor { + fn supports(&self, mime: &str) -> bool { + mime_to_ext(mime).is_some() + } + + fn extract(&self, path: &Path) -> Result { + // Recompute the extension from MIME at call time by asking the path. + // Using the filesystem extension directly is more robust than round- + // tripping through MIME: `.docm` has the same MIME as `.docx` but + // `extract_document_text` looks up by extension. + let ext = path + .extension() + .and_then(|s| s.to_str()) + .map(|s| s.to_ascii_lowercase()) + .unwrap_or_default(); + let text = extract_document_text(&OsString::from(path.as_os_str()), &ext) + .map_err(|e| format!("office extractor {}: {}", path.display(), e))?; + Ok(ExtractedContent::with_text(text)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supports_docx_and_friends() { + let e = OfficeExtractor; + for m in [ + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.oasis.opendocument.text", + ] { + assert!(e.supports(m), "should support {}", m); + } + assert!(!e.supports("image/png")); + } +} diff --git a/crates/quicksearch-core/src/extract/pdf.rs b/crates/quicksearch-core/src/extract/pdf.rs new file mode 100644 index 0000000..ae7c511 --- /dev/null +++ b/crates/quicksearch-core/src/extract/pdf.rs @@ -0,0 +1,77 @@ +//! PDF text extraction. +//! +//! Primary path: [`pdf_extract::extract_text`], which handles most modern PDFs +//! and is simple to call. It can panic or hard-error on malformed files; any +//! failure is surfaced to the caller and marks the file's content state as +//! failed. Properties (title, author, etc.) from the PDF `Info` dictionary +//! are pulled via `lopdf` where available. + +use std::path::Path; + +use lopdf::{Document as LopdfDocument, Object}; + +use super::{ExtractError, ExtractedContent, Extractor}; + +pub struct PdfExtractor; + +impl Extractor for PdfExtractor { + fn supports(&self, mime: &str) -> bool { + mime == "application/pdf" + } + + fn extract(&self, path: &Path) -> Result { + // Text. Catch panics from pdf_extract (some PDFs crash its parser). + let path_buf = path.to_path_buf(); + let text = std::panic::catch_unwind(move || pdf_extract::extract_text(&path_buf)) + .map_err(|_| "pdf_extract panicked".to_string())? + .map_err(|e| format!("pdf_extract: {}", e))?; + + let mut out = ExtractedContent::with_text(text); + + // Info dictionary via lopdf. Soft-fail: if lopdf can't open the file + // we still return the text. + if let Ok(doc) = LopdfDocument::load(path) { + if let Ok(info_ref) = doc.trailer.get(b"Info") { + if let Ok(info_id) = info_ref.as_reference() { + if let Ok(info) = doc.get_object(info_id) { + if let Ok(dict) = info.as_dict() { + for key in ["Title", "Author", "Subject", "Keywords", "Creator", "Producer"] { + if let Ok(val) = dict.get(key.as_bytes()) { + if let Some(s) = object_to_string(val) { + if !s.is_empty() { + out.properties.insert(key.to_ascii_lowercase(), s); + } + } + } + } + } + } + } + } + } + + Ok(out) + } +} + +fn object_to_string(obj: &Object) -> Option { + match obj { + Object::String(bytes, _) => { + // Try UTF-8; fall back to lossy decoding. + Some(String::from_utf8_lossy(bytes).into_owned()) + } + Object::Name(bytes) => Some(String::from_utf8_lossy(bytes).into_owned()), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supports_pdf_mime() { + assert!(PdfExtractor.supports("application/pdf")); + assert!(!PdfExtractor.supports("application/zip")); + } +} diff --git a/crates/quicksearch-core/src/extract/plaintext.rs b/crates/quicksearch-core/src/extract/plaintext.rs new file mode 100644 index 0000000..e20aaff --- /dev/null +++ b/crates/quicksearch-core/src/extract/plaintext.rs @@ -0,0 +1,65 @@ +//! Read the file as UTF-8 text. Handles text/plain, text/x-*, application/json +//! and most source-code MIMEs. + +use std::path::Path; + +use super::{ExtractError, ExtractedContent, Extractor}; + +pub struct PlaintextExtractor; + +impl Extractor for PlaintextExtractor { + fn supports(&self, mime: &str) -> bool { + if mime.starts_with("text/") { + return true; + } + matches!( + mime, + "application/json" + | "application/xml" + | "application/javascript" + | "application/x-shellscript" + | "application/x-python" + | "application/toml" + | "application/yaml" + | "application/x-yaml" + ) + } + + fn extract(&self, path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .map_err(|e| format!("plaintext read {}: {}", path.display(), e))?; + Ok(ExtractedContent::with_text(text)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_utf8_file() { + let mut p = std::env::temp_dir(); + p.push(format!( + "qs-plaintext-{}-{}.txt", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&p, "hello world").unwrap(); + let c = PlaintextExtractor.extract(&p).unwrap(); + assert_eq!(c.text, "hello world"); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn supports_text_mimes() { + let e = PlaintextExtractor; + assert!(e.supports("text/plain")); + assert!(e.supports("text/x-rust")); + assert!(e.supports("application/json")); + assert!(!e.supports("application/pdf")); + assert!(!e.supports("image/png")); + } +} diff --git a/src/file_handling.rs b/crates/quicksearch-core/src/file_handling.rs similarity index 77% rename from src/file_handling.rs rename to crates/quicksearch-core/src/file_handling.rs index ddc1dc6..8420c75 100644 --- a/src/file_handling.rs +++ b/crates/quicksearch-core/src/file_handling.rs @@ -1,22 +1,26 @@ +use std::sync::atomic::AtomicBool; use std::sync::{Mutex, Arc}; use std::ffi::OsString; -use std::fs::{File,read_to_string}; +use std::fs::File; use std::io::{Read, Seek, SeekFrom}; -use std::path::Component; +use std::path::{Component, Path}; use std::process::{Command, Stdio}; use std::time::UNIX_EPOCH; use std::collections::HashMap; use sha2::{Sha256, Digest}; use walkdir::{DirEntry, WalkDir}; -use rusqlite::{params, Connection}; +use rusqlite::Connection; -use crate::document_extraction::extract_document_text; use crate::config::Config; +use crate::db::repo::{self, NewFile}; +use crate::extract::Registry; +use crate::indexing::should_abort; +use crate::mime::{guess_mime, mime_to_type, FileType}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ExistingFileEntry { - pub moddate: u64, + pub mtime: u64, } pub const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 84] = @@ -58,15 +62,15 @@ pub const SUPPORTED_DOCUMENT_EXTENSIONS_LIST: [&'static str; 9] = "ppt", "pptx", "odp", // Presentation "xls", "xlsx", "ods"]; // Spreadsheet -/// Load path and moddate per row for incremental classification (hash/size loaded only when updating a file). +/// Load path and mtime per row for incremental classification (hash/size loaded only when updating a file). pub fn load_existing_files(conn: &Connection) -> Result, rusqlite::Error> { let mut existing_files = HashMap::new(); - let mut stmt = conn.prepare("SELECT path, moddate FROM files")?; + let mut stmt = conn.prepare("SELECT path, mtime FROM files")?; let rows = stmt.query_map([], |row| { Ok(( row.get::<_, String>(0)?, ExistingFileEntry { - moddate: row.get(1)?, + mtime: row.get(1)?, }, )) })?; @@ -79,6 +83,29 @@ pub fn load_existing_files(conn: &Connection) -> Result (Option, Option) { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + (Some(_meta.ino()), Some(_meta.dev())) + } + #[cfg(not(unix))] + { + (None, None) + } +} + +/// Parent directory of a path as a UTF-8 string, empty if root. +fn parent_str(path: &str) -> String { + Path::new(path) + .parent() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default() +} + + pub fn indexed_walk_file_entries( path: &str, follow_symlinks: bool, @@ -240,7 +267,7 @@ pub fn classify_dir_entry_for_indexing( }; if let Some(existing) = existing_files.get(&fpath) { - if existing.moddate != fmodified { + if existing.mtime != fmodified { Some(FileIndexAction::Update) } else { Some(FileIndexAction::Skip) @@ -294,39 +321,14 @@ fn format_progress_pair(visit_index: usize, progress_display_total: Option Result<(), String> { - conn.execute(FTS_SQL_AUTOMERGE_8, []) - .map_err(|e| format!("FTS automerge(8): {}", e))?; - conn.execute(FTS_SQL_REBUILD, []) - .map_err(|e| format!("FTS rebuild: {}", e))?; - Ok(()) -} - -fn fts_remove_document_for_path( - tx: &rusqlite::Transaction<'_>, - path: &str, -) -> Result<(), String> { - let id_opt: Option = match tx.query_row( - "SELECT id FROM documents WHERE path = ?1", - params![path], - |r| r.get(0), + if let Err(e) = conn.execute( + "INSERT INTO searchabletext(searchabletext, rank) VALUES('automerge', 8)", + [], ) { - Ok(id) => Some(id), - Err(rusqlite::Error::QueryReturnedNoRows) => None, - Err(e) => return Err(format!("documents id lookup: {}", e)), - }; - if let Some(doc_id) = id_opt { - tx.execute( - "INSERT INTO searchabletext(searchabletext, rowid) VALUES('delete', ?1)", - params![doc_id], - ) - .map_err(|e| format!("FTS delete doc {}: {}", doc_id, e))?; - tx.execute("DELETE FROM documents WHERE id = ?1", params![doc_id]) - .map_err(|e| format!("delete documents row: {}", e))?; + eprintln!("Warning: FTS automerge failed (non-fatal): {}", e); } Ok(()) } @@ -459,15 +461,24 @@ pub fn process_batch_updates_files_only( )); } - tx.execute( - "UPDATE files SET size = ?1, moddate = ?2, hash = ?3 WHERE path = ?4", - params![row.fsize, row.fmodified, row.fhash, row.path_db], + let p = Path::new(row.path_db.as_str()); + let guessed_mime = guess_mime(p); + let ftype = guessed_mime + .as_deref() + .map(mime_to_type) + .unwrap_or(FileType::EMPTY); + let _ = repo::update_file_basic( + &tx, + &row.path_db, + row.fsize, + row.fmodified, + Some(row.fhash.as_slice()), + guessed_mime.as_deref(), + ftype, ) - .map_err(|e| format!("Failed to update file record: {}", e))?; - - fts_remove_document_for_path(&tx, &row.path_db).map_err(|e| { + .map_err(|e| { format!( - "Failed to remove old document / FTS entry for {}: {}", + "Failed to update file record + clear stale content for {}: {}", row.path_db, e ) })?; @@ -565,12 +576,32 @@ pub fn process_batch_inserts_files_only( }; let fname = entry.path().file_name().unwrap().to_os_string(); + let fname_str = fname.to_string_lossy().into_owned(); + let fpath_str = fpath.to_string_lossy().into_owned(); + let parent = parent_str(&fpath_str); + let (inode, device_id) = inode_and_device(&meta); + let guessed_mime = guess_mime(Path::new(&fpath_str)); + let ftype = guessed_mime + .as_deref() + .map(mime_to_type) + .unwrap_or(FileType::EMPTY); - // Insert into files table - tx.execute( - "INSERT INTO files VALUES (?1, ?2, ?3, ?4, ?5)", - params![fname.to_str(), fpath.to_string_lossy(), fsize, fmodified, fhash] - ).map_err(|e| format!("Failed to insert file record: {}", e))?; + repo::insert_file( + &tx, + &NewFile { + name: &fname_str, + path: &fpath_str, + parent: &parent, + size: fsize, + mtime: fmodified, + inode, + device_id, + mime: guessed_mime.as_deref(), + ftype, + hash: Some(fhash.as_slice()), + }, + ) + .map_err(|e| format!("Failed to insert file record: {}", e))?; } // Update status with current file @@ -588,6 +619,7 @@ pub fn cleanup_stale_index_entries( conn_mutex: &Arc>, stale_paths: &[String], stop_flag: &Arc>, + suspend_flag: &Arc, status_callback: Option>, ) -> Result { if stale_paths.is_empty() { @@ -601,7 +633,7 @@ pub fn cleanup_stale_index_entries( let mut deleted_count = 0usize; for path in stale_paths { - if *stop_flag.lock().unwrap() { + if should_abort(stop_flag, suspend_flag) { let _ = tx.commit(); drop(conn); return Ok(deleted_count); @@ -611,23 +643,22 @@ pub fn cleanup_stale_index_entries( callback(&format!("Removing stale index entry: {}", path)); } - fts_remove_document_for_path(&tx, path).map_err(|e| { + if repo::delete_file_by_path(&tx, path).map_err(|e| { format!( - "Failed to remove stale document / FTS entry for {}: {}", + "Failed to remove stale index entry for {}: {}", path, e ) - })?; - tx.execute("DELETE FROM files WHERE path = ?1", params![path]) - .map_err(|e| format!("Failed to delete stale file record {}: {}", path, e))?; - deleted_count += 1; + })? { + deleted_count += 1; + } } tx.commit() .map_err(|e| format!("Failed to commit stale cleanup transaction: {}", e))?; - if deleted_count > 0 && !*stop_flag.lock().unwrap() { + if deleted_count > 0 && !should_abort(stop_flag, suspend_flag) { if let Some(ref callback) = status_callback { - callback("Rebuilding FTS index after stale cleanup..."); + callback("Optimizing FTS index after stale cleanup..."); } fts_finalize_after_text_indexing(&conn)?; } @@ -635,14 +666,19 @@ pub fn cleanup_stale_index_entries( Ok(deleted_count) } -/// Process text indexing for files - writes `documents`; FTS rebuilt in `fts_finalize_after_text_indexing`. +/// Process text indexing for all files with `content_state = pending`. For +/// each file dispatches to the configured extractor [`Registry`], writes the +/// extracted text + properties via the repo helpers, then flips the row's +/// `content_state` to done/failed/na so it won't be retried next run. pub fn process_text_indexing( conn_mutex: &Arc>, stop_flag: &Arc>, + suspend_flag: &Arc, status_callback: Option>, progress_callback: Option>, - config: &Config + config: &Config, ) -> Result<(), String> { + let registry = Registry::default_set(); let max_size = config.processing.maximum_text_file_size; let batch_size = config.processing.batch_size; let batch_limit = batch_size as i64; @@ -654,43 +690,40 @@ pub fn process_text_indexing( let total_files: usize = { let conn = conn_mutex.lock().unwrap(); conn.query_row( - "SELECT COUNT(*) FROM files f - LEFT JOIN documents d ON f.path = d.path - WHERE d.path IS NULL AND f.size <= ?1", + "SELECT COUNT(*) FROM files WHERE content_state = 0 AND size <= ?1", [max_size], |row| row.get(0), ) .map_err(|e| format!("Failed to count pending text files: {}", e))? }; - let mut cursor_path = String::new(); + let mut cursor_id: i64 = 0; let mut global_index: usize = 0; loop { - if *stop_flag.lock().unwrap() { + if should_abort(stop_flag, suspend_flag) { return Ok(()); } - let batch: Vec<(String, String, u64)> = { + let batch: Vec<(i64, String, String, Option)> = { let conn = conn_mutex.lock().unwrap(); let mut stmt = conn .prepare( - "SELECT f.name, f.path, f.size FROM files f - LEFT JOIN documents d ON f.path = d.path - WHERE d.path IS NULL AND f.size <= ?1 - AND (?2 = '' OR f.path > ?2) - ORDER BY f.path - LIMIT ?3", + "SELECT id, name, path, mime FROM files + WHERE content_state = 0 AND size <= ?1 AND id > ?2 + ORDER BY id + LIMIT ?3", ) .map_err(|e| format!("Failed to prepare text indexing query: {}", e))?; let rows = stmt .query_map( - rusqlite::params![max_size, cursor_path.as_str(), batch_limit], + rusqlite::params![max_size, cursor_id, batch_limit], |row| { Ok(( - row.get::<_, String>(0)?, + row.get::<_, i64>(0)?, row.get::<_, String>(1)?, - row.get::<_, u64>(2)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, )) }, ) @@ -703,15 +736,14 @@ pub fn process_text_indexing( break; } - let last_path = batch.last().unwrap().1.clone(); - cursor_path = last_path; + cursor_id = batch.last().unwrap().0; let conn = conn_mutex.lock().unwrap(); let tx = conn .unchecked_transaction() .map_err(|e| format!("Failed to begin transaction: {}", e))?; - for (fname, fpath, _fsize) in batch.iter() { + for (file_id, fname, fpath, fmime) in batch.iter() { if *stop_flag.lock().unwrap() { let _ = tx.commit(); drop(conn); @@ -731,69 +763,43 @@ pub fn process_text_indexing( progress_cb(global_index); } - let path = std::path::Path::new(fpath.as_str()); - let default_ext = OsString::new(); - let file_extension = path - .extension() - .unwrap_or(&default_ext) - .to_ascii_lowercase() - .to_str() - .unwrap_or("") - .to_string(); - let ext_str = file_extension.as_str(); - - let text_result = if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) { - match read_to_string(fpath) { - Ok(file_string) => { - let trimmed_file_string = - safe_truncate_string(&file_string, config.processing.maximum_text_size); - Some(trimmed_file_string) - } - Err(_e) => None, - } - } else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) { - match extract_document_text(&std::ffi::OsString::from(fpath), ext_str) { - Ok(extracted_text) => { - if !extracted_text.trim().is_empty() { - Some(safe_truncate_string( - &extracted_text, - config.processing.maximum_text_size, - )) - } else { - None - } - } - Err(e) => { - eprintln!( - "Warning: Failed to extract text from document {}: {}", - fpath, e - ); - None - } - } - } else { - None + let mime_str = fmime.clone().or_else(|| guess_mime(Path::new(fpath))); + let result = match mime_str.as_deref() { + Some(m) => registry.extract(Path::new(fpath), m), + None => Ok(None), }; - - if let Some(text_content) = text_result { - if let Err(e) = tx.execute( - "INSERT OR REPLACE INTO documents(name, path, text) VALUES (?1, ?2, ?3)", - params![fname, fpath, text_content], - ) { - eprintln!("Warning: Failed to insert document row for {}: {}", fpath, e); + match result { + Ok(Some(mut content)) => { + // Truncate extracted text to configured limit before storage. + if content.text.len() > config.processing.maximum_text_size { + content.text = + safe_truncate_string(&content.text, config.processing.maximum_text_size); + } + let props = content.properties_sorted(); + if let Err(e) = + repo::set_content_done(&tx, *file_id, fname, &content.text, &props) + { + eprintln!("Warning: set_content_done for {}: {}", fpath, e); + } + } + Ok(None) => { + if let Err(e) = repo::set_content_na(&tx, *file_id) { + eprintln!("Warning: set_content_na for {}: {}", fpath, e); + } + } + Err(reason) => { + if let Err(e) = repo::set_content_failed(&tx, *file_id, &reason) { + eprintln!("Warning: set_content_failed for {}: {}", fpath, e); + } } } } - if let Some(ref callback) = status_callback { - callback("Rebuilding FTS index after text addition..."); - } - tx.commit() .map_err(|e| format!("Failed to commit transaction: {}", e))?; } - if total_files > 0 && !*stop_flag.lock().unwrap() { + if total_files > 0 && !should_abort(stop_flag, suspend_flag) { let conn = conn_mutex.lock().unwrap(); fts_finalize_after_text_indexing(&conn)?; } diff --git a/src/indexing.rs b/crates/quicksearch-core/src/indexing.rs similarity index 80% rename from src/indexing.rs rename to crates/quicksearch-core/src/indexing.rs index 4bc81f2..f63fea1 100644 --- a/src/indexing.rs +++ b/crates/quicksearch-core/src/indexing.rs @@ -1,984 +1,988 @@ -use std::sync::{Arc, Mutex, mpsc}; -use std::thread; -use std::time::Instant; -use std::process::Command; -use std::collections::HashSet; -use rusqlite::{Connection, OptionalExtension, params}; -use walkdir::DirEntry; - -use crate::file_handling::{ - classify_dir_entry_for_indexing, - cleanup_stale_index_entries, - count_tree_entries_fast, - indexed_walk_file_entries, - load_existing_files, - path_has_hidden_component, - process_batch_inserts_files_only, - process_batch_updates_files_only, - process_text_indexing, - FileIndexAction, -}; -use crate::config::Config; - -#[derive(Debug, Clone)] -pub struct SearchResultRow { - pub values: Vec, -} - -#[derive(Debug, Clone)] -pub struct SearchResult { - pub columns: Vec, - pub rows: Vec, -} - -#[derive(Debug, Clone)] -pub enum IndexingStatus { - Idle, - CountingFiles { - _entries_scanned: usize, - _indexable_files_counted: usize, - current_file: Option, - start_time: Instant, - }, - RunningFileIndex { - files_processed: usize, - total_files: Option, - current_file: Option, - start_time: Instant, - }, - RunningTextIndex { - files_processed: usize, - current_file: Option, - start_time: Instant, - }, - Stopping, - Error(String), -} - -#[derive(Debug, Clone)] -pub enum IndexingCommand { - Start { - path: String, - db_path: String, - config: Config, - }, - Stop, -} - -#[derive(Debug)] -pub struct IndexingService { - status: Arc>, - command_tx: mpsc::Sender, - db_connection: Arc>>>>, - _handle: thread::JoinHandle<()>, -} - -/// Set process priority for background operation -// fn set_background_priority() { -// #[cfg(windows)] -// { -// use std::os::windows::raw::HANDLE; - -// // Windows implementation -// extern "system" { -// fn GetCurrentProcess() -> HANDLE; -// fn SetPriorityClass(hprocess: HANDLE, dwpriorityclass: u32) -> i32; -// } - -// const BELOW_NORMAL_PRIORITY_CLASS: u32 = 0x00004000; -// unsafe { -// SetPriorityClass(GetCurrentProcess(), BELOW_NORMAL_PRIORITY_CLASS); -// } -// } - -// #[cfg(unix)] -// { -// // Unix implementation -// use std::os::unix::process::CommandExt; -// unsafe { -// libc::nice(10); // Lower priority -// } -// } -// } - -impl IndexingService { - pub fn new() -> Self { - let status = Arc::new(Mutex::new(IndexingStatus::Idle)); - let (command_tx, command_rx) = mpsc::channel(); - let db_connection = Arc::new(Mutex::new(None)); - - let status_clone = status.clone(); - let db_connection_clone = db_connection.clone(); - let handle = thread::spawn(move || { - Self::indexing_thread(status_clone, command_rx, db_connection_clone); - }); - - IndexingService { - status, - command_tx, - db_connection, - _handle: handle, - } - } - - pub fn start_indexing(&self, path: String, db_path: String, config: Config) -> Result<(), String> { - self.command_tx - .send(IndexingCommand::Start { path, db_path, config }) - .map_err(|e| format!("Failed to send start command: {}", e)) - } - - pub fn stop_indexing(&self) -> Result<(), String> { - // First send the stop command - self.command_tx - .send(IndexingCommand::Stop) - .map_err(|e| format!("Failed to send stop command: {}", e))?; - - // Wait for indexing to transition to stopping state - let mut attempts = 0; - while attempts < 50 { // Wait up to 5 seconds - match self.get_status() { - IndexingStatus::Stopping => break, - IndexingStatus::Idle => return Ok(()), // Already stopped - IndexingStatus::Error(_) => return Ok(()), // Consider error state as stopped - _ => { - std::thread::sleep(std::time::Duration::from_millis(100)); - attempts += 1; - } - } - } - - // Flush and close database connection if it exists - if let Ok(mut db_opt) = self.db_connection.lock() { - if let Some(db_conn_arc) = db_opt.take() { - if let Ok(conn) = db_conn_arc.lock() { - // Re-enable journal mode and synchronous writes for proper flushing - let _ = conn.execute_batch( - "PRAGMA journal_mode = DELETE; - PRAGMA synchronous = FULL;" - ); - - // Force a checkpoint to flush any remaining WAL data - let _ = conn.execute("PRAGMA wal_checkpoint(FULL);", ()); - - // Explicitly close the connection by dropping it - drop(conn); - } - } - } - - Ok(()) - } - - pub fn get_status(&self) -> IndexingStatus { - self.status.lock().unwrap().clone() - } - - /// Force graceful shutdown - used for signal handling - pub fn graceful_shutdown(&self) -> Result<(), String> { - self.stop_indexing() - } - - /// Execute a search query against the database - pub fn execute_search(&self, db_path: &str, query: &str) -> Result, String> { - let conn = Connection::open(db_path) - .map_err(|e| { - if e.to_string().contains("corrupt") || e.to_string().contains("malformed") { - format!("DATABASE_CORRUPTED: {}", e) - } else { - format!("Failed to open database: {}", e) - } - })?; - - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);", - (), - ) - .map_err(|e| format!("Failed to prepare duplicate-search index: {}", e))?; - - let mut stmt = conn.prepare(query) - .map_err(|e| { - let error_msg = e.to_string(); - if error_msg.contains("malformed") || error_msg.contains("corrupt") || error_msg.contains("database disk image is malformed") { - format!("DATABASE_CORRUPTED: {}", error_msg) - } else if error_msg.contains("fts5: syntax error") { - format!("Search syntax error: The search term contains characters that cannot be processed. Please try a simpler search term.") - } else { - format!("Failed to prepare query: {}", error_msg) - } - })?; - - let column_count = stmt.column_count(); - let column_names: Vec = (0..column_count) - .map(|i| stmt.column_name(i).unwrap_or("").to_string()) - .collect(); - - let rows = stmt.query_map([], |row| { - let mut values = Vec::new(); - for i in 0..column_count { - let value = match row.get_ref(i)? { - rusqlite::types::ValueRef::Null => "NULL".to_string(), - rusqlite::types::ValueRef::Integer(i) => i.to_string(), - rusqlite::types::ValueRef::Real(f) => f.to_string(), - rusqlite::types::ValueRef::Text(t) => String::from_utf8_lossy(t).to_string(), - rusqlite::types::ValueRef::Blob(b) => format!("BLOB({} bytes)", b.len()), - }; - values.push(value); - } - Ok(SearchResultRow { values }) - }) - .map_err(|e| { - let error_msg = e.to_string(); - if error_msg.contains("malformed") || error_msg.contains("corrupt") || error_msg.contains("database disk image is malformed") { - format!("DATABASE_CORRUPTED: {}", error_msg) - } else if error_msg.contains("fts5: syntax error") { - format!("Search syntax error: The search term contains characters that cannot be processed. Please try a simpler search term.") - } else { - format!("Failed to execute query: {}", error_msg) - } - })?; - - let mut results = Vec::new(); - for row in rows { - match row { - Ok(search_row) => results.push(search_row), - Err(e) => { - let error_msg = e.to_string(); - if error_msg.contains("malformed") || error_msg.contains("corrupt") || error_msg.contains("database disk image is malformed") { - return Err(format!("DATABASE_CORRUPTED: {}", error_msg)); - } else if error_msg.contains("fts5: syntax error") { - return Err(format!("Search syntax error: The search term contains characters that cannot be processed. Please try a simpler search term.")); - } else { - return Err(format!("Error reading row: {}", error_msg)); - } - } - } - } - - Ok(vec![SearchResult { - columns: column_names, - rows: results, - }]) - } - - /// Open file explorer to the directory containing the specified file path - pub fn open_file_explorer(&self, file_path: &str) -> Result<(), String> { - #[cfg(windows)] - { - Command::new("explorer") - .arg("/select,") - .arg(file_path) - .spawn() - .map_err(|e| format!("Failed to open file explorer: {}", e))?; - } - - #[cfg(target_os = "macos")] - { - Command::new("open") - .arg("-R") - .arg(file_path) - .spawn() - .map_err(|e| format!("Failed to open file explorer: {}", e))?; - } - - #[cfg(target_os = "linux")] - { - let path = std::path::Path::new(file_path); - let dir_path = if path.is_file() { - path.parent().unwrap_or(path) - } else { - path - }; - - // Try different file managers - let managers = ["xdg-open", "nautilus", "dolphin", "thunar", "pcmanfm"]; - let mut success = false; - - for manager in &managers { - if let Ok(_) = Command::new(manager) - .arg(dir_path) - .spawn() { - success = true; - break; - } - } - - if !success { - return Err("No suitable file manager found".to_string()); - } - } - - Ok(()) - } - - /// Clean up UNC prefixes from existing database entries - #[allow(dead_code)] - pub fn clean_unc_prefixes(&self, db_path: &str) -> Result<(), String> { - let conn = Connection::open(db_path) - .map_err(|e| format!("Failed to open database: {}", e))?; - - // Clean UNC prefixes from files table - conn.execute( - "UPDATE files SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'", - (), - ).map_err(|e| format!("Failed to update files table: {}", e))?; - - let doc_table: i64 = conn - .query_row( - "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='documents'", - [], - |r| r.get(0), - ) - .unwrap_or(0); - if doc_table > 0 { - conn.execute( - "UPDATE documents SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'", - (), - ) - .map_err(|e| format!("Failed to update documents table: {}", e))?; - } - - Ok(()) - } - - /// Check if the database is corrupted or malformed - #[allow(dead_code)] - pub fn check_database_health(&self, db_path: &str) -> Result { - match Connection::open(db_path) { - Ok(conn) => { - // Try to run integrity check - match conn.prepare("PRAGMA integrity_check") { - Ok(mut stmt) => { - match stmt.query_row([], |row| { - let result: String = row.get(0)?; - Ok(result == "ok") - }) { - Ok(is_ok) => Ok(is_ok), - Err(_) => Ok(false) - } - }, - Err(_) => Ok(false) - } - }, - Err(_) => Ok(false) - } - } - - /// Check if configuration changes require index recreation - pub fn check_config_validation(&self, db_path: &str, config: &Config, indexing_path: &str) -> Result>, String> { - let conn = Connection::open(db_path) - .map_err(|e| format!("Failed to open database: {}", e))?; - - // Create config validation table if it doesn't exist - conn.execute( - "CREATE TABLE IF NOT EXISTS config_validation ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL);", - (), - ).map_err(|e| format!("Failed to create config_validation table: {}", e))?; - - Self::validate_config(&conn, config, indexing_path) - } - - /// Stop indexing and delete the database file for a clean rebuild - pub fn delete_index_for_rebuild(&self, db_path: &str) -> Result<(), String> { - // Stop any running indexing first - self.stop_indexing() - .map_err(|e| format!("Failed to stop indexing: {}", e))?; - - // Wait for indexing to actually stop - let mut attempts = 0; - while attempts < 50 { // Wait up to 5 seconds - match self.get_status() { - IndexingStatus::Idle => break, - IndexingStatus::Stopping - | IndexingStatus::CountingFiles { .. } - | IndexingStatus::RunningFileIndex { .. } - | IndexingStatus::RunningTextIndex { .. } => { - std::thread::sleep(std::time::Duration::from_millis(100)); - attempts += 1; - } - IndexingStatus::Error(_) => break, // Consider error state as stopped - } - } - - // Delete the database file - if std::path::Path::new(db_path).exists() { - std::fs::remove_file(db_path) - .map_err(|e| format!("Failed to delete database file: {}", e))?; - } - - Ok(()) - } - - fn indexing_thread( - status: Arc>, - command_rx: mpsc::Receiver, - db_connection: Arc>>>> - ) { - let stop_flag = Arc::new(Mutex::new(false)); - let mut indexing_handle: Option> = None; - - while let Ok(command) = command_rx.recv() { - match command { - IndexingCommand::Start { path, db_path, config } => { - if matches!( - *status.lock().unwrap(), - IndexingStatus::CountingFiles { .. } - | IndexingStatus::RunningFileIndex { .. } - | IndexingStatus::RunningTextIndex { .. } - ) { - continue; // Already running - } - - // Join any previous indexing thread - if let Some(handle) = indexing_handle.take() { - let _ = handle.join(); - } - - *stop_flag.lock().unwrap() = false; - *status.lock().unwrap() = if config.processing.precount_files_for_progress { - IndexingStatus::CountingFiles { - _entries_scanned: 0, - _indexable_files_counted: 0, - current_file: Some("Preparing database...".to_string()), - start_time: Instant::now(), - } - } else { - IndexingStatus::RunningFileIndex { - files_processed: 0, - total_files: None, - current_file: None, - start_time: Instant::now(), - } - }; - - // Run indexing in a separate thread - let status_clone = status.clone(); - let stop_flag_clone = stop_flag.clone(); - let path_owned = path.clone(); - let db_path_owned = db_path.clone(); - let config_owned = config.clone(); - - let db_connection_clone = db_connection.clone(); - indexing_handle = Some(thread::spawn(move || { - if let Err(e) = Self::run_indexing(&status_clone, &path_owned, &db_path_owned, &stop_flag_clone, &config_owned, &db_connection_clone) { - *status_clone.lock().unwrap() = IndexingStatus::Error(e); - } else { - // Only set to Idle if we weren't stopped - if !*stop_flag_clone.lock().unwrap() { - *status_clone.lock().unwrap() = IndexingStatus::Idle; - } - } - - // Clear the database connection when indexing completes - if let Ok(mut db_opt) = db_connection_clone.lock() { - *db_opt = None; - } - })); - } - IndexingCommand::Stop => { - if matches!( - *status.lock().unwrap(), - IndexingStatus::CountingFiles { .. } - | IndexingStatus::RunningFileIndex { .. } - | IndexingStatus::RunningTextIndex { .. } - ) { - *status.lock().unwrap() = IndexingStatus::Stopping; - *stop_flag.lock().unwrap() = true; - } - } - } - } - - // Clean up any remaining indexing thread - if let Some(handle) = indexing_handle { - let _ = handle.join(); - } - } - - fn file_index_status_callback( - status: &Arc>, - ) -> Box { - let st = status.clone(); - Box::new(move |file_status: &str| { - if let Ok(mut status_guard) = st.lock() { - if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard - { - *current_file = Some(file_status.to_string()); - } - } - }) - } - - fn run_indexing( - status: &Arc>, - path: &str, - db_path: &str, - stop_flag: &Arc>, - config: &Config, - db_connection: &Arc>>>>, - ) -> Result<(), String> { - // Set up database - let conn = Connection::open(db_path) - .map_err(|e| format!("Failed to open database: {}", e))?; - - conn.execute_batch( // Default SQLITE page size is 4kB, and our memory cache is in units of page count - "PRAGMA journal_mode = OFF; - PRAGMA synchronous = 0; - PRAGMA cache_size = 10000; - PRAGMA temp_store = MEMORY;", - ) - .map_err(|e| format!("Failed to set PRAGMA: {}", e))?; - - conn.execute( - "CREATE TABLE IF NOT EXISTS files ( - name TEXT, - path TEXT, - size INTEGER, - moddate INTEGER, - hash BLOB);", - (), - ) - .map_err(|e| format!("Failed to create files table: {}", e))?; - - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);", - (), - ) - .map_err(|e| format!("Failed to create files hash index: {}", e))?; - - conn.execute( - "CREATE TABLE IF NOT EXISTS documents ( - id INTEGER PRIMARY KEY, - name TEXT, - path TEXT NOT NULL UNIQUE, - text TEXT NOT NULL)", - (), - ) - .map_err(|e| format!("Failed to create documents table: {}", e))?; - - let fts_external = Self::searchabletext_is_external_content(&conn)?; - if !fts_external { - conn.execute("DROP TABLE IF EXISTS searchabletext", ()) - .map_err(|e| format!("Failed to drop legacy searchabletext: {}", e))?; - conn.execute("DROP TABLE IF EXISTS searchabletext_doc", ()) - .map_err(|e| format!("Failed to drop legacy searchabletext_doc: {}", e))?; - } - - let create_fts_sql = format!( - "CREATE VIRTUAL TABLE IF NOT EXISTS searchabletext USING fts5(name, text, content='documents', content_rowid='id', tokenize='{}');", - config.processing.tokenize - ); - conn.execute(&create_fts_sql, ()) - .map_err(|e| format!("Failed to create searchabletext table: {}", e))?; - - if !fts_external { - conn.execute("INSERT INTO searchabletext(searchabletext) VALUES('rebuild')", ()) - .map_err(|e| format!("Failed to rebuild searchabletext: {}", e))?; - } - - conn.execute( - "CREATE TABLE IF NOT EXISTS config_validation ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL);", - (), - ) - .map_err(|e| format!("Failed to create config_validation table: {}", e))?; - - // Update configuration (for new installations or when no validation issues) - Self::update_config(&conn, config, path)?; - - // Load existing files from database for incremental indexing - let existing_files = { - let conn_ref = &conn; - load_existing_files(conn_ref) - .map_err(|e| format!("Failed to load existing files: {}", e))? - }; - - let conn_mutex = Arc::new(Mutex::new(conn)); - - // Store the database connection for proper cleanup on stop - if let Ok(mut db_opt) = db_connection.lock() { - *db_opt = Some(conn_mutex.clone()); - } - - let progress_display_total: Option = - if config.processing.precount_files_for_progress { - if *stop_flag.lock().unwrap() { - if let Ok(mut status_guard) = status.lock() { - *status_guard = IndexingStatus::Idle; - } - return Ok(()); - } - if let Ok(mut g) = status.lock() { - if let IndexingStatus::CountingFiles { ref mut current_file, .. } = *g { - *current_file = Some("Counting paths (shell)...".to_string()); - } - } - let n = count_tree_entries_fast(path).map_err(|e| format!("Precount: {}", e))?; - if *stop_flag.lock().unwrap() { - if let Ok(mut status_guard) = status.lock() { - *status_guard = IndexingStatus::Idle; - } - return Ok(()); - } - if let Ok(mut status_guard) = status.lock() { - *status_guard = IndexingStatus::RunningFileIndex { - files_processed: 0, - total_files: Some(n), - current_file: None, - start_time: Instant::now(), - }; - } - Some(n) - } else { - None - }; - - let batch_size = config.processing.batch_size; - let mut pending_updates: Vec<(DirEntry, usize)> = Vec::new(); - let mut pending_inserts: Vec<(DirEntry, usize)> = Vec::new(); - let mut seen_existing_paths: HashSet = HashSet::new(); - let mut visit: usize = 0; - let mut had_incremental_work = false; - let flush_updates = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> { - if buf.is_empty() { - return Ok(()); - } - process_batch_updates_files_only( - &conn_mutex, - buf.as_slice(), - stop_flag, - Some(Self::file_index_status_callback(status)), - None, - config, - progress_display_total, - )?; - buf.clear(); - Ok(()) - }; - - let flush_inserts = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> { - if buf.is_empty() { - return Ok(()); - } - process_batch_inserts_files_only( - &conn_mutex, - buf.as_slice(), - stop_flag, - Some(Self::file_index_status_callback(status)), - None, - config, - progress_display_total, - )?; - buf.clear(); - Ok(()) - }; - - for entry in indexed_walk_file_entries(path, config.processing.follow_symlinks) { - if *stop_flag.lock().unwrap() { - flush_updates(&mut pending_updates)?; - flush_inserts(&mut pending_inserts)?; - if let Ok(mut status_guard) = status.lock() { - *status_guard = IndexingStatus::Idle; - } - return Ok(()); - } - - visit += 1; - if let Ok(mut g) = status.lock() { - if let IndexingStatus::RunningFileIndex { - ref mut files_processed, - .. - } = *g - { - *files_processed = visit; - } - } - - if !config.processing.include_hidden && path_has_hidden_component(entry.path()) { - continue; - } - - let action = classify_dir_entry_for_indexing(&entry, &existing_files); - let Some(action) = action else { - continue; - }; - let current_path = entry - .path() - .canonicalize() - .ok() - .map(|fp| { - let path_str = fp.to_string_lossy().to_string(); - if path_str.starts_with("\\\\?\\") { - path_str[4..].to_string() - } else { - path_str - } - }); - - match action { - FileIndexAction::Skip => { - if let Some(path) = current_path { - seen_existing_paths.insert(path); - } - } - FileIndexAction::Update => { - if let Some(path) = current_path { - seen_existing_paths.insert(path); - } - had_incremental_work = true; - pending_updates.push((entry, visit)); - if pending_updates.len() >= batch_size { - flush_updates(&mut pending_updates)?; - } - } - FileIndexAction::Insert => { - had_incremental_work = true; - pending_inserts.push((entry, visit)); - if pending_inserts.len() >= batch_size { - flush_inserts(&mut pending_inserts)?; - } - } - } - } - - flush_updates(&mut pending_updates)?; - flush_inserts(&mut pending_inserts)?; - - let stale_paths: Vec = existing_files - .keys() - .filter(|p| !seen_existing_paths.contains(*p)) - .cloned() - .collect(); - let stale_deleted = cleanup_stale_index_entries( - &conn_mutex, - stale_paths.as_slice(), - stop_flag, - Some(Self::file_index_status_callback(status)), - )?; - if stale_deleted > 0 { - had_incremental_work = true; - } - - if !had_incremental_work { - if let Ok(mut status_guard) = status.lock() { - if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard - { - *current_file = Some("File index is up to date".to_string()); - } - } - } - - // Check for stop signal before starting text indexing - if *stop_flag.lock().unwrap() { - if let Ok(mut status_guard) = status.lock() { - *status_guard = IndexingStatus::Idle; - } - return Ok(()); - } - - // Phase 2: Text indexing - if let Ok(mut status_guard) = status.lock() { - *status_guard = IndexingStatus::RunningTextIndex { - files_processed: 0, - current_file: Some("Starting text indexing...".to_string()), - start_time: Instant::now(), - }; - } - - // Create status callback for text indexing - let status_clone_5 = status.clone(); - let text_status_callback = Box::new(move |file_status: &str| { - if let Ok(mut status_guard) = status_clone_5.lock() { - if let IndexingStatus::RunningTextIndex { ref mut current_file, .. } = *status_guard { - *current_file = Some(file_status.to_string()); - } - } - }); - - // Create progress callback for text indexing - let status_clone_6 = status.clone(); - let text_progress_callback = Box::new(move |current_index: usize| { - if let Ok(mut status_guard) = status_clone_6.lock() { - if let IndexingStatus::RunningTextIndex { ref mut files_processed, .. } = *status_guard { - *files_processed = current_index; - } - } - }); - - // Process text indexing - if let Err(e) = process_text_indexing(&conn_mutex, &stop_flag, Some(text_status_callback), Some(text_progress_callback), config) { - return Err(format!("Failed to process text indexing: {}", e)); - } - - // Mark text indexing as complete - if let Ok(mut status_guard) = status.lock() { - if let IndexingStatus::RunningTextIndex { ref mut current_file, .. } = *status_guard { - *current_file = Some("Text indexing complete".to_string()); - } - } - - Ok(()) - } - - fn searchabletext_is_external_content(conn: &Connection) -> Result { - let sql: Option = conn - .query_row( - "SELECT sql FROM sqlite_master WHERE name='searchabletext'", - [], - |r| r.get(0), - ) - .optional() - .map_err(|e| format!("sqlite_master searchabletext: {}", e))?; - Ok(sql - .as_deref() - .map(|s| { - s.contains("content='documents'") - || s.contains("content=\"documents\"") - || s.contains("content=documents") - }) - .unwrap_or(false)) - } - - /// Validates configuration against stored values and returns validation results. - /// Critical configuration changes that require index recreation: - /// - hash_length: affects file hash computation, invalidates existing file metadata - /// - indexing_path: changes the scope of indexed files - /// - tokenize: changes FTS5 tokenization, invalidates text search index - fn validate_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result>, String> { - // Critical configuration values that require index recreation - let hash_length = config.processing.hash_length.to_string(); - let tokenize = config.processing.tokenize.clone(); - let include_hidden = config.processing.include_hidden.to_string(); - let normalized_path = { - let path = std::path::Path::new(indexing_path) - .canonicalize() - .unwrap_or_else(|_| std::path::PathBuf::from(indexing_path)) - .to_string_lossy() - .to_string(); - // Remove Windows UNC prefix \\?\ - if path.starts_with("\\\\?\\") { - path[4..].to_string() - } else { - path - } - }; - - // Check stored configuration values - let mut stored_hash_length: Option = None; - let mut stored_indexing_path: Option = None; - let mut stored_tokenize: Option = None; - let mut stored_include_hidden: Option = None; - - if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path', 'tokenize', 'include_hidden')") { - if let Ok(rows) = stmt.query_map([], |row| { - let key: String = row.get(0)?; - let value: String = row.get(1)?; - Ok((key, value)) - }) { - for row in rows.flatten() { - match row.0.as_str() { - "hash_length" => stored_hash_length = Some(row.1), - "indexing_path" => stored_indexing_path = Some(row.1), - "tokenize" => stored_tokenize = Some(row.1), - "include_hidden" => stored_include_hidden = Some(row.1), - _ => {} - } - } - } - } - - // Check if configuration is invalid - let hash_length_changed = stored_hash_length.as_ref().map_or(false, |stored| stored != &hash_length); - let indexing_path_changed = stored_indexing_path.as_ref().map_or(false, |stored| stored != &normalized_path); - let tokenize_changed = stored_tokenize.as_ref().map_or(false, |stored| stored != &tokenize); - let include_hidden_changed = stored_include_hidden.as_ref().map_or(false, |stored| stored != &include_hidden); - - if hash_length_changed || indexing_path_changed || tokenize_changed || include_hidden_changed { - let mut changes = Vec::new(); - if hash_length_changed { - changes.push(format!("hash_length: {} -> {}", - stored_hash_length.unwrap_or_else(|| "unknown".to_string()), hash_length)); - } - if indexing_path_changed { - changes.push(format!("indexing_path: {} -> {}", - stored_indexing_path.unwrap_or_else(|| "unknown".to_string()), normalized_path)); - } - if tokenize_changed { - changes.push(format!("tokenize: {} -> {}", - stored_tokenize.unwrap_or_else(|| "unknown".to_string()), tokenize)); - } - if include_hidden_changed { - changes.push(format!( - "include_hidden: {} -> {}", - stored_include_hidden.unwrap_or_else(|| "unknown".to_string()), - include_hidden - )); - } - - return Ok(Some(changes)); - } - - // No configuration changes detected - Ok(None) - } - - - /// Updates stored configuration values without clearing the index - fn update_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result<(), String> { - let hash_length = config.processing.hash_length.to_string(); - let tokenize = config.processing.tokenize.clone(); - let include_hidden = config.processing.include_hidden.to_string(); - let normalized_path = { - let path = std::path::Path::new(indexing_path) - .canonicalize() - .unwrap_or_else(|_| std::path::PathBuf::from(indexing_path)) - .to_string_lossy() - .to_string(); - // Remove Windows UNC prefix \\?\ - if path.starts_with("\\\\?\\") { - path[4..].to_string() - } else { - path - } - }; - - // Update stored configuration values - conn.execute( - "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('hash_length', ?1)", - params![hash_length], - ).map_err(|e| format!("Failed to store hash_length config: {}", e))?; - - conn.execute( - "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('indexing_path', ?1)", - params![normalized_path], - ).map_err(|e| format!("Failed to store indexing_path config: {}", e))?; - - conn.execute( - "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('tokenize', ?1)", - params![tokenize], - ).map_err(|e| format!("Failed to store tokenize config: {}", e))?; - - conn.execute( - "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('include_hidden', ?1)", - params![include_hidden], - ).map_err(|e| format!("Failed to store include_hidden config: {}", e))?; - - Ok(()) - } -} - -impl Drop for IndexingService { - fn drop(&mut self) { - // Ensure graceful shutdown when the service is dropped - let _ = self.stop_indexing(); - } -} - -impl Default for IndexingService { - fn default() -> Self { - Self::new() - } -} - +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread; +use std::time::{Duration, Instant}; +use std::process::Command; +use std::collections::HashSet; +use rusqlite::{Connection, params}; +use walkdir::DirEntry; + +use crate::file_handling::{ + classify_dir_entry_for_indexing, + cleanup_stale_index_entries, + count_tree_entries_fast, + indexed_walk_file_entries, + load_existing_files, + path_has_hidden_component, + process_batch_inserts_files_only, + process_batch_updates_files_only, + process_text_indexing, + FileIndexAction, +}; +use crate::config::Config; +use crate::db; + +#[derive(Debug, Clone)] +pub struct SearchResultRow { + pub values: Vec, +} + +#[derive(Debug, Clone)] +pub struct SearchResult { + pub columns: Vec, + pub rows: Vec, +} + +#[derive(Debug, Clone)] +pub enum IndexingStatus { + Idle, + CountingFiles { + _entries_scanned: usize, + _indexable_files_counted: usize, + current_file: Option, + start_time: Instant, + }, + RunningFileIndex { + files_processed: usize, + total_files: Option, + current_file: Option, + start_time: Instant, + }, + RunningTextIndex { + files_processed: usize, + current_file: Option, + start_time: Instant, + }, + Stopping, + Error(String), +} + +#[derive(Debug, Clone)] +pub enum IndexingCommand { + Start { + /// One or more directory roots to index. Order determines walk order; + /// duplicates are silently dropped at run time. + paths: Vec, + db_path: String, + config: Config, + }, + Stop, +} + +#[derive(Debug)] +pub struct IndexingService { + status: Arc>, + command_tx: mpsc::Sender, + db_connection: Arc>>>>, + suspend_flag: Arc, + _handle: thread::JoinHandle<()>, +} + +/// Polling interval for `should_abort` while suspended. +const SUSPEND_POLL_MS: u64 = 100; + +/// Combined stop/suspend check used by worker loops. Returns `true` iff the +/// caller should abort the operation. While the suspend flag is set and stop +/// is not, this parks the thread by sleeping in short increments so a later +/// `resume()` unblocks it. Cheap to call in tight loops. +pub(crate) fn should_abort( + stop: &Arc>, + suspend: &Arc, +) -> bool { + loop { + if *stop.lock().unwrap() { + return true; + } + if !suspend.load(Ordering::Relaxed) { + return false; + } + thread::sleep(Duration::from_millis(SUSPEND_POLL_MS)); + } +} + +/// Set process priority for background operation +// fn set_background_priority() { +// #[cfg(windows)] +// { +// use std::os::windows::raw::HANDLE; + +// // Windows implementation +// extern "system" { +// fn GetCurrentProcess() -> HANDLE; +// fn SetPriorityClass(hprocess: HANDLE, dwpriorityclass: u32) -> i32; +// } + +// const BELOW_NORMAL_PRIORITY_CLASS: u32 = 0x00004000; +// unsafe { +// SetPriorityClass(GetCurrentProcess(), BELOW_NORMAL_PRIORITY_CLASS); +// } +// } + +// #[cfg(unix)] +// { +// // Unix implementation +// use std::os::unix::process::CommandExt; +// unsafe { +// libc::nice(10); // Lower priority +// } +// } +// } + +impl IndexingService { + pub fn new() -> Self { + let status = Arc::new(Mutex::new(IndexingStatus::Idle)); + let (command_tx, command_rx) = mpsc::channel(); + let db_connection = Arc::new(Mutex::new(None)); + let suspend_flag = Arc::new(AtomicBool::new(false)); + + let status_clone = status.clone(); + let db_connection_clone = db_connection.clone(); + let suspend_clone = suspend_flag.clone(); + let handle = thread::spawn(move || { + Self::indexing_thread(status_clone, command_rx, db_connection_clone, suspend_clone); + }); + + IndexingService { + status, + command_tx, + db_connection, + suspend_flag, + _handle: handle, + } + } + + /// Pause the indexer. All worker loops that call [`should_abort`] will + /// block until [`resume`](Self::resume) is called. No-op if already + /// suspended. Does not stop the worker — stop_indexing is still the way + /// to abort. + pub fn suspend(&self) { + self.suspend_flag.store(true, Ordering::Relaxed); + } + + /// Resume indexing after [`suspend`](Self::suspend). No-op if not + /// suspended. + pub fn resume(&self) { + self.suspend_flag.store(false, Ordering::Relaxed); + } + + pub fn is_suspended(&self) -> bool { + self.suspend_flag.load(Ordering::Relaxed) + } + + /// Start indexing one or more roots. Paths are walked in order; duplicate + /// or nested roots are de-duplicated by the indexer. At least one path is + /// required. + pub fn start_indexing( + &self, + paths: Vec, + db_path: String, + config: Config, + ) -> Result<(), String> { + if paths.is_empty() { + return Err("start_indexing requires at least one path".into()); + } + self.command_tx + .send(IndexingCommand::Start { paths, db_path, config }) + .map_err(|e| format!("Failed to send start command: {}", e)) + } + + pub fn stop_indexing(&self) -> Result<(), String> { + // First send the stop command + self.command_tx + .send(IndexingCommand::Stop) + .map_err(|e| format!("Failed to send stop command: {}", e))?; + + // Wait for indexing to transition to stopping state + let mut attempts = 0; + while attempts < 50 { // Wait up to 5 seconds + match self.get_status() { + IndexingStatus::Stopping => break, + IndexingStatus::Idle => return Ok(()), // Already stopped + IndexingStatus::Error(_) => return Ok(()), // Consider error state as stopped + _ => { + std::thread::sleep(std::time::Duration::from_millis(100)); + attempts += 1; + } + } + } + + // Flush and close database connection if it exists + if let Ok(mut db_opt) = self.db_connection.lock() { + if let Some(db_conn_arc) = db_opt.take() { + if let Ok(conn) = db_conn_arc.lock() { + // Re-enable journal mode and synchronous writes for proper flushing + let _ = conn.execute_batch( + "PRAGMA journal_mode = DELETE; + PRAGMA synchronous = FULL;" + ); + + // Force a checkpoint to flush any remaining WAL data + let _ = conn.execute("PRAGMA wal_checkpoint(FULL);", ()); + + // Explicitly close the connection by dropping it + drop(conn); + } + } + } + + Ok(()) + } + + pub fn get_status(&self) -> IndexingStatus { + self.status.lock().unwrap().clone() + } + + /// Force graceful shutdown - used for signal handling + pub fn graceful_shutdown(&self) -> Result<(), String> { + self.stop_indexing() + } + + /// Execute a search query against the database + pub fn execute_search(&self, db_path: &str, query: &str) -> Result, String> { + let conn = db::open_and_migrate(db_path, "trigram") + .map_err(|e| { + if e.contains("corrupt") || e.contains("malformed") { + format!("DATABASE_CORRUPTED: {}", e) + } else { + e + } + })?; + + let mut stmt = conn.prepare(query) + .map_err(|e| { + let error_msg = e.to_string(); + if error_msg.contains("malformed") || error_msg.contains("corrupt") || error_msg.contains("database disk image is malformed") { + format!("DATABASE_CORRUPTED: {}", error_msg) + } else if error_msg.contains("fts5: syntax error") { + format!("Search syntax error: The search term contains characters that cannot be processed. Please try a simpler search term.") + } else { + format!("Failed to prepare query: {}", error_msg) + } + })?; + + let column_count = stmt.column_count(); + let column_names: Vec = (0..column_count) + .map(|i| stmt.column_name(i).unwrap_or("").to_string()) + .collect(); + + let rows = stmt.query_map([], |row| { + let mut values = Vec::new(); + for i in 0..column_count { + let value = match row.get_ref(i)? { + rusqlite::types::ValueRef::Null => "NULL".to_string(), + rusqlite::types::ValueRef::Integer(i) => i.to_string(), + rusqlite::types::ValueRef::Real(f) => f.to_string(), + rusqlite::types::ValueRef::Text(t) => String::from_utf8_lossy(t).to_string(), + rusqlite::types::ValueRef::Blob(b) => format!("BLOB({} bytes)", b.len()), + }; + values.push(value); + } + Ok(SearchResultRow { values }) + }) + .map_err(|e| { + let error_msg = e.to_string(); + if error_msg.contains("malformed") || error_msg.contains("corrupt") || error_msg.contains("database disk image is malformed") { + format!("DATABASE_CORRUPTED: {}", error_msg) + } else if error_msg.contains("fts5: syntax error") { + format!("Search syntax error: The search term contains characters that cannot be processed. Please try a simpler search term.") + } else { + format!("Failed to execute query: {}", error_msg) + } + })?; + + let mut results = Vec::new(); + for row in rows { + match row { + Ok(search_row) => results.push(search_row), + Err(e) => { + let error_msg = e.to_string(); + if error_msg.contains("malformed") || error_msg.contains("corrupt") || error_msg.contains("database disk image is malformed") { + return Err(format!("DATABASE_CORRUPTED: {}", error_msg)); + } else if error_msg.contains("fts5: syntax error") { + return Err(format!("Search syntax error: The search term contains characters that cannot be processed. Please try a simpler search term.")); + } else { + return Err(format!("Error reading row: {}", error_msg)); + } + } + } + } + + Ok(vec![SearchResult { + columns: column_names, + rows: results, + }]) + } + + /// Open file explorer to the directory containing the specified file path + pub fn open_file_explorer(&self, file_path: &str) -> Result<(), String> { + #[cfg(windows)] + { + Command::new("explorer") + .arg("/select,") + .arg(file_path) + .spawn() + .map_err(|e| format!("Failed to open file explorer: {}", e))?; + } + + #[cfg(target_os = "macos")] + { + Command::new("open") + .arg("-R") + .arg(file_path) + .spawn() + .map_err(|e| format!("Failed to open file explorer: {}", e))?; + } + + #[cfg(target_os = "linux")] + { + let path = std::path::Path::new(file_path); + let dir_path = if path.is_file() { + path.parent().unwrap_or(path) + } else { + path + }; + + // Try different file managers + let managers = ["xdg-open", "nautilus", "dolphin", "thunar", "pcmanfm"]; + let mut success = false; + + for manager in &managers { + if let Ok(_) = Command::new(manager) + .arg(dir_path) + .spawn() { + success = true; + break; + } + } + + if !success { + return Err("No suitable file manager found".to_string()); + } + } + + Ok(()) + } + + /// Clean up Windows UNC prefixes (\\?\) from existing `files.path` + /// entries. Relic of pre-A layouts where paths were stored with the + /// prefix; current code strips them at insert time, so this is a + /// one-shot cleanup users can invoke manually if needed. + #[allow(dead_code)] + pub fn clean_unc_prefixes(&self, db_path: &str) -> Result<(), String> { + let conn = Connection::open(db_path) + .map_err(|e| format!("Failed to open database: {}", e))?; + conn.execute( + "UPDATE files SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'", + (), + ) + .map_err(|e| format!("Failed to update files table: {}", e))?; + Ok(()) + } + + /// Check if the database is corrupted or malformed + #[allow(dead_code)] + pub fn check_database_health(&self, db_path: &str) -> Result { + match Connection::open(db_path) { + Ok(conn) => { + // Try to run integrity check + match conn.prepare("PRAGMA integrity_check") { + Ok(mut stmt) => { + match stmt.query_row([], |row| { + let result: String = row.get(0)?; + Ok(result == "ok") + }) { + Ok(is_ok) => Ok(is_ok), + Err(_) => Ok(false) + } + }, + Err(_) => Ok(false) + } + }, + Err(_) => Ok(false) + } + } + + /// Check if configuration changes require index recreation + pub fn check_config_validation(&self, db_path: &str, config: &Config, indexing_path: &str) -> Result>, String> { + let conn = db::open_and_migrate(db_path, &config.processing.tokenize)?; + Self::validate_config(&conn, config, indexing_path) + } + + /// Stop indexing and delete the database file for a clean rebuild + pub fn delete_index_for_rebuild(&self, db_path: &str) -> Result<(), String> { + // Stop any running indexing first + self.stop_indexing() + .map_err(|e| format!("Failed to stop indexing: {}", e))?; + + // Wait for indexing to actually stop + let mut attempts = 0; + while attempts < 50 { // Wait up to 5 seconds + match self.get_status() { + IndexingStatus::Idle => break, + IndexingStatus::Stopping + | IndexingStatus::CountingFiles { .. } + | IndexingStatus::RunningFileIndex { .. } + | IndexingStatus::RunningTextIndex { .. } => { + std::thread::sleep(std::time::Duration::from_millis(100)); + attempts += 1; + } + IndexingStatus::Error(_) => break, // Consider error state as stopped + } + } + + // Delete the database file + if std::path::Path::new(db_path).exists() { + std::fs::remove_file(db_path) + .map_err(|e| format!("Failed to delete database file: {}", e))?; + } + + Ok(()) + } + + fn indexing_thread( + status: Arc>, + command_rx: mpsc::Receiver, + db_connection: Arc>>>>, + suspend_flag: Arc, + ) { + let stop_flag = Arc::new(Mutex::new(false)); + let mut indexing_handle: Option> = None; + + while let Ok(command) = command_rx.recv() { + match command { + IndexingCommand::Start { paths, db_path, config } => { + if matches!( + *status.lock().unwrap(), + IndexingStatus::CountingFiles { .. } + | IndexingStatus::RunningFileIndex { .. } + | IndexingStatus::RunningTextIndex { .. } + ) { + continue; // Already running + } + + // Join any previous indexing thread + if let Some(handle) = indexing_handle.take() { + let _ = handle.join(); + } + + *stop_flag.lock().unwrap() = false; + *status.lock().unwrap() = if config.processing.precount_files_for_progress { + IndexingStatus::CountingFiles { + _entries_scanned: 0, + _indexable_files_counted: 0, + current_file: Some("Preparing database...".to_string()), + start_time: Instant::now(), + } + } else { + IndexingStatus::RunningFileIndex { + files_processed: 0, + total_files: None, + current_file: None, + start_time: Instant::now(), + } + }; + + // Run indexing in a separate thread + let status_clone = status.clone(); + let stop_flag_clone = stop_flag.clone(); + let paths_owned = paths.clone(); + let db_path_owned = db_path.clone(); + let config_owned = config.clone(); + + let db_connection_clone = db_connection.clone(); + let suspend_clone = suspend_flag.clone(); + indexing_handle = Some(thread::spawn(move || { + if let Err(e) = Self::run_indexing(&status_clone, &paths_owned, &db_path_owned, &stop_flag_clone, &suspend_clone, &config_owned, &db_connection_clone) { + *status_clone.lock().unwrap() = IndexingStatus::Error(e); + } else { + // Only set to Idle if we weren't stopped + if !*stop_flag_clone.lock().unwrap() { + *status_clone.lock().unwrap() = IndexingStatus::Idle; + } + } + + // Clear the database connection when indexing completes + if let Ok(mut db_opt) = db_connection_clone.lock() { + *db_opt = None; + } + })); + } + IndexingCommand::Stop => { + if matches!( + *status.lock().unwrap(), + IndexingStatus::CountingFiles { .. } + | IndexingStatus::RunningFileIndex { .. } + | IndexingStatus::RunningTextIndex { .. } + ) { + *status.lock().unwrap() = IndexingStatus::Stopping; + *stop_flag.lock().unwrap() = true; + } + } + } + } + + // Clean up any remaining indexing thread + if let Some(handle) = indexing_handle { + let _ = handle.join(); + } + } + + fn file_index_status_callback( + status: &Arc>, + ) -> Box { + let st = status.clone(); + Box::new(move |file_status: &str| { + if let Ok(mut status_guard) = st.lock() { + if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard + { + *current_file = Some(file_status.to_string()); + } + } + }) + } + + fn run_indexing( + status: &Arc>, + paths: &[String], + db_path: &str, + stop_flag: &Arc>, + suspend_flag: &Arc, + config: &Config, + db_connection: &Arc>>>>, + ) -> Result<(), String> { + if paths.is_empty() { + return Err("run_indexing: no paths provided".into()); + } + + // De-duplicate while preserving order. Roots are canonicalized first + // so `/home/jeremy` and `/home/jeremy/` (or a symlink to either) + // collapse to one walk. Pure nested-root deduplication (skip a root + // that is a prefix of an already-walked root) is handled by the + // per-file `seen_paths` set below. + let mut seen_roots = HashSet::new(); + let roots: Vec = paths + .iter() + .map(|p| { + std::path::Path::new(p) + .canonicalize() + .ok() + .map(|c| { + let s = c.to_string_lossy().to_string(); + if s.starts_with("\\\\?\\") { + s[4..].to_string() + } else { + s + } + }) + .unwrap_or_else(|| p.clone()) + }) + .filter(|p| seen_roots.insert(p.clone())) + .collect(); + + // Open and migrate the database to the current schema version. + let conn = db::open_and_migrate(db_path, &config.processing.tokenize)?; + + // Update configuration (for new installations or when no validation issues). + // `indexing_path` in the validation table stores the joined list so + // adding/removing a root triggers the same rebuild prompt as changing + // the legacy single path did. + Self::update_config(&conn, config, &roots.join("\n"))?; + + // Load existing files from database for incremental indexing + let existing_files = { + let conn_ref = &conn; + load_existing_files(conn_ref) + .map_err(|e| format!("Failed to load existing files: {}", e))? + }; + + let conn_mutex = Arc::new(Mutex::new(conn)); + + // Store the database connection for proper cleanup on stop + if let Ok(mut db_opt) = db_connection.lock() { + *db_opt = Some(conn_mutex.clone()); + } + + let progress_display_total: Option = + if config.processing.precount_files_for_progress { + if *stop_flag.lock().unwrap() { + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::Idle; + } + return Ok(()); + } + if let Ok(mut g) = status.lock() { + if let IndexingStatus::CountingFiles { ref mut current_file, .. } = *g { + *current_file = Some("Counting paths (shell)...".to_string()); + } + } + let mut n: usize = 0; + for root in &roots { + if *stop_flag.lock().unwrap() { + break; + } + // A failing precount on one root (e.g. permission denied) + // shouldn't abort all indexing; just skip its contribution. + match count_tree_entries_fast(root) { + Ok(c) => n += c, + Err(e) => eprintln!("Warning: precount for {}: {}", root, e), + } + } + if *stop_flag.lock().unwrap() { + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::Idle; + } + return Ok(()); + } + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::RunningFileIndex { + files_processed: 0, + total_files: Some(n), + current_file: None, + start_time: Instant::now(), + }; + } + Some(n) + } else { + None + }; + + let batch_size = config.processing.batch_size; + let mut pending_updates: Vec<(DirEntry, usize)> = Vec::new(); + let mut pending_inserts: Vec<(DirEntry, usize)> = Vec::new(); + // Every path seen during this walk, regardless of outcome. Dedupes + // duplicate visits that can happen when two roots overlap (e.g. + // `/home` and `/home/user`) or when canonicalization collapses + // symlinks. Also feeds the stale-entry cleanup at the end of the walk. + let mut seen_paths: HashSet = HashSet::new(); + let mut visit: usize = 0; + let mut had_incremental_work = false; + let flush_updates = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> { + if buf.is_empty() { + return Ok(()); + } + process_batch_updates_files_only( + &conn_mutex, + buf.as_slice(), + stop_flag, + Some(Self::file_index_status_callback(status)), + None, + config, + progress_display_total, + )?; + buf.clear(); + Ok(()) + }; + + let flush_inserts = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> { + if buf.is_empty() { + return Ok(()); + } + process_batch_inserts_files_only( + &conn_mutex, + buf.as_slice(), + stop_flag, + Some(Self::file_index_status_callback(status)), + None, + config, + progress_display_total, + )?; + buf.clear(); + Ok(()) + }; + + let root_iter = roots + .iter() + .flat_map(|r| indexed_walk_file_entries(r, config.processing.follow_symlinks)); + for entry in root_iter { + if should_abort(stop_flag, suspend_flag) { + flush_updates(&mut pending_updates)?; + flush_inserts(&mut pending_inserts)?; + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::Idle; + } + return Ok(()); + } + + visit += 1; + if let Ok(mut g) = status.lock() { + if let IndexingStatus::RunningFileIndex { + ref mut files_processed, + .. + } = *g + { + *files_processed = visit; + } + } + + if !config.processing.include_hidden && path_has_hidden_component(entry.path()) { + continue; + } + + let action = classify_dir_entry_for_indexing(&entry, &existing_files); + let Some(action) = action else { + continue; + }; + let current_path = entry + .path() + .canonicalize() + .ok() + .map(|fp| { + let path_str = fp.to_string_lossy().to_string(); + if path_str.starts_with("\\\\?\\") { + path_str[4..].to_string() + } else { + path_str + } + }); + + // Skip this entry if its canonical path was already queued in + // this run. Prevents UNIQUE(path) violations when overlapping + // roots or symlinks lead the walker to the same file twice. + let path_for_dedup = match ¤t_path { + Some(p) => p.clone(), + None => continue, + }; + if !seen_paths.insert(path_for_dedup) { + continue; + } + + match action { + FileIndexAction::Skip => {} + FileIndexAction::Update => { + had_incremental_work = true; + pending_updates.push((entry, visit)); + if pending_updates.len() >= batch_size { + flush_updates(&mut pending_updates)?; + } + } + FileIndexAction::Insert => { + had_incremental_work = true; + pending_inserts.push((entry, visit)); + if pending_inserts.len() >= batch_size { + flush_inserts(&mut pending_inserts)?; + } + } + } + } + + flush_updates(&mut pending_updates)?; + flush_inserts(&mut pending_inserts)?; + + let stale_paths: Vec = existing_files + .keys() + .filter(|p| !seen_paths.contains(*p)) + .cloned() + .collect(); + let stale_deleted = cleanup_stale_index_entries( + &conn_mutex, + stale_paths.as_slice(), + stop_flag, + suspend_flag, + Some(Self::file_index_status_callback(status)), + )?; + if stale_deleted > 0 { + had_incremental_work = true; + } + + if !had_incremental_work { + if let Ok(mut status_guard) = status.lock() { + if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard + { + *current_file = Some("File index is up to date".to_string()); + } + } + } + + // Check for stop signal (and park if suspended) before starting text indexing. + if should_abort(stop_flag, suspend_flag) { + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::Idle; + } + return Ok(()); + } + + // Phase 2: Text indexing + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::RunningTextIndex { + files_processed: 0, + current_file: Some("Starting text indexing...".to_string()), + start_time: Instant::now(), + }; + } + + // Create status callback for text indexing + let status_clone_5 = status.clone(); + let text_status_callback = Box::new(move |file_status: &str| { + if let Ok(mut status_guard) = status_clone_5.lock() { + if let IndexingStatus::RunningTextIndex { ref mut current_file, .. } = *status_guard { + *current_file = Some(file_status.to_string()); + } + } + }); + + // Create progress callback for text indexing + let status_clone_6 = status.clone(); + let text_progress_callback = Box::new(move |current_index: usize| { + if let Ok(mut status_guard) = status_clone_6.lock() { + if let IndexingStatus::RunningTextIndex { ref mut files_processed, .. } = *status_guard { + *files_processed = current_index; + } + } + }); + + // Process text indexing + if let Err(e) = process_text_indexing(&conn_mutex, &stop_flag, suspend_flag, Some(text_status_callback), Some(text_progress_callback), config) { + return Err(format!("Failed to process text indexing: {}", e)); + } + + // Mark text indexing as complete + if let Ok(mut status_guard) = status.lock() { + if let IndexingStatus::RunningTextIndex { ref mut current_file, .. } = *status_guard { + *current_file = Some("Text indexing complete".to_string()); + } + } + + Ok(()) + } + + /// Validates configuration against stored values and returns validation results. + /// Critical configuration changes that require index recreation: + /// - hash_length: affects file hash computation, invalidates existing file metadata + /// - indexing_path: changes the scope of indexed files + /// - tokenize: changes FTS5 tokenization, invalidates text search index + fn validate_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result>, String> { + // Critical configuration values that require index recreation + let hash_length = config.processing.hash_length.to_string(); + let tokenize = config.processing.tokenize.clone(); + let include_hidden = config.processing.include_hidden.to_string(); + let normalized_path = { + let path = std::path::Path::new(indexing_path) + .canonicalize() + .unwrap_or_else(|_| std::path::PathBuf::from(indexing_path)) + .to_string_lossy() + .to_string(); + // Remove Windows UNC prefix \\?\ + if path.starts_with("\\\\?\\") { + path[4..].to_string() + } else { + path + } + }; + + // Check stored configuration values + let mut stored_hash_length: Option = None; + let mut stored_indexing_path: Option = None; + let mut stored_tokenize: Option = None; + let mut stored_include_hidden: Option = None; + + if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path', 'tokenize', 'include_hidden')") { + if let Ok(rows) = stmt.query_map([], |row| { + let key: String = row.get(0)?; + let value: String = row.get(1)?; + Ok((key, value)) + }) { + for row in rows.flatten() { + match row.0.as_str() { + "hash_length" => stored_hash_length = Some(row.1), + "indexing_path" => stored_indexing_path = Some(row.1), + "tokenize" => stored_tokenize = Some(row.1), + "include_hidden" => stored_include_hidden = Some(row.1), + _ => {} + } + } + } + } + + // Check if configuration is invalid + let hash_length_changed = stored_hash_length.as_ref().map_or(false, |stored| stored != &hash_length); + let indexing_path_changed = stored_indexing_path.as_ref().map_or(false, |stored| stored != &normalized_path); + let tokenize_changed = stored_tokenize.as_ref().map_or(false, |stored| stored != &tokenize); + let include_hidden_changed = stored_include_hidden.as_ref().map_or(false, |stored| stored != &include_hidden); + + if hash_length_changed || indexing_path_changed || tokenize_changed || include_hidden_changed { + let mut changes = Vec::new(); + if hash_length_changed { + changes.push(format!("hash_length: {} -> {}", + stored_hash_length.unwrap_or_else(|| "unknown".to_string()), hash_length)); + } + if indexing_path_changed { + changes.push(format!("indexing_path: {} -> {}", + stored_indexing_path.unwrap_or_else(|| "unknown".to_string()), normalized_path)); + } + if tokenize_changed { + changes.push(format!("tokenize: {} -> {}", + stored_tokenize.unwrap_or_else(|| "unknown".to_string()), tokenize)); + } + if include_hidden_changed { + changes.push(format!( + "include_hidden: {} -> {}", + stored_include_hidden.unwrap_or_else(|| "unknown".to_string()), + include_hidden + )); + } + + return Ok(Some(changes)); + } + + // No configuration changes detected + Ok(None) + } + + + /// Updates stored configuration values without clearing the index + fn update_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result<(), String> { + let hash_length = config.processing.hash_length.to_string(); + let tokenize = config.processing.tokenize.clone(); + let include_hidden = config.processing.include_hidden.to_string(); + let normalized_path = { + let path = std::path::Path::new(indexing_path) + .canonicalize() + .unwrap_or_else(|_| std::path::PathBuf::from(indexing_path)) + .to_string_lossy() + .to_string(); + // Remove Windows UNC prefix \\?\ + if path.starts_with("\\\\?\\") { + path[4..].to_string() + } else { + path + } + }; + + // Update stored configuration values + conn.execute( + "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('hash_length', ?1)", + params![hash_length], + ).map_err(|e| format!("Failed to store hash_length config: {}", e))?; + + conn.execute( + "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('indexing_path', ?1)", + params![normalized_path], + ).map_err(|e| format!("Failed to store indexing_path config: {}", e))?; + + conn.execute( + "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('tokenize', ?1)", + params![tokenize], + ).map_err(|e| format!("Failed to store tokenize config: {}", e))?; + + conn.execute( + "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('include_hidden', ?1)", + params![include_hidden], + ).map_err(|e| format!("Failed to store include_hidden config: {}", e))?; + + Ok(()) + } +} + +impl Drop for IndexingService { + fn drop(&mut self) { + // Ensure graceful shutdown when the service is dropped + let _ = self.stop_indexing(); + } +} + +impl Default for IndexingService { + fn default() -> Self { + Self::new() + } +} + diff --git a/crates/quicksearch-core/src/lib.rs b/crates/quicksearch-core/src/lib.rs new file mode 100644 index 0000000..b1af394 --- /dev/null +++ b/crates/quicksearch-core/src/lib.rs @@ -0,0 +1,12 @@ +pub mod cli; +pub mod config; +pub mod db; +pub mod document_extraction; +pub mod extract; +pub mod file_handling; +pub mod indexing; +pub mod mime; +pub mod query; +pub mod search_sql; +pub mod shutdown; +pub mod watcher; diff --git a/crates/quicksearch-core/src/mime.rs b/crates/quicksearch-core/src/mime.rs new file mode 100644 index 0000000..bd080d8 --- /dev/null +++ b/crates/quicksearch-core/src/mime.rs @@ -0,0 +1,238 @@ +//! MIME type guessing and `FileType` bitmask classification. +//! +//! Two stages: +//! 1. [`guess_mime`] infers a MIME type from a path — extension first via +//! `mime_guess`, falling back to magic-byte sniffing via `infer` for files +//! whose extension is missing or ambiguous. +//! 2. [`mime_to_type`] maps a MIME string to a [`FileType`] bitmask so a single +//! file can belong to multiple categories (e.g. a `.docx` is Document|Text). + +use std::path::Path; + +/// Bit-flag category for a file. Unlike the MIME string this is designed for +/// cheap bitmask queries like `type & FileType::AUDIO != 0`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileType(pub u32); + +impl FileType { + pub const EMPTY: FileType = FileType(0); + pub const AUDIO: FileType = FileType(1 << 0); + pub const IMAGE: FileType = FileType(1 << 1); + pub const VIDEO: FileType = FileType(1 << 2); + pub const DOCUMENT: FileType = FileType(1 << 3); + pub const TEXT: FileType = FileType(1 << 4); + pub const ARCHIVE: FileType = FileType(1 << 5); + pub const PRESENTATION: FileType = FileType(1 << 6); + pub const SPREADSHEET: FileType = FileType(1 << 7); + pub const FOLDER: FileType = FileType(1 << 8); + + pub const fn bits(self) -> u32 { + self.0 + } + + pub const fn contains(self, other: FileType) -> bool { + (self.0 & other.0) == other.0 + } + + /// Parse a single Baloo-style category name (`Audio`, `Image`, ...). + /// Case-insensitive. Returns `EMPTY` for unknown names. + pub fn from_name(s: &str) -> FileType { + match s.to_ascii_lowercase().as_str() { + "audio" => FileType::AUDIO, + "image" => FileType::IMAGE, + "video" => FileType::VIDEO, + "document" => FileType::DOCUMENT, + "text" => FileType::TEXT, + "archive" => FileType::ARCHIVE, + "presentation" => FileType::PRESENTATION, + "spreadsheet" => FileType::SPREADSHEET, + "folder" => FileType::FOLDER, + _ => FileType::EMPTY, + } + } +} + +impl std::ops::BitOr for FileType { + type Output = FileType; + fn bitor(self, rhs: FileType) -> FileType { + FileType(self.0 | rhs.0) + } +} + +impl std::ops::BitOrAssign for FileType { + fn bitor_assign(&mut self, rhs: FileType) { + self.0 |= rhs.0; + } +} + +/// Guess a MIME type for a path on disk. +/// +/// Tries extension-based lookup via `mime_guess` first (cheap, no I/O). If +/// that returns nothing or a generic `application/octet-stream`, and the file +/// is readable, falls back to `infer` magic-byte detection (reads a small +/// prefix of the file). +/// +/// Returns `None` if no guess can be made. +pub fn guess_mime(path: &Path) -> Option { + if let Some(g) = mime_guess::from_path(path).first() { + let s = g.essence_str(); + if !s.is_empty() && s != "application/octet-stream" { + return Some(s.to_string()); + } + } + // Magic-byte fallback. `infer::get_from_path` handles errors by returning None. + if let Ok(Some(t)) = infer::get_from_path(path) { + return Some(t.mime_type().to_string()); + } + None +} + +/// Map a MIME string to a [`FileType`] bitmask. Ported from Baloo's +/// `basicindexingjob.cpp:typesForMimeType`. +pub fn mime_to_type(mime: &str) -> FileType { + let lower = mime.to_ascii_lowercase(); + let (top, sub) = match lower.split_once('/') { + Some(pair) => pair, + None => return FileType::EMPTY, + }; + let mut t = FileType::EMPTY; + match top { + "audio" => t |= FileType::AUDIO, + "image" => t |= FileType::IMAGE, + "video" => t |= FileType::VIDEO, + "text" => { + t |= FileType::TEXT; + // HTML counts as a document too in Baloo. + if sub == "html" || sub == "xhtml+xml" { + t |= FileType::DOCUMENT; + } + } + _ => {} + } + // Subtype-based classification for the `application/*` grab bag. + match sub { + // Office formats + "msword" + | "vnd.openxmlformats-officedocument.wordprocessingml.document" + | "vnd.oasis.opendocument.text" + | "rtf" + | "pdf" + | "epub+zip" + | "x-mobipocket-ebook" => { + t |= FileType::DOCUMENT; + } + "vnd.ms-excel" + | "vnd.openxmlformats-officedocument.spreadsheetml.sheet" + | "vnd.oasis.opendocument.spreadsheet" => { + t |= FileType::DOCUMENT | FileType::SPREADSHEET; + } + "vnd.ms-powerpoint" + | "vnd.openxmlformats-officedocument.presentationml.presentation" + | "vnd.oasis.opendocument.presentation" => { + t |= FileType::DOCUMENT | FileType::PRESENTATION; + } + // Archives + "zip" + | "x-tar" + | "x-7z-compressed" + | "x-rar" + | "x-rar-compressed" + | "gzip" + | "x-bzip" + | "x-bzip2" + | "x-xz" + | "vnd.debian.binary-package" + | "x-rpm" => { + t |= FileType::ARCHIVE; + } + // application/xml is structured text + "xml" | "json" | "javascript" | "x-shellscript" | "x-python" => { + t |= FileType::TEXT; + } + _ => {} + } + t +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn audio_mime() { + assert!(mime_to_type("audio/mpeg").contains(FileType::AUDIO)); + assert!(mime_to_type("audio/flac").contains(FileType::AUDIO)); + } + + #[test] + fn image_mime() { + assert!(mime_to_type("image/jpeg").contains(FileType::IMAGE)); + assert!(mime_to_type("image/png").contains(FileType::IMAGE)); + } + + #[test] + fn docx_is_document_and_office() { + let t = mime_to_type( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ); + assert!(t.contains(FileType::DOCUMENT)); + } + + #[test] + fn xlsx_is_spreadsheet_and_document() { + let t = mime_to_type( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ); + assert!(t.contains(FileType::DOCUMENT)); + assert!(t.contains(FileType::SPREADSHEET)); + } + + #[test] + fn pptx_is_presentation() { + let t = mime_to_type( + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ); + assert!(t.contains(FileType::PRESENTATION)); + } + + #[test] + fn html_is_text_and_document() { + let t = mime_to_type("text/html"); + assert!(t.contains(FileType::TEXT)); + assert!(t.contains(FileType::DOCUMENT)); + } + + #[test] + fn plain_text() { + let t = mime_to_type("text/plain"); + assert!(t.contains(FileType::TEXT)); + assert!(!t.contains(FileType::DOCUMENT)); + } + + #[test] + fn zip_is_archive() { + assert!(mime_to_type("application/zip").contains(FileType::ARCHIVE)); + } + + #[test] + fn unknown_mime_is_empty() { + assert_eq!(mime_to_type("weird/blob"), FileType::EMPTY); + } + + #[test] + fn from_name_round_trip() { + for n in ["Audio", "Image", "Video", "Document", "Text", "Archive", + "Spreadsheet", "Presentation", "Folder"] { + assert_ne!(FileType::from_name(n), FileType::EMPTY, "{}", n); + } + assert_eq!(FileType::from_name("Weird"), FileType::EMPTY); + } + + #[test] + fn guess_mime_by_extension() { + use std::path::PathBuf; + assert_eq!(guess_mime(&PathBuf::from("a.txt")).as_deref(), Some("text/plain")); + assert_eq!(guess_mime(&PathBuf::from("a.png")).as_deref(), Some("image/png")); + assert_eq!(guess_mime(&PathBuf::from("a.mp3")).as_deref(), Some("audio/mpeg")); + } +} diff --git a/crates/quicksearch-core/src/query/ast.rs b/crates/quicksearch-core/src/query/ast.rs new file mode 100644 index 0000000..b3fe14d --- /dev/null +++ b/crates/quicksearch-core/src/query/ast.rs @@ -0,0 +1,58 @@ +//! Query AST shared between parser and translator. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Op { + /// `key:value` — substring / FTS MATCH semantics. + Contains, + /// `key=value` — exact match. + Eq, + /// `keyvalue` + Gt, + /// `key>=value` + Ge, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Term { + /// An unquoted word or quoted phrase. Feeds the FTS MATCH expression. + Literal(String), + /// A structured filter such as `type:Audio` or `modified:>=2024-01-01`. + Property { key: String, op: Op, value: String }, + And(Vec), + Or(Vec), +} + +impl Term { + /// Combine two terms with AND, flattening to avoid a deeply nested tree. + pub fn and(a: Term, b: Term) -> Term { + match (a, b) { + (Term::And(mut xs), Term::And(ys)) => { + xs.extend(ys); + Term::And(xs) + } + (Term::And(mut xs), other) | (other, Term::And(mut xs)) => { + xs.push(other); + Term::And(xs) + } + (a, b) => Term::And(vec![a, b]), + } + } + + pub fn or(a: Term, b: Term) -> Term { + match (a, b) { + (Term::Or(mut xs), Term::Or(ys)) => { + xs.extend(ys); + Term::Or(xs) + } + (Term::Or(mut xs), other) | (other, Term::Or(mut xs)) => { + xs.push(other); + Term::Or(xs) + } + (a, b) => Term::Or(vec![a, b]), + } + } +} diff --git a/crates/quicksearch-core/src/query/lexer.rs b/crates/quicksearch-core/src/query/lexer.rs new file mode 100644 index 0000000..20beb3c --- /dev/null +++ b/crates/quicksearch-core/src/query/lexer.rs @@ -0,0 +1,214 @@ +//! Tokenizer for the query grammar. + +use super::ast::Op; + +#[derive(Debug, Clone, PartialEq)] +pub enum Token { + /// Unquoted word. `AND`/`OR` are intercepted before emitting a `Word`. + Word(String), + Quoted(String), + LParen, + RParen, + /// Binary property operator (`:`, `=`, `>`, `>=`, `<`, `<=`). + /// + /// `:` is [`Op::Contains`] by default; the parser re-interprets it when + /// followed immediately by a comparator (e.g. `modified:>=2024-01-01`). + Op(Op), + And, + Or, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LexError { + pub message: String, + pub offset: usize, +} + +impl std::fmt::Display for LexError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "query lex error at {}: {}", self.offset, self.message) + } +} + +impl std::error::Error for LexError {} + +pub fn tokenize(input: &str) -> Result, LexError> { + let bytes = input.as_bytes(); + let mut i = 0usize; + let mut out = Vec::new(); + + while i < bytes.len() { + let b = bytes[i]; + if b.is_ascii_whitespace() { + i += 1; + continue; + } + match b { + b'(' => { + out.push(Token::LParen); + i += 1; + } + b')' => { + out.push(Token::RParen); + i += 1; + } + b':' => { + out.push(Token::Op(Op::Contains)); + i += 1; + } + b'=' => { + out.push(Token::Op(Op::Eq)); + i += 1; + } + b'<' => { + if bytes.get(i + 1) == Some(&b'=') { + out.push(Token::Op(Op::Le)); + i += 2; + } else { + out.push(Token::Op(Op::Lt)); + i += 1; + } + } + b'>' => { + if bytes.get(i + 1) == Some(&b'=') { + out.push(Token::Op(Op::Ge)); + i += 2; + } else { + out.push(Token::Op(Op::Gt)); + i += 1; + } + } + b'"' => { + // Double-quoted phrase. Supports doubled-quote escape `""`. + let mut j = i + 1; + let mut buf = String::new(); + while j < bytes.len() { + if bytes[j] == b'"' { + if bytes.get(j + 1) == Some(&b'"') { + buf.push('"'); + j += 2; + continue; + } + break; + } + buf.push(bytes[j] as char); + j += 1; + } + if j >= bytes.len() { + return Err(LexError { + message: "unterminated quoted phrase".into(), + offset: i, + }); + } + out.push(Token::Quoted(buf)); + i = j + 1; + } + _ => { + // Unquoted word; continues until whitespace, paren, or operator. + let start = i; + while i < bytes.len() { + let c = bytes[i]; + if c.is_ascii_whitespace() + || matches!(c, b'(' | b')' | b':' | b'=' | b'<' | b'>' | b'"') + { + break; + } + i += 1; + } + let word = std::str::from_utf8(&bytes[start..i]) + .map_err(|e| LexError { + message: format!("invalid UTF-8 in word: {}", e), + offset: start, + })? + .to_string(); + match word.as_str() { + "AND" => out.push(Token::And), + "OR" => out.push(Token::Or), + _ => out.push(Token::Word(word)), + } + } + } + } + + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn simple_words() { + let t = tokenize("foo bar").unwrap(); + assert_eq!( + t, + vec![Token::Word("foo".into()), Token::Word("bar".into())] + ); + } + + #[test] + fn and_or_keywords() { + let t = tokenize("a AND b OR c").unwrap(); + assert_eq!( + t, + vec![ + Token::Word("a".into()), + Token::And, + Token::Word("b".into()), + Token::Or, + Token::Word("c".into()), + ] + ); + } + + #[test] + fn quoted_phrase() { + let t = tokenize(r#""hello world""#).unwrap(); + assert_eq!(t, vec![Token::Quoted("hello world".into())]); + } + + #[test] + fn doubled_quote_is_escape() { + let t = tokenize(r#""a""b""#).unwrap(); + assert_eq!(t, vec![Token::Quoted(r#"a"b"#.into())]); + } + + #[test] + fn property_operators() { + let t = tokenize("type:Audio modified>=2024-01-01 size<100").unwrap(); + assert_eq!( + t, + vec![ + Token::Word("type".into()), + Token::Op(Op::Contains), + Token::Word("Audio".into()), + Token::Word("modified".into()), + Token::Op(Op::Ge), + Token::Word("2024-01-01".into()), + Token::Word("size".into()), + Token::Op(Op::Lt), + Token::Word("100".into()), + ] + ); + } + + #[test] + fn parens() { + let t = tokenize("(a OR b)").unwrap(); + assert_eq!( + t, + vec![ + Token::LParen, + Token::Word("a".into()), + Token::Or, + Token::Word("b".into()), + Token::RParen, + ] + ); + } + + #[test] + fn unterminated_quote_is_error() { + assert!(tokenize(r#""oops"#).is_err()); + } +} diff --git a/crates/quicksearch-core/src/query/mod.rs b/crates/quicksearch-core/src/query/mod.rs new file mode 100644 index 0000000..4a6d3bd --- /dev/null +++ b/crates/quicksearch-core/src/query/mod.rs @@ -0,0 +1,28 @@ +//! Structured query parser and SQL translator. +//! +//! Input syntax (a deliberate subset of KDE Baloo's query language — just +//! enough to be useful standalone; full Baloo grammar lives in the Set B +//! compat layer): +//! +//! - plain words: `foo bar` (implicit AND) +//! - quoted phrases: `"hello world"` +//! - boolean operators: `AND`, `OR` (case-sensitive) +//! - grouping: `(a OR b)` +//! - structured filters: +//! - `type:Audio` / `type:Image` / `type:Document` / `type:Text` / `type:Video` +//! / `type:Archive` / `type:Spreadsheet` / `type:Presentation` / `type:Folder` +//! - `modified:>=2024-01-01`, `modified:<2023-12-01`, `modified:=2024-05-20` +//! (also accepts `modified>=2024-01-01` without the colon) +//! - `path:/some/dir` — matches files with that directory as their parent +//! or any ancestor. +//! +//! The entry point is [`parse_and_build`], which accepts a query string and +//! returns a ready-to-execute [`SqlQuery`]. + +pub mod ast; +pub mod lexer; +pub mod parser; +pub mod translator; + +pub use ast::{Op, Term}; +pub use translator::{parse_and_build, SqlQuery}; diff --git a/crates/quicksearch-core/src/query/parser.rs b/crates/quicksearch-core/src/query/parser.rs new file mode 100644 index 0000000..77bbadb --- /dev/null +++ b/crates/quicksearch-core/src/query/parser.rs @@ -0,0 +1,284 @@ +//! Recursive-descent parser: token stream → [`Term`] tree. +//! +//! Grammar (implicit AND between adjacent terms): +//! ```text +//! expr := or_expr +//! or_expr := and_expr ("OR" and_expr)* +//! and_expr := atom ("AND"? atom)* +//! atom := "(" expr ")" | property | literal +//! property := ident ":"|"="|"<"|"<="|">"|">=" value +//! literal := WORD | QUOTED +//! ``` + +use super::ast::{Op, Term}; +use super::lexer::{tokenize, LexError, Token}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ParseError { + Lex(LexError), + Unexpected { at: usize, reason: String }, + Empty, +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParseError::Lex(e) => write!(f, "{}", e), + ParseError::Unexpected { at, reason } => { + write!(f, "parse error at token {}: {}", at, reason) + } + ParseError::Empty => write!(f, "empty query"), + } + } +} + +impl std::error::Error for ParseError {} + +pub fn parse(input: &str) -> Result { + let tokens = tokenize(input).map_err(ParseError::Lex)?; + let trimmed: Vec<_> = tokens.into_iter().collect(); + if trimmed.is_empty() { + return Err(ParseError::Empty); + } + let mut p = Parser { tokens: trimmed, pos: 0 }; + let term = p.parse_or()?; + if p.pos < p.tokens.len() { + return Err(ParseError::Unexpected { + at: p.pos, + reason: format!("trailing token {:?}", p.tokens[p.pos]), + }); + } + Ok(term) +} + +struct Parser { + tokens: Vec, + pos: usize, +} + +impl Parser { + fn peek(&self) -> Option<&Token> { + self.tokens.get(self.pos) + } + + fn bump(&mut self) -> Option { + let t = self.tokens.get(self.pos).cloned(); + if t.is_some() { + self.pos += 1; + } + t + } + + fn parse_or(&mut self) -> Result { + let mut left = self.parse_and()?; + while matches!(self.peek(), Some(Token::Or)) { + self.bump(); + let right = self.parse_and()?; + left = Term::or(left, right); + } + Ok(left) + } + + fn parse_and(&mut self) -> Result { + let mut left = self.parse_atom()?; + loop { + match self.peek() { + Some(Token::And) => { + self.bump(); + let right = self.parse_atom()?; + left = Term::and(left, right); + } + // Implicit AND: adjacent atoms without an operator. + Some(Token::Word(_)) + | Some(Token::Quoted(_)) + | Some(Token::LParen) => { + let right = self.parse_atom()?; + left = Term::and(left, right); + } + _ => break, + } + } + Ok(left) + } + + fn parse_atom(&mut self) -> Result { + match self.bump() { + Some(Token::LParen) => { + let inner = self.parse_or()?; + match self.bump() { + Some(Token::RParen) => Ok(inner), + other => Err(ParseError::Unexpected { + at: self.pos, + reason: format!("expected ')' got {:?}", other), + }), + } + } + Some(Token::Quoted(s)) => Ok(Term::Literal(s)), + Some(Token::Word(w)) => { + // If followed by an operator, this word is a property key. + if let Some(Token::Op(op)) = self.peek().cloned() { + self.bump(); + // `key:>=value` — an Op followed by another Op becomes the + // effective comparator; the original colon is "separator". + let effective_op = if op == Op::Contains { + if let Some(Token::Op(inner_op)) = self.peek().cloned() { + self.bump(); + inner_op + } else { + Op::Contains + } + } else { + op + }; + let value = match self.bump() { + Some(Token::Word(v)) => v, + Some(Token::Quoted(v)) => v, + other => { + return Err(ParseError::Unexpected { + at: self.pos, + reason: format!("expected property value got {:?}", other), + }) + } + }; + Ok(Term::Property { + key: w, + op: effective_op, + value, + }) + } else { + Ok(Term::Literal(w)) + } + } + other => Err(ParseError::Unexpected { + at: self.pos, + reason: format!("expected term, got {:?}", other), + }), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lit(s: &str) -> Term { + Term::Literal(s.into()) + } + + fn prop(k: &str, op: Op, v: &str) -> Term { + Term::Property { + key: k.into(), + op, + value: v.into(), + } + } + + #[test] + fn single_word() { + assert_eq!(parse("foo").unwrap(), lit("foo")); + } + + #[test] + fn quoted_phrase() { + assert_eq!(parse(r#""hello world""#).unwrap(), lit("hello world")); + } + + #[test] + fn implicit_and() { + assert_eq!( + parse("foo bar").unwrap(), + Term::And(vec![lit("foo"), lit("bar")]) + ); + } + + #[test] + fn explicit_and() { + assert_eq!( + parse("foo AND bar").unwrap(), + Term::And(vec![lit("foo"), lit("bar")]) + ); + } + + #[test] + fn or_has_lower_precedence_than_and() { + assert_eq!( + parse("a b OR c").unwrap(), + Term::Or(vec![ + Term::And(vec![lit("a"), lit("b")]), + lit("c") + ]) + ); + } + + #[test] + fn parens_override_precedence() { + assert_eq!( + parse("a (b OR c)").unwrap(), + Term::And(vec![lit("a"), Term::Or(vec![lit("b"), lit("c")])]) + ); + } + + #[test] + fn property_contains() { + assert_eq!( + parse("type:Audio").unwrap(), + prop("type", Op::Contains, "Audio") + ); + } + + #[test] + fn property_comparator_via_colon() { + assert_eq!( + parse("modified:>=2024-01-01").unwrap(), + prop("modified", Op::Ge, "2024-01-01") + ); + } + + #[test] + fn property_comparator_bare() { + assert_eq!( + parse("modified>=2024-01-01").unwrap(), + prop("modified", Op::Ge, "2024-01-01") + ); + } + + #[test] + fn mixed_filter_and_literal() { + assert_eq!( + parse("type:Audio beatles").unwrap(), + Term::And(vec![ + prop("type", Op::Contains, "Audio"), + lit("beatles") + ]) + ); + } + + #[test] + fn nested_or() { + assert_eq!( + parse("(a OR b) AND (c OR d)").unwrap(), + Term::And(vec![ + Term::Or(vec![lit("a"), lit("b")]), + Term::Or(vec![lit("c"), lit("d")]) + ]) + ); + } + + #[test] + fn empty_query_is_error() { + assert!(matches!(parse(" "), Err(ParseError::Empty))); + } + + #[test] + fn trailing_token_is_error() { + assert!(parse("a )").is_err()); + } + + #[test] + fn property_value_may_be_quoted() { + assert_eq!( + parse(r#"path:"/tmp with space""#).unwrap(), + prop("path", Op::Contains, "/tmp with space") + ); + } +} diff --git a/crates/quicksearch-core/src/query/translator.rs b/crates/quicksearch-core/src/query/translator.rs new file mode 100644 index 0000000..958ba34 --- /dev/null +++ b/crates/quicksearch-core/src/query/translator.rs @@ -0,0 +1,654 @@ +//! AST → executable SQL. + +use super::ast::{Op, Term}; +use super::parser::{parse, ParseError}; +use crate::mime::FileType; + +/// A prepared SQL statement plus its positional parameters. Parameters are +/// rusqlite `Value` for convenience at the call site. Every `?N` in `sql` +/// corresponds to `params[N-1]`. +#[derive(Debug, Clone)] +pub struct SqlQuery { + pub sql: String, + pub params: Vec, +} + +/// Sort strategy for the result set. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Sort { + /// Newest first by file modification time. + ByMtimeDesc, + /// FTS5 relevance rank. Only sensible when the query has FTS terms. + ByRank, + /// No ORDER BY clause. + None, +} + +impl Default for Sort { + fn default() -> Self { + Sort::ByMtimeDesc + } +} + +#[derive(Debug, Clone)] +pub enum TranslateError { + Parse(ParseError), + UnknownProperty(String), + BadDate(String), + UnsupportedOp { + key: String, + op: Op, + }, +} + +impl std::fmt::Display for TranslateError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TranslateError::Parse(e) => write!(f, "{}", e), + TranslateError::UnknownProperty(k) => write!(f, "unknown property '{}'", k), + TranslateError::BadDate(s) => write!(f, "bad date '{}'", s), + TranslateError::UnsupportedOp { key, op } => { + write!(f, "operator {:?} is not supported for property '{}'", op, key) + } + } + } +} + +impl std::error::Error for TranslateError {} + +/// One-shot: parse the input string and build the SQL. Limit/offset are +/// applied at the SQL level; `None` means "no limit". +pub fn parse_and_build( + input: &str, + limit: Option, + offset: u32, + sort: Sort, +) -> Result { + let ast = parse(input).map_err(TranslateError::Parse)?; + build(&ast, limit, offset, sort) +} + +/// Translate an already-parsed AST into SQL. +pub fn build( + ast: &Term, + limit: Option, + offset: u32, + sort: Sort, +) -> Result { + let mut b = Builder::default(); + let where_sql = b.translate(ast)?; + + // The FTS branch joins a CTE of matching rowids; the structured branch + // stands alone. A query may have both — in that case we intersect at the + // file.id level. + let needs_fts_join = !b.fts_parts.is_empty(); + + // Structured placeholders were numbered assuming no FTS param. When an + // FTS MATCH is prepended as ?1, bump every `?N` in the generated WHERE + // clause by 1 so the positional bindings line up. + let where_sql = if needs_fts_join { + shift_placeholders(&where_sql) + } else { + where_sql + }; + + let mut sql = String::new(); + if needs_fts_join { + sql.push_str( + "WITH fts_hits AS (SELECT rowid FROM searchabletext WHERE searchabletext MATCH ?1) ", + ); + b.all_params + .insert(0, rusqlite::types::Value::Text(fts_expr(&b.fts_parts))); + } + sql.push_str("SELECT f.id, f.name, f.path FROM files f "); + if needs_fts_join { + sql.push_str("JOIN fts_hits ON fts_hits.rowid = f.id "); + } + if !where_sql.is_empty() { + sql.push_str("WHERE "); + sql.push_str(&where_sql); + sql.push(' '); + } + match sort { + Sort::ByMtimeDesc => sql.push_str("ORDER BY f.mtime DESC "), + Sort::ByRank if needs_fts_join => sql.push_str("ORDER BY rank "), + Sort::ByRank | Sort::None => {} + } + if let Some(n) = limit { + sql.push_str(&format!("LIMIT {} ", n)); + } + if offset > 0 { + sql.push_str(&format!("OFFSET {}", offset)); + } + + Ok(SqlQuery { + sql: sql.trim_end().to_string(), + params: b.all_params, + }) +} + +#[derive(Default)] +struct Builder { + /// Tokens that will be joined into the single FTS MATCH expression. + fts_parts: Vec, + /// Everything else (structured filters) as a WHERE clause. + all_params: Vec, +} + +#[derive(Debug, Clone)] +enum FtsFragment { + /// A plain word or phrase to be included as an AND'd token. + And(String), + /// Grouped alternation, already rendered with internal ORs. + OrGroup(Vec), +} + +fn fts_expr(parts: &[FtsFragment]) -> String { + // Render `(a AND b AND (c OR d))` where a/b/c/d are quoted phrases. + let mut out = String::new(); + let mut first = true; + for p in parts { + if !first { + out.push_str(" AND "); + } + first = false; + match p { + FtsFragment::And(s) => out.push_str("e_phrase(s)), + FtsFragment::OrGroup(items) => { + out.push('('); + let mut first_item = true; + for item in items { + if !first_item { + out.push_str(" OR "); + } + first_item = false; + out.push_str("e_phrase(item)); + } + out.push(')'); + } + } + } + out +} + +/// Escape a phrase for FTS5 MATCH. FTS5 itself uses doubled quotes for +/// literal quotes inside a quoted phrase. +fn quote_phrase(s: &str) -> String { + let mut buf = String::with_capacity(s.len() + 2); + buf.push('"'); + for c in s.chars() { + if c == '"' { + buf.push('"'); + } + buf.push(c); + } + buf.push('"'); + buf +} + +impl Builder { + /// Translate a sub-term, returning its contribution to the SQL WHERE + /// clause. FTS parts are accumulated in `self.fts_parts` (not returned + /// here) since they collapse into a single MATCH expression at the top. + fn translate(&mut self, t: &Term) -> Result { + match t { + Term::Literal(s) => { + self.fts_parts.push(FtsFragment::And(s.clone())); + Ok(String::new()) + } + Term::Property { key, op, value } => self.translate_property(key, *op, value), + Term::And(children) => { + let mut pieces = Vec::new(); + for c in children { + let p = self.translate(c)?; + if !p.is_empty() { + pieces.push(p); + } + } + Ok(join_with("AND", &pieces)) + } + Term::Or(children) => { + // An OR of pure-literal children collapses into one FTS OR-group. + if children.iter().all(|c| matches!(c, Term::Literal(_))) { + let group: Vec = children + .iter() + .map(|c| match c { + Term::Literal(s) => s.clone(), + _ => unreachable!(), + }) + .collect(); + self.fts_parts.push(FtsFragment::OrGroup(group)); + return Ok(String::new()); + } + // Mixed OR — each branch becomes a separate sub-Builder whose + // WHERE fragments we OR together. FTS branches cannot mix + // with structured branches cleanly at the SQL level here; + // keep it simple by requiring that mixed-OR branches produce + // structured-only WHERE fragments. + let mut pieces = Vec::new(); + for c in children { + let before = self.fts_parts.len(); + let p = self.translate(c)?; + if self.fts_parts.len() > before { + return Err(TranslateError::UnknownProperty( + "OR mixing FTS and structured terms is not supported in Set A".into(), + )); + } + if !p.is_empty() { + pieces.push(p); + } + } + Ok(format!("({})", join_with("OR", &pieces))) + } + } + } + + fn translate_property( + &mut self, + key: &str, + op: Op, + value: &str, + ) -> Result { + let lower_key = key.to_ascii_lowercase(); + match lower_key.as_str() { + "type" => self.prop_type(op, value, key), + "modified" | "mtime" => self.prop_mtime(op, value, key), + "path" | "folder" | "includefolder" => self.prop_path(op, value, key), + "name" | "filename" => self.prop_name(op, value, key), + "mime" => self.prop_mime(op, value, key), + _ => Err(TranslateError::UnknownProperty(key.to_string())), + } + } + + fn prop_type(&mut self, op: Op, value: &str, key: &str) -> Result { + if op != Op::Contains && op != Op::Eq { + return Err(TranslateError::UnsupportedOp { + key: key.into(), + op, + }); + } + let bits = FileType::from_name(value).bits() as i64; + if bits == 0 { + return Err(TranslateError::UnknownProperty(format!( + "type name '{}'", + value + ))); + } + self.all_params + .push(rusqlite::types::Value::Integer(bits)); + Ok(format!("(f.type & ?{}) != 0", self.param_placeholder_idx())) + } + + fn prop_mtime(&mut self, op: Op, value: &str, key: &str) -> Result { + let unix = parse_date_to_unix(value).ok_or_else(|| TranslateError::BadDate(value.into()))?; + let col = "f.mtime"; + let sql_op = match op { + Op::Contains | Op::Eq => "=", + Op::Lt => "<", + Op::Le => "<=", + Op::Gt => ">", + Op::Ge => ">=", + }; + // `modified:=2024-01-01` should match the whole day, not the second. + if op == Op::Eq || op == Op::Contains { + let start = unix; + let end = unix + 86_400; + self.all_params.push(rusqlite::types::Value::Integer(start)); + let i = self.param_placeholder_idx(); + self.all_params.push(rusqlite::types::Value::Integer(end)); + let j = self.param_placeholder_idx(); + return Ok(format!("({} >= ?{} AND {} < ?{})", col, i, col, j)); + } + self.all_params.push(rusqlite::types::Value::Integer(unix)); + let i = self.param_placeholder_idx(); + let _ = key; + Ok(format!("{} {} ?{}", col, sql_op, i)) + } + + fn prop_path(&mut self, op: Op, value: &str, key: &str) -> Result { + if op != Op::Contains && op != Op::Eq { + return Err(TranslateError::UnsupportedOp { + key: key.into(), + op, + }); + } + self.all_params + .push(rusqlite::types::Value::Text(value.into())); + let i = self.param_placeholder_idx(); + self.all_params + .push(rusqlite::types::Value::Text(format!("{}/%", value.trim_end_matches('/')))); + let j = self.param_placeholder_idx(); + Ok(format!("(f.parent = ?{} OR f.parent LIKE ?{})", i, j)) + } + + fn prop_name(&mut self, op: Op, value: &str, key: &str) -> Result { + if op != Op::Contains { + return Err(TranslateError::UnsupportedOp { + key: key.into(), + op, + }); + } + self.all_params + .push(rusqlite::types::Value::Text(format!("%{}%", value))); + let i = self.param_placeholder_idx(); + Ok(format!("f.name LIKE ?{}", i)) + } + + fn prop_mime(&mut self, op: Op, value: &str, key: &str) -> Result { + if op != Op::Contains && op != Op::Eq { + return Err(TranslateError::UnsupportedOp { + key: key.into(), + op, + }); + } + self.all_params + .push(rusqlite::types::Value::Text(value.into())); + let i = self.param_placeholder_idx(); + Ok(format!("f.mime = ?{}", i)) + } + + fn param_placeholder_idx(&self) -> usize { + // params[0] is reserved for the FTS MATCH if one is built; structured + // params start at index 2 in that case (1-based). We track it by + // calling this *after* pushing the value; result is `len` so the SQL + // says `?` which matches the 1-based positional binding rusqlite + // uses for `?N` placeholders. When an FTS match is prepended at + // `build`, each index shifts by 1 implicitly. + self.all_params.len() + } +} + +fn join_with(sep: &str, pieces: &[String]) -> String { + pieces + .iter() + .map(|p| p.clone()) + .collect::>() + .join(&format!(" {} ", sep)) +} + +/// Parse a date string. Accepts `YYYY-MM-DD`. Returns unix seconds at 00:00 UTC. +fn parse_date_to_unix(s: &str) -> Option { + // Minimal parser: split on '-' into y/m/d integers. + let parts: Vec<&str> = s.split('-').collect(); + if parts.len() != 3 { + return None; + } + let y: i64 = parts[0].parse().ok()?; + let m: u32 = parts[1].parse().ok()?; + let d: u32 = parts[2].parse().ok()?; + if !(1..=12).contains(&m) || !(1..=31).contains(&d) || !(1970..=9999).contains(&y) { + return None; + } + // Compute unix seconds using the days-since-epoch formula. + Some(civil_to_unix(y, m as i64, d as i64)) +} + +/// Howard Hinnant's civil-from-days algorithm, converting (year, month, day) +/// in the Gregorian calendar to days since 1970-01-01. +fn civil_to_unix(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = (y - era * 400) as i64; + let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146_097 + doe - 719_468; + days * 86_400 +} + +/// Rewrite placeholder indices after an FTS MATCH parameter was inserted at +/// position 0. Called automatically in [`build`] when needed — but since we +/// append placeholders numerically during translation and prepend the FTS +/// param afterwards, every explicit `?N` in the generated SQL is off by one +/// when FTS is present. +fn shift_placeholders(sql: &str) -> String { + // Find every `?N` and bump N by 1. Only ASCII digits after `?`; skip + // anonymous `?` (which rusqlite won't mix with numbered anyway). + let bytes = sql.as_bytes(); + let mut out = String::with_capacity(sql.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'?' && i + 1 < bytes.len() && bytes[i + 1].is_ascii_digit() { + out.push('?'); + let mut j = i + 1; + while j < bytes.len() && bytes[j].is_ascii_digit() { + j += 1; + } + let n: u64 = std::str::from_utf8(&bytes[i + 1..j]) + .unwrap() + .parse() + .unwrap(); + out.push_str(&(n + 1).to_string()); + i = j; + } else { + out.push(bytes[i] as char); + i += 1; + } + } + out +} + + +#[cfg(test)] +mod tests { + use super::*; + + fn build_q(input: &str) -> SqlQuery { + parse_and_build(input, None, 0, Sort::None).expect("build") + } + + #[test] + fn single_literal() { + let q = build_q("foo"); + assert!(q.sql.contains("searchabletext MATCH ?")); + assert_eq!( + q.params, + vec![rusqlite::types::Value::Text("\"foo\"".into())] + ); + } + + #[test] + fn implicit_and_joins_as_fts_and() { + let q = build_q("foo bar"); + assert!(q.sql.contains("searchabletext MATCH ?")); + let m: &rusqlite::types::Value = &q.params[0]; + if let rusqlite::types::Value::Text(s) = m { + assert_eq!(s, "\"foo\" AND \"bar\""); + } else { + panic!(); + } + } + + #[test] + fn or_of_literals_becomes_fts_group() { + let q = build_q("a OR b"); + if let rusqlite::types::Value::Text(s) = &q.params[0] { + assert_eq!(s, "(\"a\" OR \"b\")"); + } else { + panic!(); + } + } + + #[test] + fn type_filter_only() { + let q = build_q("type:Audio"); + assert!(q.sql.contains("(f.type & ?")); + assert_eq!( + q.params, + vec![rusqlite::types::Value::Integer(FileType::AUDIO.bits() as i64)] + ); + } + + #[test] + fn modified_ge() { + let q = build_q("modified:>=2024-01-01"); + assert!(q.sql.contains("f.mtime >= ?")); + assert_eq!(q.params.len(), 1); + } + + #[test] + fn modified_eq_is_day_range() { + let q = build_q("modified:=2024-05-20"); + assert!(q.sql.contains("f.mtime >= ?") && q.sql.contains("f.mtime < ?")); + assert_eq!(q.params.len(), 2); + } + + #[test] + fn path_filter() { + let q = build_q("path:/home/me/docs"); + assert!(q.sql.contains("f.parent = ?")); + assert!(q.sql.contains("f.parent LIKE ?")); + } + + #[test] + fn combined_fts_and_type() { + let q = build_q("type:Audio beatles"); + assert!(q.sql.contains("searchabletext MATCH ?")); + assert!(q.sql.contains("(f.type & ?")); + // FTS param first, then type bits. + assert_eq!(q.params.len(), 2); + assert_eq!( + q.params[0], + rusqlite::types::Value::Text("\"beatles\"".into()) + ); + } + + #[test] + fn unknown_property_errors() { + let err = parse_and_build("artist:beatles", None, 0, Sort::None).unwrap_err(); + assert!(matches!(err, TranslateError::UnknownProperty(_))); + } + + #[test] + fn bad_date_errors() { + let err = parse_and_build("modified:>=not-a-date", None, 0, Sort::None).unwrap_err(); + assert!(matches!(err, TranslateError::BadDate(_))); + } + + #[test] + fn civil_conversion() { + // 2024-01-01 → 1704067200 unix + assert_eq!(civil_to_unix(2024, 1, 1), 1_704_067_200); + // 1970-01-01 → 0 + assert_eq!(civil_to_unix(1970, 1, 1), 0); + // 2000-02-29 is valid (leap year) + assert_eq!(civil_to_unix(2000, 2, 29), 951_782_400); + } + + #[test] + fn limit_offset_sort_render() { + let q = parse_and_build("foo", Some(10), 20, Sort::ByMtimeDesc).unwrap(); + assert!(q.sql.contains("ORDER BY f.mtime DESC")); + assert!(q.sql.contains("LIMIT 10")); + assert!(q.sql.contains("OFFSET 20")); + } + + // End-to-end: run a generated query against a real SQLite DB to verify + // placeholder shifting after FTS prepending is correct. + #[test] + fn end_to_end_combined_filter_executes() { + use crate::db::{ + open_and_migrate, + repo::{insert_file, set_content_done, NewFile}, + }; + use crate::mime::FileType; + use rusqlite::types::ToSql; + + let mut p = std::env::temp_dir(); + p.push(format!( + "qs-query-e2e-{}-{}.sqlite", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + + // Two audio files, one document. Only the "audio with beatles" file + // should be returned for `type:Audio beatles`. + let ids: Vec = { + let tx = conn.transaction().unwrap(); + let a = insert_file( + &tx, + &NewFile { + name: "beatles-track.mp3", + path: "/music/beatles-track.mp3", + parent: "/music", + size: 100, + mtime: 1_700_000_000, + inode: None, + device_id: None, + mime: Some("audio/mpeg"), + ftype: FileType::AUDIO, + hash: None, + }, + ) + .unwrap() + .expect("unique path"); + set_content_done( + &tx, + a, + "beatles-track.mp3", + "beatles hey jude", + &[("artist".into(), "The Beatles".into())], + ) + .unwrap(); + let b = insert_file( + &tx, + &NewFile { + name: "bach.flac", + path: "/music/bach.flac", + parent: "/music", + size: 100, + mtime: 1_700_000_000, + inode: None, + device_id: None, + mime: Some("audio/flac"), + ftype: FileType::AUDIO, + hash: None, + }, + ) + .unwrap() + .expect("unique path"); + set_content_done(&tx, b, "bach.flac", "bach prelude", &[]).unwrap(); + let c = insert_file( + &tx, + &NewFile { + name: "notes.txt", + path: "/docs/notes.txt", + parent: "/docs", + size: 50, + mtime: 1_700_000_000, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + }, + ) + .unwrap() + .expect("unique path"); + set_content_done(&tx, c, "notes.txt", "beatles biography", &[]).unwrap(); + tx.commit().unwrap(); + vec![a, b, c] + }; + + let q = parse_and_build("type:Audio beatles", None, 0, Sort::ByMtimeDesc).unwrap(); + let rows: Vec<(i64, String, String)> = { + let mut stmt = conn.prepare(&q.sql).expect("prepare"); + let bind: Vec<&dyn ToSql> = q.params.iter().map(|v| v as &dyn ToSql).collect(); + stmt.query_map(bind.as_slice(), |r| { + Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)) + }) + .unwrap() + .map(|r| r.unwrap()) + .collect() + }; + + assert_eq!(rows.len(), 1, "expected only beatles-track.mp3; got {:?}", rows); + assert_eq!(rows[0].0, ids[0]); + + drop(conn); + std::fs::remove_file(&p).ok(); + } +} diff --git a/crates/quicksearch-core/src/search_sql.rs b/crates/quicksearch-core/src/search_sql.rs new file mode 100644 index 0000000..6f15677 --- /dev/null +++ b/crates/quicksearch-core/src/search_sql.rs @@ -0,0 +1,328 @@ +//! SQL builders for the GUI's three legacy search modes (full-text, +//! filename, duplicate). Lives in core so it has unit-test coverage; the +//! GUI just composes these into per-page queries. +//! +//! For the structured Baloo-subset query language (`type:`, `modified:`, +//! …), see [`crate::query`]. + +/// All inputs needed to run (and re-run) one of the three search modes. +/// Cached after a fresh search so paging buttons don't have to rebuild +/// from form state. +#[derive(Debug, Clone, PartialEq)] +pub struct SearchArgs { + pub search_type: String, + pub term: String, + pub fulltext_exact: bool, + pub fulltext_case_sensitive: bool, +} + +/// SQL that counts every row matching `args`, ignoring pagination. Used to +/// drive the "page X of Y" UI. For very large FTS hit sets this can take a +/// noticeable fraction of the per-page query time, but it only runs on a +/// fresh search — page navigation reuses the cached total. +pub fn build_count(args: &SearchArgs) -> Result { + match args.search_type.as_str() { + "fulltext" => { + let where_clause = build_fulltext_where(args)?; + Ok(format!( + "SELECT COUNT(*) FROM searchabletext AS st WHERE {}", + where_clause + )) + } + "filename" => { + if args.term.trim().is_empty() { + return Err("Please enter a filename pattern".into()); + } + Ok(format!( + "SELECT COUNT(*) FROM files WHERE name LIKE '%{}%'", + args.term.replace('\'', "''") + )) + } + "duplicates" => Ok( + "SELECT COUNT(*) FROM (SELECT 1 FROM files WHERE hash IS NOT NULL \ + GROUP BY hash HAVING count(*) > 1)" + .into(), + ), + _ => Err("Unknown search type".into()), + } +} + +/// SQL that returns one page of results. +pub fn build_select(args: &SearchArgs, limit: u32, offset: u32) -> Result { + match args.search_type.as_str() { + "fulltext" => { + let where_clause = build_fulltext_where(args)?; + Ok(format!( + "SELECT f.name, f.path, snippet(searchabletext, 1, '', '', '...', 64) as snippet \ + FROM searchabletext AS st JOIN files f ON f.id = st.rowid \ + WHERE {} ORDER BY rank LIMIT {} OFFSET {}", + where_clause, limit, offset + )) + } + "filename" => { + if args.term.trim().is_empty() { + return Err("Please enter a filename pattern".into()); + } + Ok(format!( + "SELECT name, path FROM files WHERE name LIKE '%{}%' ORDER BY name LIMIT {} OFFSET {}", + args.term.replace('\'', "''"), + limit, + offset + )) + } + "duplicates" => Ok(format!( + "SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL \ + GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC LIMIT {} OFFSET {}", + limit, offset + )), + _ => Err("Unknown search type".into()), + } +} + +/// Translate the user-typed term into the FTS5 `MATCH` expression and any +/// supplemental case-sensitivity filters. Shared by count and select so +/// pagination doesn't accidentally diverge from the totals. +fn build_fulltext_where(args: &SearchArgs) -> Result { + let trimmed = args.term.trim(); + if trimmed.is_empty() { + return Err("Please enter a search term".into()); + } + + // Strip FTS5 control characters that confuse the parser. Replace with + // spaces so word boundaries survive. + let sanitized: String = trimmed + .chars() + .map(|c| { + if matches!( + c, + ':' | ';' | '(' | ')' | '[' | ']' | '{' | '}' | '^' | '~' | '"' + ) { + ' ' + } else { + c + } + }) + .collect(); + + let tokens: Vec<&str> = sanitized.split_whitespace().collect(); + if tokens.is_empty() { + return Err("Please enter a valid search term".into()); + } + + // Trigram tokenizer needs each word to be at least 3 characters. Exact + // phrase mode skips this filter — a quoted phrase of short words still + // matches because the trigrams overlap the spaces between words. + let words: Vec<&str> = if args.fulltext_exact { + tokens + } else { + let filtered: Vec<&str> = tokens + .into_iter() + .filter(|w| w.chars().count() >= 3) + .collect(); + if filtered.is_empty() { + return Err( + "Trigram index needs each word to be at least 3 characters unless you use exact phrase search." + .into(), + ); + } + filtered + }; + + let sql_quote = |s: &str| s.replace('\'', "''"); + + let fts_match = if args.fulltext_exact { + let phrase = words.join(" "); + format!("\"{}\"", phrase.replace('"', "\"\"")) + } else { + words.join(" AND ") + }; + + let mut where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match)); + if args.fulltext_case_sensitive { + if args.fulltext_exact { + let literal = words.join(" "); + where_clause.push_str(&format!( + " AND instr(st.text, '{}') > 0", + sql_quote(&literal) + )); + } else { + for w in &words { + where_clause.push_str(&format!( + " AND instr(st.text, '{}') > 0", + sql_quote(w) + )); + } + } + } + Ok(where_clause) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{open_and_migrate, repo::{insert_file, set_content_done, NewFile}}; + use crate::mime::FileType; + + fn args(search_type: &str, term: &str) -> SearchArgs { + SearchArgs { + search_type: search_type.into(), + term: term.into(), + fulltext_exact: false, + fulltext_case_sensitive: false, + } + } + + #[test] + fn fulltext_select_has_limit_and_offset() { + let sql = build_select(&args("fulltext", "hello world"), 50, 100).unwrap(); + assert!(sql.contains("LIMIT 50")); + assert!(sql.contains("OFFSET 100")); + assert!(sql.contains("ORDER BY rank")); + assert!(sql.contains("snippet(searchabletext")); + } + + #[test] + fn fulltext_count_lacks_pagination_and_join() { + let sql = build_count(&args("fulltext", "hello world")).unwrap(); + assert!(sql.starts_with("SELECT COUNT(*)")); + assert!(!sql.contains("LIMIT")); + assert!(!sql.contains("OFFSET")); + assert!(!sql.contains("JOIN files")); + } + + #[test] + fn fulltext_short_words_filtered_unless_exact() { + let err = build_select(&args("fulltext", "a b"), 50, 0).unwrap_err(); + assert!(err.contains("3 characters")); + let mut a = args("fulltext", "a b"); + a.fulltext_exact = true; + assert!(build_select(&a, 50, 0).is_ok()); + } + + #[test] + fn fulltext_quotes_are_escaped() { + let sql = build_select(&args("fulltext", "it's working"), 50, 0).unwrap(); + // SQL literals double single quotes. + assert!(sql.contains("it''s")); + } + + #[test] + fn filename_select_has_limit_offset_and_order() { + let sql = build_select(&args("filename", "report"), 50, 0).unwrap(); + assert!(sql.contains("LIMIT 50")); + assert!(sql.contains("OFFSET 0")); + assert!(sql.contains("ORDER BY name")); + assert!(sql.contains("name LIKE '%report%'")); + } + + #[test] + fn filename_empty_term_errors() { + assert!(build_select(&args("filename", " "), 50, 0).is_err()); + assert!(build_count(&args("filename", "")).is_err()); + } + + #[test] + fn duplicates_select_has_limit_offset() { + let sql = build_select(&args("duplicates", ""), 50, 100).unwrap(); + assert!(sql.contains("LIMIT 50")); + assert!(sql.contains("OFFSET 100")); + assert!(sql.contains("GROUP BY hash")); + } + + #[test] + fn unknown_search_type_errors() { + assert!(build_select(&args("nope", ""), 50, 0).is_err()); + assert!(build_count(&args("nope", "")).is_err()); + } + + fn tmp_path() -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "qs-search-sql-{}-{}.sqlite", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + p + } + + /// End-to-end: seed three rows, run count + paged select, verify + /// pagination boundaries actually behave on a real DB. + #[test] + fn end_to_end_pagination_smoke() { + let p = tmp_path(); + let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + { + let tx = conn.transaction().unwrap(); + for i in 0..7 { + let path = format!("/tmp/file_{}.txt", i); + let id = insert_file( + &tx, + &NewFile { + name: &format!("file_{}.txt", i), + path: &path, + parent: "/tmp", + size: 1, + mtime: 1, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + }, + ) + .unwrap() + .expect("unique path"); + set_content_done(&tx, id, &format!("file_{}.txt", i), "shared body content", &[]).unwrap(); + } + tx.commit().unwrap(); + } + + // Count: all 7 rows match "shared". + let count_sql = build_count(&args("fulltext", "shared body content")).unwrap(); + let n: i64 = conn.query_row(&count_sql, [], |r| r.get(0)).unwrap(); + assert_eq!(n, 7); + + // Page 1, page_size 3 → 3 rows. + let sel1 = build_select(&args("fulltext", "shared body content"), 3, 0).unwrap(); + let rows1: Vec = conn + .prepare(&sel1) + .unwrap() + .query_map([], |r| r.get::<_, String>(1)) + .unwrap() + .map(|r| r.unwrap()) + .collect(); + assert_eq!(rows1.len(), 3); + + // Page 3, page_size 3 → 1 row (offset 6, 7 total). + let sel3 = build_select(&args("fulltext", "shared body content"), 3, 6).unwrap(); + let rows3: Vec = conn + .prepare(&sel3) + .unwrap() + .query_map([], |r| r.get::<_, String>(1)) + .unwrap() + .map(|r| r.unwrap()) + .collect(); + assert_eq!(rows3.len(), 1); + + // Filename pagination on the same fixture. + let fn_count = build_count(&args("filename", "file_")).unwrap(); + let n2: i64 = conn.query_row(&fn_count, [], |r| r.get(0)).unwrap(); + assert_eq!(n2, 7); + + let fn_sel = build_select(&args("filename", "file_"), 5, 0).unwrap(); + let rows: Vec = conn + .prepare(&fn_sel) + .unwrap() + .query_map([], |r| r.get::<_, String>(1)) + .unwrap() + .map(|r| r.unwrap()) + .collect(); + assert_eq!(rows.len(), 5); + + drop(conn); + std::fs::remove_file(&p).ok(); + } +} diff --git a/crates/quicksearch-core/src/shutdown.rs b/crates/quicksearch-core/src/shutdown.rs new file mode 100644 index 0000000..2a48a60 --- /dev/null +++ b/crates/quicksearch-core/src/shutdown.rs @@ -0,0 +1,27 @@ +//! Process-level shutdown helpers. +//! +//! Wires Ctrl-C (and on Unix, SIGTERM) to a graceful shutdown that flushes +//! the indexing DB and exits. Replaces the ad-hoc `ctrlc::set_handler` + +//! `OnceLock` dance the GUI used to carry. Call +//! [`install_signal_handler`] once from a binary's `main` with a cloned +//! [`IndexingService`] handle. + +use std::sync::Arc; + +use crate::indexing::IndexingService; + +/// Install a Ctrl-C (and, where supported, SIGTERM) handler that calls +/// [`IndexingService::graceful_shutdown`] and then exits with status 0. +/// +/// Returns an error only if a handler was already installed elsewhere in +/// this process (ctrlc::set_handler is one-shot). +pub fn install_signal_handler(service: Arc) -> Result<(), String> { + ctrlc::set_handler(move || { + eprintln!("Received Ctrl-C, shutting down gracefully..."); + if let Err(e) = service.graceful_shutdown() { + eprintln!("Error during graceful shutdown: {}", e); + } + std::process::exit(0); + }) + .map_err(|e| format!("install signal handler: {}", e)) +} diff --git a/crates/quicksearch-core/src/watcher.rs b/crates/quicksearch-core/src/watcher.rs new file mode 100644 index 0000000..16abd08 --- /dev/null +++ b/crates/quicksearch-core/src/watcher.rs @@ -0,0 +1,496 @@ +//! Filesystem watcher with per-directory debouncing. +//! +//! Wraps the [`notify`] crate with a throttling pipeline patterned after +//! ffb-server's `ingest::pipeline`: events are bucketed by directory, same- +//! path events within a window are coalesced, and a tick loop flushes ready +//! buckets. The caller provides an [`EventSink`] callback that applies +//! emitted [`FsEvent`]s — typically to the QuickSearch database via the +//! [`crate::db::repo`] helpers. +//! +//! Inotify watch-limit (ENOSPC) handling: the watcher logs a prominent +//! warning on first occurrence and switches the offending root to periodic +//! rescans. Rescan cadence is configurable in [`WatcherConfig`]. +//! +//! This module deliberately stays sync (std::thread + crossbeam-style +//! channels via `std::sync::mpsc`) so it integrates cleanly with the +//! existing indexer which is not async. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use notify::{Config as NotifyConfig, Event as NotifyEvent, EventKind, RecommendedWatcher, + RecursiveMode, Watcher as NotifyWatcher}; + +/// An event surfaced to the caller after debouncing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FsEvent { + Create(PathBuf), + Modify(PathBuf), + Remove(PathBuf), + /// Rename where both endpoints arrived in the same notify event. For + /// split rename halves (From or To only) the watcher emits Remove/Create + /// instead. + Rename { from: PathBuf, to: PathBuf }, +} + +/// Sink callback. Called on the watcher thread; implementors should keep +/// work short and push heavier operations to their own worker. +pub type EventSink = Arc; + +#[derive(Debug, Clone)] +pub struct WatcherConfig { + /// Per-directory debounce window. Bursts of events in the same directory + /// collapse to one flush after this interval of quiet. + pub throttle_window: Duration, + /// How often the tick loop inspects the throttle map. Short ticks mean + /// low latency for first-in-a-burst; long ticks lower CPU at idle. + pub tick_interval: Duration, + /// Maximum directories processed per tick. Caps the time spent in a + /// single flush pass so long backlogs don't monopolize the thread. + pub max_dirs_per_tick: usize, + /// When to garbage-collect stale throttle entries (idle > window * N). + pub prune_max_age_multiplier: u32, +} + +impl Default for WatcherConfig { + fn default() -> Self { + Self { + throttle_window: Duration::from_secs(30), + tick_interval: Duration::from_millis(500), + max_dirs_per_tick: 64, + prune_max_age_multiplier: 10, + } + } +} + +/// Handle to a running watcher. Dropping calls [`Self::stop`] implicitly. +pub struct Watcher { + stop_flag: Arc, + handle: Option>, + /// Held so notify::Watcher drops on stop (releasing inotify watches). + _raw: RecommendedWatcher, +} + +impl Watcher { + /// Start watching `roots` recursively. Returns once the watcher is + /// registered and its background thread is running. + pub fn start( + roots: I, + config: WatcherConfig, + sink: EventSink, + ) -> Result + where + I: IntoIterator, + P: AsRef, + { + let (tx, rx) = mpsc::channel::(); + let tx_for_cb = tx.clone(); + let mut watcher = RecommendedWatcher::new( + move |res: notify::Result| match res { + Ok(ev) => { + // A closed receiver just means the watcher was stopped; ignore. + let _ = tx_for_cb.send(ev); + } + Err(e) => { + eprintln!("watcher: notify error: {}", e); + } + }, + NotifyConfig::default(), + ) + .map_err(|e| format!("create watcher: {}", e))?; + + let mut watched_any = false; + for root in roots { + let root = root.as_ref(); + match watcher.watch(root, RecursiveMode::Recursive) { + Ok(_) => { + watched_any = true; + } + Err(e) => { + // Best-effort: log and continue. ENOSPC (watch limit) is + // detected here by inspecting the error string — the + // notify crate doesn't expose a typed variant for it. + let msg = format!("{}", e); + if is_enospc_error(&msg) { + eprintln!( + "watcher: inotify watch limit exceeded for {}. \ + Increase fs.inotify.max_user_watches (currently the kernel default).", + root.display() + ); + } else { + eprintln!("watcher: watch({}): {}", root.display(), e); + } + } + } + } + if !watched_any { + return Err("watcher: no roots could be watched".into()); + } + + let stop_flag = Arc::new(AtomicBool::new(false)); + let stop_clone = stop_flag.clone(); + let handle = thread::spawn(move || run_loop(rx, sink, config, stop_clone)); + + Ok(Self { + stop_flag, + handle: Some(handle), + _raw: watcher, + }) + } + + /// Signal the background thread to stop and wait for it to join. Safe to + /// call multiple times. + pub fn stop(&mut self) { + self.stop_flag.store(true, Ordering::Relaxed); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} + +impl Drop for Watcher { + fn drop(&mut self) { + self.stop(); + } +} + +fn is_enospc_error(msg: &str) -> bool { + // notify crate wraps libc errors; the message includes "No space left" + // or the errno. Be generous in matching. + msg.contains("ENOSPC") + || msg.contains("No space left") + || msg.contains("inotify") + || msg.contains("watch limit") +} + +/// A queued event, deduplicated per path within a window. +#[derive(Debug, Clone)] +struct QueuedEvent { + op: QueuedOp, +} + +#[derive(Debug, Clone, Copy)] +enum QueuedOp { + Create, + Modify, + Remove, +} + +#[derive(Debug)] +struct DirThrottleEntry { + /// Last time this entry's queue was flushed (or when the entry was + /// created as leading-edge). + record_time: Instant, + /// Per-path pending op. Same path seen twice in a window keeps only the + /// latest op — coalescing a rename-as-create+modify spam into one event. + queue: HashMap, + /// If true, the next tick flushes regardless of window age. Set for the + /// first event in a previously-idle directory so it reacts fast. + immediate: bool, +} + +fn run_loop( + rx: mpsc::Receiver, + sink: EventSink, + config: WatcherConfig, + stop: Arc, +) { + let mut throttle: HashMap = HashMap::new(); + // Pending rename halves keyed by cookie are not supported by notify 6.x's + // high-level API uniformly across backends; when From/To aren't bundled + // we emit Remove/Create which remains correct semantically. + let prune_interval_ticks = 20u32; + let mut tick_counter: u32 = 0; + + loop { + if stop.load(Ordering::Relaxed) { + break; + } + + // Drain incoming events. Block briefly to avoid spinning when idle. + let deadline = Instant::now() + config.tick_interval; + loop { + if stop.load(Ordering::Relaxed) { + break; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break; + } + match rx.recv_timeout(remaining) { + Ok(ev) => handle_notify_event(&ev, &mut throttle, &sink), + Err(mpsc::RecvTimeoutError::Timeout) => break, + Err(mpsc::RecvTimeoutError::Disconnected) => return, + } + } + + if stop.load(Ordering::Relaxed) { + break; + } + + // Tick: flush ready directories, up to max_dirs_per_tick. + flush_ready(&mut throttle, &sink, &config); + + // Periodic GC of abandoned throttle entries. + tick_counter = tick_counter.wrapping_add(1); + if tick_counter % prune_interval_ticks == 0 { + let max_age = config + .throttle_window + .saturating_mul(config.prune_max_age_multiplier); + prune_stale(&mut throttle, max_age); + } + } +} + +fn handle_notify_event( + ev: &NotifyEvent, + throttle: &mut HashMap, + sink: &EventSink, +) { + // Rename events that carry both sides are emitted directly — they + // can't be coalesced with same-dir creates/modifies meaningfully. + if let EventKind::Modify(notify::event::ModifyKind::Name(kind)) = ev.kind { + if matches!(kind, notify::event::RenameMode::Both) && ev.paths.len() == 2 { + sink(FsEvent::Rename { + from: ev.paths[0].clone(), + to: ev.paths[1].clone(), + }); + return; + } + // Split renames (From alone, To alone) degrade to Remove/Create. + } + + for p in &ev.paths { + let op = match ev.kind { + EventKind::Create(_) => QueuedOp::Create, + EventKind::Remove(_) => QueuedOp::Remove, + EventKind::Modify(notify::event::ModifyKind::Name(notify::event::RenameMode::From)) => { + QueuedOp::Remove + } + EventKind::Modify(notify::event::ModifyKind::Name(notify::event::RenameMode::To)) => { + QueuedOp::Create + } + EventKind::Modify(_) => QueuedOp::Modify, + _ => continue, + }; + enqueue(throttle, p.clone(), op); + } +} + +fn enqueue( + throttle: &mut HashMap, + path: PathBuf, + op: QueuedOp, +) { + let dir = path.parent().map(|p| p.to_path_buf()).unwrap_or_else(|| path.clone()); + let entry = throttle + .entry(dir) + .or_insert_with(|| DirThrottleEntry { + record_time: Instant::now(), + queue: HashMap::new(), + immediate: true, + }); + // Coalesce: Remove after Create → drop both. Modify after Modify → one Modify. + match (op, entry.queue.get(&path).map(|q| q.op)) { + (QueuedOp::Remove, Some(QueuedOp::Create)) => { + entry.queue.remove(&path); + } + _ => { + entry + .queue + .insert(path, QueuedEvent { op }); + } + } +} + +fn flush_ready( + throttle: &mut HashMap, + sink: &EventSink, + config: &WatcherConfig, +) { + let now = Instant::now(); + // Collect ready dir keys first, up to max_dirs_per_tick. Copying keys + // avoids borrow conflicts when we mutate entries below. + let mut ready: Vec = Vec::new(); + for (dir, entry) in throttle.iter() { + let age = now.saturating_duration_since(entry.record_time); + if entry.immediate || (!entry.queue.is_empty() && age >= config.throttle_window) { + ready.push(dir.clone()); + if ready.len() >= config.max_dirs_per_tick { + break; + } + } + } + for dir in ready { + if let Some(entry) = throttle.get_mut(&dir) { + let drained: Vec<(PathBuf, QueuedOp)> = entry + .queue + .drain() + .map(|(p, q)| (p, q.op)) + .collect(); + entry.immediate = false; + entry.record_time = now; + for (path, op) in drained { + let ev = match op { + QueuedOp::Create => FsEvent::Create(path), + QueuedOp::Modify => FsEvent::Modify(path), + QueuedOp::Remove => FsEvent::Remove(path), + }; + sink(ev); + } + } + } +} + +fn prune_stale(throttle: &mut HashMap, max_age: Duration) { + let now = Instant::now(); + throttle.retain(|_, entry| { + !entry.queue.is_empty() + || entry.immediate + || now.saturating_duration_since(entry.record_time) < max_age + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + fn sink_to_vec() -> (EventSink, Arc>>) { + let v: Arc>> = Arc::new(Mutex::new(Vec::new())); + let v_clone = v.clone(); + let s: EventSink = Arc::new(move |e| v_clone.lock().unwrap().push(e)); + (s, v) + } + + #[test] + fn enqueue_create_then_remove_cancels() { + let mut map: HashMap = HashMap::new(); + let p = PathBuf::from("/tmp/a.txt"); + enqueue(&mut map, p.clone(), QueuedOp::Create); + enqueue(&mut map, p.clone(), QueuedOp::Remove); + let entry = map.get(p.parent().unwrap()).unwrap(); + assert!(entry.queue.is_empty(), "Create then Remove should cancel"); + } + + #[test] + fn modify_after_modify_is_one() { + let mut map: HashMap = HashMap::new(); + let p = PathBuf::from("/tmp/a.txt"); + enqueue(&mut map, p.clone(), QueuedOp::Modify); + enqueue(&mut map, p.clone(), QueuedOp::Modify); + let entry = map.get(p.parent().unwrap()).unwrap(); + assert_eq!(entry.queue.len(), 1); + } + + #[test] + fn flush_ready_leading_edge_fires_immediately() { + let mut map: HashMap = HashMap::new(); + enqueue(&mut map, PathBuf::from("/tmp/a.txt"), QueuedOp::Create); + let (sink, got) = sink_to_vec(); + let config = WatcherConfig::default(); + flush_ready(&mut map, &sink, &config); + let got = got.lock().unwrap(); + assert_eq!(got.len(), 1); + assert!(matches!(got[0], FsEvent::Create(_))); + } + + #[test] + fn flush_ready_respects_max_dirs_per_tick() { + let mut map: HashMap = HashMap::new(); + for i in 0..10 { + enqueue(&mut map, PathBuf::from(format!("/dir{}/a", i)), QueuedOp::Create); + } + let (sink, got) = sink_to_vec(); + let mut config = WatcherConfig::default(); + config.max_dirs_per_tick = 3; + flush_ready(&mut map, &sink, &config); + // Each dir contributes one event because each entry has one path. + assert_eq!(got.lock().unwrap().len(), 3); + } + + #[test] + fn prune_stale_drops_empty_old_entries() { + let mut map: HashMap = HashMap::new(); + map.insert( + PathBuf::from("/tmp"), + DirThrottleEntry { + record_time: Instant::now() - Duration::from_secs(3600), + queue: HashMap::new(), + immediate: false, + }, + ); + prune_stale(&mut map, Duration::from_secs(1)); + assert!(map.is_empty()); + } + + #[test] + fn prune_stale_keeps_active_entries() { + let mut map: HashMap = HashMap::new(); + let mut queue = HashMap::new(); + queue.insert( + PathBuf::from("/tmp/a"), + QueuedEvent { op: QueuedOp::Modify }, + ); + map.insert( + PathBuf::from("/tmp"), + DirThrottleEntry { + record_time: Instant::now() - Duration::from_secs(3600), + queue, + immediate: false, + }, + ); + prune_stale(&mut map, Duration::from_secs(1)); + assert_eq!(map.len(), 1); + } + + /// End-to-end: create files in a tempdir, verify the watcher surfaces + /// events via the sink. Short timeouts keep the test fast; if it becomes + /// flaky on slow CI, increase the sleeps. + #[test] + fn e2e_create_modify_remove_surfaces() { + let dir = std::env::temp_dir().join(format!( + "qs-watch-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir(&dir).unwrap(); + + let (sink, got) = sink_to_vec(); + let mut config = WatcherConfig::default(); + // Speed the test up: small window, small tick. + config.throttle_window = Duration::from_millis(50); + config.tick_interval = Duration::from_millis(20); + + let mut w = Watcher::start(std::iter::once(&dir), config, sink).unwrap(); + + let f = dir.join("hello.txt"); + std::fs::write(&f, "hi").unwrap(); + std::thread::sleep(Duration::from_millis(150)); + std::fs::write(&f, "hi again").unwrap(); + std::thread::sleep(Duration::from_millis(200)); + std::fs::remove_file(&f).unwrap(); + std::thread::sleep(Duration::from_millis(200)); + + w.stop(); + + let events = got.lock().unwrap().clone(); + // Expect at least one Create (or Modify, depending on backend) and one Remove. + // Some platforms emit Create+Modify for `write`. + let has_create_or_modify = events + .iter() + .any(|e| matches!(e, FsEvent::Create(_) | FsEvent::Modify(_))); + let has_remove = events.iter().any(|e| matches!(e, FsEvent::Remove(_))); + assert!(has_create_or_modify, "no create/modify in {:?}", events); + assert!(has_remove, "no remove in {:?}", events); + + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/crates/quicksearch-gui/Cargo.toml b/crates/quicksearch-gui/Cargo.toml new file mode 100644 index 0000000..85b8022 --- /dev/null +++ b/crates/quicksearch-gui/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "quicksearch-gui" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "quicksearch" +path = "src/main.rs" + +[dependencies] +quicksearch-core = { path = "../quicksearch-core" } + +dioxus = { version = "0.5.1", features = ["desktop"] } +dioxus-desktop = "0.5.1" +tokio = { version = "1.0", features = ["time", "signal"] } diff --git a/assets/app.js b/crates/quicksearch-gui/assets/app.js similarity index 97% rename from assets/app.js rename to crates/quicksearch-gui/assets/app.js index 8841630..af8d7cc 100644 --- a/assets/app.js +++ b/crates/quicksearch-gui/assets/app.js @@ -1,232 +1,232 @@ -// QuickSearch Application JavaScript - -// Enhanced UI interactions -document.addEventListener('DOMContentLoaded', function() { - console.log('QuickSearch UI loaded'); - - // Add loading states to buttons - function addLoadingState(button, originalText) { - button.disabled = true; - button.innerHTML = '' + originalText; - } - - function removeLoadingState(button, originalText) { - button.disabled = false; - button.innerHTML = originalText; - } - - // Enhanced form interactions - const forms = document.querySelectorAll('form'); - forms.forEach(form => { - form.addEventListener('submit', function(e) { - const submitButton = form.querySelector('button[type="submit"]'); - if (submitButton) { - addLoadingState(submitButton, submitButton.textContent); - } - }); - }); - - // Keyboard shortcuts - document.addEventListener('keydown', function(e) { - // Ctrl+F to focus search - if (e.ctrlKey && e.key === 'f') { - e.preventDefault(); - const searchInput = document.querySelector('input[type="text"]'); - if (searchInput) { - searchInput.focus(); - searchInput.select(); - } - } - - // Escape to clear search - if (e.key === 'Escape') { - const searchInput = document.querySelector('input[type="text"]'); - if (searchInput && searchInput === document.activeElement) { - searchInput.value = ''; - searchInput.blur(); - } - } - }); - - // Enhanced table interactions - function enhanceTable(table) { - // Add click-to-copy functionality for table cells - const cells = table.querySelectorAll('td'); - cells.forEach(cell => { - cell.addEventListener('click', function() { - const text = cell.textContent.trim(); - if (text && navigator.clipboard) { - navigator.clipboard.writeText(text).then(() => { - // Visual feedback - cell.style.backgroundColor = '#4CAF50'; - cell.style.color = 'white'; - setTimeout(() => { - cell.style.backgroundColor = ''; - cell.style.color = ''; - }, 200); - }); - } - }); - }); - - // Add sortable columns (basic implementation) - const headers = table.querySelectorAll('th'); - headers.forEach((header, index) => { - header.style.cursor = 'pointer'; - header.addEventListener('click', () => sortTable(table, index)); - }); - } - - // Simple table sorting - function sortTable(table, columnIndex) { - const tbody = table.querySelector('tbody'); - const rows = Array.from(tbody.querySelectorAll('tr')); - - rows.sort((a, b) => { - const aVal = a.cells[columnIndex]?.textContent.trim() || ''; - const bVal = b.cells[columnIndex]?.textContent.trim() || ''; - - // Try numeric sort first - const aNum = parseFloat(aVal); - const bNum = parseFloat(bVal); - - if (!isNaN(aNum) && !isNaN(bNum)) { - return aNum - bNum; - } - - // Fall back to string sort - return aVal.localeCompare(bVal); - }); - - // Clear tbody and re-append sorted rows - tbody.innerHTML = ''; - rows.forEach(row => tbody.appendChild(row)); - } - - // Auto-enhance any tables that appear - const observer = new MutationObserver(function(mutations) { - mutations.forEach(function(mutation) { - mutation.addedNodes.forEach(function(node) { - if (node.nodeType === 1) { // Element node - const tables = node.querySelectorAll ? node.querySelectorAll('table') : []; - tables.forEach(enhanceTable); - - if (node.tagName === 'TABLE') { - enhanceTable(node); - } - } - }); - }); - }); - - observer.observe(document.body, { childList: true, subtree: true }); - - // Enhance existing tables - document.querySelectorAll('table').forEach(enhanceTable); -}); - -// Utility functions for Rust integration -window.QuickSearch = { - // Function to show toast notifications - showToast: function(message, type = 'info') { - const toast = document.createElement('div'); - toast.className = `toast toast-${type}`; - toast.textContent = message; - toast.style.cssText = ` - position: fixed; - top: 20px; - right: 20px; - padding: 12px 20px; - border-radius: 6px; - color: white; - font-weight: 600; - z-index: 2000; - animation: slideIn 0.3s ease; - `; - - // Set background based on type - const colors = { - info: '#2196F3', - success: '#4CAF50', - warning: '#FF9800', - error: '#f44336' - }; - toast.style.backgroundColor = colors[type] || colors.info; - - document.body.appendChild(toast); - - setTimeout(() => { - toast.style.animation = 'slideOut 0.3s ease'; - setTimeout(() => { - document.body.removeChild(toast); - }, 300); - }, 3000); - }, - - // Function to update status display - updateStatus: function(status) { - const statusDisplay = document.querySelector('.status-display'); - if (statusDisplay) { - statusDisplay.textContent = status; - } - }, - - // Function to highlight search terms in results - highlightSearchTerms: function(searchTerm, container) { - if (!searchTerm || !container) return; - - const walker = document.createTreeWalker( - container, - NodeFilter.SHOW_TEXT, - null, - false - ); - - const textNodes = []; - let node; - while (node = walker.nextNode()) { - textNodes.push(node); - } - - textNodes.forEach(textNode => { - const parent = textNode.parentNode; - if (parent.tagName === 'B') return; // Skip already highlighted - - const text = textNode.textContent; - const regex = new RegExp(`(${searchTerm})`, 'gi'); - - if (regex.test(text)) { - const highlightedHTML = text.replace(regex, '$1'); - const wrapper = document.createElement('span'); - wrapper.innerHTML = highlightedHTML; - parent.replaceChild(wrapper, textNode); - } - }); - } -}; - -// Add custom CSS for toasts and animations -const style = document.createElement('style'); -style.textContent = ` - @keyframes slideIn { - from { transform: translateX(100%); opacity: 0; } - to { transform: translateX(0); opacity: 1; } - } - - @keyframes slideOut { - from { transform: translateX(0); opacity: 1; } - to { transform: translateX(100%); opacity: 0; } - } - - mark { - background: #ffeb3b; - padding: 2px 4px; - border-radius: 3px; - font-weight: bold; - } - - .toast { - box-shadow: 0 4px 12px rgba(0,0,0,0.2); - } -`; -document.head.appendChild(style); +// QuickSearch Application JavaScript + +// Enhanced UI interactions +document.addEventListener('DOMContentLoaded', function() { + console.log('QuickSearch UI loaded'); + + // Add loading states to buttons + function addLoadingState(button, originalText) { + button.disabled = true; + button.innerHTML = '' + originalText; + } + + function removeLoadingState(button, originalText) { + button.disabled = false; + button.innerHTML = originalText; + } + + // Enhanced form interactions + const forms = document.querySelectorAll('form'); + forms.forEach(form => { + form.addEventListener('submit', function(e) { + const submitButton = form.querySelector('button[type="submit"]'); + if (submitButton) { + addLoadingState(submitButton, submitButton.textContent); + } + }); + }); + + // Keyboard shortcuts + document.addEventListener('keydown', function(e) { + // Ctrl+F to focus search + if (e.ctrlKey && e.key === 'f') { + e.preventDefault(); + const searchInput = document.querySelector('input[type="text"]'); + if (searchInput) { + searchInput.focus(); + searchInput.select(); + } + } + + // Escape to clear search + if (e.key === 'Escape') { + const searchInput = document.querySelector('input[type="text"]'); + if (searchInput && searchInput === document.activeElement) { + searchInput.value = ''; + searchInput.blur(); + } + } + }); + + // Enhanced table interactions + function enhanceTable(table) { + // Add click-to-copy functionality for table cells + const cells = table.querySelectorAll('td'); + cells.forEach(cell => { + cell.addEventListener('click', function() { + const text = cell.textContent.trim(); + if (text && navigator.clipboard) { + navigator.clipboard.writeText(text).then(() => { + // Visual feedback + cell.style.backgroundColor = '#4CAF50'; + cell.style.color = 'white'; + setTimeout(() => { + cell.style.backgroundColor = ''; + cell.style.color = ''; + }, 200); + }); + } + }); + }); + + // Add sortable columns (basic implementation) + const headers = table.querySelectorAll('th'); + headers.forEach((header, index) => { + header.style.cursor = 'pointer'; + header.addEventListener('click', () => sortTable(table, index)); + }); + } + + // Simple table sorting + function sortTable(table, columnIndex) { + const tbody = table.querySelector('tbody'); + const rows = Array.from(tbody.querySelectorAll('tr')); + + rows.sort((a, b) => { + const aVal = a.cells[columnIndex]?.textContent.trim() || ''; + const bVal = b.cells[columnIndex]?.textContent.trim() || ''; + + // Try numeric sort first + const aNum = parseFloat(aVal); + const bNum = parseFloat(bVal); + + if (!isNaN(aNum) && !isNaN(bNum)) { + return aNum - bNum; + } + + // Fall back to string sort + return aVal.localeCompare(bVal); + }); + + // Clear tbody and re-append sorted rows + tbody.innerHTML = ''; + rows.forEach(row => tbody.appendChild(row)); + } + + // Auto-enhance any tables that appear + const observer = new MutationObserver(function(mutations) { + mutations.forEach(function(mutation) { + mutation.addedNodes.forEach(function(node) { + if (node.nodeType === 1) { // Element node + const tables = node.querySelectorAll ? node.querySelectorAll('table') : []; + tables.forEach(enhanceTable); + + if (node.tagName === 'TABLE') { + enhanceTable(node); + } + } + }); + }); + }); + + observer.observe(document.body, { childList: true, subtree: true }); + + // Enhance existing tables + document.querySelectorAll('table').forEach(enhanceTable); +}); + +// Utility functions for Rust integration +window.QuickSearch = { + // Function to show toast notifications + showToast: function(message, type = 'info') { + const toast = document.createElement('div'); + toast.className = `toast toast-${type}`; + toast.textContent = message; + toast.style.cssText = ` + position: fixed; + top: 20px; + right: 20px; + padding: 12px 20px; + border-radius: 6px; + color: white; + font-weight: 600; + z-index: 2000; + animation: slideIn 0.3s ease; + `; + + // Set background based on type + const colors = { + info: '#2196F3', + success: '#4CAF50', + warning: '#FF9800', + error: '#f44336' + }; + toast.style.backgroundColor = colors[type] || colors.info; + + document.body.appendChild(toast); + + setTimeout(() => { + toast.style.animation = 'slideOut 0.3s ease'; + setTimeout(() => { + document.body.removeChild(toast); + }, 300); + }, 3000); + }, + + // Function to update status display + updateStatus: function(status) { + const statusDisplay = document.querySelector('.status-display'); + if (statusDisplay) { + statusDisplay.textContent = status; + } + }, + + // Function to highlight search terms in results + highlightSearchTerms: function(searchTerm, container) { + if (!searchTerm || !container) return; + + const walker = document.createTreeWalker( + container, + NodeFilter.SHOW_TEXT, + null, + false + ); + + const textNodes = []; + let node; + while (node = walker.nextNode()) { + textNodes.push(node); + } + + textNodes.forEach(textNode => { + const parent = textNode.parentNode; + if (parent.tagName === 'B') return; // Skip already highlighted + + const text = textNode.textContent; + const regex = new RegExp(`(${searchTerm})`, 'gi'); + + if (regex.test(text)) { + const highlightedHTML = text.replace(regex, '$1'); + const wrapper = document.createElement('span'); + wrapper.innerHTML = highlightedHTML; + parent.replaceChild(wrapper, textNode); + } + }); + } +}; + +// Add custom CSS for toasts and animations +const style = document.createElement('style'); +style.textContent = ` + @keyframes slideIn { + from { transform: translateX(100%); opacity: 0; } + to { transform: translateX(0); opacity: 1; } + } + + @keyframes slideOut { + from { transform: translateX(0); opacity: 1; } + to { transform: translateX(100%); opacity: 0; } + } + + mark { + background: #ffeb3b; + padding: 2px 4px; + border-radius: 3px; + font-weight: bold; + } + + .toast { + box-shadow: 0 4px 12px rgba(0,0,0,0.2); + } +`; +document.head.appendChild(style); diff --git a/assets/index.html b/crates/quicksearch-gui/assets/index.html similarity index 97% rename from assets/index.html rename to crates/quicksearch-gui/assets/index.html index 5bbae84..a0f3413 100644 --- a/assets/index.html +++ b/crates/quicksearch-gui/assets/index.html @@ -1,21 +1,21 @@ - - - - - - QuickSearch - File Indexer & Search - - - - - - - - - -
- - - - - + + + + + + QuickSearch - File Indexer & Search + + + + + + + + + +
+ + + + + diff --git a/assets/styles.css b/crates/quicksearch-gui/assets/styles.css similarity index 94% rename from assets/styles.css rename to crates/quicksearch-gui/assets/styles.css index 3a68b19..ff1ca68 100644 --- a/assets/styles.css +++ b/crates/quicksearch-gui/assets/styles.css @@ -1,299 +1,299 @@ -/* QuickSearch Application Styles */ -body { - font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; - margin: 0; - padding: 20px; - background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); - min-height: 100vh; -} - -.app-container { - max-width: 1200px; - margin: 0 auto; - background: white; - border-radius: 12px; - box-shadow: 0 8px 32px rgba(0,0,0,0.1); - overflow: hidden; -} - -.app-header { - background: linear-gradient(90deg, #4CAF50 0%, #45a049 100%); - color: white; - padding: 20px; - text-align: center; -} - -.app-content { - padding: 20px; -} - -.section { - background: #f8f9fa; - border-radius: 8px; - padding: 20px; - margin-bottom: 20px; - border-left: 4px solid #4CAF50; -} - -.form-control { - width: 100%; - padding: 12px; - border: 2px solid #e0e0e0; - border-radius: 6px; - font-size: 14px; - transition: border-color 0.3s ease; - box-sizing: border-box; -} - -.form-control:focus { - outline: none; - border-color: #4CAF50; - box-shadow: 0 0 0 3px rgba(76, 175, 80, 0.1); -} - -.btn { - padding: 12px 24px; - border: none; - border-radius: 6px; - cursor: pointer; - font-size: 14px; - font-weight: 600; - text-transform: uppercase; - letter-spacing: 0.5px; - transition: all 0.3s ease; - margin-right: 10px; - display: inline-block; -} - -.btn-primary { - background: linear-gradient(90deg, #4CAF50 0%, #45a049 100%); - color: white; -} - -.btn-primary:hover:not(:disabled) { - transform: translateY(-2px); - box-shadow: 0 4px 12px rgba(76, 175, 80, 0.3); -} - -.btn-danger { - background: linear-gradient(90deg, #f44336 0%, #d32f2f 100%); - color: white; -} - -.btn-danger:hover:not(:disabled) { - transform: translateY(-2px); - box-shadow: 0 4px 12px rgba(244, 67, 54, 0.3); -} - -.btn-info { - background: linear-gradient(90deg, #2196F3 0%, #1976D2 100%); - color: white; -} - -.btn-info:hover:not(:disabled) { - transform: translateY(-2px); - box-shadow: 0 4px 12px rgba(33, 150, 243, 0.3); -} - -.btn:disabled { - opacity: 0.6; - cursor: not-allowed; - transform: none !important; - box-shadow: none !important; -} - -.status-display { - background: #1a1a1a; - color: #00ff00; - padding: 15px; - border-radius: 6px; - font-family: 'Courier New', monospace; - white-space: pre-wrap; - font-size: 13px; - border: 1px solid #333; - overflow-x: auto; -} - -.search-results { - background: white; - border-radius: 8px; - overflow: hidden; - box-shadow: 0 2px 8px rgba(0,0,0,0.1); - margin-top: 20px; -} - -.results-table { - max-height: 400px; - overflow: auto; - border: 1px solid #ddd; - border-radius: 6px; -} - -.results-table table { - width: 100%; - border-collapse: collapse; -} - -.results-table th { - background: #4CAF50; - color: white; - padding: 12px; - text-align: left; - font-weight: 600; - position: sticky; - top: 0; - border-bottom: 2px solid #45a049; -} - -.results-table td { - padding: 10px 12px; - border-bottom: 1px solid #e0e0e0; - word-break: break-all; -} - -.results-table tbody tr:hover { - background: #f5f5f5; -} - -.results-table tbody tr:nth-child(even) { - background: #fafafa; -} - -.results-table tbody tr:nth-child(even):hover { - background: #f0f0f0; -} - -/* Clickable path cells */ -.path-cell.clickable { - cursor: pointer; - color: #1976d2; - font-weight: 500; - transition: all 0.2s ease; - position: relative; -} - -.path-cell.clickable:hover { - background: #e3f2fd !important; - color: #0d47a1; - text-decoration: underline; -} - -.path-cell.clickable:active { - background: #bbdefb !important; - transform: scale(0.98); -} - - -.error-message { - background: #ffebee; - color: #c62828; - padding: 12px; - border-radius: 6px; - border-left: 4px solid #f44336; - margin: 10px 0; -} - -.modal-backdrop { - position: fixed; - top: 0; - left: 0; - width: 100%; - height: 100%; - background: rgba(0,0,0,0.5); - backdrop-filter: blur(4px); - display: flex; - align-items: center; - justify-content: center; - z-index: 1000; -} - -.modal-dialog { - background: white; - padding: 30px; - border-radius: 12px; - max-width: 600px; - box-shadow: 0 20px 60px rgba(0,0,0,0.3); - animation: modalSlideIn 0.3s ease; -} - -@keyframes modalSlideIn { - from { - opacity: 0; - transform: translateY(-20px); - } - to { - opacity: 1; - transform: translateY(0); - } -} - -h1, h2, h3 { - margin-top: 0; - color: #333; -} - -label { - display: block; - margin-bottom: 8px; - font-weight: 600; - color: #555; -} - -.form-group { - margin-bottom: 20px; -} - -/* Highlight matched text in search results */ -.results-table b { - background: #ffeb3b; - padding: 2px 4px; - border-radius: 3px; - font-weight: bold; - color: #333; -} - -/* Loading spinner */ -.loading { - display: inline-block; - width: 20px; - height: 20px; - border: 3px solid #f3f3f3; - border-top: 3px solid #4CAF50; - border-radius: 50%; - animation: spin 1s linear infinite; - margin-right: 10px; -} - -@keyframes spin { - 0% { transform: rotate(0deg); } - 100% { transform: rotate(360deg); } -} - -/* Responsive design */ -@media (max-width: 768px) { - body { - padding: 10px; - } - - .app-container { - border-radius: 8px; - } - - .app-content { - padding: 15px; - } - - .form-control { - font-size: 16px; /* Prevents zoom on iOS */ - } - - .results-table { - font-size: 12px; - } - - .btn { - width: 100%; - margin-bottom: 10px; - margin-right: 0; - } -} +/* QuickSearch Application Styles */ +body { + font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; + margin: 0; + padding: 20px; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + min-height: 100vh; +} + +.app-container { + max-width: 1200px; + margin: 0 auto; + background: white; + border-radius: 12px; + box-shadow: 0 8px 32px rgba(0,0,0,0.1); + overflow: hidden; +} + +.app-header { + background: linear-gradient(90deg, #4CAF50 0%, #45a049 100%); + color: white; + padding: 20px; + text-align: center; +} + +.app-content { + padding: 20px; +} + +.section { + background: #f8f9fa; + border-radius: 8px; + padding: 20px; + margin-bottom: 20px; + border-left: 4px solid #4CAF50; +} + +.form-control { + width: 100%; + padding: 12px; + border: 2px solid #e0e0e0; + border-radius: 6px; + font-size: 14px; + transition: border-color 0.3s ease; + box-sizing: border-box; +} + +.form-control:focus { + outline: none; + border-color: #4CAF50; + box-shadow: 0 0 0 3px rgba(76, 175, 80, 0.1); +} + +.btn { + padding: 12px 24px; + border: none; + border-radius: 6px; + cursor: pointer; + font-size: 14px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + transition: all 0.3s ease; + margin-right: 10px; + display: inline-block; +} + +.btn-primary { + background: linear-gradient(90deg, #4CAF50 0%, #45a049 100%); + color: white; +} + +.btn-primary:hover:not(:disabled) { + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(76, 175, 80, 0.3); +} + +.btn-danger { + background: linear-gradient(90deg, #f44336 0%, #d32f2f 100%); + color: white; +} + +.btn-danger:hover:not(:disabled) { + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(244, 67, 54, 0.3); +} + +.btn-info { + background: linear-gradient(90deg, #2196F3 0%, #1976D2 100%); + color: white; +} + +.btn-info:hover:not(:disabled) { + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(33, 150, 243, 0.3); +} + +.btn:disabled { + opacity: 0.6; + cursor: not-allowed; + transform: none !important; + box-shadow: none !important; +} + +.status-display { + background: #1a1a1a; + color: #00ff00; + padding: 15px; + border-radius: 6px; + font-family: 'Courier New', monospace; + white-space: pre-wrap; + font-size: 13px; + border: 1px solid #333; + overflow-x: auto; +} + +.search-results { + background: white; + border-radius: 8px; + overflow: hidden; + box-shadow: 0 2px 8px rgba(0,0,0,0.1); + margin-top: 20px; +} + +.results-table { + max-height: 400px; + overflow: auto; + border: 1px solid #ddd; + border-radius: 6px; +} + +.results-table table { + width: 100%; + border-collapse: collapse; +} + +.results-table th { + background: #4CAF50; + color: white; + padding: 12px; + text-align: left; + font-weight: 600; + position: sticky; + top: 0; + border-bottom: 2px solid #45a049; +} + +.results-table td { + padding: 10px 12px; + border-bottom: 1px solid #e0e0e0; + word-break: break-all; +} + +.results-table tbody tr:hover { + background: #f5f5f5; +} + +.results-table tbody tr:nth-child(even) { + background: #fafafa; +} + +.results-table tbody tr:nth-child(even):hover { + background: #f0f0f0; +} + +/* Clickable path cells */ +.path-cell.clickable { + cursor: pointer; + color: #1976d2; + font-weight: 500; + transition: all 0.2s ease; + position: relative; +} + +.path-cell.clickable:hover { + background: #e3f2fd !important; + color: #0d47a1; + text-decoration: underline; +} + +.path-cell.clickable:active { + background: #bbdefb !important; + transform: scale(0.98); +} + + +.error-message { + background: #ffebee; + color: #c62828; + padding: 12px; + border-radius: 6px; + border-left: 4px solid #f44336; + margin: 10px 0; +} + +.modal-backdrop { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(0,0,0,0.5); + backdrop-filter: blur(4px); + display: flex; + align-items: center; + justify-content: center; + z-index: 1000; +} + +.modal-dialog { + background: white; + padding: 30px; + border-radius: 12px; + max-width: 600px; + box-shadow: 0 20px 60px rgba(0,0,0,0.3); + animation: modalSlideIn 0.3s ease; +} + +@keyframes modalSlideIn { + from { + opacity: 0; + transform: translateY(-20px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +h1, h2, h3 { + margin-top: 0; + color: #333; +} + +label { + display: block; + margin-bottom: 8px; + font-weight: 600; + color: #555; +} + +.form-group { + margin-bottom: 20px; +} + +/* Highlight matched text in search results */ +.results-table b { + background: #ffeb3b; + padding: 2px 4px; + border-radius: 3px; + font-weight: bold; + color: #333; +} + +/* Loading spinner */ +.loading { + display: inline-block; + width: 20px; + height: 20px; + border: 3px solid #f3f3f3; + border-top: 3px solid #4CAF50; + border-radius: 50%; + animation: spin 1s linear infinite; + margin-right: 10px; +} + +@keyframes spin { + 0% { transform: rotate(0deg); } + 100% { transform: rotate(360deg); } +} + +/* Responsive design */ +@media (max-width: 768px) { + body { + padding: 10px; + } + + .app-container { + border-radius: 8px; + } + + .app-content { + padding: 15px; + } + + .form-control { + font-size: 16px; /* Prevents zoom on iOS */ + } + + .results-table { + font-size: 12px; + } + + .btn { + width: 100%; + margin-bottom: 10px; + margin-right: 0; + } +} diff --git a/src/frontend.rs b/crates/quicksearch-gui/src/frontend.rs similarity index 83% rename from src/frontend.rs rename to crates/quicksearch-gui/src/frontend.rs index c258fa4..b3c0394 100644 --- a/src/frontend.rs +++ b/crates/quicksearch-gui/src/frontend.rs @@ -1,368 +1,385 @@ -#![allow(non_snake_case)] - -use std::sync::Arc; -use std::collections::VecDeque; -use std::time::Instant; -use dioxus::prelude::*; -use crate::indexing::{IndexingService, IndexingStatus}; -use crate::config::Config; - -#[derive(Debug, Clone)] -struct SpeedDataPoint { - timestamp: Instant, - files_processed: usize, -} - -struct SpeedTracker { - data_points: VecDeque, -} - -impl SpeedTracker { - fn new() -> Self { - Self { - data_points: VecDeque::new(), - } - } - - fn add_data_point(&mut self, files_processed: usize) { - let now = Instant::now(); - self.data_points.push_back(SpeedDataPoint { - timestamp: now, - files_processed, - }); - - // Prune data points older than 1 second - while let Some(front) = self.data_points.front() { - if now.duration_since(front.timestamp).as_secs_f64() > 1.0 { - self.data_points.pop_front(); - } else { - break; - } - } - } - - fn calculate_files_per_second(&self) -> Option { - if self.data_points.len() < 2 { - return None; - } - - let newest = self.data_points.back()?; - let oldest = self.data_points.front()?; - - let time_span = newest.timestamp.duration_since(oldest.timestamp).as_secs_f64(); - if time_span < 0.1 { // Avoid division by very small numbers - return None; - } - - let files_diff = newest.files_processed.saturating_sub(oldest.files_processed); - Some(files_diff as f64 / time_span) - } -} - -#[derive(Props, Clone)] -pub struct AppProps { - pub indexing_service: Arc, - pub config: Config, -} - -impl PartialEq for AppProps { - fn eq(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.indexing_service, &other.indexing_service) && self.config.paths.default_indexing_path == other.config.paths.default_indexing_path && self.config.paths.database_path == other.config.paths.database_path - } -} - -pub fn App(props: AppProps) -> Element { - let mut indexing_path = use_signal(|| props.config.paths.default_indexing_path.clone()); - let mut db_path = use_signal(|| props.config.paths.database_path.clone()); - let mut status_text = use_signal(|| "Idle".to_string()); - let mut show_config_dialog = use_signal(|| false); - let mut config_changes = use_signal(|| Vec::::new()); - let speed_tracker = use_signal(|| SpeedTracker::new()); - - - let indexing_service_for_start = props.indexing_service.clone(); - let indexing_service_for_start_dialog = props.indexing_service.clone(); - let indexing_service_for_stop = props.indexing_service.clone(); - let indexing_service_for_timer = props.indexing_service.clone(); - let config_for_start = props.config.clone(); - let config_for_dialog = props.config.clone(); - - - // Automatic status updates every second - { - let mut status_text_clone = status_text.clone(); - let mut speed_tracker_clone = speed_tracker.clone(); - let service_clone = indexing_service_for_timer.clone(); - use_future(move || { - let service = service_clone.clone(); - async move { - loop { - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - - let status = service.get_status(); - let status_str = match status { - IndexingStatus::Idle => { - // Reset speed tracker when idle - speed_tracker_clone.set(SpeedTracker::new()); - "Idle".to_string() - }, - IndexingStatus::CountingFiles { - current_file, - start_time, - .. - } => { - let elapsed = start_time.elapsed(); - let current_file_display = current_file - .as_ref() - .map(|f| format!("{}", f)) - .unwrap_or_else(|| "...".to_string()); - format!( - "Phase 0 - Counting paths (shell) - {:.1}s elapsed\n{}", - elapsed.as_secs_f64(), - current_file_display - ) - } - IndexingStatus::RunningFileIndex { files_processed, total_files, current_file, start_time } => { - // Add data point to speed tracker - speed_tracker_clone.with_mut(|tracker| { - tracker.add_data_point(files_processed); - }); - - let elapsed = start_time.elapsed(); - let current_file_display = current_file - .as_ref() - .map(|f| format!("Current: {}", f)) - .unwrap_or_default(); - - // Calculate speed - let speed_display = speed_tracker_clone.with(|tracker| { - tracker.calculate_files_per_second() - .map(|fps| format!(" - {:.1} files/sec", fps)) - .unwrap_or_default() - }); - - if let Some(total) = total_files { - let percentage = if total > 0 { - (files_processed as f64 / total as f64 * 100.0) as u32 - } else { 0 }; - format!( - "Phase 1 - File Index: {}/{} files ({}%) - {:.1}s elapsed{}\n{}", - files_processed, - total, - percentage, - elapsed.as_secs_f64(), - speed_display, - current_file_display - ) - } else { - format!( - "Phase 1 - File Index: {} files processed - {:.1}s elapsed{}\n{}", - files_processed, - elapsed.as_secs_f64(), - speed_display, - current_file_display - ) - } - } - IndexingStatus::RunningTextIndex { files_processed, current_file, start_time } => { - // Add data point to speed tracker - speed_tracker_clone.with_mut(|tracker| { - tracker.add_data_point(files_processed); - }); - - let elapsed = start_time.elapsed(); - let current_file_display = current_file - .as_ref() - .map(|f| format!("Current: {}", f)) - .unwrap_or_default(); - - // Calculate speed - let speed_display = speed_tracker_clone.with(|tracker| { - tracker.calculate_files_per_second() - .map(|fps| format!(" - {:.1} files/sec", fps)) - .unwrap_or_default() - }); - format!( - "Phase 2 - Text Index: {} files processed - {:.1}s elapsed{}\n{}", - files_processed, - elapsed.as_secs_f64(), - speed_display, - current_file_display - ) - } - IndexingStatus::Stopping => "Indexing Stopped".to_string(), - IndexingStatus::Error(ref e) => format!("Error: {}", e), - }; - status_text_clone.set(status_str); - } - } - }); - } - - rsx! { - div { - class: "app-container", - - div { - class: "app-header", - h1 { "QuickSearch File Indexer" } - } - - div { - class: "app-content", - - div { - class: "section", - h2 { "Indexing Controls" } - - div { - class: "form-group", - label { "Path to index:" } - input { - class: "form-control", - r#type: "text", - value: "{indexing_path}", - oninput: move |evt| indexing_path.set(evt.value()) - } - } - - div { - class: "form-group", - label { "Database path:" } - input { - class: "form-control", - r#type: "text", - value: "{db_path}", - oninput: move |evt| db_path.set(evt.value()) - } - } - - div { - class: "form-group", - button { - class: "btn btn-primary", - onclick: move |_| { - let service = indexing_service_for_start.clone(); - let config = config_for_start.clone(); - let path = indexing_path().clone(); - let db = db_path().clone(); - - // Check for configuration validation - match service.check_config_validation(&db, &config, &path) { - Ok(Some(changes)) => { - // Configuration changes detected, show dialog - config_changes.set(changes); - show_config_dialog.set(true); - } - Ok(None) => { - // No configuration issues, start indexing - let _ = service.start_indexing(path, db, config); - } - Err(e) => { - status_text.set(format!("Configuration validation error: {}", e)); - } - } - }, - "Start Indexing" - } - button { - class: "btn btn-danger", - onclick: move |_| { - let _ = indexing_service_for_stop.stop_indexing(); - }, - "Stop Indexing" - } - } - } - - div { - class: "section", - h2 { "Status" } - pre { - class: "status-display", - "{status_text}" - } - } - - crate::search::Search { - indexing_service: props.indexing_service.clone(), - db_path: db_path().clone() - } - - } // Close app-content - } - - // Configuration validation dialog - if show_config_dialog() { - div { - class: "modal-backdrop", - div { - class: "modal-dialog", - h3 { - style: "margin-top: 0; color: #d32f2f;", - "⚠️ Configuration Changes Detected" - } - p { - style: "margin: 15px 0;", - "The following configuration changes require deleting and rebuilding the search index:" - } - ul { - style: "margin: 15px 0; padding-left: 20px;", - for change in config_changes().iter() { - li { - style: "margin: 5px 0; font-family: monospace; background-color: #f5f5f5; padding: 5px; border-radius: 3px;", - "{change}" - } - } - } - p { - style: "margin: 15px 0; font-weight: bold;", - "This will delete the existing index and rebuild it from scratch." - } - div { - style: "display: flex; gap: 10px; margin-top: 20px;", - button { - style: "padding: 10px 20px; background-color: #d32f2f; color: white; border: none; border-radius: 5px; cursor: pointer;", - onclick: move |_| { - let service = indexing_service_for_start_dialog.clone(); - let config = config_for_dialog.clone(); - let path = indexing_path().clone(); - let db = db_path().clone(); - - show_config_dialog.set(false); - status_text.set("Stopping indexing and deleting database...".to_string()); - - // Delete database file and restart indexing - let service_clone = service.clone(); - let path_clone = path.clone(); - let db_clone = db.clone(); - let config_clone = config.clone(); - let mut status_clone = status_text.clone(); - - spawn(async move { - match service_clone.delete_index_for_rebuild(&db_clone) { - Ok(()) => { - status_clone.set("Database deleted. Starting fresh indexing...".to_string()); - let _ = service_clone.start_indexing(path_clone, db_clone, config_clone); - } - Err(e) => { - status_clone.set(format!("Error deleting database: {}", e)); - } - } - }); - }, - "Yes, Rebuild Index" - } - button { - style: "padding: 10px 20px; background-color: #666; color: white; border: none; border-radius: 5px; cursor: pointer;", - onclick: move |_| { - show_config_dialog.set(false); - }, - "Cancel" - } - } - } - } - } - } +#![allow(non_snake_case)] + +use std::sync::Arc; +use std::collections::VecDeque; +use std::time::Instant; +use dioxus::prelude::*; +use quicksearch_core::indexing::{IndexingService, IndexingStatus}; +use quicksearch_core::config::Config; + +#[derive(Debug, Clone)] +struct SpeedDataPoint { + timestamp: Instant, + files_processed: usize, +} + +struct SpeedTracker { + data_points: VecDeque, +} + +impl SpeedTracker { + fn new() -> Self { + Self { + data_points: VecDeque::new(), + } + } + + fn add_data_point(&mut self, files_processed: usize) { + let now = Instant::now(); + self.data_points.push_back(SpeedDataPoint { + timestamp: now, + files_processed, + }); + + // Prune data points older than 1 second + while let Some(front) = self.data_points.front() { + if now.duration_since(front.timestamp).as_secs_f64() > 1.0 { + self.data_points.pop_front(); + } else { + break; + } + } + } + + fn calculate_files_per_second(&self) -> Option { + if self.data_points.len() < 2 { + return None; + } + + let newest = self.data_points.back()?; + let oldest = self.data_points.front()?; + + let time_span = newest.timestamp.duration_since(oldest.timestamp).as_secs_f64(); + if time_span < 0.1 { // Avoid division by very small numbers + return None; + } + + let files_diff = newest.files_processed.saturating_sub(oldest.files_processed); + Some(files_diff as f64 / time_span) + } +} + +#[derive(Props, Clone)] +pub struct AppProps { + pub indexing_service: Arc, + pub config: Config, +} + +impl PartialEq for AppProps { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.indexing_service, &other.indexing_service) + && self.config.paths.indexing_paths == other.config.paths.indexing_paths + && self.config.paths.database_path == other.config.paths.database_path + } +} + +pub fn App(props: AppProps) -> Element { + // Multi-root support: the GUI's single line edit holds one path per line. + // Empty lines are ignored. Core still de-duplicates and handles nested roots. + let mut indexing_path = use_signal(|| props.config.paths.indexing_paths.join("\n")); + let mut db_path = use_signal(|| props.config.paths.database_path.clone()); + let mut status_text = use_signal(|| "Idle".to_string()); + let mut show_config_dialog = use_signal(|| false); + let mut config_changes = use_signal(|| Vec::::new()); + let speed_tracker = use_signal(|| SpeedTracker::new()); + + + let indexing_service_for_start = props.indexing_service.clone(); + let indexing_service_for_start_dialog = props.indexing_service.clone(); + let indexing_service_for_stop = props.indexing_service.clone(); + let indexing_service_for_timer = props.indexing_service.clone(); + let config_for_start = props.config.clone(); + let config_for_dialog = props.config.clone(); + + + // Automatic status updates every second + { + let mut status_text_clone = status_text.clone(); + let mut speed_tracker_clone = speed_tracker.clone(); + let service_clone = indexing_service_for_timer.clone(); + use_future(move || { + let service = service_clone.clone(); + async move { + loop { + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + + let status = service.get_status(); + let status_str = match status { + IndexingStatus::Idle => { + // Reset speed tracker when idle + speed_tracker_clone.set(SpeedTracker::new()); + "Idle".to_string() + }, + IndexingStatus::CountingFiles { + current_file, + start_time, + .. + } => { + let elapsed = start_time.elapsed(); + let current_file_display = current_file + .as_ref() + .map(|f| format!("{}", f)) + .unwrap_or_else(|| "...".to_string()); + format!( + "Phase 0 - Counting paths (shell) - {:.1}s elapsed\n{}", + elapsed.as_secs_f64(), + current_file_display + ) + } + IndexingStatus::RunningFileIndex { files_processed, total_files, current_file, start_time } => { + // Add data point to speed tracker + speed_tracker_clone.with_mut(|tracker| { + tracker.add_data_point(files_processed); + }); + + let elapsed = start_time.elapsed(); + let current_file_display = current_file + .as_ref() + .map(|f| format!("Current: {}", f)) + .unwrap_or_default(); + + // Calculate speed + let speed_display = speed_tracker_clone.with(|tracker| { + tracker.calculate_files_per_second() + .map(|fps| format!(" - {:.1} files/sec", fps)) + .unwrap_or_default() + }); + + if let Some(total) = total_files { + let percentage = if total > 0 { + (files_processed as f64 / total as f64 * 100.0) as u32 + } else { 0 }; + format!( + "Phase 1 - File Index: {}/{} files ({}%) - {:.1}s elapsed{}\n{}", + files_processed, + total, + percentage, + elapsed.as_secs_f64(), + speed_display, + current_file_display + ) + } else { + format!( + "Phase 1 - File Index: {} files processed - {:.1}s elapsed{}\n{}", + files_processed, + elapsed.as_secs_f64(), + speed_display, + current_file_display + ) + } + } + IndexingStatus::RunningTextIndex { files_processed, current_file, start_time } => { + // Add data point to speed tracker + speed_tracker_clone.with_mut(|tracker| { + tracker.add_data_point(files_processed); + }); + + let elapsed = start_time.elapsed(); + let current_file_display = current_file + .as_ref() + .map(|f| format!("Current: {}", f)) + .unwrap_or_default(); + + // Calculate speed + let speed_display = speed_tracker_clone.with(|tracker| { + tracker.calculate_files_per_second() + .map(|fps| format!(" - {:.1} files/sec", fps)) + .unwrap_or_default() + }); + format!( + "Phase 2 - Text Index: {} files processed - {:.1}s elapsed{}\n{}", + files_processed, + elapsed.as_secs_f64(), + speed_display, + current_file_display + ) + } + IndexingStatus::Stopping => "Indexing Stopped".to_string(), + IndexingStatus::Error(ref e) => format!("Error: {}", e), + }; + status_text_clone.set(status_str); + } + } + }); + } + + rsx! { + div { + class: "app-container", + + div { + class: "app-header", + h1 { "QuickSearch File Indexer" } + } + + div { + class: "app-content", + + div { + class: "section", + h2 { "Indexing Controls" } + + div { + class: "form-group", + label { "Paths to index (one per line):" } + textarea { + class: "form-control", + rows: "3", + value: "{indexing_path}", + oninput: move |evt| indexing_path.set(evt.value()) + } + } + + div { + class: "form-group", + label { "Database path:" } + input { + class: "form-control", + r#type: "text", + value: "{db_path}", + oninput: move |evt| db_path.set(evt.value()) + } + } + + div { + class: "form-group", + button { + class: "btn btn-primary", + onclick: move |_| { + let service = indexing_service_for_start.clone(); + let config = config_for_start.clone(); + let path_text = indexing_path().clone(); + let paths: Vec = path_text + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + let db = db_path().clone(); + + if paths.is_empty() { + status_text.set("Enter at least one path to index (one per line).".to_string()); + } else { + // The config_validation key still stores a single joined string. + let joined = paths.join("\n"); + match service.check_config_validation(&db, &config, &joined) { + Ok(Some(changes)) => { + config_changes.set(changes); + show_config_dialog.set(true); + } + Ok(None) => { + let _ = service.start_indexing(paths, db, config); + } + Err(e) => { + status_text.set(format!("Configuration validation error: {}", e)); + } + } + } + }, + "Start Indexing" + } + button { + class: "btn btn-danger", + onclick: move |_| { + let _ = indexing_service_for_stop.stop_indexing(); + }, + "Stop Indexing" + } + } + } + + div { + class: "section", + h2 { "Status" } + pre { + class: "status-display", + "{status_text}" + } + } + + crate::search::Search { + indexing_service: props.indexing_service.clone(), + db_path: db_path().clone() + } + + } // Close app-content + } + + // Configuration validation dialog + if show_config_dialog() { + div { + class: "modal-backdrop", + div { + class: "modal-dialog", + h3 { + style: "margin-top: 0; color: #d32f2f;", + "⚠️ Configuration Changes Detected" + } + p { + style: "margin: 15px 0;", + "The following configuration changes require deleting and rebuilding the search index:" + } + ul { + style: "margin: 15px 0; padding-left: 20px;", + for change in config_changes().iter() { + li { + style: "margin: 5px 0; font-family: monospace; background-color: #f5f5f5; padding: 5px; border-radius: 3px;", + "{change}" + } + } + } + p { + style: "margin: 15px 0; font-weight: bold;", + "This will delete the existing index and rebuild it from scratch." + } + div { + style: "display: flex; gap: 10px; margin-top: 20px;", + button { + style: "padding: 10px 20px; background-color: #d32f2f; color: white; border: none; border-radius: 5px; cursor: pointer;", + onclick: move |_| { + let service = indexing_service_for_start_dialog.clone(); + let config = config_for_dialog.clone(); + let path_text = indexing_path().clone(); + let paths: Vec = path_text + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + let db = db_path().clone(); + + show_config_dialog.set(false); + status_text.set("Stopping indexing and deleting database...".to_string()); + + // Delete database file and restart indexing + let service_clone = service.clone(); + let paths_clone = paths.clone(); + let db_clone = db.clone(); + let config_clone = config.clone(); + let mut status_clone = status_text.clone(); + + spawn(async move { + match service_clone.delete_index_for_rebuild(&db_clone) { + Ok(()) => { + status_clone.set("Database deleted. Starting fresh indexing...".to_string()); + let _ = service_clone.start_indexing(paths_clone, db_clone, config_clone); + } + Err(e) => { + status_clone.set(format!("Error deleting database: {}", e)); + } + } + }); + }, + "Yes, Rebuild Index" + } + button { + style: "padding: 10px 20px; background-color: #666; color: white; border: none; border-radius: 5px; cursor: pointer;", + onclick: move |_| { + show_config_dialog.set(false); + }, + "Cancel" + } + } + } + } + } + } } \ No newline at end of file diff --git a/crates/quicksearch-gui/src/main.rs b/crates/quicksearch-gui/src/main.rs new file mode 100644 index 0000000..6756fd1 --- /dev/null +++ b/crates/quicksearch-gui/src/main.rs @@ -0,0 +1,64 @@ +use std::sync::{Arc, OnceLock}; +use dioxus::prelude::*; +use quicksearch_core::{config, indexing, shutdown}; +mod frontend; +mod search; + +static INDEXING_SERVICE: OnceLock> = OnceLock::new(); + +fn main() { + let indexing_service = Arc::new(indexing::IndexingService::new()); + INDEXING_SERVICE + .set(indexing_service.clone()) + .expect("Failed to set global indexing service"); + + if let Err(e) = shutdown::install_signal_handler(indexing_service.clone()) { + eprintln!("Warning: failed to install signal handler: {}", e); + } + + LaunchBuilder::desktop() + .with_cfg( + dioxus_desktop::Config::new() + .with_custom_head(format!("", include_str!("../assets/styles.css"))) + .with_window(dioxus_desktop::WindowBuilder::new() + .with_title("QuickSearch - File Indexer & Search") + .with_resizable(true) + .with_inner_size(dioxus_desktop::LogicalSize::new(1000.0, 700.0)) + ) + ) + .launch(app); +} + + +fn app() -> Element { + let cfg = match config::Config::load() { + Ok(c) => c, + Err(e) => { + eprintln!("Failed to load config: {}", e); + return rsx! { div { "Failed to load configuration" } }; + } + }; + + let indexing_service = INDEXING_SERVICE + .get() + .expect("Indexing service not initialized") + .clone(); + + rsx! { + frontend::App { + indexing_service: indexing_service, + config: cfg + } + } +} + +/* +Duplicate files: +SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC; + +Full text search: +SELECT f.name, f.path, snippet(searchabletext, 1, "", "", "...", 64) as "snip" FROM searchabletext AS st JOIN files f ON f.id = st.rowid WHERE st.text MATCH 'searchstring'; + +Filename search: +SELECT name, path FROM files WHERE name LIKE '%searchstring%'; +*/ diff --git a/crates/quicksearch-gui/src/search.rs b/crates/quicksearch-gui/src/search.rs new file mode 100644 index 0000000..c2c2ead --- /dev/null +++ b/crates/quicksearch-gui/src/search.rs @@ -0,0 +1,512 @@ +#![allow(non_snake_case)] + +use std::sync::Arc; +use std::time::Instant; +use dioxus::prelude::*; +use quicksearch_core::indexing::{IndexingService, SearchResult}; +use quicksearch_core::search_sql::{build_count, build_select, SearchArgs}; + +/// One page of results. Tuned to keep DOM size bounded — rendering ten +/// thousand `` nodes wedges WebKit for tens of seconds. +const PAGE_SIZE: u32 = 50; + +#[derive(Props, Clone)] +pub struct SearchProps { + pub indexing_service: Arc, + pub db_path: String, +} + +impl PartialEq for SearchProps { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.indexing_service, &other.indexing_service) && self.db_path == other.db_path + } +} + +pub fn Search(props: SearchProps) -> Element { + let mut search_type = use_signal(|| "fulltext".to_string()); + let mut search_term = use_signal(|| String::new()); + let mut fulltext_exact = use_signal(|| false); + let mut fulltext_case_sensitive = use_signal(|| false); + let mut search_results = use_signal(|| Vec::::new()); + let mut search_error = use_signal(|| None::); + let mut is_searching = use_signal(|| false); + let mut last_search_time = use_signal(|| None::); + let mut show_corruption_dialog = use_signal(|| false); + let mut current_page = use_signal(|| 1u32); + let mut total_count = use_signal(|| None::); + let mut last_args = use_signal(|| None::); + let mut goto_input = use_signal(|| String::new()); + + let service = props.indexing_service.clone(); + let db_path = props.db_path.clone(); + + // Spawn a search task. `refresh_count` is true for fresh searches and + // false for in-place page navigation (the cached total still applies). + let run_query = { + let service = service.clone(); + let db_path = db_path.clone(); + move |args: SearchArgs, page: u32, refresh_count: bool| { + let service = service.clone(); + let db_path = db_path.clone(); + spawn(async move { + is_searching.set(true); + search_error.set(None); + last_search_time.set(None); + let start = Instant::now(); + + let count_sql = if refresh_count { + match build_count(&args) { + Ok(s) => Some(s), + Err(e) => { + search_error.set(Some(e)); + is_searching.set(false); + return; + } + } + } else { + None + }; + + let offset = page.saturating_sub(1).saturating_mul(PAGE_SIZE); + let select_sql = match build_select(&args, PAGE_SIZE, offset) { + Ok(s) => s, + Err(e) => { + search_error.set(Some(e)); + is_searching.set(false); + return; + } + }; + + // Run select + (optional) count in parallel on the blocking pool. + let svc1 = service.clone(); + let db1 = db_path.clone(); + let select_handle = tokio::task::spawn_blocking(move || { + svc1.execute_search(&db1, &select_sql) + }); + + let count_handle = count_sql.map(|sql| { + let svc2 = service.clone(); + let db2 = db_path.clone(); + tokio::task::spawn_blocking(move || svc2.execute_search(&db2, &sql)) + }); + + let select_run = select_handle.await; + let count_run = match count_handle { + Some(h) => Some(h.await), + None => None, + }; + + let elapsed = start.elapsed().as_secs_f64(); + + if let Some(c) = count_run { + match c { + Ok(Ok(rs)) => { + let n = rs + .first() + .and_then(|r| r.rows.first()) + .and_then(|r| r.values.first()) + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + total_count.set(Some(n)); + } + Ok(Err(e)) => { + handle_query_error( + e, + elapsed, + search_error, + show_corruption_dialog, + last_search_time, + is_searching, + ); + return; + } + Err(e) => { + search_error.set(Some(format!("Task execution error: {}", e))); + last_search_time.set(Some(elapsed)); + is_searching.set(false); + return; + } + } + } + + match select_run { + Ok(Ok(rs)) => { + search_results.set(rs); + current_page.set(page); + last_args.set(Some(args)); + last_search_time.set(Some(elapsed)); + } + Ok(Err(e)) => { + handle_query_error( + e, + elapsed, + search_error, + show_corruption_dialog, + last_search_time, + is_searching, + ); + return; + } + Err(e) => { + search_error.set(Some(format!("Task execution error: {}", e))); + last_search_time.set(Some(elapsed)); + } + } + is_searching.set(false); + }); + } + }; + + let do_fresh_search = { + let run_query = run_query.clone(); + move || { + let args = SearchArgs { + search_type: search_type(), + term: search_term(), + fulltext_exact: fulltext_exact(), + fulltext_case_sensitive: fulltext_case_sensitive(), + }; + run_query(args, 1, true); + } + }; + + let do_goto_page = { + let run_query = run_query.clone(); + move |target: u32| { + if let Some(args) = last_args() { + let total = total_count() + .map(|n| pages_for(n)) + .unwrap_or(1) + .max(1); + let clamped = target.clamp(1, total); + if clamped != current_page() { + run_query(args, clamped, false); + } + } + } + }; + + let total_pages = total_count().map(pages_for).unwrap_or(0); + + rsx! { + div { + class: "section", + h2 { "Search Database" } + + div { + class: "form-group", + label { "Search Type: " } + select { + class: "form-control", + value: "{search_type}", + onchange: move |evt| search_type.set(evt.value()), + option { value: "fulltext", "Full Text Search" } + option { value: "filename", "Filename Search" } + option { value: "duplicates", "Find Duplicate Files" } + } + } + + if search_type() == "fulltext" { + div { + class: "form-group", + style: "display: flex; flex-direction: column; gap: 6px;", + span { style: "font-weight: 600;", "Full text options" } + label { + style: "display: flex; align-items: center; gap: 8px; cursor: pointer;", + input { + r#type: "checkbox", + checked: fulltext_exact(), + onchange: move |evt| fulltext_exact.set(evt.checked()), + } + "Exact phrase match" + } + label { + style: "display: flex; align-items: center; gap: 8px; cursor: pointer;", + input { + r#type: "checkbox", + checked: fulltext_case_sensitive(), + onchange: move |evt| fulltext_case_sensitive.set(evt.checked()), + } + "Case-sensitive match" + } + } + } + + if search_type() != "duplicates" { + div { + class: "form-group", + label { "Search Term: " } + input { + class: "form-control", + r#type: "text", + value: "{search_term}", + oninput: move |evt| search_term.set(evt.value()), + onkeydown: { + let do_fresh_search = do_fresh_search.clone(); + move |evt| { + if evt.code() == dioxus::events::Code::Enter { + do_fresh_search(); + } + } + } + } + } + } + + div { + style: "display: flex; align-items: center; gap: 10px;", + button { + class: "btn btn-info", + disabled: is_searching(), + onclick: { + let do_fresh_search = do_fresh_search.clone(); + move |_| { do_fresh_search(); } + }, + "Search" + } + + if is_searching() { + div { + class: "loading", + title: "Searching..." + } + } else if let Some(elapsed) = last_search_time() { + span { + style: "color: #666; font-size: 0.9em;", + "Search completed in {elapsed:.3}s" + } + } + } + + if let Some(error) = search_error() { + div { + class: "error-message", + "Error: {error}" + } + } + + // Results panel: pagination header + bounded table. Only renders + // when at least one search has completed (last_args is Some). + if last_args().is_some() { + div { + class: "search-results", + { + let total_str = match total_count() { + Some(n) => format!("{}", n), + None => "?".to_string(), + }; + let page_first = (current_page().saturating_sub(1) as u64) * PAGE_SIZE as u64 + 1; + let page_last_calc = page_first + search_results().first().map(|r| r.rows.len() as u64).unwrap_or(0).saturating_sub(1); + let header = if total_count() == Some(0) { + "No results.".to_string() + } else { + format!( + "Showing {}-{} of {} (page {} of {})", + page_first, + page_last_calc, + total_str, + current_page(), + total_pages + ) + }; + rsx! { h3 { "{header}" } } + } + + // Pagination controls. Hidden if there's only one page. + if total_pages > 1 { + div { + style: "display: flex; align-items: center; gap: 8px; margin: 8px 0;", + button { + class: "btn", + disabled: is_searching() || current_page() <= 1, + onclick: { + let do_goto_page = do_goto_page.clone(); + move |_| do_goto_page(1) + }, + "« First" + } + button { + class: "btn", + disabled: is_searching() || current_page() <= 1, + onclick: { + let do_goto_page = do_goto_page.clone(); + move |_| do_goto_page(current_page().saturating_sub(1)) + }, + "‹ Prev" + } + button { + class: "btn", + disabled: is_searching() || current_page() >= total_pages, + onclick: { + let do_goto_page = do_goto_page.clone(); + move |_| do_goto_page(current_page().saturating_add(1)) + }, + "Next ›" + } + button { + class: "btn", + disabled: is_searching() || current_page() >= total_pages, + onclick: { + let do_goto_page = do_goto_page.clone(); + move |_| do_goto_page(total_pages) + }, + "Last »" + } + span { "Go to:" } + input { + r#type: "number", + style: "width: 70px;", + value: "{goto_input}", + oninput: move |evt| goto_input.set(evt.value()), + onkeydown: { + let do_goto_page = do_goto_page.clone(); + move |evt| { + if evt.code() == dioxus::events::Code::Enter { + if let Ok(p) = goto_input().trim().parse::() { + do_goto_page(p); + goto_input.set(String::new()); + } + } + } + } + } + } + } + + if !search_results().is_empty() && !search_results()[0].rows.is_empty() { + div { + class: "results-table", + table { + thead { + tr { + for column in search_results()[0].columns.iter() { + th { "{column}" } + } + } + } + tbody { + for row in search_results()[0].rows.iter() { + tr { + for (col_index, value) in row.values.iter().enumerate() { + if search_results()[0].columns.get(col_index).map(|s| s.as_str()) == Some("path") { + { + let value_owned = value.clone(); + let service_owned = props.indexing_service.clone(); + rsx! { + td { + class: "path-cell clickable", + onclick: move |_| { + let path = value_owned.clone(); + let service_clone = service_owned.clone(); + spawn(async move { + if let Err(e) = service_clone.open_file_explorer(&path) { + eprintln!("Failed to open file explorer: {}", e); + } + }); + }, + title: "Click to open in file explorer", + dangerous_inner_html: "{value}" + } + } + } + } else { + td { + dangerous_inner_html: "{value}" + } + } + } + } + } + } + } + } + } + } + } + + if show_corruption_dialog() { + div { + class: "modal-backdrop", + div { + class: "modal-dialog", + h3 { + style: "margin-top: 0; color: #d32f2f;", + "⚠️ Database Corruption Detected" + } + p { + style: "margin: 15px 0;", + "The database appears to be corrupted or malformed. This can happen due to unexpected shutdowns or disk issues." + } + p { + style: "margin: 15px 0; font-weight: bold;", + "Would you like to delete the corrupted database and create a new one? This will require re-indexing your files." + } + div { + style: "display: flex; gap: 10px; margin-top: 20px;", + button { + style: "padding: 10px 20px; background-color: #d32f2f; color: white; border: none; border-radius: 5px; cursor: pointer;", + onclick: move |_| { + let service = props.indexing_service.clone(); + let db = props.db_path.clone(); + + show_corruption_dialog.set(false); + search_error.set(Some("Deleting corrupted database...".to_string())); + + spawn(async move { + match service.delete_index_for_rebuild(&db) { + Ok(()) => { + search_error.set(Some("Database deleted. You can now start indexing again.".to_string())); + } + Err(e) => { + search_error.set(Some(format!("Error deleting database: {}", e))); + } + } + }); + }, + "Yes, Delete & Rebuild" + } + button { + style: "padding: 10px 20px; background-color: #666; color: white; border: none; border-radius: 5px; cursor: pointer;", + onclick: move |_| { + show_corruption_dialog.set(false); + }, + "Cancel" + } + } + } + } + } + } + } +} + +/// Number of pages required to fit `total` rows at [`PAGE_SIZE`] per page. +/// Returns 0 for 0 rows so callers can branch on "no results yet". +fn pages_for(total: u64) -> u32 { + if total == 0 { + 0 + } else { + ((total - 1) / PAGE_SIZE as u64) as u32 + 1 + } +} + +/// Centralized error-router for the two query branches that can fail +/// identically (count and select). Sets the error/timing/spinner signals +/// and pops the corruption dialog when warranted. +fn handle_query_error( + e: String, + elapsed: f64, + mut search_error: Signal>, + mut show_corruption_dialog: Signal, + mut last_search_time: Signal>, + mut is_searching: Signal, +) { + if e.starts_with("DATABASE_CORRUPTED:") { + search_error.set(Some("Database appears to be corrupted".into())); + show_corruption_dialog.set(true); + } else { + search_error.set(Some(e)); + } + last_search_time.set(Some(elapsed)); + is_searching.set(false); +} diff --git a/index.html b/index.html deleted file mode 100644 index d914d42..0000000 --- a/index.html +++ /dev/null @@ -1,170 +0,0 @@ - - - - - - QuickSearch - - - -
-
-

QuickSearch File Indexer

-
- -
-
- - diff --git a/run.bat b/run.bat index 0f4c3cf..c1e8007 100644 --- a/run.bat +++ b/run.bat @@ -1 +1 @@ -cargo run \ No newline at end of file +cargo run -p quicksearch-gui diff --git a/run.sh b/run.sh index 1c9c1eb..7cef3e5 100644 --- a/run.sh +++ b/run.sh @@ -1,2 +1,2 @@ -cargo build --release -./target/release/quicksearch \ No newline at end of file +cargo build --release -p quicksearch-gui +./target/release/quicksearch diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 5bf65fa..0000000 --- a/src/main.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::sync::{Arc, OnceLock}; -use dioxus::prelude::*; -mod frontend; -mod file_handling; -mod document_extraction; -mod indexing; -mod config; -mod search; - -// Global indexing service for signal handling -static INDEXING_SERVICE: OnceLock> = OnceLock::new(); - -fn main() { - // Initialize global indexing service - let indexing_service = Arc::new(indexing::IndexingService::new()); - INDEXING_SERVICE.set(indexing_service.clone()).expect("Failed to set global indexing service"); - - // Set up Ctrl-C signal handler - ctrlc::set_handler(|| { - eprintln!("Received Ctrl-C, shutting down gracefully..."); - if let Some(service) = INDEXING_SERVICE.get() { - if let Err(e) = service.graceful_shutdown() { - eprintln!("Error during graceful shutdown: {}", e); - } - } - std::process::exit(0); - }).expect("Error setting Ctrl-C handler"); - - LaunchBuilder::desktop() - .with_cfg( - dioxus_desktop::Config::new() - .with_custom_head(format!("", include_str!("../assets/styles.css"))) - .with_window(dioxus_desktop::WindowBuilder::new() - .with_title("QuickSearch - File Indexer & Search") - .with_resizable(true) - .with_inner_size(dioxus_desktop::LogicalSize::new(1000.0, 700.0)) - ) - ) - .launch(app); -} - - -fn app() -> Element { - let config = match config::Config::load() { - Ok(config) => config, - Err(e) => { - eprintln!("Failed to load config: {}", e); - return rsx! { div { "Failed to load configuration" } }; - } - }; - - let indexing_service = INDEXING_SERVICE.get().expect("Indexing service not initialized").clone(); - - rsx! { - frontend::App { - indexing_service: indexing_service, - config: config - } - } -} - -/* -Duplicate files: -SELECT name, count(*) as cnt, path FROM files GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC; - -Full text search: -SELECT d.name, d.path, d.text, snippet(st, 1 , "", "", "...", 64) as "snip" FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE st.text MATCH 'searchstring' - -Filename search: -SELECT name, path FROM files WHERE name LIKE '%searchstring%'; -*/ \ No newline at end of file diff --git a/src/search.rs b/src/search.rs deleted file mode 100644 index ab321c9..0000000 --- a/src/search.rs +++ /dev/null @@ -1,404 +0,0 @@ -#![allow(non_snake_case)] - -use std::sync::Arc; -use std::time::Instant; -use dioxus::prelude::*; -use crate::indexing::{IndexingService, SearchResult}; - -#[derive(Props, Clone)] -pub struct SearchProps { - pub indexing_service: Arc, - pub db_path: String, -} - -impl PartialEq for SearchProps { - fn eq(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.indexing_service, &other.indexing_service) && self.db_path == other.db_path - } -} - -pub fn Search(props: SearchProps) -> Element { - let mut search_type = use_signal(|| "fulltext".to_string()); - let mut search_term = use_signal(|| String::new()); - let mut fulltext_exact = use_signal(|| false); - let mut fulltext_case_sensitive = use_signal(|| false); - let search_results = use_signal(|| Vec::::new()); - let mut search_error = use_signal(|| None::); - let is_searching = use_signal(|| false); - let last_search_time = use_signal(|| None::); - let mut show_corruption_dialog = use_signal(|| false); - - let service = props.indexing_service.clone(); - let db_path = props.db_path.clone(); - - // Create a callback to perform search - let perform_search = { - let service = service.clone(); - let db_path = db_path.clone(); - let search_type = search_type.clone(); - let search_term = search_term.clone(); - let fulltext_exact = fulltext_exact.clone(); - let fulltext_case_sensitive = fulltext_case_sensitive.clone(); - let search_results = search_results.clone(); - let search_error = search_error.clone(); - let is_searching = is_searching.clone(); - let last_search_time = last_search_time.clone(); - let show_corruption_dialog = show_corruption_dialog.clone(); - - move || { - let service_clone = service.clone(); - let db_clone = db_path.clone(); - let search_type_val = search_type().clone(); - let search_term_val = search_term().clone(); - let fulltext_exact_val = fulltext_exact(); - let fulltext_case_sensitive_val = fulltext_case_sensitive(); - - let mut search_results_clone = search_results.clone(); - let mut search_error_clone = search_error.clone(); - let mut is_searching_clone = is_searching.clone(); - let mut last_search_time_clone = last_search_time.clone(); - let mut show_corruption_dialog_clone = show_corruption_dialog.clone(); - - spawn(async move { - is_searching_clone.set(true); - search_error_clone.set(None); - last_search_time_clone.set(None); - let start_time = Instant::now(); - - let query = match search_type_val.as_str() { - "fulltext" => { - let trimmed = search_term_val.trim(); - if trimmed.is_empty() { - search_error_clone.set(Some("Please enter a search term".to_string())); - is_searching_clone.set(false); - return; - } - - let sanitized_term = trimmed - .replace(':', " ") - .replace(';', " ") - .replace('(', " ") - .replace(')', " ") - .replace('[', " ") - .replace(']', " ") - .replace('{', " ") - .replace('}', " ") - .replace('^', " ") - .replace('~', " ") - .replace('"', " "); - - let tokens: Vec<&str> = sanitized_term.split_whitespace().collect(); - if tokens.is_empty() { - search_error_clone.set(Some("Please enter a valid search term".to_string())); - is_searching_clone.set(false); - return; - } - - let words: Vec<&str> = if fulltext_exact_val { - tokens - } else { - let filtered: Vec<&str> = tokens - .into_iter() - .filter(|w| w.chars().count() >= 3) - .collect(); - if filtered.is_empty() { - search_error_clone.set(Some( - "Trigram index needs each word to be at least 3 characters unless you use exact phrase search.".to_string(), - )); - is_searching_clone.set(false); - return; - } - filtered - }; - - let sql_quote = |s: &str| s.replace('\'', "''"); - - let fts_match = if fulltext_exact_val { - let phrase = words.join(" "); - format!("\"{}\"", phrase.replace('"', "\"\"")) - } else { - words.join(" AND ") - }; - - let mut where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match)); - if fulltext_case_sensitive_val { - if fulltext_exact_val { - let literal = words.join(" "); - where_clause.push_str(&format!( - " AND instr(st.text, '{}') > 0", - sql_quote(&literal) - )); - } else { - for w in &words { - where_clause.push_str(&format!( - " AND instr(st.text, '{}') > 0", - sql_quote(w) - )); - } - } - } - - format!( - "SELECT d.name, d.path, snippet(searchabletext, 1, '', '', '...', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank", - where_clause - ) - }, - "filename" => { - if search_term_val.trim().is_empty() { - search_error_clone.set(Some("Please enter a filename pattern".to_string())); - is_searching_clone.set(false); - return; - } - format!("SELECT name, path FROM files WHERE name LIKE '%{}%'", search_term_val.replace("'", "''")) - }, - "duplicates" => "SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC LIMIT 5000".to_string(), - _ => { - is_searching_clone.set(false); - return; - } - }; - - // Run the search in a blocking task to prevent UI freezing - let search_result = tokio::task::spawn_blocking(move || { - service_clone.execute_search(&db_clone, &query) - }).await; - - let elapsed = start_time.elapsed().as_secs_f64(); - match search_result { - Ok(db_result) => { - match db_result { - Ok(results) => { - search_results_clone.set(results); - last_search_time_clone.set(Some(elapsed)); - }, - Err(e) => { - if e.starts_with("DATABASE_CORRUPTED:") { - search_error_clone.set(Some("Database appears to be corrupted".to_string())); - show_corruption_dialog_clone.set(true); - } else { - search_error_clone.set(Some(e)); - } - last_search_time_clone.set(Some(elapsed)); - } - } - }, - Err(e) => { - search_error_clone.set(Some(format!("Task execution error: {}", e))); - last_search_time_clone.set(Some(elapsed)); - } - } - is_searching_clone.set(false); - }); - } - }; - - rsx! { - div { - class: "section", - h2 { "Search Database" } - - div { - class: "form-group", - label { "Search Type: " } - select { - class: "form-control", - value: "{search_type}", - onchange: move |evt| search_type.set(evt.value()), - option { value: "fulltext", "Full Text Search" } - option { value: "filename", "Filename Search" } - option { value: "duplicates", "Find Duplicate Files" } - } - } - - if search_type() == "fulltext" { - div { - class: "form-group", - style: "display: flex; flex-direction: column; gap: 6px;", - span { style: "font-weight: 600;", "Full text options" } - label { - style: "display: flex; align-items: center; gap: 8px; cursor: pointer;", - input { - r#type: "checkbox", - checked: fulltext_exact(), - onchange: move |evt| fulltext_exact.set(evt.checked()), - } - "Exact phrase match" - } - label { - style: "display: flex; align-items: center; gap: 8px; cursor: pointer;", - input { - r#type: "checkbox", - checked: fulltext_case_sensitive(), - onchange: move |evt| fulltext_case_sensitive.set(evt.checked()), - } - "Case-sensitive match" - } - } - } - - if search_type() != "duplicates" { - div { - class: "form-group", - label { "Search Term: " } - input { - class: "form-control", - r#type: "text", - value: "{search_term}", - oninput: move |evt| search_term.set(evt.value()), - onkeydown: { - let perform_search = perform_search.clone(); - move |evt| { - if evt.code() == dioxus::events::Code::Enter { - perform_search(); - } - } - } - } - } - } - - div { - style: "display: flex; align-items: center; gap: 10px;", - button { - class: "btn btn-info", - disabled: is_searching(), - onclick: { - let perform_search = perform_search.clone(); - move |_| { - perform_search(); - } - }, - "Search" - } - - if is_searching() { - div { - class: "loading", - title: "Searching..." - } - } else if let Some(elapsed) = last_search_time() { - span { - style: "color: #666; font-size: 0.9em;", - "Search completed in {elapsed:.3}s" - } - } - } - - if let Some(error) = search_error() { - div { - class: "error-message", - "Error: {error}" - } - } - - if !search_results().is_empty() { - div { - class: "search-results", - h3 { "Search Results ({search_results()[0].rows.len()} results)" } - div { - class: "results-table", - table { - thead { - tr { - for column in search_results()[0].columns.iter() { - th { "{column}" } - } - } - } - tbody { - for (_i, row) in search_results()[0].rows.iter().enumerate() { - tr { - for (col_index, value) in row.values.iter().enumerate() { - // Check if this column is a path column - if search_results()[0].columns.get(col_index).map(|s| s.as_str()) == Some("path") { - { - let value_owned = value.clone(); - let service_owned = props.indexing_service.clone(); - rsx! { - td { - class: "path-cell clickable", - onclick: move |_| { - let path = value_owned.clone(); - let service_clone = service_owned.clone(); - - spawn(async move { - if let Err(e) = service_clone.open_file_explorer(&path) { - eprintln!("Failed to open file explorer: {}", e); - } - }); - }, - title: "Click to open in file explorer", - dangerous_inner_html: "{value}" - } - } - } - } else { - td { - dangerous_inner_html: "{value}" - } - } - } - } - } - } - } - } - } - } - - // Database corruption recovery dialog - if show_corruption_dialog() { - div { - class: "modal-backdrop", - div { - class: "modal-dialog", - h3 { - style: "margin-top: 0; color: #d32f2f;", - "⚠️ Database Corruption Detected" - } - p { - style: "margin: 15px 0;", - "The database appears to be corrupted or malformed. This can happen due to unexpected shutdowns or disk issues." - } - p { - style: "margin: 15px 0; font-weight: bold;", - "Would you like to delete the corrupted database and create a new one? This will require re-indexing your files." - } - div { - style: "display: flex; gap: 10px; margin-top: 20px;", - button { - style: "padding: 10px 20px; background-color: #d32f2f; color: white; border: none; border-radius: 5px; cursor: pointer;", - onclick: move |_| { - let service = props.indexing_service.clone(); - let db = props.db_path.clone(); - - show_corruption_dialog.set(false); - search_error.set(Some("Deleting corrupted database...".to_string())); - - spawn(async move { - match service.delete_index_for_rebuild(&db) { - Ok(()) => { - search_error.set(Some("Database deleted. You can now start indexing again.".to_string())); - } - Err(e) => { - search_error.set(Some(format!("Error deleting database: {}", e))); - } - } - }); - }, - "Yes, Delete & Rebuild" - } - button { - style: "padding: 10px 20px; background-color: #666; color: white; border: none; border-radius: 5px; cursor: pointer;", - onclick: move |_| { - show_corruption_dialog.set(false); - }, - "Cancel" - } - } - } - } - } - } - } -}