jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

Added pagination to search results, handling of more file types, and several structural changes working towards baloo integration.

Browse source
This commit is contained in:
= 2026-04-21 23:00:47 -04:00
parent f0edf3963b
commit e1e4c19ef8
41 changed files with 7750 additions and 3281 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

626
Cargo.lock generated
View file

@ -17,6 +17,15 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "adobe-cmap-parser"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3"
dependencies = [
"pom",
]
[[package]] [[package]]
name = "aes" name = "aes"
version = "0.8.4" version = "0.8.4"
@ -56,10 +65,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
[[package]] [[package]]
name = "anyhow" name = "android_system_properties"
version = "1.0.86" version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "ashpd" name = "ashpd"
@ -490,6 +502,17 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
dependencies = [
"iana-time-zone",
"num-traits",
"windows-link",
]
[[package]] [[package]]
name = "ciborium" name = "ciborium"
version = "0.2.2" version = "0.2.2"
@ -681,19 +704,6 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "crossbeam"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-queue",
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-channel" name = "crossbeam-channel"
version = "0.5.13" version = "0.5.13"
@ -705,9 +715,9 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-deque" name = "crossbeam-deque"
version = "0.8.5" version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [ dependencies = [
"crossbeam-epoch", "crossbeam-epoch",
"crossbeam-utils", "crossbeam-utils",
@ -722,46 +732,12 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "crossbeam-queue"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
dependencies = [
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.20" version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "crossterm"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
dependencies = [
"bitflags 1.3.2",
"crossterm_winapi",
"libc",
"mio",
"parking_lot",
"signal-hook",
"signal-hook-mio",
"winapi",
]
[[package]]
name = "crossterm_winapi"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "crunchy" name = "crunchy"
version = "0.2.4" version = "0.2.4"
@ -863,6 +839,12 @@ dependencies = [
"parking_lot_core", "parking_lot_core",
] ]
[[package]]
name = "data-encoding"
version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
[[package]] [[package]]
name = "deranged" name = "deranged"
version = "0.5.3" version = "0.5.3"
@ -1004,7 +986,7 @@ dependencies = [
"futures-util", "futures-util",
"generational-box", "generational-box",
"global-hotkey", "global-hotkey",
"infer", "infer 0.11.0",
"muda", "muda",
"objc", "objc",
"objc_id", "objc_id",
@ -1086,7 +1068,7 @@ dependencies = [
"dioxus-core", "dioxus-core",
"dioxus-html-internal-macro", "dioxus-html-internal-macro",
"enumset", "enumset",
"euclid", "euclid 0.22.10",
"futures-channel", "futures-channel",
"generational-box", "generational-box",
"keyboard-types", "keyboard-types",
@ -1222,15 +1204,6 @@ dependencies = [
"syn 2.0.66", "syn 2.0.66",
] ]
[[package]]
name = "dpc-pariter"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a6f60e0061d01135c7e9b77486a7f5a8e63ba14acbdbca73e42007cfd8a1c91"
dependencies = [
"pariter",
]
[[package]] [[package]]
name = "dtoa" name = "dtoa"
version = "1.0.9" version = "1.0.9"
@ -1252,6 +1225,21 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b"
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "encoding_rs"
version = "0.8.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
dependencies = [
"cfg-if",
]
[[package]] [[package]]
name = "endi" name = "endi"
version = "1.1.0" version = "1.1.0"
@ -1316,6 +1304,15 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "euclid"
version = "0.20.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad"
dependencies = [
"num-traits",
]
[[package]] [[package]]
name = "euclid" name = "euclid"
version = "0.22.10" version = "0.22.10"
@ -1384,6 +1381,17 @@ dependencies = [
"rustc_version", "rustc_version",
] ]
[[package]]
name = "filetime"
version = "0.2.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db"
dependencies = [
"cfg-if",
"libc",
"libredox",
]
[[package]] [[package]]
name = "fixedbitset" name = "fixedbitset"
version = "0.4.2" version = "0.4.2"
@ -1442,6 +1450,15 @@ dependencies = [
"percent-encoding", "percent-encoding",
] ]
[[package]]
name = "fsevent-sys"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "futf" name = "futf"
version = "0.1.5" version = "0.1.5"
@ -2015,6 +2032,30 @@ dependencies = [
"itoa 1.0.11", "itoa 1.0.11",
] ]
[[package]]
name = "iana-time-zone"
version = "0.1.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.62.1",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]] [[package]]
name = "icu_collections" name = "icu_collections"
version = "1.5.0" version = "1.5.0"
@ -2180,6 +2221,35 @@ dependencies = [
"cfb", "cfb",
] ]
[[package]]
name = "infer"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb33622da908807a06f9513c19b3c1ad50fab3e4137d82a78107d502075aa199"
dependencies = [
"cfb",
]
[[package]]
name = "inotify"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff"
dependencies = [
"bitflags 1.3.2",
"inotify-sys",
"libc",
]
[[package]]
name = "inotify-sys"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "inout" name = "inout"
version = "0.1.4" version = "0.1.4"
@ -2309,6 +2379,15 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "kamadak-exif"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef4fc70d0ab7e5b6bafa30216a6b48705ea964cdfc29c050f2412295eba58077"
dependencies = [
"mutate_once",
]
[[package]] [[package]]
name = "keyboard-types" name = "keyboard-types"
version = "0.7.0" version = "0.7.0"
@ -2320,6 +2399,26 @@ dependencies = [
"unicode-segmentation", "unicode-segmentation",
] ]
[[package]]
name = "kqueue"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a"
dependencies = [
"kqueue-sys",
"libc",
]
[[package]]
name = "kqueue-sys"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b"
dependencies = [
"bitflags 1.3.2",
"libc",
]
[[package]] [[package]]
name = "krates" name = "krates"
version = "0.16.10" version = "0.16.10"
@ -2358,6 +2457,18 @@ version = "0.2.175"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
[[package]]
name = "libredox"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c"
dependencies = [
"bitflags 2.5.0",
"libc",
"plain",
"redox_syscall 0.7.4",
]
[[package]] [[package]]
name = "libsqlite3-sys" name = "libsqlite3-sys"
version = "0.28.0" version = "0.28.0"
@ -2388,6 +2499,12 @@ dependencies = [
"x11", "x11",
] ]
[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.4.14" version = "0.4.14"
@ -2410,6 +2527,32 @@ dependencies = [
"scopeguard", "scopeguard",
] ]
[[package]]
name = "lofty"
version = "0.19.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee8436c21042327836a52493b8cb2e1bb8a8f44c723a838d8c8b01f0bb657ea2"
dependencies = [
"byteorder",
"data-encoding",
"flate2",
"lofty_attr",
"log",
"ogg_pager",
"paste",
]
[[package]]
name = "lofty_attr"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83e2dcdba64a01c3ce936b02401ab17b52db3c72720cf484c9df3387ca7517d3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.21" version = "0.4.21"
@ -2422,6 +2565,43 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86" checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86"
[[package]]
name = "lopdf"
version = "0.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e775e4ee264e8a87d50a9efef7b67b4aa988cf94e75630859875fc347e6c872b"
dependencies = [
"chrono",
"encoding_rs",
"flate2",
"itoa 1.0.11",
"linked-hash-map",
"log",
"md5",
"nom",
"rayon",
"time",
"weezl",
]
[[package]]
name = "lopdf"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5c8ecfc6c72051981c0459f75ccc585e7ff67c70829560cda8e647882a9abff"
dependencies = [
"encoding_rs",
"flate2",
"indexmap 2.2.6",
"itoa 1.0.11",
"log",
"md-5",
"nom",
"rangemap",
"time",
"weezl",
]
[[package]] [[package]]
name = "lru" name = "lru"
version = "0.12.3" version = "0.12.3"
@ -2466,6 +2646,16 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest",
]
[[package]] [[package]]
name = "md5" name = "md5"
version = "0.7.0" version = "0.7.0"
@ -2487,6 +2677,28 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "mime"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]] [[package]]
name = "miniz_oxide" name = "miniz_oxide"
version = "0.7.3" version = "0.7.3"
@ -2527,6 +2739,12 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "mutate_once"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af"
[[package]] [[package]]
name = "ndk" name = "ndk"
version = "0.7.0" version = "0.7.0"
@ -2592,6 +2810,35 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
[[package]]
name = "nom"
version = "7.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "notify"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
dependencies = [
"bitflags 2.5.0",
"crossbeam-channel",
"filetime",
"fsevent-sys",
"inotify",
"kqueue",
"libc",
"log",
"mio",
"walkdir",
"windows-sys 0.48.0",
]
[[package]] [[package]]
name = "nu-ansi-term" name = "nu-ansi-term"
version = "0.46.0" version = "0.46.0"
@ -2696,6 +2943,15 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "ogg_pager"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87b0bef808533c5890ab77279538212efdbbbd9aa4ef1ccdfcfbf77a42f7e6fa"
dependencies = [
"byteorder",
]
[[package]] [[package]]
name = "once_cell" name = "once_cell"
version = "1.19.0" version = "1.19.0"
@ -2752,17 +3008,6 @@ dependencies = [
"system-deps", "system-deps",
] ]
[[package]]
name = "pariter"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "324a62b9e7b5f270c0acc92a2040f8028bb643f959f9c068f11a7864f327e3d9"
dependencies = [
"crossbeam",
"crossbeam-channel",
"num_cpus",
]
[[package]] [[package]]
name = "parking" name = "parking"
version = "2.2.0" version = "2.2.0"
@ -2787,7 +3032,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"libc", "libc",
"redox_syscall", "redox_syscall 0.5.1",
"smallvec", "smallvec",
"windows-targets 0.52.5", "windows-targets 0.52.5",
] ]
@ -2803,6 +3048,12 @@ dependencies = [
"subtle", "subtle",
] ]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]] [[package]]
name = "pbkdf2" name = "pbkdf2"
version = "0.11.0" version = "0.11.0"
@ -2815,6 +3066,21 @@ dependencies = [
"sha2", "sha2",
] ]
[[package]]
name = "pdf-extract"
version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbb3a5387b94b9053c1e69d8abfd4dd6dae7afda65a5c5279bc1f42ab39df575"
dependencies = [
"adobe-cmap-parser",
"encoding_rs",
"euclid 0.20.14",
"lopdf 0.34.0",
"postscript",
"type1-encoding-parser",
"unicode-normalization",
]
[[package]] [[package]]
name = "percent-encoding" name = "percent-encoding"
version = "2.3.1" version = "2.3.1"
@ -2972,6 +3238,12 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
[[package]]
name = "plain"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]] [[package]]
name = "png" name = "png"
version = "0.17.13" version = "0.17.13"
@ -3006,6 +3278,18 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22686f4785f02a4fcc856d3b3bb19bf6c8160d103f7a99cc258bddd0251dc7f2" checksum = "22686f4785f02a4fcc856d3b3bb19bf6c8160d103f7a99cc258bddd0251dc7f2"
[[package]]
name = "pom"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6"
[[package]]
name = "postscript"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306"
[[package]] [[package]]
name = "powerfmt" name = "powerfmt"
version = "0.2.0" version = "0.2.0"
@ -3103,24 +3387,36 @@ dependencies = [
] ]
[[package]] [[package]]
name = "quicksearch" name = "quicksearch-core"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"ctrlc", "ctrlc",
"dioxus", "infer 0.15.0",
"dioxus-desktop", "kamadak-exif",
"dpc-pariter", "lofty",
"lopdf 0.32.0",
"mime_guess",
"notify",
"pdf-extract",
"quick-xml", "quick-xml",
"rusqlite", "rusqlite",
"serde", "serde",
"sha2", "sha2",
"tokio",
"toml", "toml",
"tqdm",
"walkdir", "walkdir",
"zip", "zip",
] ]
[[package]]
name = "quicksearch-gui"
version = "0.1.0"
dependencies = [
"dioxus",
"dioxus-desktop",
"quicksearch-core",
"tokio",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.36" version = "1.0.36"
@ -3211,6 +3507,12 @@ dependencies = [
"rand_core 0.5.1", "rand_core 0.5.1",
] ]
[[package]]
name = "rangemap"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
[[package]] [[package]]
name = "raw-window-handle" name = "raw-window-handle"
version = "0.5.2" version = "0.5.2"
@ -3223,6 +3525,26 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539"
[[package]]
name = "rayon"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.5.1" version = "0.5.1"
@ -3232,6 +3554,15 @@ dependencies = [
"bitflags 2.5.0", "bitflags 2.5.0",
] ]
[[package]]
name = "redox_syscall"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a"
dependencies = [
"bitflags 2.5.0",
]
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.10.5" version = "1.10.5"
@ -3566,17 +3897,6 @@ dependencies = [
"signal-hook-registry", "signal-hook-registry",
] ]
[[package]]
name = "signal-hook-mio"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
dependencies = [
"libc",
"mio",
"signal-hook",
]
[[package]] [[package]]
name = "signal-hook-registry" name = "signal-hook-registry"
version = "1.4.2" version = "1.4.2"
@ -3893,6 +4213,7 @@ dependencies = [
"powerfmt", "powerfmt",
"serde", "serde",
"time-core", "time-core",
"time-macros",
] ]
[[package]] [[package]]
@ -3901,6 +4222,16 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b"
[[package]]
name = "time-macros"
version = "0.2.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3"
dependencies = [
"num-conv",
"time-core",
]
[[package]] [[package]]
name = "tinystr" name = "tinystr"
version = "0.7.6" version = "0.7.6"
@ -3911,6 +4242,21 @@ dependencies = [
"zerovec", "zerovec",
] ]
[[package]]
name = "tinyvec"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]] [[package]]
name = "to_method" name = "to_method"
version = "1.1.0" version = "1.1.0"
@ -3990,17 +4336,6 @@ dependencies = [
"winnow", "winnow",
] ]
[[package]]
name = "tqdm"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa2d2932240205a99b65f15d9861992c95fbb8c9fb280b3a1f17a92db6dc611f"
dependencies = [
"anyhow",
"crossterm",
"once_cell",
]
[[package]] [[package]]
name = "tracing" name = "tracing"
version = "0.1.40" version = "0.1.40"
@ -4058,6 +4393,15 @@ dependencies = [
"tracing-log", "tracing-log",
] ]
[[package]]
name = "type1-encoding-parser"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa10c302f5a53b7ad27fd42a3996e23d096ba39b5b8dd6d9e683a05b01bee749"
dependencies = [
"pom",
]
[[package]] [[package]]
name = "typenum" name = "typenum"
version = "1.17.0" version = "1.17.0"
@ -4075,12 +4419,27 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]] [[package]]
name = "unicode-ident" name = "unicode-ident"
version = "1.0.12" version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "unicode-normalization"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
dependencies = [
"tinyvec",
]
[[package]] [[package]]
name = "unicode-segmentation" name = "unicode-segmentation"
version = "1.11.0" version = "1.11.0"
@ -4367,6 +4726,12 @@ dependencies = [
"windows-core 0.52.0", "windows-core 0.52.0",
] ]
[[package]]
name = "weezl"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"
@ -4437,10 +4802,23 @@ version = "0.54.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65"
dependencies = [ dependencies = [
"windows-result", "windows-result 0.1.2",
"windows-targets 0.52.5", "windows-targets 0.52.5",
] ]
[[package]]
name = "windows-core"
version = "0.62.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6844ee5416b285084d3d3fffd743b925a6c9385455f64f6d4fa3031c4c2749a9"
dependencies = [
"windows-implement 0.60.2",
"windows-interface 0.59.3",
"windows-link",
"windows-result 0.4.0",
"windows-strings",
]
[[package]] [[package]]
name = "windows-implement" name = "windows-implement"
version = "0.52.0" version = "0.52.0"
@ -4463,6 +4841,17 @@ dependencies = [
"syn 2.0.66", "syn 2.0.66",
] ]
[[package]]
name = "windows-implement"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]] [[package]]
name = "windows-interface" name = "windows-interface"
version = "0.52.0" version = "0.52.0"
@ -4485,6 +4874,17 @@ dependencies = [
"syn 2.0.66", "syn 2.0.66",
] ]
[[package]]
name = "windows-interface"
version = "0.59.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]] [[package]]
name = "windows-link" name = "windows-link"
version = "0.2.0" version = "0.2.0"
@ -4500,6 +4900,24 @@ dependencies = [
"windows-targets 0.52.5", "windows-targets 0.52.5",
] ]
[[package]]
name = "windows-result"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-strings"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda"
dependencies = [
"windows-link",
]
[[package]] [[package]]
name = "windows-sys" name = "windows-sys"
version = "0.45.0" version = "0.45.0"

25
Cargo.toml
View file

@ -1,19 +1,10 @@
[package] [workspace]
name = "quicksearch" resolver = "2"
members = [
"crates/quicksearch-core",
"crates/quicksearch-gui",
]
[workspace.package]
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"
[dependencies]
dioxus = { version = "0.5.1", features = ["desktop"] }
dioxus-desktop = "0.5.1"
dpc-pariter = "0.5.1"
rusqlite = { version = "0.31.0", features = ["bundled"] }
sha2 = "0.10.8"
tqdm = "0.7.0"
walkdir = "2.5.0"
zip = "0.6"
quick-xml = "0.31"
tokio = { version = "1.0", features = ["time", "signal"] }
serde = { version = "1.0", features = ["derive"] }
toml = "0.8"
ctrlc = "3.4"

4
config_example.toml
View file

@ -1,5 +1,7 @@
[paths] [paths]
default_indexing_path = "C:\\" # One or more directory roots to index. Walked in order; duplicate and
# nested roots are de-duplicated automatically.
indexing_paths = ["C:\\"]
database_path = "QuickSearch.db" database_path = "QuickSearch.db"
[processing] [processing]

25
crates/quicksearch-core/Cargo.toml Normal file
View file

@ -0,0 +1,25 @@
[package]
name = "quicksearch-core"
version.workspace = true
edition.workspace = true
[lib]
name = "quicksearch_core"
path = "src/lib.rs"
[dependencies]
rusqlite = { version = "0.31.0", features = ["bundled"] }
sha2 = "0.10.8"
walkdir = "2.5.0"
zip = "0.6"
quick-xml = "0.31"
serde = { version = "1.0", features = ["derive"] }
toml = "0.8"
mime_guess = "2.0"
infer = "0.15"
pdf-extract = "0.7"
lopdf = "0.32"
lofty = "0.19"
kamadak-exif = "0.5"
notify = "6.1"
ctrlc = "3.4"

283
crates/quicksearch-core/src/cli.rs Normal file
View file

@ -0,0 +1,283 @@
//! Programmatic read-only query helpers.
//!
//! Pure functions that open a DB, run a query, and return structured data.
//! No stdout, no CLI framing — callers (GUI, future CLI binaries, Set B
//! `balooctl`) format the result as they see fit. Mutating operations live
//! on [`crate::indexing::IndexingService`] since they require a running
//! worker thread.
use rusqlite::{params, OptionalExtension};
use crate::db::open_and_migrate;
use crate::db::repo::{STATE_DONE, STATE_FAILED, STATE_NA, STATE_PENDING};
/// Per-file indexing status, mirroring Baloo's multi-state reporting.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IndexState {
NotIndexed,
Pending,
Done,
Failed,
NotApplicable,
}
impl From<i64> for IndexState {
fn from(v: i64) -> Self {
match v {
x if x == STATE_PENDING => IndexState::Pending,
x if x == STATE_DONE => IndexState::Done,
x if x == STATE_FAILED => IndexState::Failed,
x if x == STATE_NA => IndexState::NotApplicable,
_ => IndexState::Pending,
}
}
}
/// Indexing status for a single file. `basic` is the metadata row state
/// (indexed or not); `content` is the extractor state.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileStatus {
pub path: String,
pub basic: IndexState,
pub content: IndexState,
pub failure_reason: Option<String>,
}
/// Per-file entry returned by [`list_failed`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FailedEntry {
pub file_id: i64,
pub path: String,
pub reason: Option<String>,
pub ts: i64,
}
/// Storage footprint report. "Partitions" correspond to SQL tables for our
/// SQLite layout (Baloo's LMDB has named sub-DBs; our equivalent is per-table
/// row/size counts).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SizeReport {
pub file_size_bytes: u64,
pub files_row_count: i64,
pub properties_row_count: i64,
pub failed_files_row_count: i64,
pub searchabletext_row_count: i64,
}
/// Query the per-file indexing status. Returns `FileStatus` with
/// `basic == NotIndexed` if the path isn't in the database.
pub fn status_for_path(db_path: &str, path: &str) -> Result<FileStatus, String> {
let conn = open_and_migrate(db_path, "trigram")?;
let row: Option<(i64, i64, Option<String>)> = conn
.query_row(
"SELECT basic_state, content_state, failure_msg FROM files WHERE path = ?1",
params![path],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)
.optional()
.map_err(|e| format!("status_for_path({}): {}", path, e))?;
Ok(match row {
None => FileStatus {
path: path.to_string(),
basic: IndexState::NotIndexed,
content: IndexState::NotIndexed,
failure_reason: None,
},
Some((basic, content, reason)) => FileStatus {
path: path.to_string(),
basic: IndexState::from(basic),
content: IndexState::from(content),
failure_reason: reason,
},
})
}
/// Return every file that failed content extraction, newest first.
pub fn list_failed(db_path: &str, limit: Option<u32>) -> Result<Vec<FailedEntry>, String> {
let conn = open_and_migrate(db_path, "trigram")?;
let limit_sql = match limit {
Some(n) => format!(" LIMIT {}", n),
None => String::new(),
};
let sql = format!(
"SELECT ff.file_id, f.path, ff.reason, ff.ts \
FROM failed_files ff \
JOIN files f ON f.id = ff.file_id \
ORDER BY ff.ts DESC{}",
limit_sql
);
let mut stmt = conn
.prepare(&sql)
.map_err(|e| format!("list_failed prepare: {}", e))?;
let rows = stmt
.query_map([], |r| {
Ok(FailedEntry {
file_id: r.get(0)?,
path: r.get(1)?,
reason: r.get(2)?,
ts: r.get(3)?,
})
})
.map_err(|e| format!("list_failed query: {}", e))?;
rows.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("list_failed row: {}", e))
}
/// Return a rough size breakdown of the database on disk and by table.
pub fn index_size_breakdown(db_path: &str) -> Result<SizeReport, String> {
let file_size_bytes = std::fs::metadata(db_path)
.map(|m| m.len())
.unwrap_or(0);
let conn = open_and_migrate(db_path, "trigram")?;
let count = |table: &str| -> Result<i64, String> {
conn.query_row(&format!("SELECT COUNT(*) FROM {}", table), [], |r| r.get(0))
.map_err(|e| format!("count {}: {}", table, e))
};
Ok(SizeReport {
file_size_bytes,
files_row_count: count("files")?,
properties_row_count: count("properties")?,
failed_files_row_count: count("failed_files")?,
searchabletext_row_count: count("searchabletext")?,
})
}
/// Remove a single file from the index. Returns whether a row was deleted.
/// Keeps FTS/documents/properties in sync via the repo helpers.
pub fn clear_path(db_path: &str, path: &str) -> Result<bool, String> {
let mut conn = open_and_migrate(db_path, "trigram")?;
let tx = conn
.transaction()
.map_err(|e| format!("clear_path begin tx: {}", e))?;
let removed = crate::db::repo::delete_file_by_path(&tx, path)?;
tx.commit()
.map_err(|e| format!("clear_path commit: {}", e))?;
Ok(removed)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::repo::{insert_file, set_content_done, set_content_failed, NewFile};
use crate::mime::FileType;
fn tmp_path() -> std::path::PathBuf {
let mut p = std::env::temp_dir();
p.push(format!(
"qs-cli-test-{}-{}.sqlite",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
p
}
fn seed_fixture(db_path: &str) -> (i64, i64) {
let mut conn = open_and_migrate(db_path, "trigram").unwrap();
let (a, b) = {
let tx = conn.transaction().unwrap();
let a = insert_file(
&tx,
&NewFile {
name: "a.txt",
path: "/tmp/a.txt",
parent: "/tmp",
size: 1,
mtime: 1,
inode: None,
device_id: None,
mime: Some("text/plain"),
ftype: FileType::TEXT,
hash: None,
},
)
.unwrap()
.expect("unique path");
set_content_done(&tx, a, "a.txt", "hello", &[]).unwrap();
let b = insert_file(
&tx,
&NewFile {
name: "b.bin",
path: "/tmp/b.bin",
parent: "/tmp",
size: 1,
mtime: 1,
inode: None,
device_id: None,
mime: None,
ftype: FileType::EMPTY,
hash: None,
},
)
.unwrap()
.expect("unique path");
set_content_failed(&tx, b, "bad extract").unwrap();
tx.commit().unwrap();
(a, b)
};
drop(conn);
(a, b)
}
#[test]
fn status_for_path_returns_states() {
let p = tmp_path();
let (_a, _b) = seed_fixture(p.to_str().unwrap());
let st_a = status_for_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap();
assert_eq!(st_a.basic, IndexState::Done);
assert_eq!(st_a.content, IndexState::Done);
let st_b = status_for_path(p.to_str().unwrap(), "/tmp/b.bin").unwrap();
assert_eq!(st_b.basic, IndexState::Done);
assert_eq!(st_b.content, IndexState::Failed);
assert_eq!(st_b.failure_reason.as_deref(), Some("bad extract"));
let st_missing = status_for_path(p.to_str().unwrap(), "/tmp/never.txt").unwrap();
assert_eq!(st_missing.basic, IndexState::NotIndexed);
std::fs::remove_file(&p).ok();
}
#[test]
fn list_failed_returns_failed_rows() {
let p = tmp_path();
let (_a, b) = seed_fixture(p.to_str().unwrap());
let failed = list_failed(p.to_str().unwrap(), None).unwrap();
assert_eq!(failed.len(), 1);
assert_eq!(failed[0].file_id, b);
assert_eq!(failed[0].path, "/tmp/b.bin");
assert_eq!(failed[0].reason.as_deref(), Some("bad extract"));
std::fs::remove_file(&p).ok();
}
#[test]
fn index_size_breakdown_counts_rows() {
let p = tmp_path();
let _ = seed_fixture(p.to_str().unwrap());
let r = index_size_breakdown(p.to_str().unwrap()).unwrap();
assert!(r.file_size_bytes > 0);
assert_eq!(r.files_row_count, 2);
assert_eq!(r.failed_files_row_count, 1);
std::fs::remove_file(&p).ok();
}
#[test]
fn clear_path_removes_file_and_returns_true() {
let p = tmp_path();
let _ = seed_fixture(p.to_str().unwrap());
assert!(clear_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap());
let st = status_for_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap();
assert_eq!(st.basic, IndexState::NotIndexed);
assert!(!clear_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap());
std::fs::remove_file(&p).ok();
}
}

17
src/config.rs → crates/quicksearch-core/src/config.rs
View file

@ -10,7 +10,10 @@ pub struct Config {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PathConfig { pub struct PathConfig {
pub default_indexing_path: String, /// One or more directory roots to index. Indexing walks each root
/// independently; duplicates and nested roots are de-duplicated by the
/// indexer at run time. Must contain at least one entry.
pub indexing_paths: Vec<String>,
pub database_path: String, pub database_path: String,
} }
@ -18,6 +21,16 @@ fn default_fts_update_batch_size() -> usize {
1000 1000
} }
/// Platform-sensible default for the first indexing root when no config
/// exists. `$HOME` on Unix, `%USERPROFILE%` on Windows; falls back to the
/// current directory.
fn default_home_path() -> String {
if let Some(home) = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) {
return home.to_string_lossy().into_owned();
}
".".to_string()
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ProcessingConfig { pub struct ProcessingConfig {
pub hash_length: usize, pub hash_length: usize,
@ -39,7 +52,7 @@ impl Default for Config {
fn default() -> Self { fn default() -> Self {
Config { Config {
paths: PathConfig { paths: PathConfig {
default_indexing_path: "C:\\".to_string(), indexing_paths: vec![default_home_path()],
database_path: "QuickSearch.db".to_string(), database_path: "QuickSearch.db".to_string(),
}, },
processing: ProcessingConfig { processing: ProcessingConfig {

281
crates/quicksearch-core/src/db/migrate.rs Normal file
View file

@ -0,0 +1,281 @@
//! Schema version detection and upgrade.
//!
//! Callers should always enter the DB via [`open_and_migrate`]. It applies
//! pragmas, detects the on-disk schema version, and upgrades or recreates as
//! required.
//!
//! Current policy: Set A introduces schema v1 and is the first versioned
//! release. Any pre-A database (has the legacy `files(name, path, size,
//! moddate, hash)` shape and no `schema_info` table) is wiped and rebuilt —
//! the user will re-index. A prominent log line is printed so the behavior is
//! not silent. Future migrations should prefer ALTER TABLE.
use std::path::Path;
use rusqlite::{params, Connection, OptionalExtension};
use super::schema::{fts_create_sql, PRAGMAS_FAST, SCHEMA_CURRENT};
pub const CURRENT_SCHEMA_VERSION: u32 = 2;
/// Open the database at `db_path`, apply pragmas, and ensure the schema is at
/// [`CURRENT_SCHEMA_VERSION`]. Recreates the DB if a pre-versioned layout is
/// detected.
///
/// `tokenizer` is used when (re)creating the FTS5 virtual table. It has no
/// effect on an already-current DB.
pub fn open_and_migrate(db_path: &str, tokenizer: &str) -> Result<Connection, String> {
let path_for_rebuild = Path::new(db_path).to_path_buf();
let mut conn = Connection::open(db_path)
.map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?;
conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas: {}", e))?;
let version = read_schema_version(&conn)?;
match version {
Some(v) if v == CURRENT_SCHEMA_VERSION => {}
Some(v) if v > CURRENT_SCHEMA_VERSION => {
return Err(format!(
"Database schema version {} is newer than this build ({}). \
Use a newer QuickSearch or move the database aside.",
v, CURRENT_SCHEMA_VERSION
));
}
Some(v) => {
// Older schema version. Set A's upgrade policy: wipe and rebuild.
// When we start adding ALTER-based migrations this match arm
// will gain a proper stepwise runner.
eprintln!(
"QuickSearch: database at {} is schema v{}; rebuilding to v{}. \
Existing rows will be re-scanned.",
db_path, v, CURRENT_SCHEMA_VERSION
);
conn = wipe_and_reopen(conn, &path_for_rebuild)?;
apply_current_schema(&conn, tokenizer)?;
}
None => {
// No schema_info row. Either an empty DB (good — just create) or a
// legacy pre-A layout (detected by presence of the old `files`
// table). Legacy layouts are wiped.
if has_legacy_layout(&conn)? {
eprintln!(
"QuickSearch: legacy database detected at {}; rebuilding index with schema v{}. \
Existing file rows will be re-scanned.",
db_path, CURRENT_SCHEMA_VERSION
);
conn = wipe_and_reopen(conn, &path_for_rebuild)?;
}
apply_current_schema(&conn, tokenizer)?;
}
}
Ok(conn)
}
fn wipe_and_reopen(
conn: Connection,
path_for_rebuild: &std::path::Path,
) -> Result<Connection, String> {
drop(conn);
std::fs::remove_file(path_for_rebuild)
.map_err(|e| format!("Failed to remove old database: {}", e))?;
// Remove WAL/SHM/journal sidecars defensively even though journal_mode=OFF.
for suffix in ["-wal", "-shm", "-journal"] {
let sidecar = path_for_rebuild.with_file_name(format!(
"{}{}",
path_for_rebuild
.file_name()
.and_then(|s| s.to_str())
.unwrap_or(""),
suffix
));
let _ = std::fs::remove_file(sidecar);
}
let conn = Connection::open(path_for_rebuild)
.map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?;
conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?;
Ok(conn)
}
fn read_schema_version(conn: &Connection) -> Result<Option<u32>, String> {
let has_info: bool = conn
.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_info'",
[],
|_| Ok(true),
)
.optional()
.map_err(|e| format!("sqlite_master schema_info: {}", e))?
.unwrap_or(false);
if !has_info {
return Ok(None);
}
let v: Option<String> = conn
.query_row(
"SELECT value FROM schema_info WHERE key = 'version'",
[],
|r| r.get(0),
)
.optional()
.map_err(|e| format!("read schema_info.version: {}", e))?;
match v {
Some(s) => s
.parse::<u32>()
.map(Some)
.map_err(|e| format!("invalid schema_info.version {:?}: {}", s, e)),
None => Ok(None),
}
}
fn has_legacy_layout(conn: &Connection) -> Result<bool, String> {
// Old layout has a `files` table without an `id INTEGER PRIMARY KEY`.
let has_files: bool = conn
.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='files'",
[],
|_| Ok(true),
)
.optional()
.map_err(|e| format!("sqlite_master files: {}", e))?
.unwrap_or(false);
if !has_files {
return Ok(false);
}
// Check whether the columns match the legacy shape.
let mut stmt = conn
.prepare("PRAGMA table_info(files)")
.map_err(|e| format!("pragma table_info: {}", e))?;
let has_id = stmt
.query_map([], |row| row.get::<_, String>(1))
.map_err(|e| format!("table_info query: {}", e))?
.filter_map(|r| r.ok())
.any(|name| name == "id");
Ok(!has_id)
}
fn apply_current_schema(conn: &Connection, tokenizer: &str) -> Result<(), String> {
conn.execute_batch(SCHEMA_CURRENT)
.map_err(|e| format!("Failed to create current schema tables: {}", e))?;
let fts = fts_create_sql(tokenizer);
conn.execute_batch(&fts)
.map_err(|e| format!("Failed to create searchabletext: {}", e))?;
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
conn.execute(
"INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)",
params![CURRENT_SCHEMA_VERSION.to_string(), now.to_string(), tokenizer],
)
.map_err(|e| format!("Failed to seed schema_info: {}", e))?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn tmp_db_path() -> std::path::PathBuf {
let mut p = std::env::temp_dir();
p.push(format!(
"quicksearch-test-{}-{}.sqlite",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
p
}
#[test]
fn fresh_db_gets_current_version() {
let p = tmp_db_path();
let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0))
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn reopen_is_idempotent() {
let p = tmp_db_path();
{
let _ = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
}
let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0))
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn older_versioned_db_is_wiped_and_recreated() {
// Simulate a DB that was created at a previous schema version.
let p = tmp_db_path();
{
let conn = Connection::open(&p).unwrap();
conn.execute("CREATE TABLE schema_info (key TEXT PRIMARY KEY, value TEXT NOT NULL)", []).unwrap();
conn.execute(
"INSERT INTO schema_info(key,value) VALUES('version','1')",
[],
)
.unwrap();
conn.execute("CREATE TABLE files (id INTEGER PRIMARY KEY, name TEXT)", []).unwrap();
conn.execute("INSERT INTO files(name) VALUES('a.txt')", []).unwrap();
}
let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0))
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0, "old rows should be wiped");
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn legacy_db_is_wiped_and_recreated() {
let p = tmp_db_path();
{
// Simulate a pre-A database.
let conn = Connection::open(&p).unwrap();
conn.execute(
"CREATE TABLE files (name TEXT, path TEXT, size INTEGER, moddate INTEGER, hash BLOB)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO files VALUES ('a.txt', '/tmp/a.txt', 1, 2, X'00')",
[],
)
.unwrap();
}
let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
// Old row should be gone.
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0);
// New columns should exist.
let _ = conn
.query_row("SELECT basic_state, content_state, type, mime FROM files LIMIT 0", [], |_| Ok(()))
.or_else(|e| if matches!(e, rusqlite::Error::QueryReturnedNoRows) { Ok(()) } else { Err(e) })
.unwrap();
drop(conn);
std::fs::remove_file(&p).ok();
}
}

13
crates/quicksearch-core/src/db/mod.rs Normal file
View file

@ -0,0 +1,13 @@
//! SQLite schema, migrations, and row-level repository helpers.
//!
//! The only "live" schema is `CURRENT_SCHEMA_VERSION` (see [`schema`]). Older
//! databases are detected in [`migrate::open_and_migrate`] and recreated from
//! scratch — Set A of the QuickSearch → Baloo work is the first schema bump
//! and carries no rows we'd want to preserve. Subsequent migrations should
//! prefer `ALTER TABLE` and versioned steps.
pub mod migrate;
pub mod repo;
pub mod schema;
pub use migrate::{open_and_migrate, CURRENT_SCHEMA_VERSION};

495
crates/quicksearch-core/src/db/repo.rs Normal file
View file

@ -0,0 +1,495 @@
//! Row-level write helpers that keep the FTS5 contentless table in sync with
//! `files`/`documents`/`properties`.
//!
//! FTS5 contentless tables store postings only. Updating them requires the
//! old row values to compute which terms to remove. These helpers centralize
//! that bookkeeping so callers never have to remember the order of operations.
//!
//! States (mirrors `basic_state` / `content_state` columns):
//!
//! | value | meaning |
//! |------:|---------|
//! | 0 | pending |
//! | 1 | done |
//! | 2 | failed |
//! | 3 | not applicable (content only) |
use rusqlite::{params, Connection, OptionalExtension, Transaction};
use crate::mime::FileType;
pub const STATE_PENDING: i64 = 0;
pub const STATE_DONE: i64 = 1;
pub const STATE_FAILED: i64 = 2;
pub const STATE_NA: i64 = 3;
/// Everything needed to insert a fresh file row.
#[derive(Debug, Clone)]
pub struct NewFile<'a> {
pub name: &'a str,
pub path: &'a str,
pub parent: &'a str,
pub size: u64,
pub mtime: u64,
pub inode: Option<u64>,
pub device_id: Option<u64>,
pub mime: Option<&'a str>,
pub ftype: FileType,
pub hash: Option<&'a [u8]>,
}
/// Insert a new file row, returning its id. `basic_state` is set to DONE
/// (row existing *is* the basic-index state); `content_state` is PENDING
/// unless the MIME maps to a type we won't extract text from, in which case
/// the caller can later set it to NA.
///
/// Uses `INSERT OR IGNORE` so a UNIQUE(path) collision (which indicates the
/// caller fed the same path twice in one run) becomes a silent no-op
/// returning `None` rather than aborting the whole batch. The walker is
/// expected to dedupe visits upstream; this is a defense-in-depth backstop.
pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> {
let rows = tx
.execute(
"INSERT OR IGNORE INTO files (
name, path, parent, size, mtime, inode, device_id,
mime, type, basic_state, content_state, hash
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
params![
f.name,
f.path,
f.parent,
f.size as i64,
f.mtime as i64,
f.inode.map(|x| x as i64),
f.device_id.map(|x| x as i64),
f.mime,
f.ftype.bits() as i64,
STATE_DONE,
STATE_PENDING,
f.hash,
],
)
.map_err(|e| format!("insert file {}: {}", f.path, e))?;
if rows == 0 {
// Existing row with the same path (e.g. duplicate visit within the run).
return Ok(None);
}
Ok(Some(tx.last_insert_rowid()))
}
/// Update a file's metadata in place (same path, changed size/mtime/hash).
/// Clears any extracted content so the text-indexing pass re-processes it.
pub fn update_file_basic(
tx: &Transaction<'_>,
path: &str,
size: u64,
mtime: u64,
hash: Option<&[u8]>,
mime: Option<&str>,
ftype: FileType,
) -> Result<Option<i64>, String> {
let id: Option<i64> = tx
.query_row(
"SELECT id FROM files WHERE path = ?1",
params![path],
|r| r.get(0),
)
.optional()
.map_err(|e| format!("lookup file id {}: {}", path, e))?;
let Some(id) = id else {
return Ok(None);
};
tx.execute(
"UPDATE files
SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5,
content_state = ?6, failure_msg = NULL
WHERE id = ?7",
params![
size as i64,
mtime as i64,
hash,
mime,
ftype.bits() as i64,
STATE_PENDING,
id,
],
)
.map_err(|e| format!("update file {}: {}", path, e))?;
// Any prior extracted content is stale — remove it along with its FTS row.
remove_content_for_id(tx, id)?;
Ok(Some(id))
}
/// Mark a file's content indexing as complete and write the extracted text +
/// properties atomically. FTS5 stores the canonical text for this file;
/// `properties` are stored both as a structured side-table (for exact
/// retrieval) and concatenated into the FTS `properties` column (for MATCH).
pub fn set_content_done(
tx: &Transaction<'_>,
file_id: i64,
name: &str,
text: &str,
properties: &[(String, String)],
) -> Result<(), String> {
// Clear any previous extraction (in case of re-run).
remove_content_for_id(tx, file_id)?;
for (k, v) in properties {
tx.execute(
"INSERT INTO properties(file_id, key, value) VALUES (?1, ?2, ?3)",
params![file_id, k, v],
)
.map_err(|e| format!("insert property {}={}: {}", k, v, e))?;
}
let props_blob = encode_properties_for_fts(properties);
tx.execute(
"INSERT INTO searchabletext(rowid, name, text, properties) VALUES (?1, ?2, ?3, ?4)",
params![file_id, name, text, props_blob],
)
.map_err(|e| format!("insert FTS row {}: {}", file_id, e))?;
tx.execute(
"UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2",
params![STATE_DONE, file_id],
)
.map_err(|e| format!("update content_state DONE {}: {}", file_id, e))?;
// Clear any prior failed-file record.
tx.execute("DELETE FROM failed_files WHERE file_id = ?1", params![file_id])
.map_err(|e| format!("clear failed_files {}: {}", file_id, e))?;
Ok(())
}
/// Mark a file's content extraction as failed. Keeps the basic row in place.
pub fn set_content_failed(
tx: &Transaction<'_>,
file_id: i64,
reason: &str,
) -> Result<(), String> {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0) as i64;
tx.execute(
"UPDATE files SET content_state = ?1, failure_msg = ?2 WHERE id = ?3",
params![STATE_FAILED, reason, file_id],
)
.map_err(|e| format!("update content_state FAILED {}: {}", file_id, e))?;
tx.execute(
"INSERT OR REPLACE INTO failed_files(file_id, reason, ts) VALUES (?1, ?2, ?3)",
params![file_id, reason, now],
)
.map_err(|e| format!("insert failed_files {}: {}", file_id, e))?;
Ok(())
}
/// Mark content extraction as not applicable (e.g. binary format we don't
/// support). The file row still contributes to filename search.
pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
tx.execute(
"UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2",
params![STATE_NA, file_id],
)
.map_err(|e| format!("update content_state NA {}: {}", file_id, e))?;
tx.execute("DELETE FROM failed_files WHERE file_id = ?1", params![file_id])
.map_err(|e| format!("clear failed_files {}: {}", file_id, e))?;
Ok(())
}
/// Delete a file row by path, keeping FTS in sync. Returns whether a row was
/// removed.
pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result<bool, String> {
let id: Option<i64> = tx
.query_row(
"SELECT id FROM files WHERE path = ?1",
params![path],
|r| r.get(0),
)
.optional()
.map_err(|e| format!("lookup {} for delete: {}", path, e))?;
let Some(id) = id else { return Ok(false) };
remove_content_for_id(tx, id)?;
tx.execute("DELETE FROM files WHERE id = ?1", params![id])
.map_err(|e| format!("delete file {}: {}", path, e))?;
Ok(true)
}
/// Remove the FTS row and any `properties` rows for a given file id. Does
/// not touch the `files` row itself. Idempotent — a missing FTS row is fine.
pub fn remove_content_for_id(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
// Regular FTS5 supports a plain DELETE by rowid; no need to supply the
// old column values the way a contentless table would require.
tx.execute(
"DELETE FROM searchabletext WHERE rowid = ?1",
params![file_id],
)
.map_err(|e| format!("FTS delete row {}: {}", file_id, e))?;
tx.execute("DELETE FROM properties WHERE file_id = ?1", params![file_id])
.map_err(|e| format!("delete properties {}: {}", file_id, e))?;
Ok(())
}
/// Serialize properties for the FTS `properties` column. `key:value` pairs
/// separated by spaces so `MATCH 'properties:artist:beatles'` works.
fn encode_properties_for_fts(props: &[(String, String)]) -> String {
let mut buf = String::new();
for (i, (k, v)) in props.iter().enumerate() {
if i > 0 {
buf.push(' ');
}
buf.push_str(k);
buf.push(':');
buf.push_str(v);
}
buf
}
/// Flush and close a connection, restoring durable PRAGMAs. Call on clean
/// shutdown so the next open sees a consistent DB.
pub fn checkpoint_and_close(conn: Connection) {
let _ = conn.execute_batch(
"PRAGMA journal_mode = DELETE; \
PRAGMA synchronous = FULL; \
PRAGMA wal_checkpoint(FULL);",
);
drop(conn);
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::open_and_migrate;
fn tmp_path() -> std::path::PathBuf {
let mut p = std::env::temp_dir();
p.push(format!(
"quicksearch-repo-{}-{}.sqlite",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
p
}
#[test]
fn insert_update_delete_round_trip() {
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
{
let tx = conn.transaction().unwrap();
let id = insert_file(
&tx,
&NewFile {
name: "a.txt",
path: "/tmp/a.txt",
parent: "/tmp",
size: 42,
mtime: 1_700_000_000,
inode: Some(7),
device_id: Some(64768),
mime: Some("text/plain"),
ftype: FileType::TEXT,
hash: Some(&[1, 2, 3]),
},
)
.unwrap()
.expect("unique path");
set_content_done(
&tx,
id,
"a.txt",
"hello world",
&[("title".to_string(), "hi".to_string())],
)
.unwrap();
tx.commit().unwrap();
}
// Text is findable via FTS.
let hit: i64 = conn
.query_row(
"SELECT rowid FROM searchabletext WHERE searchabletext MATCH 'hello'",
[],
|r| r.get(0),
)
.unwrap();
assert!(hit > 0);
// Delete cleans up.
{
let tx = conn.transaction().unwrap();
assert!(delete_file_by_path(&tx, "/tmp/a.txt").unwrap());
tx.commit().unwrap();
}
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0);
let fts_count: i64 = conn
.query_row("SELECT COUNT(*) FROM searchabletext", [], |r| r.get(0))
.unwrap();
assert_eq!(fts_count, 0);
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn update_resets_content_state() {
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let id = {
let tx = conn.transaction().unwrap();
let id = insert_file(
&tx,
&NewFile {
name: "a.txt",
path: "/tmp/a.txt",
parent: "/tmp",
size: 10,
mtime: 1,
inode: None,
device_id: None,
mime: None,
ftype: FileType::EMPTY,
hash: None,
},
)
.unwrap()
.expect("unique path");
set_content_done(&tx, id, "a.txt", "old text", &[]).unwrap();
tx.commit().unwrap();
id
};
{
let tx = conn.transaction().unwrap();
let got = update_file_basic(
&tx,
"/tmp/a.txt",
20,
2,
None,
Some("text/plain"),
FileType::TEXT,
)
.unwrap();
assert_eq!(got, Some(id));
tx.commit().unwrap();
}
let (state, content): (i64, i64) = conn
.query_row(
"SELECT basic_state, content_state FROM files WHERE id = ?1",
params![id],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
assert_eq!(state, STATE_DONE);
assert_eq!(content, STATE_PENDING);
// FTS row for the stale content should be gone.
let fts_hits: i64 = conn
.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'old'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(fts_hits, 0);
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn insert_file_twice_on_same_path_is_idempotent() {
// Defense-in-depth: when the walker visits a canonical path twice
// (overlapping roots, symlink resolution quirks), the second INSERT
// must be a silent no-op, not a run-ending error.
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let tx = conn.transaction().unwrap();
let row = NewFile {
name: "dup.txt",
path: "/tmp/dup.txt",
parent: "/tmp",
size: 1,
mtime: 1,
inode: None,
device_id: None,
mime: Some("text/plain"),
ftype: FileType::TEXT,
hash: None,
};
let id1 = insert_file(&tx, &row).unwrap().expect("first insert");
let id2 = insert_file(&tx, &row).unwrap();
assert!(id2.is_none(), "second insert of same path must return None");
// Only one row exists.
let count: i64 = tx
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 1);
let (id_read,): (i64,) = tx
.query_row(
"SELECT id FROM files WHERE path = ?1",
params!["/tmp/dup.txt"],
|r| Ok((r.get(0)?,)),
)
.unwrap();
assert_eq!(id_read, id1);
tx.commit().unwrap();
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn set_content_failed_writes_failed_table() {
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let id = {
let tx = conn.transaction().unwrap();
let id = insert_file(
&tx,
&NewFile {
name: "oops.bin",
path: "/tmp/oops.bin",
parent: "/tmp",
size: 0,
mtime: 1,
inode: None,
device_id: None,
mime: None,
ftype: FileType::EMPTY,
hash: None,
},
)
.unwrap()
.expect("unique path");
set_content_failed(&tx, id, "bad parse").unwrap();
tx.commit().unwrap();
id
};
let reason: String = conn
.query_row(
"SELECT reason FROM failed_files WHERE file_id = ?1",
params![id],
|r| r.get(0),
)
.unwrap();
assert_eq!(reason, "bad parse");
let content_state: i64 = conn
.query_row(
"SELECT content_state FROM files WHERE id = ?1",
params![id],
|r| r.get(0),
)
.unwrap();
assert_eq!(content_state, STATE_FAILED);
drop(conn);
std::fs::remove_file(&p).ok();
}
}

84
crates/quicksearch-core/src/db/schema.rs Normal file
View file

@ -0,0 +1,84 @@
//! SQL strings for the current schema. Versioned; [`migrate`](super::migrate)
//! drives the upgrade path.
/// Pragmas applied on every connection open. Tuned for write throughput during
/// indexing; a clean shutdown re-enables journal_mode/synchronous via
/// [`super::repo::checkpoint_and_close`].
pub const PRAGMAS_FAST: &str = "
PRAGMA journal_mode = OFF;
PRAGMA synchronous = 0;
PRAGMA cache_size = 10000;
PRAGMA temp_store = MEMORY;
PRAGMA foreign_keys = ON;
";
/// The full current schema. Applied by [`migrate::open_and_migrate`] when
/// the DB is fresh or has been wiped during upgrade.
///
/// FTS5 is a *regular* (non-contentless) virtual table so `snippet()` and
/// `highlight()` can read the stored text. This also means there is no
/// separate `documents` table — FTS5 *is* the text store.
pub const SCHEMA_CURRENT: &str = r#"
CREATE TABLE schema_info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE files (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
path TEXT NOT NULL UNIQUE,
parent TEXT NOT NULL,
size INTEGER NOT NULL,
mtime INTEGER NOT NULL,
inode INTEGER,
device_id INTEGER,
mime TEXT,
type INTEGER NOT NULL DEFAULT 0,
basic_state INTEGER NOT NULL DEFAULT 0, -- 0=pending 1=done 2=failed
content_state INTEGER NOT NULL DEFAULT 0, -- 0=pending 1=done 2=failed 3=n/a
failure_msg TEXT,
hash BLOB
);
CREATE INDEX idx_files_parent ON files(parent);
CREATE INDEX idx_files_mtime ON files(mtime);
CREATE INDEX idx_files_type ON files(type);
CREATE INDEX idx_files_mime ON files(mime);
CREATE INDEX idx_files_hash ON files(hash);
CREATE INDEX idx_files_content_pending ON files(id) WHERE content_state = 0;
CREATE TABLE properties (
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
key TEXT NOT NULL,
value TEXT NOT NULL,
PRIMARY KEY (file_id, key)
);
CREATE TABLE failed_files (
file_id INTEGER PRIMARY KEY REFERENCES files(id) ON DELETE CASCADE,
reason TEXT,
ts INTEGER NOT NULL
);
CREATE TABLE config_validation (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
"#;
/// FTS5 virtual table DDL. Separate because the tokenizer is config-driven.
///
/// Regular (not contentless, not external-content) FTS5: the table stores
/// its own text, which enables `snippet()`/`highlight()` and makes row-level
/// INSERT/UPDATE/DELETE work with normal SQL semantics. `rowid` is supplied
/// by the caller and must equal `files.id`.
pub fn fts_create_sql(tokenizer: &str) -> String {
format!(
"CREATE VIRTUAL TABLE searchabletext USING fts5(\
name, text, properties, \
tokenize='{}'\
);",
tokenizer.replace('\'', "''")
)
}

0
src/document_extraction.rs → crates/quicksearch-core/src/document_extraction.rs
View file

96
crates/quicksearch-core/src/extract/audio.rs Normal file
View file

@ -0,0 +1,96 @@
//! Audio tag extraction via [`lofty`]. Pulls title/artist/album/genre/year/
//! track/duration into properties; concatenates tag values into `text` so
//! full-text search works across them.
use std::path::Path;
use lofty::{
file::{AudioFile, TaggedFileExt},
probe::Probe,
tag::{Accessor, ItemKey},
};
use super::{ExtractError, ExtractedContent, Extractor};
pub struct AudioExtractor;
impl Extractor for AudioExtractor {
fn supports(&self, mime: &str) -> bool {
mime.starts_with("audio/")
}
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
let tagged = Probe::open(path)
.map_err(|e| format!("lofty probe {}: {}", path.display(), e))?
.read()
.map_err(|e| format!("lofty read {}: {}", path.display(), e))?;
let mut out = ExtractedContent::default();
// Duration in seconds.
let duration_secs = tagged.properties().duration().as_secs();
if duration_secs > 0 {
out.properties
.insert("duration".to_string(), duration_secs.to_string());
}
if let Some(tag) = tagged.primary_tag().or_else(|| tagged.first_tag()) {
for (key, item_key) in [
("title", ItemKey::TrackTitle),
("artist", ItemKey::TrackArtist),
("album", ItemKey::AlbumTitle),
("genre", ItemKey::Genre),
("year", ItemKey::Year),
("track", ItemKey::TrackNumber),
("comment", ItemKey::Comment),
] {
if let Some(v) = tag.get_string(&item_key) {
if !v.is_empty() {
out.properties.insert(key.to_string(), v.to_string());
}
}
}
// Accessor shortcuts for common fields if the ItemKey lookup missed.
if !out.properties.contains_key("title") {
if let Some(t) = tag.title() {
out.properties.insert("title".to_string(), t.to_string());
}
}
if !out.properties.contains_key("artist") {
if let Some(a) = tag.artist() {
out.properties.insert("artist".to_string(), a.to_string());
}
}
if !out.properties.contains_key("album") {
if let Some(a) = tag.album() {
out.properties.insert("album".to_string(), a.to_string());
}
}
}
// Join the searchable tag values into one blob so FTS hits on any of them.
let mut pieces: Vec<&str> = Vec::new();
for k in ["title", "artist", "album", "genre", "comment"] {
if let Some(v) = out.properties.get(k) {
pieces.push(v.as_str());
}
}
out.text = pieces.join(" ");
Ok(out)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn supports_audio_mimes() {
let e = AudioExtractor;
assert!(e.supports("audio/mpeg"));
assert!(e.supports("audio/flac"));
assert!(!e.supports("video/mp4"));
assert!(!e.supports("image/png"));
}
}

96
crates/quicksearch-core/src/extract/image.rs Normal file
View file

@ -0,0 +1,96 @@
//! Image metadata extraction via [`kamadak_exif`]. Reads EXIF tags (camera
//! make/model, date, GPS, dimensions) into properties. `text` is left empty
//! — this extractor does not OCR.
use std::fs::File;
use std::io::BufReader;
use std::path::Path;
use exif::{In, Reader, Tag, Value};
use super::{ExtractError, ExtractedContent, Extractor};
pub struct ImageExtractor;
impl Extractor for ImageExtractor {
fn supports(&self, mime: &str) -> bool {
mime.starts_with("image/")
}
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
let file = File::open(path)
.map_err(|e| format!("image open {}: {}", path.display(), e))?;
let mut bufreader = BufReader::new(&file);
let mut out = ExtractedContent::default();
// Not every image has EXIF (e.g. PNGs usually don't). Treat "no EXIF"
// as a successful no-op rather than a failure.
let exif = match Reader::new().read_from_container(&mut bufreader) {
Ok(exif) => exif,
Err(_) => return Ok(out),
};
for (key, tag) in [
("make", Tag::Make),
("model", Tag::Model),
("date_taken", Tag::DateTimeOriginal),
("software", Tag::Software),
("orientation", Tag::Orientation),
("width", Tag::PixelXDimension),
("height", Tag::PixelYDimension),
("iso", Tag::PhotographicSensitivity),
("f_number", Tag::FNumber),
("exposure", Tag::ExposureTime),
("focal_length", Tag::FocalLength),
("gps_latitude", Tag::GPSLatitude),
("gps_longitude", Tag::GPSLongitude),
] {
if let Some(field) = exif.get_field(tag, In::PRIMARY) {
let s = field_to_string(&field.value);
if !s.is_empty() {
out.properties.insert(key.to_string(), s);
}
}
}
Ok(out)
}
}
fn field_to_string(value: &Value) -> String {
match value {
Value::Ascii(bytes_vec) => {
let mut parts = Vec::new();
for bytes in bytes_vec {
let s: String = bytes
.iter()
.take_while(|&&b| b != 0)
.map(|&b| b as char)
.collect();
if !s.is_empty() {
parts.push(s);
}
}
parts.join(" ")
}
other => {
// The EXIF crate's Display formatter handles integer/rational
// types. Use it for any value we didn't handle explicitly.
format!("{}", other.display_as(exif::Tag::Copyright))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn supports_image_mimes() {
let e = ImageExtractor;
assert!(e.supports("image/jpeg"));
assert!(e.supports("image/png"));
assert!(!e.supports("video/mp4"));
assert!(!e.supports("text/plain"));
}
}

149
crates/quicksearch-core/src/extract/mod.rs Normal file
View file

@ -0,0 +1,149 @@
//! Content extractors: text plus structured properties (title, artist, EXIF, …).
//!
//! An [`Extractor`] decides whether it can handle a given MIME type and, if
//! so, produces [`ExtractedContent`] for the file. The [`Registry`] picks the
//! first registered extractor that accepts the MIME and runs it. Callers can
//! also fall back to an extension-based match for files with no detected MIME.
use std::collections::HashMap;
use std::path::Path;
pub mod audio;
pub mod image;
pub mod office;
pub mod pdf;
pub mod plaintext;
/// Result of a successful extraction. `text` feeds the FTS5 `text` column;
/// `properties` feeds both the `properties` FTS5 column (as `key:value`
/// tokens) and the structured `properties` table for later retrieval.
///
/// Extractors may return an empty `text` when the file has no narrative
/// content (e.g. an image where only EXIF matters). Filename search still
/// works in that case.
#[derive(Debug, Default, Clone)]
pub struct ExtractedContent {
pub text: String,
pub properties: HashMap<String, String>,
}
impl ExtractedContent {
pub fn with_text(text: impl Into<String>) -> Self {
Self {
text: text.into(),
properties: HashMap::new(),
}
}
pub fn with_property(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.properties.insert(key.into(), value.into());
self
}
/// Convert properties into the `Vec<(String, String)>` shape expected by
/// [`crate::db::repo::set_content_done`]. Keys are sorted for determinism
/// in tests and snapshots.
pub fn properties_sorted(&self) -> Vec<(String, String)> {
let mut v: Vec<(String, String)> = self
.properties
.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
v.sort_by(|a, b| a.0.cmp(&b.0));
v
}
}
/// Boxed error type for extractor failures. A string reason is stored on the
/// file row (see [`crate::db::repo::set_content_failed`]), so extractors
/// should surface human-readable messages.
pub type ExtractError = String;
/// A pluggable content extractor. Stateless; implementors should not hold
/// file handles across calls.
pub trait Extractor: Send + Sync {
/// Whether this extractor can handle the given MIME type. `mime` is
/// normalized to lowercase before dispatch.
fn supports(&self, mime: &str) -> bool;
/// Read the file at `path` and return its extracted content. Return an
/// [`ExtractError`] to mark the file's content state as failed (so it
/// won't be retried every run).
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError>;
}
/// An ordered dispatch table of extractors. The first extractor whose
/// [`supports`](Extractor::supports) returns true for the MIME is used.
pub struct Registry {
extractors: Vec<Box<dyn Extractor>>,
}
impl Registry {
pub fn new() -> Self {
Self { extractors: Vec::new() }
}
pub fn with(mut self, e: impl Extractor + 'static) -> Self {
self.extractors.push(Box::new(e));
self
}
/// Look up a handler for `mime` and run it against `path`. Returns
/// `Ok(None)` if no extractor claims the MIME — the caller should then
/// decide whether the file is "not applicable" (text state NA).
pub fn extract(
&self,
path: &Path,
mime: &str,
) -> Result<Option<ExtractedContent>, ExtractError> {
let lower = mime.to_ascii_lowercase();
for e in &self.extractors {
if e.supports(&lower) {
return e.extract(path).map(Some);
}
}
Ok(None)
}
/// The default set wired up for Set A: plaintext, office docs, PDF,
/// audio tags, image EXIF.
pub fn default_set() -> Self {
Self::new()
.with(plaintext::PlaintextExtractor)
.with(office::OfficeExtractor)
.with(pdf::PdfExtractor)
.with(audio::AudioExtractor)
.with(image::ImageExtractor)
}
}
impl Default for Registry {
fn default() -> Self {
Self::default_set()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn empty_registry_returns_none() {
let r = Registry::new();
let out = r
.extract(Path::new("/tmp/x"), "text/plain")
.expect("no error");
assert!(out.is_none());
}
#[test]
fn properties_sorted_is_deterministic() {
let c = ExtractedContent::with_text("hi")
.with_property("b", "2")
.with_property("a", "1");
assert_eq!(
c.properties_sorted(),
vec![("a".to_string(), "1".to_string()), ("b".to_string(), "2".to_string())]
);
}
}

69
crates/quicksearch-core/src/extract/office.rs Normal file
View file

@ -0,0 +1,69 @@
//! Office document extraction: DOCX, XLSX, PPTX, ODT, ODP, ODS.
//!
//! Delegates to [`crate::document_extraction`] — a single entry point based
//! on file extension rather than MIME. We translate the MIME to the
//! extension expected by that module.
use std::ffi::OsString;
use std::path::Path;
use crate::document_extraction::extract_document_text;
use super::{ExtractError, ExtractedContent, Extractor};
pub struct OfficeExtractor;
fn mime_to_ext(mime: &str) -> Option<&'static str> {
match mime {
"application/msword" => Some("doc"),
"application/vnd.openxmlformats-officedocument.wordprocessingml.document" => Some("docx"),
"application/vnd.ms-excel" => Some("xls"),
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => Some("xlsx"),
"application/vnd.ms-powerpoint" => Some("ppt"),
"application/vnd.openxmlformats-officedocument.presentationml.presentation" => Some("pptx"),
"application/vnd.oasis.opendocument.text" => Some("odt"),
"application/vnd.oasis.opendocument.spreadsheet" => Some("ods"),
"application/vnd.oasis.opendocument.presentation" => Some("odp"),
_ => None,
}
}
impl Extractor for OfficeExtractor {
fn supports(&self, mime: &str) -> bool {
mime_to_ext(mime).is_some()
}
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
// Recompute the extension from MIME at call time by asking the path.
// Using the filesystem extension directly is more robust than round-
// tripping through MIME: `.docm` has the same MIME as `.docx` but
// `extract_document_text` looks up by extension.
let ext = path
.extension()
.and_then(|s| s.to_str())
.map(|s| s.to_ascii_lowercase())
.unwrap_or_default();
let text = extract_document_text(&OsString::from(path.as_os_str()), &ext)
.map_err(|e| format!("office extractor {}: {}", path.display(), e))?;
Ok(ExtractedContent::with_text(text))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn supports_docx_and_friends() {
let e = OfficeExtractor;
for m in [
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.oasis.opendocument.text",
] {
assert!(e.supports(m), "should support {}", m);
}
assert!(!e.supports("image/png"));
}
}

77
crates/quicksearch-core/src/extract/pdf.rs Normal file
View file

@ -0,0 +1,77 @@
//! PDF text extraction.
//!
//! Primary path: [`pdf_extract::extract_text`], which handles most modern PDFs
//! and is simple to call. It can panic or hard-error on malformed files; any
//! failure is surfaced to the caller and marks the file's content state as
//! failed. Properties (title, author, etc.) from the PDF `Info` dictionary
//! are pulled via `lopdf` where available.
use std::path::Path;
use lopdf::{Document as LopdfDocument, Object};
use super::{ExtractError, ExtractedContent, Extractor};
pub struct PdfExtractor;
impl Extractor for PdfExtractor {
fn supports(&self, mime: &str) -> bool {
mime == "application/pdf"
}
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
// Text. Catch panics from pdf_extract (some PDFs crash its parser).
let path_buf = path.to_path_buf();
let text = std::panic::catch_unwind(move || pdf_extract::extract_text(&path_buf))
.map_err(|_| "pdf_extract panicked".to_string())?
.map_err(|e| format!("pdf_extract: {}", e))?;
let mut out = ExtractedContent::with_text(text);
// Info dictionary via lopdf. Soft-fail: if lopdf can't open the file
// we still return the text.
if let Ok(doc) = LopdfDocument::load(path) {
if let Ok(info_ref) = doc.trailer.get(b"Info") {
if let Ok(info_id) = info_ref.as_reference() {
if let Ok(info) = doc.get_object(info_id) {
if let Ok(dict) = info.as_dict() {
for key in ["Title", "Author", "Subject", "Keywords", "Creator", "Producer"] {
if let Ok(val) = dict.get(key.as_bytes()) {
if let Some(s) = object_to_string(val) {
if !s.is_empty() {
out.properties.insert(key.to_ascii_lowercase(), s);
}
}
}
}
}
}
}
}
}
Ok(out)
}
}
fn object_to_string(obj: &Object) -> Option<String> {
match obj {
Object::String(bytes, _) => {
// Try UTF-8; fall back to lossy decoding.
Some(String::from_utf8_lossy(bytes).into_owned())
}
Object::Name(bytes) => Some(String::from_utf8_lossy(bytes).into_owned()),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn supports_pdf_mime() {
assert!(PdfExtractor.supports("application/pdf"));
assert!(!PdfExtractor.supports("application/zip"));
}
}

65
crates/quicksearch-core/src/extract/plaintext.rs Normal file
View file

@ -0,0 +1,65 @@
//! Read the file as UTF-8 text. Handles text/plain, text/x-*, application/json
//! and most source-code MIMEs.
use std::path::Path;
use super::{ExtractError, ExtractedContent, Extractor};
pub struct PlaintextExtractor;
impl Extractor for PlaintextExtractor {
fn supports(&self, mime: &str) -> bool {
if mime.starts_with("text/") {
return true;
}
matches!(
mime,
"application/json"
| "application/xml"
| "application/javascript"
| "application/x-shellscript"
| "application/x-python"
| "application/toml"
| "application/yaml"
| "application/x-yaml"
)
}
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
let text = std::fs::read_to_string(path)
.map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
Ok(ExtractedContent::with_text(text))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_utf8_file() {
let mut p = std::env::temp_dir();
p.push(format!(
"qs-plaintext-{}-{}.txt",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::write(&p, "hello world").unwrap();
let c = PlaintextExtractor.extract(&p).unwrap();
assert_eq!(c.text, "hello world");
std::fs::remove_file(&p).ok();
}
#[test]
fn supports_text_mimes() {
let e = PlaintextExtractor;
assert!(e.supports("text/plain"));
assert!(e.supports("text/x-rust"));
assert!(e.supports("application/json"));
assert!(!e.supports("application/pdf"));
assert!(!e.supports("image/png"));
}
}

276
src/file_handling.rs → crates/quicksearch-core/src/file_handling.rs
View file

@ -1,22 +1,26 @@
use std::sync::atomic::AtomicBool;
use std::sync::{Mutex, Arc}; use std::sync::{Mutex, Arc};
use std::ffi::OsString; use std::ffi::OsString;
use std::fs::{File,read_to_string}; use std::fs::File;
use std::io::{Read, Seek, SeekFrom}; use std::io::{Read, Seek, SeekFrom};
use std::path::Component; use std::path::{Component, Path};
use std::process::{Command, Stdio}; use std::process::{Command, Stdio};
use std::time::UNIX_EPOCH; use std::time::UNIX_EPOCH;
use std::collections::HashMap; use std::collections::HashMap;
use sha2::{Sha256, Digest}; use sha2::{Sha256, Digest};
use walkdir::{DirEntry, WalkDir}; use walkdir::{DirEntry, WalkDir};
use rusqlite::{params, Connection}; use rusqlite::Connection;
use crate::document_extraction::extract_document_text;
use crate::config::Config; use crate::config::Config;
use crate::db::repo::{self, NewFile};
use crate::extract::Registry;
use crate::indexing::should_abort;
use crate::mime::{guess_mime, mime_to_type, FileType};
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ExistingFileEntry { pub struct ExistingFileEntry {
pub moddate: u64, pub mtime: u64,
} }
pub const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 84] = pub const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 84] =
@ -58,15 +62,15 @@ pub const SUPPORTED_DOCUMENT_EXTENSIONS_LIST: [&'static str; 9] =
"ppt", "pptx", "odp", // Presentation "ppt", "pptx", "odp", // Presentation
"xls", "xlsx", "ods"]; // Spreadsheet "xls", "xlsx", "ods"]; // Spreadsheet
/// Load path and moddate per row for incremental classification (hash/size loaded only when updating a file). /// Load path and mtime per row for incremental classification (hash/size loaded only when updating a file).
pub fn load_existing_files(conn: &Connection) -> Result<HashMap<String, ExistingFileEntry>, rusqlite::Error> { pub fn load_existing_files(conn: &Connection) -> Result<HashMap<String, ExistingFileEntry>, rusqlite::Error> {
let mut existing_files = HashMap::new(); let mut existing_files = HashMap::new();
let mut stmt = conn.prepare("SELECT path, moddate FROM files")?; let mut stmt = conn.prepare("SELECT path, mtime FROM files")?;
let rows = stmt.query_map([], |row| { let rows = stmt.query_map([], |row| {
Ok(( Ok((
row.get::<_, String>(0)?, row.get::<_, String>(0)?,
ExistingFileEntry { ExistingFileEntry {
moddate: row.get(1)?, mtime: row.get(1)?,
}, },
)) ))
})?; })?;
@ -79,6 +83,29 @@ pub fn load_existing_files(conn: &Connection) -> Result<HashMap<String, Existing
Ok(existing_files) Ok(existing_files)
} }
/// Derive (inode, device_id) from a `std::fs::Metadata` on platforms that
/// expose them. Returns `(None, None)` on Windows and other non-Unix targets.
fn inode_and_device(_meta: &std::fs::Metadata) -> (Option<u64>, Option<u64>) {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
(Some(_meta.ino()), Some(_meta.dev()))
}
#[cfg(not(unix))]
{
(None, None)
}
}
/// Parent directory of a path as a UTF-8 string, empty if root.
fn parent_str(path: &str) -> String {
Path::new(path)
.parent()
.map(|p| p.to_string_lossy().into_owned())
.unwrap_or_default()
}
pub fn indexed_walk_file_entries( pub fn indexed_walk_file_entries(
path: &str, path: &str,
follow_symlinks: bool, follow_symlinks: bool,
@ -240,7 +267,7 @@ pub fn classify_dir_entry_for_indexing(
}; };
if let Some(existing) = existing_files.get(&fpath) { if let Some(existing) = existing_files.get(&fpath) {
if existing.moddate != fmodified { if existing.mtime != fmodified {
Some(FileIndexAction::Update) Some(FileIndexAction::Update)
} else { } else {
Some(FileIndexAction::Skip) Some(FileIndexAction::Skip)
@ -294,39 +321,14 @@ fn format_progress_pair(visit_index: usize, progress_display_total: Option<usize
} }
} }
const FTS_SQL_AUTOMERGE_8: &str = /// Nudge FTS5 to merge its index segments. Best-effort optimization; any
"INSERT INTO searchabletext(searchabletext, rank) VALUES('automerge', 8)"; /// error is logged but not fatal.
const FTS_SQL_REBUILD: &str = "INSERT INTO searchabletext(searchabletext) VALUES('rebuild')";
pub fn fts_finalize_after_text_indexing(conn: &Connection) -> Result<(), String> { pub fn fts_finalize_after_text_indexing(conn: &Connection) -> Result<(), String> {
conn.execute(FTS_SQL_AUTOMERGE_8, []) if let Err(e) = conn.execute(
.map_err(|e| format!("FTS automerge(8): {}", e))?; "INSERT INTO searchabletext(searchabletext, rank) VALUES('automerge', 8)",
conn.execute(FTS_SQL_REBUILD, []) [],
.map_err(|e| format!("FTS rebuild: {}", e))?;
Ok(())
}
fn fts_remove_document_for_path(
tx: &rusqlite::Transaction<'_>,
path: &str,
) -> Result<(), String> {
let id_opt: Option<i64> = match tx.query_row(
"SELECT id FROM documents WHERE path = ?1",
params![path],
|r| r.get(0),
) { ) {
Ok(id) => Some(id), eprintln!("Warning: FTS automerge failed (non-fatal): {}", e);
Err(rusqlite::Error::QueryReturnedNoRows) => None,
Err(e) => return Err(format!("documents id lookup: {}", e)),
};
if let Some(doc_id) = id_opt {
tx.execute(
"INSERT INTO searchabletext(searchabletext, rowid) VALUES('delete', ?1)",
params![doc_id],
)
.map_err(|e| format!("FTS delete doc {}: {}", doc_id, e))?;
tx.execute("DELETE FROM documents WHERE id = ?1", params![doc_id])
.map_err(|e| format!("delete documents row: {}", e))?;
} }
Ok(()) Ok(())
} }
@ -459,15 +461,24 @@ pub fn process_batch_updates_files_only(
)); ));
} }
tx.execute( let p = Path::new(row.path_db.as_str());
"UPDATE files SET size = ?1, moddate = ?2, hash = ?3 WHERE path = ?4", let guessed_mime = guess_mime(p);
params![row.fsize, row.fmodified, row.fhash, row.path_db], let ftype = guessed_mime
.as_deref()
.map(mime_to_type)
.unwrap_or(FileType::EMPTY);
let _ = repo::update_file_basic(
&tx,
&row.path_db,
row.fsize,
row.fmodified,
Some(row.fhash.as_slice()),
guessed_mime.as_deref(),
ftype,
) )
.map_err(|e| format!("Failed to update file record: {}", e))?; .map_err(|e| {
fts_remove_document_for_path(&tx, &row.path_db).map_err(|e| {
format!( format!(
"Failed to remove old document / FTS entry for {}: {}", "Failed to update file record + clear stale content for {}: {}",
row.path_db, e row.path_db, e
) )
})?; })?;
@ -565,12 +576,32 @@ pub fn process_batch_inserts_files_only(
}; };
let fname = entry.path().file_name().unwrap().to_os_string(); let fname = entry.path().file_name().unwrap().to_os_string();
let fname_str = fname.to_string_lossy().into_owned();
let fpath_str = fpath.to_string_lossy().into_owned();
let parent = parent_str(&fpath_str);
let (inode, device_id) = inode_and_device(&meta);
let guessed_mime = guess_mime(Path::new(&fpath_str));
let ftype = guessed_mime
.as_deref()
.map(mime_to_type)
.unwrap_or(FileType::EMPTY);
// Insert into files table repo::insert_file(
tx.execute( &tx,
"INSERT INTO files VALUES (?1, ?2, ?3, ?4, ?5)", &NewFile {
params![fname.to_str(), fpath.to_string_lossy(), fsize, fmodified, fhash] name: &fname_str,
).map_err(|e| format!("Failed to insert file record: {}", e))?; path: &fpath_str,
parent: &parent,
size: fsize,
mtime: fmodified,
inode,
device_id,
mime: guessed_mime.as_deref(),
ftype,
hash: Some(fhash.as_slice()),
},
)
.map_err(|e| format!("Failed to insert file record: {}", e))?;
} }
// Update status with current file // Update status with current file
@ -588,6 +619,7 @@ pub fn cleanup_stale_index_entries(
conn_mutex: &Arc<Mutex<Connection>>, conn_mutex: &Arc<Mutex<Connection>>,
stale_paths: &[String], stale_paths: &[String],
stop_flag: &Arc<Mutex<bool>>, stop_flag: &Arc<Mutex<bool>>,
suspend_flag: &Arc<AtomicBool>,
status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>, status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
) -> Result<usize, String> { ) -> Result<usize, String> {
if stale_paths.is_empty() { if stale_paths.is_empty() {
@ -601,7 +633,7 @@ pub fn cleanup_stale_index_entries(
let mut deleted_count = 0usize; let mut deleted_count = 0usize;
for path in stale_paths { for path in stale_paths {
if *stop_flag.lock().unwrap() { if should_abort(stop_flag, suspend_flag) {
let _ = tx.commit(); let _ = tx.commit();
drop(conn); drop(conn);
return Ok(deleted_count); return Ok(deleted_count);
@ -611,23 +643,22 @@ pub fn cleanup_stale_index_entries(
callback(&format!("Removing stale index entry: {}", path)); callback(&format!("Removing stale index entry: {}", path));
} }
fts_remove_document_for_path(&tx, path).map_err(|e| { if repo::delete_file_by_path(&tx, path).map_err(|e| {
format!( format!(
"Failed to remove stale document / FTS entry for {}: {}", "Failed to remove stale index entry for {}: {}",
path, e path, e
) )
})?; })? {
tx.execute("DELETE FROM files WHERE path = ?1", params![path]) deleted_count += 1;
.map_err(|e| format!("Failed to delete stale file record {}: {}", path, e))?; }
deleted_count += 1;
} }
tx.commit() tx.commit()
.map_err(|e| format!("Failed to commit stale cleanup transaction: {}", e))?; .map_err(|e| format!("Failed to commit stale cleanup transaction: {}", e))?;
if deleted_count > 0 && !*stop_flag.lock().unwrap() { if deleted_count > 0 && !should_abort(stop_flag, suspend_flag) {
if let Some(ref callback) = status_callback { if let Some(ref callback) = status_callback {
callback("Rebuilding FTS index after stale cleanup..."); callback("Optimizing FTS index after stale cleanup...");
} }
fts_finalize_after_text_indexing(&conn)?; fts_finalize_after_text_indexing(&conn)?;
} }
@ -635,14 +666,19 @@ pub fn cleanup_stale_index_entries(
Ok(deleted_count) Ok(deleted_count)
} }
/// Process text indexing for files - writes `documents`; FTS rebuilt in `fts_finalize_after_text_indexing`. /// Process text indexing for all files with `content_state = pending`. For
/// each file dispatches to the configured extractor [`Registry`], writes the
/// extracted text + properties via the repo helpers, then flips the row's
/// `content_state` to done/failed/na so it won't be retried next run.
pub fn process_text_indexing( pub fn process_text_indexing(
conn_mutex: &Arc<Mutex<Connection>>, conn_mutex: &Arc<Mutex<Connection>>,
stop_flag: &Arc<Mutex<bool>>, stop_flag: &Arc<Mutex<bool>>,
suspend_flag: &Arc<AtomicBool>,
status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>, status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>, progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>,
config: &Config config: &Config,
) -> Result<(), String> { ) -> Result<(), String> {
let registry = Registry::default_set();
let max_size = config.processing.maximum_text_file_size; let max_size = config.processing.maximum_text_file_size;
let batch_size = config.processing.batch_size; let batch_size = config.processing.batch_size;
let batch_limit = batch_size as i64; let batch_limit = batch_size as i64;
@ -654,43 +690,40 @@ pub fn process_text_indexing(
let total_files: usize = { let total_files: usize = {
let conn = conn_mutex.lock().unwrap(); let conn = conn_mutex.lock().unwrap();
conn.query_row( conn.query_row(
"SELECT COUNT(*) FROM files f "SELECT COUNT(*) FROM files WHERE content_state = 0 AND size <= ?1",
LEFT JOIN documents d ON f.path = d.path
WHERE d.path IS NULL AND f.size <= ?1",
[max_size], [max_size],
|row| row.get(0), |row| row.get(0),
) )
.map_err(|e| format!("Failed to count pending text files: {}", e))? .map_err(|e| format!("Failed to count pending text files: {}", e))?
}; };
let mut cursor_path = String::new(); let mut cursor_id: i64 = 0;
let mut global_index: usize = 0; let mut global_index: usize = 0;
loop { loop {
if *stop_flag.lock().unwrap() { if should_abort(stop_flag, suspend_flag) {
return Ok(()); return Ok(());
} }
let batch: Vec<(String, String, u64)> = { let batch: Vec<(i64, String, String, Option<String>)> = {
let conn = conn_mutex.lock().unwrap(); let conn = conn_mutex.lock().unwrap();
let mut stmt = conn let mut stmt = conn
.prepare( .prepare(
"SELECT f.name, f.path, f.size FROM files f "SELECT id, name, path, mime FROM files
LEFT JOIN documents d ON f.path = d.path WHERE content_state = 0 AND size <= ?1 AND id > ?2
WHERE d.path IS NULL AND f.size <= ?1 ORDER BY id
AND (?2 = '' OR f.path > ?2) LIMIT ?3",
ORDER BY f.path
LIMIT ?3",
) )
.map_err(|e| format!("Failed to prepare text indexing query: {}", e))?; .map_err(|e| format!("Failed to prepare text indexing query: {}", e))?;
let rows = stmt let rows = stmt
.query_map( .query_map(
rusqlite::params![max_size, cursor_path.as_str(), batch_limit], rusqlite::params![max_size, cursor_id, batch_limit],
|row| { |row| {
Ok(( Ok((
row.get::<_, String>(0)?, row.get::<_, i64>(0)?,
row.get::<_, String>(1)?, row.get::<_, String>(1)?,
row.get::<_, u64>(2)?, row.get::<_, String>(2)?,
row.get::<_, Option<String>>(3)?,
)) ))
}, },
) )
@ -703,15 +736,14 @@ pub fn process_text_indexing(
break; break;
} }
let last_path = batch.last().unwrap().1.clone(); cursor_id = batch.last().unwrap().0;
cursor_path = last_path;
let conn = conn_mutex.lock().unwrap(); let conn = conn_mutex.lock().unwrap();
let tx = conn let tx = conn
.unchecked_transaction() .unchecked_transaction()
.map_err(|e| format!("Failed to begin transaction: {}", e))?; .map_err(|e| format!("Failed to begin transaction: {}", e))?;
for (fname, fpath, _fsize) in batch.iter() { for (file_id, fname, fpath, fmime) in batch.iter() {
if *stop_flag.lock().unwrap() { if *stop_flag.lock().unwrap() {
let _ = tx.commit(); let _ = tx.commit();
drop(conn); drop(conn);
@ -731,69 +763,43 @@ pub fn process_text_indexing(
progress_cb(global_index); progress_cb(global_index);
} }
let path = std::path::Path::new(fpath.as_str()); let mime_str = fmime.clone().or_else(|| guess_mime(Path::new(fpath)));
let default_ext = OsString::new(); let result = match mime_str.as_deref() {
let file_extension = path Some(m) => registry.extract(Path::new(fpath), m),
.extension() None => Ok(None),
.unwrap_or(&default_ext)
.to_ascii_lowercase()
.to_str()
.unwrap_or("")
.to_string();
let ext_str = file_extension.as_str();
let text_result = if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) {
match read_to_string(fpath) {
Ok(file_string) => {
let trimmed_file_string =
safe_truncate_string(&file_string, config.processing.maximum_text_size);
Some(trimmed_file_string)
}
Err(_e) => None,
}
} else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) {
match extract_document_text(&std::ffi::OsString::from(fpath), ext_str) {
Ok(extracted_text) => {
if !extracted_text.trim().is_empty() {
Some(safe_truncate_string(
&extracted_text,
config.processing.maximum_text_size,
))
} else {
None
}
}
Err(e) => {
eprintln!(
"Warning: Failed to extract text from document {}: {}",
fpath, e
);
None
}
}
} else {
None
}; };
match result {
if let Some(text_content) = text_result { Ok(Some(mut content)) => {
if let Err(e) = tx.execute( // Truncate extracted text to configured limit before storage.
"INSERT OR REPLACE INTO documents(name, path, text) VALUES (?1, ?2, ?3)", if content.text.len() > config.processing.maximum_text_size {
params![fname, fpath, text_content], content.text =
) { safe_truncate_string(&content.text, config.processing.maximum_text_size);
eprintln!("Warning: Failed to insert document row for {}: {}", fpath, e); }
let props = content.properties_sorted();
if let Err(e) =
repo::set_content_done(&tx, *file_id, fname, &content.text, &props)
{
eprintln!("Warning: set_content_done for {}: {}", fpath, e);
}
}
Ok(None) => {
if let Err(e) = repo::set_content_na(&tx, *file_id) {
eprintln!("Warning: set_content_na for {}: {}", fpath, e);
}
}
Err(reason) => {
if let Err(e) = repo::set_content_failed(&tx, *file_id, &reason) {
eprintln!("Warning: set_content_failed for {}: {}", fpath, e);
}
} }
} }
} }
if let Some(ref callback) = status_callback {
callback("Rebuilding FTS index after text addition...");
}
tx.commit() tx.commit()
.map_err(|e| format!("Failed to commit transaction: {}", e))?; .map_err(|e| format!("Failed to commit transaction: {}", e))?;
} }
if total_files > 0 && !*stop_flag.lock().unwrap() { if total_files > 0 && !should_abort(stop_flag, suspend_flag) {
let conn = conn_mutex.lock().unwrap(); let conn = conn_mutex.lock().unwrap();
fts_finalize_after_text_indexing(&conn)?; fts_finalize_after_text_indexing(&conn)?;
} }

306
src/indexing.rs → crates/quicksearch-core/src/indexing.rs
View file

@ -1,9 +1,10 @@
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, mpsc}; use std::sync::{Arc, Mutex, mpsc};
use std::thread; use std::thread;
use std::time::Instant; use std::time::{Duration, Instant};
use std::process::Command; use std::process::Command;
use std::collections::HashSet; use std::collections::HashSet;
use rusqlite::{Connection, OptionalExtension, params}; use rusqlite::{Connection, params};
use walkdir::DirEntry; use walkdir::DirEntry;
use crate::file_handling::{ use crate::file_handling::{
@ -19,6 +20,7 @@ use crate::file_handling::{
FileIndexAction, FileIndexAction,
}; };
use crate::config::Config; use crate::config::Config;
use crate::db;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct SearchResultRow { pub struct SearchResultRow {
@ -58,7 +60,9 @@ pub enum IndexingStatus {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum IndexingCommand { pub enum IndexingCommand {
Start { Start {
path: String, /// One or more directory roots to index. Order determines walk order;
/// duplicates are silently dropped at run time.
paths: Vec<String>,
db_path: String, db_path: String,
config: Config, config: Config,
}, },
@ -70,9 +74,32 @@ pub struct IndexingService {
status: Arc<Mutex<IndexingStatus>>, status: Arc<Mutex<IndexingStatus>>,
command_tx: mpsc::Sender<IndexingCommand>, command_tx: mpsc::Sender<IndexingCommand>,
db_connection: Arc<Mutex<Option<Arc<Mutex<Connection>>>>>, db_connection: Arc<Mutex<Option<Arc<Mutex<Connection>>>>>,
suspend_flag: Arc<AtomicBool>,
_handle: thread::JoinHandle<()>, _handle: thread::JoinHandle<()>,
} }
/// Polling interval for `should_abort` while suspended.
const SUSPEND_POLL_MS: u64 = 100;
/// Combined stop/suspend check used by worker loops. Returns `true` iff the
/// caller should abort the operation. While the suspend flag is set and stop
/// is not, this parks the thread by sleeping in short increments so a later
/// `resume()` unblocks it. Cheap to call in tight loops.
pub(crate) fn should_abort(
stop: &Arc<Mutex<bool>>,
suspend: &Arc<AtomicBool>,
) -> bool {
loop {
if *stop.lock().unwrap() {
return true;
}
if !suspend.load(Ordering::Relaxed) {
return false;
}
thread::sleep(Duration::from_millis(SUSPEND_POLL_MS));
}
}
/// Set process priority for background operation /// Set process priority for background operation
// fn set_background_priority() { // fn set_background_priority() {
// #[cfg(windows)] // #[cfg(windows)]
@ -106,24 +133,56 @@ impl IndexingService {
let status = Arc::new(Mutex::new(IndexingStatus::Idle)); let status = Arc::new(Mutex::new(IndexingStatus::Idle));
let (command_tx, command_rx) = mpsc::channel(); let (command_tx, command_rx) = mpsc::channel();
let db_connection = Arc::new(Mutex::new(None)); let db_connection = Arc::new(Mutex::new(None));
let suspend_flag = Arc::new(AtomicBool::new(false));
let status_clone = status.clone(); let status_clone = status.clone();
let db_connection_clone = db_connection.clone(); let db_connection_clone = db_connection.clone();
let suspend_clone = suspend_flag.clone();
let handle = thread::spawn(move || { let handle = thread::spawn(move || {
Self::indexing_thread(status_clone, command_rx, db_connection_clone); Self::indexing_thread(status_clone, command_rx, db_connection_clone, suspend_clone);
}); });
IndexingService { IndexingService {
status, status,
command_tx, command_tx,
db_connection, db_connection,
suspend_flag,
_handle: handle, _handle: handle,
} }
} }
pub fn start_indexing(&self, path: String, db_path: String, config: Config) -> Result<(), String> { /// Pause the indexer. All worker loops that call [`should_abort`] will
/// block until [`resume`](Self::resume) is called. No-op if already
/// suspended. Does not stop the worker — stop_indexing is still the way
/// to abort.
pub fn suspend(&self) {
self.suspend_flag.store(true, Ordering::Relaxed);
}
/// Resume indexing after [`suspend`](Self::suspend). No-op if not
/// suspended.
pub fn resume(&self) {
self.suspend_flag.store(false, Ordering::Relaxed);
}
pub fn is_suspended(&self) -> bool {
self.suspend_flag.load(Ordering::Relaxed)
}
/// Start indexing one or more roots. Paths are walked in order; duplicate
/// or nested roots are de-duplicated by the indexer. At least one path is
/// required.
pub fn start_indexing(
&self,
paths: Vec<String>,
db_path: String,
config: Config,
) -> Result<(), String> {
if paths.is_empty() {
return Err("start_indexing requires at least one path".into());
}
self.command_tx self.command_tx
.send(IndexingCommand::Start { path, db_path, config }) .send(IndexingCommand::Start { paths, db_path, config })
.map_err(|e| format!("Failed to send start command: {}", e)) .map_err(|e| format!("Failed to send start command: {}", e))
} }
@ -180,21 +239,15 @@ impl IndexingService {
/// Execute a search query against the database /// Execute a search query against the database
pub fn execute_search(&self, db_path: &str, query: &str) -> Result<Vec<SearchResult>, String> { pub fn execute_search(&self, db_path: &str, query: &str) -> Result<Vec<SearchResult>, String> {
let conn = Connection::open(db_path) let conn = db::open_and_migrate(db_path, "trigram")
.map_err(|e| { .map_err(|e| {
if e.to_string().contains("corrupt") || e.to_string().contains("malformed") { if e.contains("corrupt") || e.contains("malformed") {
format!("DATABASE_CORRUPTED: {}", e) format!("DATABASE_CORRUPTED: {}", e)
} else { } else {
format!("Failed to open database: {}", e) e
} }
})?; })?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);",
(),
)
.map_err(|e| format!("Failed to prepare duplicate-search index: {}", e))?;
let mut stmt = conn.prepare(query) let mut stmt = conn.prepare(query)
.map_err(|e| { .map_err(|e| {
let error_msg = e.to_string(); let error_msg = e.to_string();
@ -310,33 +363,19 @@ impl IndexingService {
Ok(()) Ok(())
} }
/// Clean up UNC prefixes from existing database entries /// Clean up Windows UNC prefixes (\\?\) from existing `files.path`
/// entries. Relic of pre-A layouts where paths were stored with the
/// prefix; current code strips them at insert time, so this is a
/// one-shot cleanup users can invoke manually if needed.
#[allow(dead_code)] #[allow(dead_code)]
pub fn clean_unc_prefixes(&self, db_path: &str) -> Result<(), String> { pub fn clean_unc_prefixes(&self, db_path: &str) -> Result<(), String> {
let conn = Connection::open(db_path) let conn = Connection::open(db_path)
.map_err(|e| format!("Failed to open database: {}", e))?; .map_err(|e| format!("Failed to open database: {}", e))?;
// Clean UNC prefixes from files table
conn.execute( conn.execute(
"UPDATE files SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'", "UPDATE files SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'",
(), (),
).map_err(|e| format!("Failed to update files table: {}", e))?; )
.map_err(|e| format!("Failed to update files table: {}", e))?;
let doc_table: i64 = conn
.query_row(
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='documents'",
[],
|r| r.get(0),
)
.unwrap_or(0);
if doc_table > 0 {
conn.execute(
"UPDATE documents SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'",
(),
)
.map_err(|e| format!("Failed to update documents table: {}", e))?;
}
Ok(()) Ok(())
} }
@ -365,17 +404,7 @@ impl IndexingService {
/// Check if configuration changes require index recreation /// Check if configuration changes require index recreation
pub fn check_config_validation(&self, db_path: &str, config: &Config, indexing_path: &str) -> Result<Option<Vec<String>>, String> { pub fn check_config_validation(&self, db_path: &str, config: &Config, indexing_path: &str) -> Result<Option<Vec<String>>, String> {
let conn = Connection::open(db_path) let conn = db::open_and_migrate(db_path, &config.processing.tokenize)?;
.map_err(|e| format!("Failed to open database: {}", e))?;
// Create config validation table if it doesn't exist
conn.execute(
"CREATE TABLE IF NOT EXISTS config_validation (
key TEXT PRIMARY KEY,
value TEXT NOT NULL);",
(),
).map_err(|e| format!("Failed to create config_validation table: {}", e))?;
Self::validate_config(&conn, config, indexing_path) Self::validate_config(&conn, config, indexing_path)
} }
@ -413,14 +442,15 @@ impl IndexingService {
fn indexing_thread( fn indexing_thread(
status: Arc<Mutex<IndexingStatus>>, status: Arc<Mutex<IndexingStatus>>,
command_rx: mpsc::Receiver<IndexingCommand>, command_rx: mpsc::Receiver<IndexingCommand>,
db_connection: Arc<Mutex<Option<Arc<Mutex<Connection>>>>> db_connection: Arc<Mutex<Option<Arc<Mutex<Connection>>>>>,
suspend_flag: Arc<AtomicBool>,
) { ) {
let stop_flag = Arc::new(Mutex::new(false)); let stop_flag = Arc::new(Mutex::new(false));
let mut indexing_handle: Option<thread::JoinHandle<()>> = None; let mut indexing_handle: Option<thread::JoinHandle<()>> = None;
while let Ok(command) = command_rx.recv() { while let Ok(command) = command_rx.recv() {
match command { match command {
IndexingCommand::Start { path, db_path, config } => { IndexingCommand::Start { paths, db_path, config } => {
if matches!( if matches!(
*status.lock().unwrap(), *status.lock().unwrap(),
IndexingStatus::CountingFiles { .. } IndexingStatus::CountingFiles { .. }
@ -455,13 +485,14 @@ impl IndexingService {
// Run indexing in a separate thread // Run indexing in a separate thread
let status_clone = status.clone(); let status_clone = status.clone();
let stop_flag_clone = stop_flag.clone(); let stop_flag_clone = stop_flag.clone();
let path_owned = path.clone(); let paths_owned = paths.clone();
let db_path_owned = db_path.clone(); let db_path_owned = db_path.clone();
let config_owned = config.clone(); let config_owned = config.clone();
let db_connection_clone = db_connection.clone(); let db_connection_clone = db_connection.clone();
let suspend_clone = suspend_flag.clone();
indexing_handle = Some(thread::spawn(move || { indexing_handle = Some(thread::spawn(move || {
if let Err(e) = Self::run_indexing(&status_clone, &path_owned, &db_path_owned, &stop_flag_clone, &config_owned, &db_connection_clone) { if let Err(e) = Self::run_indexing(&status_clone, &paths_owned, &db_path_owned, &stop_flag_clone, &suspend_clone, &config_owned, &db_connection_clone) {
*status_clone.lock().unwrap() = IndexingStatus::Error(e); *status_clone.lock().unwrap() = IndexingStatus::Error(e);
} else { } else {
// Only set to Idle if we weren't stopped // Only set to Idle if we weren't stopped
@ -512,81 +543,50 @@ impl IndexingService {
fn run_indexing( fn run_indexing(
status: &Arc<Mutex<IndexingStatus>>, status: &Arc<Mutex<IndexingStatus>>,
path: &str, paths: &[String],
db_path: &str, db_path: &str,
stop_flag: &Arc<Mutex<bool>>, stop_flag: &Arc<Mutex<bool>>,
suspend_flag: &Arc<AtomicBool>,
config: &Config, config: &Config,
db_connection: &Arc<Mutex<Option<Arc<Mutex<Connection>>>>>, db_connection: &Arc<Mutex<Option<Arc<Mutex<Connection>>>>>,
) -> Result<(), String> { ) -> Result<(), String> {
// Set up database if paths.is_empty() {
let conn = Connection::open(db_path) return Err("run_indexing: no paths provided".into());
.map_err(|e| format!("Failed to open database: {}", e))?;
conn.execute_batch( // Default SQLITE page size is 4kB, and our memory cache is in units of page count
"PRAGMA journal_mode = OFF;
PRAGMA synchronous = 0;
PRAGMA cache_size = 10000;
PRAGMA temp_store = MEMORY;",
)
.map_err(|e| format!("Failed to set PRAGMA: {}", e))?;
conn.execute(
"CREATE TABLE IF NOT EXISTS files (
name TEXT,
path TEXT,
size INTEGER,
moddate INTEGER,
hash BLOB);",
(),
)
.map_err(|e| format!("Failed to create files table: {}", e))?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);",
(),
)
.map_err(|e| format!("Failed to create files hash index: {}", e))?;
conn.execute(
"CREATE TABLE IF NOT EXISTS documents (
id INTEGER PRIMARY KEY,
name TEXT,
path TEXT NOT NULL UNIQUE,
text TEXT NOT NULL)",
(),
)
.map_err(|e| format!("Failed to create documents table: {}", e))?;
let fts_external = Self::searchabletext_is_external_content(&conn)?;
if !fts_external {
conn.execute("DROP TABLE IF EXISTS searchabletext", ())
.map_err(|e| format!("Failed to drop legacy searchabletext: {}", e))?;
conn.execute("DROP TABLE IF EXISTS searchabletext_doc", ())
.map_err(|e| format!("Failed to drop legacy searchabletext_doc: {}", e))?;
} }
let create_fts_sql = format!( // De-duplicate while preserving order. Roots are canonicalized first
"CREATE VIRTUAL TABLE IF NOT EXISTS searchabletext USING fts5(name, text, content='documents', content_rowid='id', tokenize='{}');", // so `/home/jeremy` and `/home/jeremy/` (or a symlink to either)
config.processing.tokenize // collapse to one walk. Pure nested-root deduplication (skip a root
); // that is a prefix of an already-walked root) is handled by the
conn.execute(&create_fts_sql, ()) // per-file `seen_paths` set below.
.map_err(|e| format!("Failed to create searchabletext table: {}", e))?; let mut seen_roots = HashSet::new();
let roots: Vec<String> = paths
.iter()
.map(|p| {
std::path::Path::new(p)
.canonicalize()
.ok()
.map(|c| {
let s = c.to_string_lossy().to_string();
if s.starts_with("\\\\?\\") {
s[4..].to_string()
} else {
s
}
})
.unwrap_or_else(|| p.clone())
})
.filter(|p| seen_roots.insert(p.clone()))
.collect();
if !fts_external { // Open and migrate the database to the current schema version.
conn.execute("INSERT INTO searchabletext(searchabletext) VALUES('rebuild')", ()) let conn = db::open_and_migrate(db_path, &config.processing.tokenize)?;
.map_err(|e| format!("Failed to rebuild searchabletext: {}", e))?;
}
conn.execute( // Update configuration (for new installations or when no validation issues).
"CREATE TABLE IF NOT EXISTS config_validation ( // `indexing_path` in the validation table stores the joined list so
key TEXT PRIMARY KEY, // adding/removing a root triggers the same rebuild prompt as changing
value TEXT NOT NULL);", // the legacy single path did.
(), Self::update_config(&conn, config, &roots.join("\n"))?;
)
.map_err(|e| format!("Failed to create config_validation table: {}", e))?;
// Update configuration (for new installations or when no validation issues)
Self::update_config(&conn, config, path)?;
// Load existing files from database for incremental indexing // Load existing files from database for incremental indexing
let existing_files = { let existing_files = {
@ -615,7 +615,18 @@ impl IndexingService {
*current_file = Some("Counting paths (shell)...".to_string()); *current_file = Some("Counting paths (shell)...".to_string());
} }
} }
let n = count_tree_entries_fast(path).map_err(|e| format!("Precount: {}", e))?; let mut n: usize = 0;
for root in &roots {
if *stop_flag.lock().unwrap() {
break;
}
// A failing precount on one root (e.g. permission denied)
// shouldn't abort all indexing; just skip its contribution.
match count_tree_entries_fast(root) {
Ok(c) => n += c,
Err(e) => eprintln!("Warning: precount for {}: {}", root, e),
}
}
if *stop_flag.lock().unwrap() { if *stop_flag.lock().unwrap() {
if let Ok(mut status_guard) = status.lock() { if let Ok(mut status_guard) = status.lock() {
*status_guard = IndexingStatus::Idle; *status_guard = IndexingStatus::Idle;
@ -638,7 +649,11 @@ impl IndexingService {
let batch_size = config.processing.batch_size; let batch_size = config.processing.batch_size;
let mut pending_updates: Vec<(DirEntry, usize)> = Vec::new(); let mut pending_updates: Vec<(DirEntry, usize)> = Vec::new();
let mut pending_inserts: Vec<(DirEntry, usize)> = Vec::new(); let mut pending_inserts: Vec<(DirEntry, usize)> = Vec::new();
let mut seen_existing_paths: HashSet<String> = HashSet::new(); // Every path seen during this walk, regardless of outcome. Dedupes
// duplicate visits that can happen when two roots overlap (e.g.
// `/home` and `/home/user`) or when canonicalization collapses
// symlinks. Also feeds the stale-entry cleanup at the end of the walk.
let mut seen_paths: HashSet<String> = HashSet::new();
let mut visit: usize = 0; let mut visit: usize = 0;
let mut had_incremental_work = false; let mut had_incremental_work = false;
let flush_updates = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> { let flush_updates = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> {
@ -675,8 +690,11 @@ impl IndexingService {
Ok(()) Ok(())
}; };
for entry in indexed_walk_file_entries(path, config.processing.follow_symlinks) { let root_iter = roots
if *stop_flag.lock().unwrap() { .iter()
.flat_map(|r| indexed_walk_file_entries(r, config.processing.follow_symlinks));
for entry in root_iter {
if should_abort(stop_flag, suspend_flag) {
flush_updates(&mut pending_updates)?; flush_updates(&mut pending_updates)?;
flush_inserts(&mut pending_inserts)?; flush_inserts(&mut pending_inserts)?;
if let Ok(mut status_guard) = status.lock() { if let Ok(mut status_guard) = status.lock() {
@ -717,16 +735,20 @@ impl IndexingService {
} }
}); });
// Skip this entry if its canonical path was already queued in
// this run. Prevents UNIQUE(path) violations when overlapping
// roots or symlinks lead the walker to the same file twice.
let path_for_dedup = match &current_path {
Some(p) => p.clone(),
None => continue,
};
if !seen_paths.insert(path_for_dedup) {
continue;
}
match action { match action {
FileIndexAction::Skip => { FileIndexAction::Skip => {}
if let Some(path) = current_path {
seen_existing_paths.insert(path);
}
}
FileIndexAction::Update => { FileIndexAction::Update => {
if let Some(path) = current_path {
seen_existing_paths.insert(path);
}
had_incremental_work = true; had_incremental_work = true;
pending_updates.push((entry, visit)); pending_updates.push((entry, visit));
if pending_updates.len() >= batch_size { if pending_updates.len() >= batch_size {
@ -748,13 +770,14 @@ impl IndexingService {
let stale_paths: Vec<String> = existing_files let stale_paths: Vec<String> = existing_files
.keys() .keys()
.filter(|p| !seen_existing_paths.contains(*p)) .filter(|p| !seen_paths.contains(*p))
.cloned() .cloned()
.collect(); .collect();
let stale_deleted = cleanup_stale_index_entries( let stale_deleted = cleanup_stale_index_entries(
&conn_mutex, &conn_mutex,
stale_paths.as_slice(), stale_paths.as_slice(),
stop_flag, stop_flag,
suspend_flag,
Some(Self::file_index_status_callback(status)), Some(Self::file_index_status_callback(status)),
)?; )?;
if stale_deleted > 0 { if stale_deleted > 0 {
@ -770,8 +793,8 @@ impl IndexingService {
} }
} }
// Check for stop signal before starting text indexing // Check for stop signal (and park if suspended) before starting text indexing.
if *stop_flag.lock().unwrap() { if should_abort(stop_flag, suspend_flag) {
if let Ok(mut status_guard) = status.lock() { if let Ok(mut status_guard) = status.lock() {
*status_guard = IndexingStatus::Idle; *status_guard = IndexingStatus::Idle;
} }
@ -808,7 +831,7 @@ impl IndexingService {
}); });
// Process text indexing // Process text indexing
if let Err(e) = process_text_indexing(&conn_mutex, &stop_flag, Some(text_status_callback), Some(text_progress_callback), config) { if let Err(e) = process_text_indexing(&conn_mutex, &stop_flag, suspend_flag, Some(text_status_callback), Some(text_progress_callback), config) {
return Err(format!("Failed to process text indexing: {}", e)); return Err(format!("Failed to process text indexing: {}", e));
} }
@ -822,25 +845,6 @@ impl IndexingService {
Ok(()) Ok(())
} }
fn searchabletext_is_external_content(conn: &Connection) -> Result<bool, String> {
let sql: Option<String> = conn
.query_row(
"SELECT sql FROM sqlite_master WHERE name='searchabletext'",
[],
|r| r.get(0),
)
.optional()
.map_err(|e| format!("sqlite_master searchabletext: {}", e))?;
Ok(sql
.as_deref()
.map(|s| {
s.contains("content='documents'")
|| s.contains("content=\"documents\"")
|| s.contains("content=documents")
})
.unwrap_or(false))
}
/// Validates configuration against stored values and returns validation results. /// Validates configuration against stored values and returns validation results.
/// Critical configuration changes that require index recreation: /// Critical configuration changes that require index recreation:
/// - hash_length: affects file hash computation, invalidates existing file metadata /// - hash_length: affects file hash computation, invalidates existing file metadata

12
crates/quicksearch-core/src/lib.rs Normal file
View file

@ -0,0 +1,12 @@
pub mod cli;
pub mod config;
pub mod db;
pub mod document_extraction;
pub mod extract;
pub mod file_handling;
pub mod indexing;
pub mod mime;
pub mod query;
pub mod search_sql;
pub mod shutdown;
pub mod watcher;

238
crates/quicksearch-core/src/mime.rs Normal file
View file

@ -0,0 +1,238 @@
//! MIME type guessing and `FileType` bitmask classification.
//!
//! Two stages:
//! 1. [`guess_mime`] infers a MIME type from a path — extension first via
//! `mime_guess`, falling back to magic-byte sniffing via `infer` for files
//! whose extension is missing or ambiguous.
//! 2. [`mime_to_type`] maps a MIME string to a [`FileType`] bitmask so a single
//! file can belong to multiple categories (e.g. a `.docx` is Document|Text).
use std::path::Path;
/// Bit-flag category for a file. Unlike the MIME string this is designed for
/// cheap bitmask queries like `type & FileType::AUDIO != 0`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FileType(pub u32);
impl FileType {
pub const EMPTY: FileType = FileType(0);
pub const AUDIO: FileType = FileType(1 << 0);
pub const IMAGE: FileType = FileType(1 << 1);
pub const VIDEO: FileType = FileType(1 << 2);
pub const DOCUMENT: FileType = FileType(1 << 3);
pub const TEXT: FileType = FileType(1 << 4);
pub const ARCHIVE: FileType = FileType(1 << 5);
pub const PRESENTATION: FileType = FileType(1 << 6);
pub const SPREADSHEET: FileType = FileType(1 << 7);
pub const FOLDER: FileType = FileType(1 << 8);
pub const fn bits(self) -> u32 {
self.0
}
pub const fn contains(self, other: FileType) -> bool {
(self.0 & other.0) == other.0
}
/// Parse a single Baloo-style category name (`Audio`, `Image`, ...).
/// Case-insensitive. Returns `EMPTY` for unknown names.
pub fn from_name(s: &str) -> FileType {
match s.to_ascii_lowercase().as_str() {
"audio" => FileType::AUDIO,
"image" => FileType::IMAGE,
"video" => FileType::VIDEO,
"document" => FileType::DOCUMENT,
"text" => FileType::TEXT,
"archive" => FileType::ARCHIVE,
"presentation" => FileType::PRESENTATION,
"spreadsheet" => FileType::SPREADSHEET,
"folder" => FileType::FOLDER,
_ => FileType::EMPTY,
}
}
}
impl std::ops::BitOr for FileType {
type Output = FileType;
fn bitor(self, rhs: FileType) -> FileType {
FileType(self.0 | rhs.0)
}
}
impl std::ops::BitOrAssign for FileType {
fn bitor_assign(&mut self, rhs: FileType) {
self.0 |= rhs.0;
}
}
/// Guess a MIME type for a path on disk.
///
/// Tries extension-based lookup via `mime_guess` first (cheap, no I/O). If
/// that returns nothing or a generic `application/octet-stream`, and the file
/// is readable, falls back to `infer` magic-byte detection (reads a small
/// prefix of the file).
///
/// Returns `None` if no guess can be made.
pub fn guess_mime(path: &Path) -> Option<String> {
if let Some(g) = mime_guess::from_path(path).first() {
let s = g.essence_str();
if !s.is_empty() && s != "application/octet-stream" {
return Some(s.to_string());
}
}
// Magic-byte fallback. `infer::get_from_path` handles errors by returning None.
if let Ok(Some(t)) = infer::get_from_path(path) {
return Some(t.mime_type().to_string());
}
None
}
/// Map a MIME string to a [`FileType`] bitmask. Ported from Baloo's
/// `basicindexingjob.cpp:typesForMimeType`.
pub fn mime_to_type(mime: &str) -> FileType {
let lower = mime.to_ascii_lowercase();
let (top, sub) = match lower.split_once('/') {
Some(pair) => pair,
None => return FileType::EMPTY,
};
let mut t = FileType::EMPTY;
match top {
"audio" => t |= FileType::AUDIO,
"image" => t |= FileType::IMAGE,
"video" => t |= FileType::VIDEO,
"text" => {
t |= FileType::TEXT;
// HTML counts as a document too in Baloo.
if sub == "html" || sub == "xhtml+xml" {
t |= FileType::DOCUMENT;
}
}
_ => {}
}
// Subtype-based classification for the `application/*` grab bag.
match sub {
// Office formats
"msword"
| "vnd.openxmlformats-officedocument.wordprocessingml.document"
| "vnd.oasis.opendocument.text"
| "rtf"
| "pdf"
| "epub+zip"
| "x-mobipocket-ebook" => {
t |= FileType::DOCUMENT;
}
"vnd.ms-excel"
| "vnd.openxmlformats-officedocument.spreadsheetml.sheet"
| "vnd.oasis.opendocument.spreadsheet" => {
t |= FileType::DOCUMENT | FileType::SPREADSHEET;
}
"vnd.ms-powerpoint"
| "vnd.openxmlformats-officedocument.presentationml.presentation"
| "vnd.oasis.opendocument.presentation" => {
t |= FileType::DOCUMENT | FileType::PRESENTATION;
}
// Archives
"zip"
| "x-tar"
| "x-7z-compressed"
| "x-rar"
| "x-rar-compressed"
| "gzip"
| "x-bzip"
| "x-bzip2"
| "x-xz"
| "vnd.debian.binary-package"
| "x-rpm" => {
t |= FileType::ARCHIVE;
}
// application/xml is structured text
"xml" | "json" | "javascript" | "x-shellscript" | "x-python" => {
t |= FileType::TEXT;
}
_ => {}
}
t
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn audio_mime() {
assert!(mime_to_type("audio/mpeg").contains(FileType::AUDIO));
assert!(mime_to_type("audio/flac").contains(FileType::AUDIO));
}
#[test]
fn image_mime() {
assert!(mime_to_type("image/jpeg").contains(FileType::IMAGE));
assert!(mime_to_type("image/png").contains(FileType::IMAGE));
}
#[test]
fn docx_is_document_and_office() {
let t = mime_to_type(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
);
assert!(t.contains(FileType::DOCUMENT));
}
#[test]
fn xlsx_is_spreadsheet_and_document() {
let t = mime_to_type(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
);
assert!(t.contains(FileType::DOCUMENT));
assert!(t.contains(FileType::SPREADSHEET));
}
#[test]
fn pptx_is_presentation() {
let t = mime_to_type(
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
);
assert!(t.contains(FileType::PRESENTATION));
}
#[test]
fn html_is_text_and_document() {
let t = mime_to_type("text/html");
assert!(t.contains(FileType::TEXT));
assert!(t.contains(FileType::DOCUMENT));
}
#[test]
fn plain_text() {
let t = mime_to_type("text/plain");
assert!(t.contains(FileType::TEXT));
assert!(!t.contains(FileType::DOCUMENT));
}
#[test]
fn zip_is_archive() {
assert!(mime_to_type("application/zip").contains(FileType::ARCHIVE));
}
#[test]
fn unknown_mime_is_empty() {
assert_eq!(mime_to_type("weird/blob"), FileType::EMPTY);
}
#[test]
fn from_name_round_trip() {
for n in ["Audio", "Image", "Video", "Document", "Text", "Archive",
"Spreadsheet", "Presentation", "Folder"] {
assert_ne!(FileType::from_name(n), FileType::EMPTY, "{}", n);
}
assert_eq!(FileType::from_name("Weird"), FileType::EMPTY);
}
#[test]
fn guess_mime_by_extension() {
use std::path::PathBuf;
assert_eq!(guess_mime(&PathBuf::from("a.txt")).as_deref(), Some("text/plain"));
assert_eq!(guess_mime(&PathBuf::from("a.png")).as_deref(), Some("image/png"));
assert_eq!(guess_mime(&PathBuf::from("a.mp3")).as_deref(), Some("audio/mpeg"));
}
}

58
crates/quicksearch-core/src/query/ast.rs Normal file
View file

@ -0,0 +1,58 @@
//! Query AST shared between parser and translator.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Op {
/// `key:value` — substring / FTS MATCH semantics.
Contains,
/// `key=value` — exact match.
Eq,
/// `key<value`
Lt,
/// `key<=value`
Le,
/// `key>value`
Gt,
/// `key>=value`
Ge,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Term {
/// An unquoted word or quoted phrase. Feeds the FTS MATCH expression.
Literal(String),
/// A structured filter such as `type:Audio` or `modified:>=2024-01-01`.
Property { key: String, op: Op, value: String },
And(Vec<Term>),
Or(Vec<Term>),
}
impl Term {
/// Combine two terms with AND, flattening to avoid a deeply nested tree.
pub fn and(a: Term, b: Term) -> Term {
match (a, b) {
(Term::And(mut xs), Term::And(ys)) => {
xs.extend(ys);
Term::And(xs)
}
(Term::And(mut xs), other) | (other, Term::And(mut xs)) => {
xs.push(other);
Term::And(xs)
}
(a, b) => Term::And(vec![a, b]),
}
}
pub fn or(a: Term, b: Term) -> Term {
match (a, b) {
(Term::Or(mut xs), Term::Or(ys)) => {
xs.extend(ys);
Term::Or(xs)
}
(Term::Or(mut xs), other) | (other, Term::Or(mut xs)) => {
xs.push(other);
Term::Or(xs)
}
(a, b) => Term::Or(vec![a, b]),
}
}
}

214
crates/quicksearch-core/src/query/lexer.rs Normal file
View file

@ -0,0 +1,214 @@
//! Tokenizer for the query grammar.
use super::ast::Op;
#[derive(Debug, Clone, PartialEq)]
pub enum Token {
/// Unquoted word. `AND`/`OR` are intercepted before emitting a `Word`.
Word(String),
Quoted(String),
LParen,
RParen,
/// Binary property operator (`:`, `=`, `>`, `>=`, `<`, `<=`).
///
/// `:` is [`Op::Contains`] by default; the parser re-interprets it when
/// followed immediately by a comparator (e.g. `modified:>=2024-01-01`).
Op(Op),
And,
Or,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LexError {
pub message: String,
pub offset: usize,
}
impl std::fmt::Display for LexError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "query lex error at {}: {}", self.offset, self.message)
}
}
impl std::error::Error for LexError {}
pub fn tokenize(input: &str) -> Result<Vec<Token>, LexError> {
let bytes = input.as_bytes();
let mut i = 0usize;
let mut out = Vec::new();
while i < bytes.len() {
let b = bytes[i];
if b.is_ascii_whitespace() {
i += 1;
continue;
}
match b {
b'(' => {
out.push(Token::LParen);
i += 1;
}
b')' => {
out.push(Token::RParen);
i += 1;
}
b':' => {
out.push(Token::Op(Op::Contains));
i += 1;
}
b'=' => {
out.push(Token::Op(Op::Eq));
i += 1;
}
b'<' => {
if bytes.get(i + 1) == Some(&b'=') {
out.push(Token::Op(Op::Le));
i += 2;
} else {
out.push(Token::Op(Op::Lt));
i += 1;
}
}
b'>' => {
if bytes.get(i + 1) == Some(&b'=') {
out.push(Token::Op(Op::Ge));
i += 2;
} else {
out.push(Token::Op(Op::Gt));
i += 1;
}
}
b'"' => {
// Double-quoted phrase. Supports doubled-quote escape `""`.
let mut j = i + 1;
let mut buf = String::new();
while j < bytes.len() {
if bytes[j] == b'"' {
if bytes.get(j + 1) == Some(&b'"') {
buf.push('"');
j += 2;
continue;
}
break;
}
buf.push(bytes[j] as char);
j += 1;
}
if j >= bytes.len() {
return Err(LexError {
message: "unterminated quoted phrase".into(),
offset: i,
});
}
out.push(Token::Quoted(buf));
i = j + 1;
}
_ => {
// Unquoted word; continues until whitespace, paren, or operator.
let start = i;
while i < bytes.len() {
let c = bytes[i];
if c.is_ascii_whitespace()
|| matches!(c, b'(' | b')' | b':' | b'=' | b'<' | b'>' | b'"')
{
break;
}
i += 1;
}
let word = std::str::from_utf8(&bytes[start..i])
.map_err(|e| LexError {
message: format!("invalid UTF-8 in word: {}", e),
offset: start,
})?
.to_string();
match word.as_str() {
"AND" => out.push(Token::And),
"OR" => out.push(Token::Or),
_ => out.push(Token::Word(word)),
}
}
}
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn simple_words() {
let t = tokenize("foo bar").unwrap();
assert_eq!(
t,
vec![Token::Word("foo".into()), Token::Word("bar".into())]
);
}
#[test]
fn and_or_keywords() {
let t = tokenize("a AND b OR c").unwrap();
assert_eq!(
t,
vec![
Token::Word("a".into()),
Token::And,
Token::Word("b".into()),
Token::Or,
Token::Word("c".into()),
]
);
}
#[test]
fn quoted_phrase() {
let t = tokenize(r#""hello world""#).unwrap();
assert_eq!(t, vec![Token::Quoted("hello world".into())]);
}
#[test]
fn doubled_quote_is_escape() {
let t = tokenize(r#""a""b""#).unwrap();
assert_eq!(t, vec![Token::Quoted(r#"a"b"#.into())]);
}
#[test]
fn property_operators() {
let t = tokenize("type:Audio modified>=2024-01-01 size<100").unwrap();
assert_eq!(
t,
vec![
Token::Word("type".into()),
Token::Op(Op::Contains),
Token::Word("Audio".into()),
Token::Word("modified".into()),
Token::Op(Op::Ge),
Token::Word("2024-01-01".into()),
Token::Word("size".into()),
Token::Op(Op::Lt),
Token::Word("100".into()),
]
);
}
#[test]
fn parens() {
let t = tokenize("(a OR b)").unwrap();
assert_eq!(
t,
vec![
Token::LParen,
Token::Word("a".into()),
Token::Or,
Token::Word("b".into()),
Token::RParen,
]
);
}
#[test]
fn unterminated_quote_is_error() {
assert!(tokenize(r#""oops"#).is_err());
}
}

28
crates/quicksearch-core/src/query/mod.rs Normal file
View file

@ -0,0 +1,28 @@
//! Structured query parser and SQL translator.
//!
//! Input syntax (a deliberate subset of KDE Baloo's query language — just
//! enough to be useful standalone; full Baloo grammar lives in the Set B
//! compat layer):
//!
//! - plain words: `foo bar` (implicit AND)
//! - quoted phrases: `"hello world"`
//! - boolean operators: `AND`, `OR` (case-sensitive)
//! - grouping: `(a OR b)`
//! - structured filters:
//! - `type:Audio` / `type:Image` / `type:Document` / `type:Text` / `type:Video`
//! / `type:Archive` / `type:Spreadsheet` / `type:Presentation` / `type:Folder`
//! - `modified:>=2024-01-01`, `modified:<2023-12-01`, `modified:=2024-05-20`
//! (also accepts `modified>=2024-01-01` without the colon)
//! - `path:/some/dir` — matches files with that directory as their parent
//! or any ancestor.
//!
//! The entry point is [`parse_and_build`], which accepts a query string and
//! returns a ready-to-execute [`SqlQuery`].
pub mod ast;
pub mod lexer;
pub mod parser;
pub mod translator;
pub use ast::{Op, Term};
pub use translator::{parse_and_build, SqlQuery};

284
crates/quicksearch-core/src/query/parser.rs Normal file
View file

@ -0,0 +1,284 @@
//! Recursive-descent parser: token stream → [`Term`] tree.
//!
//! Grammar (implicit AND between adjacent terms):
//! ```text
//! expr := or_expr
//! or_expr := and_expr ("OR" and_expr)*
//! and_expr := atom ("AND"? atom)*
//! atom := "(" expr ")" | property | literal
//! property := ident ":"|"="|"<"|"<="|">"|">=" value
//! literal := WORD | QUOTED
//! ```
use super::ast::{Op, Term};
use super::lexer::{tokenize, LexError, Token};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ParseError {
Lex(LexError),
Unexpected { at: usize, reason: String },
Empty,
}
impl std::fmt::Display for ParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ParseError::Lex(e) => write!(f, "{}", e),
ParseError::Unexpected { at, reason } => {
write!(f, "parse error at token {}: {}", at, reason)
}
ParseError::Empty => write!(f, "empty query"),
}
}
}
impl std::error::Error for ParseError {}
pub fn parse(input: &str) -> Result<Term, ParseError> {
let tokens = tokenize(input).map_err(ParseError::Lex)?;
let trimmed: Vec<_> = tokens.into_iter().collect();
if trimmed.is_empty() {
return Err(ParseError::Empty);
}
let mut p = Parser { tokens: trimmed, pos: 0 };
let term = p.parse_or()?;
if p.pos < p.tokens.len() {
return Err(ParseError::Unexpected {
at: p.pos,
reason: format!("trailing token {:?}", p.tokens[p.pos]),
});
}
Ok(term)
}
struct Parser {
tokens: Vec<Token>,
pos: usize,
}
impl Parser {
fn peek(&self) -> Option<&Token> {
self.tokens.get(self.pos)
}
fn bump(&mut self) -> Option<Token> {
let t = self.tokens.get(self.pos).cloned();
if t.is_some() {
self.pos += 1;
}
t
}
fn parse_or(&mut self) -> Result<Term, ParseError> {
let mut left = self.parse_and()?;
while matches!(self.peek(), Some(Token::Or)) {
self.bump();
let right = self.parse_and()?;
left = Term::or(left, right);
}
Ok(left)
}
fn parse_and(&mut self) -> Result<Term, ParseError> {
let mut left = self.parse_atom()?;
loop {
match self.peek() {
Some(Token::And) => {
self.bump();
let right = self.parse_atom()?;
left = Term::and(left, right);
}
// Implicit AND: adjacent atoms without an operator.
Some(Token::Word(_))
| Some(Token::Quoted(_))
| Some(Token::LParen) => {
let right = self.parse_atom()?;
left = Term::and(left, right);
}
_ => break,
}
}
Ok(left)
}
fn parse_atom(&mut self) -> Result<Term, ParseError> {
match self.bump() {
Some(Token::LParen) => {
let inner = self.parse_or()?;
match self.bump() {
Some(Token::RParen) => Ok(inner),
other => Err(ParseError::Unexpected {
at: self.pos,
reason: format!("expected ')' got {:?}", other),
}),
}
}
Some(Token::Quoted(s)) => Ok(Term::Literal(s)),
Some(Token::Word(w)) => {
// If followed by an operator, this word is a property key.
if let Some(Token::Op(op)) = self.peek().cloned() {
self.bump();
// `key:>=value` — an Op followed by another Op becomes the
// effective comparator; the original colon is "separator".
let effective_op = if op == Op::Contains {
if let Some(Token::Op(inner_op)) = self.peek().cloned() {
self.bump();
inner_op
} else {
Op::Contains
}
} else {
op
};
let value = match self.bump() {
Some(Token::Word(v)) => v,
Some(Token::Quoted(v)) => v,
other => {
return Err(ParseError::Unexpected {
at: self.pos,
reason: format!("expected property value got {:?}", other),
})
}
};
Ok(Term::Property {
key: w,
op: effective_op,
value,
})
} else {
Ok(Term::Literal(w))
}
}
other => Err(ParseError::Unexpected {
at: self.pos,
reason: format!("expected term, got {:?}", other),
}),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn lit(s: &str) -> Term {
Term::Literal(s.into())
}
fn prop(k: &str, op: Op, v: &str) -> Term {
Term::Property {
key: k.into(),
op,
value: v.into(),
}
}
#[test]
fn single_word() {
assert_eq!(parse("foo").unwrap(), lit("foo"));
}
#[test]
fn quoted_phrase() {
assert_eq!(parse(r#""hello world""#).unwrap(), lit("hello world"));
}
#[test]
fn implicit_and() {
assert_eq!(
parse("foo bar").unwrap(),
Term::And(vec![lit("foo"), lit("bar")])
);
}
#[test]
fn explicit_and() {
assert_eq!(
parse("foo AND bar").unwrap(),
Term::And(vec![lit("foo"), lit("bar")])
);
}
#[test]
fn or_has_lower_precedence_than_and() {
assert_eq!(
parse("a b OR c").unwrap(),
Term::Or(vec![
Term::And(vec![lit("a"), lit("b")]),
lit("c")
])
);
}
#[test]
fn parens_override_precedence() {
assert_eq!(
parse("a (b OR c)").unwrap(),
Term::And(vec![lit("a"), Term::Or(vec![lit("b"), lit("c")])])
);
}
#[test]
fn property_contains() {
assert_eq!(
parse("type:Audio").unwrap(),
prop("type", Op::Contains, "Audio")
);
}
#[test]
fn property_comparator_via_colon() {
assert_eq!(
parse("modified:>=2024-01-01").unwrap(),
prop("modified", Op::Ge, "2024-01-01")
);
}
#[test]
fn property_comparator_bare() {
assert_eq!(
parse("modified>=2024-01-01").unwrap(),
prop("modified", Op::Ge, "2024-01-01")
);
}
#[test]
fn mixed_filter_and_literal() {
assert_eq!(
parse("type:Audio beatles").unwrap(),
Term::And(vec![
prop("type", Op::Contains, "Audio"),
lit("beatles")
])
);
}
#[test]
fn nested_or() {
assert_eq!(
parse("(a OR b) AND (c OR d)").unwrap(),
Term::And(vec![
Term::Or(vec![lit("a"), lit("b")]),
Term::Or(vec![lit("c"), lit("d")])
])
);
}
#[test]
fn empty_query_is_error() {
assert!(matches!(parse(" "), Err(ParseError::Empty)));
}
#[test]
fn trailing_token_is_error() {
assert!(parse("a )").is_err());
}
#[test]
fn property_value_may_be_quoted() {
assert_eq!(
parse(r#"path:"/tmp with space""#).unwrap(),
prop("path", Op::Contains, "/tmp with space")
);
}
}

654
crates/quicksearch-core/src/query/translator.rs Normal file
View file

@ -0,0 +1,654 @@
//! AST → executable SQL.
use super::ast::{Op, Term};
use super::parser::{parse, ParseError};
use crate::mime::FileType;
/// A prepared SQL statement plus its positional parameters. Parameters are
/// rusqlite `Value` for convenience at the call site. Every `?N` in `sql`
/// corresponds to `params[N-1]`.
#[derive(Debug, Clone)]
pub struct SqlQuery {
pub sql: String,
pub params: Vec<rusqlite::types::Value>,
}
/// Sort strategy for the result set.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Sort {
/// Newest first by file modification time.
ByMtimeDesc,
/// FTS5 relevance rank. Only sensible when the query has FTS terms.
ByRank,
/// No ORDER BY clause.
None,
}
impl Default for Sort {
fn default() -> Self {
Sort::ByMtimeDesc
}
}
#[derive(Debug, Clone)]
pub enum TranslateError {
Parse(ParseError),
UnknownProperty(String),
BadDate(String),
UnsupportedOp {
key: String,
op: Op,
},
}
impl std::fmt::Display for TranslateError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TranslateError::Parse(e) => write!(f, "{}", e),
TranslateError::UnknownProperty(k) => write!(f, "unknown property '{}'", k),
TranslateError::BadDate(s) => write!(f, "bad date '{}'", s),
TranslateError::UnsupportedOp { key, op } => {
write!(f, "operator {:?} is not supported for property '{}'", op, key)
}
}
}
}
impl std::error::Error for TranslateError {}
/// One-shot: parse the input string and build the SQL. Limit/offset are
/// applied at the SQL level; `None` means "no limit".
pub fn parse_and_build(
input: &str,
limit: Option<u32>,
offset: u32,
sort: Sort,
) -> Result<SqlQuery, TranslateError> {
let ast = parse(input).map_err(TranslateError::Parse)?;
build(&ast, limit, offset, sort)
}
/// Translate an already-parsed AST into SQL.
pub fn build(
ast: &Term,
limit: Option<u32>,
offset: u32,
sort: Sort,
) -> Result<SqlQuery, TranslateError> {
let mut b = Builder::default();
let where_sql = b.translate(ast)?;
// The FTS branch joins a CTE of matching rowids; the structured branch
// stands alone. A query may have both — in that case we intersect at the
// file.id level.
let needs_fts_join = !b.fts_parts.is_empty();
// Structured placeholders were numbered assuming no FTS param. When an
// FTS MATCH is prepended as ?1, bump every `?N` in the generated WHERE
// clause by 1 so the positional bindings line up.
let where_sql = if needs_fts_join {
shift_placeholders(&where_sql)
} else {
where_sql
};
let mut sql = String::new();
if needs_fts_join {
sql.push_str(
"WITH fts_hits AS (SELECT rowid FROM searchabletext WHERE searchabletext MATCH ?1) ",
);
b.all_params
.insert(0, rusqlite::types::Value::Text(fts_expr(&b.fts_parts)));
}
sql.push_str("SELECT f.id, f.name, f.path FROM files f ");
if needs_fts_join {
sql.push_str("JOIN fts_hits ON fts_hits.rowid = f.id ");
}
if !where_sql.is_empty() {
sql.push_str("WHERE ");
sql.push_str(&where_sql);
sql.push(' ');
}
match sort {
Sort::ByMtimeDesc => sql.push_str("ORDER BY f.mtime DESC "),
Sort::ByRank if needs_fts_join => sql.push_str("ORDER BY rank "),
Sort::ByRank | Sort::None => {}
}
if let Some(n) = limit {
sql.push_str(&format!("LIMIT {} ", n));
}
if offset > 0 {
sql.push_str(&format!("OFFSET {}", offset));
}
Ok(SqlQuery {
sql: sql.trim_end().to_string(),
params: b.all_params,
})
}
#[derive(Default)]
struct Builder {
/// Tokens that will be joined into the single FTS MATCH expression.
fts_parts: Vec<FtsFragment>,
/// Everything else (structured filters) as a WHERE clause.
all_params: Vec<rusqlite::types::Value>,
}
#[derive(Debug, Clone)]
enum FtsFragment {
/// A plain word or phrase to be included as an AND'd token.
And(String),
/// Grouped alternation, already rendered with internal ORs.
OrGroup(Vec<String>),
}
fn fts_expr(parts: &[FtsFragment]) -> String {
// Render `(a AND b AND (c OR d))` where a/b/c/d are quoted phrases.
let mut out = String::new();
let mut first = true;
for p in parts {
if !first {
out.push_str(" AND ");
}
first = false;
match p {
FtsFragment::And(s) => out.push_str(&quote_phrase(s)),
FtsFragment::OrGroup(items) => {
out.push('(');
let mut first_item = true;
for item in items {
if !first_item {
out.push_str(" OR ");
}
first_item = false;
out.push_str(&quote_phrase(item));
}
out.push(')');
}
}
}
out
}
/// Escape a phrase for FTS5 MATCH. FTS5 itself uses doubled quotes for
/// literal quotes inside a quoted phrase.
fn quote_phrase(s: &str) -> String {
let mut buf = String::with_capacity(s.len() + 2);
buf.push('"');
for c in s.chars() {
if c == '"' {
buf.push('"');
}
buf.push(c);
}
buf.push('"');
buf
}
impl Builder {
/// Translate a sub-term, returning its contribution to the SQL WHERE
/// clause. FTS parts are accumulated in `self.fts_parts` (not returned
/// here) since they collapse into a single MATCH expression at the top.
fn translate(&mut self, t: &Term) -> Result<String, TranslateError> {
match t {
Term::Literal(s) => {
self.fts_parts.push(FtsFragment::And(s.clone()));
Ok(String::new())
}
Term::Property { key, op, value } => self.translate_property(key, *op, value),
Term::And(children) => {
let mut pieces = Vec::new();
for c in children {
let p = self.translate(c)?;
if !p.is_empty() {
pieces.push(p);
}
}
Ok(join_with("AND", &pieces))
}
Term::Or(children) => {
// An OR of pure-literal children collapses into one FTS OR-group.
if children.iter().all(|c| matches!(c, Term::Literal(_))) {
let group: Vec<String> = children
.iter()
.map(|c| match c {
Term::Literal(s) => s.clone(),
_ => unreachable!(),
})
.collect();
self.fts_parts.push(FtsFragment::OrGroup(group));
return Ok(String::new());
}
// Mixed OR — each branch becomes a separate sub-Builder whose
// WHERE fragments we OR together. FTS branches cannot mix
// with structured branches cleanly at the SQL level here;
// keep it simple by requiring that mixed-OR branches produce
// structured-only WHERE fragments.
let mut pieces = Vec::new();
for c in children {
let before = self.fts_parts.len();
let p = self.translate(c)?;
if self.fts_parts.len() > before {
return Err(TranslateError::UnknownProperty(
"OR mixing FTS and structured terms is not supported in Set A".into(),
));
}
if !p.is_empty() {
pieces.push(p);
}
}
Ok(format!("({})", join_with("OR", &pieces)))
}
}
}
fn translate_property(
&mut self,
key: &str,
op: Op,
value: &str,
) -> Result<String, TranslateError> {
let lower_key = key.to_ascii_lowercase();
match lower_key.as_str() {
"type" => self.prop_type(op, value, key),
"modified" | "mtime" => self.prop_mtime(op, value, key),
"path" | "folder" | "includefolder" => self.prop_path(op, value, key),
"name" | "filename" => self.prop_name(op, value, key),
"mime" => self.prop_mime(op, value, key),
_ => Err(TranslateError::UnknownProperty(key.to_string())),
}
}
fn prop_type(&mut self, op: Op, value: &str, key: &str) -> Result<String, TranslateError> {
if op != Op::Contains && op != Op::Eq {
return Err(TranslateError::UnsupportedOp {
key: key.into(),
op,
});
}
let bits = FileType::from_name(value).bits() as i64;
if bits == 0 {
return Err(TranslateError::UnknownProperty(format!(
"type name '{}'",
value
)));
}
self.all_params
.push(rusqlite::types::Value::Integer(bits));
Ok(format!("(f.type & ?{}) != 0", self.param_placeholder_idx()))
}
fn prop_mtime(&mut self, op: Op, value: &str, key: &str) -> Result<String, TranslateError> {
let unix = parse_date_to_unix(value).ok_or_else(|| TranslateError::BadDate(value.into()))?;
let col = "f.mtime";
let sql_op = match op {
Op::Contains | Op::Eq => "=",
Op::Lt => "<",
Op::Le => "<=",
Op::Gt => ">",
Op::Ge => ">=",
};
// `modified:=2024-01-01` should match the whole day, not the second.
if op == Op::Eq || op == Op::Contains {
let start = unix;
let end = unix + 86_400;
self.all_params.push(rusqlite::types::Value::Integer(start));
let i = self.param_placeholder_idx();
self.all_params.push(rusqlite::types::Value::Integer(end));
let j = self.param_placeholder_idx();
return Ok(format!("({} >= ?{} AND {} < ?{})", col, i, col, j));
}
self.all_params.push(rusqlite::types::Value::Integer(unix));
let i = self.param_placeholder_idx();
let _ = key;
Ok(format!("{} {} ?{}", col, sql_op, i))
}
fn prop_path(&mut self, op: Op, value: &str, key: &str) -> Result<String, TranslateError> {
if op != Op::Contains && op != Op::Eq {
return Err(TranslateError::UnsupportedOp {
key: key.into(),
op,
});
}
self.all_params
.push(rusqlite::types::Value::Text(value.into()));
let i = self.param_placeholder_idx();
self.all_params
.push(rusqlite::types::Value::Text(format!("{}/%", value.trim_end_matches('/'))));
let j = self.param_placeholder_idx();
Ok(format!("(f.parent = ?{} OR f.parent LIKE ?{})", i, j))
}
fn prop_name(&mut self, op: Op, value: &str, key: &str) -> Result<String, TranslateError> {
if op != Op::Contains {
return Err(TranslateError::UnsupportedOp {
key: key.into(),
op,
});
}
self.all_params
.push(rusqlite::types::Value::Text(format!("%{}%", value)));
let i = self.param_placeholder_idx();
Ok(format!("f.name LIKE ?{}", i))
}
fn prop_mime(&mut self, op: Op, value: &str, key: &str) -> Result<String, TranslateError> {
if op != Op::Contains && op != Op::Eq {
return Err(TranslateError::UnsupportedOp {
key: key.into(),
op,
});
}
self.all_params
.push(rusqlite::types::Value::Text(value.into()));
let i = self.param_placeholder_idx();
Ok(format!("f.mime = ?{}", i))
}
fn param_placeholder_idx(&self) -> usize {
// params[0] is reserved for the FTS MATCH if one is built; structured
// params start at index 2 in that case (1-based). We track it by
// calling this *after* pushing the value; result is `len` so the SQL
// says `?<len>` which matches the 1-based positional binding rusqlite
// uses for `?N` placeholders. When an FTS match is prepended at
// `build`, each index shifts by 1 implicitly.
self.all_params.len()
}
}
fn join_with(sep: &str, pieces: &[String]) -> String {
pieces
.iter()
.map(|p| p.clone())
.collect::<Vec<_>>()
.join(&format!(" {} ", sep))
}
/// Parse a date string. Accepts `YYYY-MM-DD`. Returns unix seconds at 00:00 UTC.
fn parse_date_to_unix(s: &str) -> Option<i64> {
// Minimal parser: split on '-' into y/m/d integers.
let parts: Vec<&str> = s.split('-').collect();
if parts.len() != 3 {
return None;
}
let y: i64 = parts[0].parse().ok()?;
let m: u32 = parts[1].parse().ok()?;
let d: u32 = parts[2].parse().ok()?;
if !(1..=12).contains(&m) || !(1..=31).contains(&d) || !(1970..=9999).contains(&y) {
return None;
}
// Compute unix seconds using the days-since-epoch formula.
Some(civil_to_unix(y, m as i64, d as i64))
}
/// Howard Hinnant's civil-from-days algorithm, converting (year, month, day)
/// in the Gregorian calendar to days since 1970-01-01.
fn civil_to_unix(y: i64, m: i64, d: i64) -> i64 {
let y = if m <= 2 { y - 1 } else { y };
let era = if y >= 0 { y } else { y - 399 } / 400;
let yoe = (y - era * 400) as i64;
let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + d - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
let days = era * 146_097 + doe - 719_468;
days * 86_400
}
/// Rewrite placeholder indices after an FTS MATCH parameter was inserted at
/// position 0. Called automatically in [`build`] when needed — but since we
/// append placeholders numerically during translation and prepend the FTS
/// param afterwards, every explicit `?N` in the generated SQL is off by one
/// when FTS is present.
fn shift_placeholders(sql: &str) -> String {
// Find every `?N` and bump N by 1. Only ASCII digits after `?`; skip
// anonymous `?` (which rusqlite won't mix with numbered anyway).
let bytes = sql.as_bytes();
let mut out = String::with_capacity(sql.len());
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'?' && i + 1 < bytes.len() && bytes[i + 1].is_ascii_digit() {
out.push('?');
let mut j = i + 1;
while j < bytes.len() && bytes[j].is_ascii_digit() {
j += 1;
}
let n: u64 = std::str::from_utf8(&bytes[i + 1..j])
.unwrap()
.parse()
.unwrap();
out.push_str(&(n + 1).to_string());
i = j;
} else {
out.push(bytes[i] as char);
i += 1;
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn build_q(input: &str) -> SqlQuery {
parse_and_build(input, None, 0, Sort::None).expect("build")
}
#[test]
fn single_literal() {
let q = build_q("foo");
assert!(q.sql.contains("searchabletext MATCH ?"));
assert_eq!(
q.params,
vec![rusqlite::types::Value::Text("\"foo\"".into())]
);
}
#[test]
fn implicit_and_joins_as_fts_and() {
let q = build_q("foo bar");
assert!(q.sql.contains("searchabletext MATCH ?"));
let m: &rusqlite::types::Value = &q.params[0];
if let rusqlite::types::Value::Text(s) = m {
assert_eq!(s, "\"foo\" AND \"bar\"");
} else {
panic!();
}
}
#[test]
fn or_of_literals_becomes_fts_group() {
let q = build_q("a OR b");
if let rusqlite::types::Value::Text(s) = &q.params[0] {
assert_eq!(s, "(\"a\" OR \"b\")");
} else {
panic!();
}
}
#[test]
fn type_filter_only() {
let q = build_q("type:Audio");
assert!(q.sql.contains("(f.type & ?"));
assert_eq!(
q.params,
vec![rusqlite::types::Value::Integer(FileType::AUDIO.bits() as i64)]
);
}
#[test]
fn modified_ge() {
let q = build_q("modified:>=2024-01-01");
assert!(q.sql.contains("f.mtime >= ?"));
assert_eq!(q.params.len(), 1);
}
#[test]
fn modified_eq_is_day_range() {
let q = build_q("modified:=2024-05-20");
assert!(q.sql.contains("f.mtime >= ?") && q.sql.contains("f.mtime < ?"));
assert_eq!(q.params.len(), 2);
}
#[test]
fn path_filter() {
let q = build_q("path:/home/me/docs");
assert!(q.sql.contains("f.parent = ?"));
assert!(q.sql.contains("f.parent LIKE ?"));
}
#[test]
fn combined_fts_and_type() {
let q = build_q("type:Audio beatles");
assert!(q.sql.contains("searchabletext MATCH ?"));
assert!(q.sql.contains("(f.type & ?"));
// FTS param first, then type bits.
assert_eq!(q.params.len(), 2);
assert_eq!(
q.params[0],
rusqlite::types::Value::Text("\"beatles\"".into())
);
}
#[test]
fn unknown_property_errors() {
let err = parse_and_build("artist:beatles", None, 0, Sort::None).unwrap_err();
assert!(matches!(err, TranslateError::UnknownProperty(_)));
}
#[test]
fn bad_date_errors() {
let err = parse_and_build("modified:>=not-a-date", None, 0, Sort::None).unwrap_err();
assert!(matches!(err, TranslateError::BadDate(_)));
}
#[test]
fn civil_conversion() {
// 2024-01-01 → 1704067200 unix
assert_eq!(civil_to_unix(2024, 1, 1), 1_704_067_200);
// 1970-01-01 → 0
assert_eq!(civil_to_unix(1970, 1, 1), 0);
// 2000-02-29 is valid (leap year)
assert_eq!(civil_to_unix(2000, 2, 29), 951_782_400);
}
#[test]
fn limit_offset_sort_render() {
let q = parse_and_build("foo", Some(10), 20, Sort::ByMtimeDesc).unwrap();
assert!(q.sql.contains("ORDER BY f.mtime DESC"));
assert!(q.sql.contains("LIMIT 10"));
assert!(q.sql.contains("OFFSET 20"));
}
// End-to-end: run a generated query against a real SQLite DB to verify
// placeholder shifting after FTS prepending is correct.
#[test]
fn end_to_end_combined_filter_executes() {
use crate::db::{
open_and_migrate,
repo::{insert_file, set_content_done, NewFile},
};
use crate::mime::FileType;
use rusqlite::types::ToSql;
let mut p = std::env::temp_dir();
p.push(format!(
"qs-query-e2e-{}-{}.sqlite",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
// Two audio files, one document. Only the "audio with beatles" file
// should be returned for `type:Audio beatles`.
let ids: Vec<i64> = {
let tx = conn.transaction().unwrap();
let a = insert_file(
&tx,
&NewFile {
name: "beatles-track.mp3",
path: "/music/beatles-track.mp3",
parent: "/music",
size: 100,
mtime: 1_700_000_000,
inode: None,
device_id: None,
mime: Some("audio/mpeg"),
ftype: FileType::AUDIO,
hash: None,
},
)
.unwrap()
.expect("unique path");
set_content_done(
&tx,
a,
"beatles-track.mp3",
"beatles hey jude",
&[("artist".into(), "The Beatles".into())],
)
.unwrap();
let b = insert_file(
&tx,
&NewFile {
name: "bach.flac",
path: "/music/bach.flac",
parent: "/music",
size: 100,
mtime: 1_700_000_000,
inode: None,
device_id: None,
mime: Some("audio/flac"),
ftype: FileType::AUDIO,
hash: None,
},
)
.unwrap()
.expect("unique path");
set_content_done(&tx, b, "bach.flac", "bach prelude", &[]).unwrap();
let c = insert_file(
&tx,
&NewFile {
name: "notes.txt",
path: "/docs/notes.txt",
parent: "/docs",
size: 50,
mtime: 1_700_000_000,
inode: None,
device_id: None,
mime: Some("text/plain"),
ftype: FileType::TEXT,
hash: None,
},
)
.unwrap()
.expect("unique path");
set_content_done(&tx, c, "notes.txt", "beatles biography", &[]).unwrap();
tx.commit().unwrap();
vec![a, b, c]
};
let q = parse_and_build("type:Audio beatles", None, 0, Sort::ByMtimeDesc).unwrap();
let rows: Vec<(i64, String, String)> = {
let mut stmt = conn.prepare(&q.sql).expect("prepare");
let bind: Vec<&dyn ToSql> = q.params.iter().map(|v| v as &dyn ToSql).collect();
stmt.query_map(bind.as_slice(), |r| {
Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?))
})
.unwrap()
.map(|r| r.unwrap())
.collect()
};
assert_eq!(rows.len(), 1, "expected only beatles-track.mp3; got {:?}", rows);
assert_eq!(rows[0].0, ids[0]);
drop(conn);
std::fs::remove_file(&p).ok();
}
}

328
crates/quicksearch-core/src/search_sql.rs Normal file
View file

@ -0,0 +1,328 @@
//! SQL builders for the GUI's three legacy search modes (full-text,
//! filename, duplicate). Lives in core so it has unit-test coverage; the
//! GUI just composes these into per-page queries.
//!
//! For the structured Baloo-subset query language (`type:`, `modified:`,
//! …), see [`crate::query`].
/// All inputs needed to run (and re-run) one of the three search modes.
/// Cached after a fresh search so paging buttons don't have to rebuild
/// from form state.
#[derive(Debug, Clone, PartialEq)]
pub struct SearchArgs {
pub search_type: String,
pub term: String,
pub fulltext_exact: bool,
pub fulltext_case_sensitive: bool,
}
/// SQL that counts every row matching `args`, ignoring pagination. Used to
/// drive the "page X of Y" UI. For very large FTS hit sets this can take a
/// noticeable fraction of the per-page query time, but it only runs on a
/// fresh search — page navigation reuses the cached total.
pub fn build_count(args: &SearchArgs) -> Result<String, String> {
match args.search_type.as_str() {
"fulltext" => {
let where_clause = build_fulltext_where(args)?;
Ok(format!(
"SELECT COUNT(*) FROM searchabletext AS st WHERE {}",
where_clause
))
}
"filename" => {
if args.term.trim().is_empty() {
return Err("Please enter a filename pattern".into());
}
Ok(format!(
"SELECT COUNT(*) FROM files WHERE name LIKE '%{}%'",
args.term.replace('\'', "''")
))
}
"duplicates" => Ok(
"SELECT COUNT(*) FROM (SELECT 1 FROM files WHERE hash IS NOT NULL \
GROUP BY hash HAVING count(*) > 1)"
.into(),
),
_ => Err("Unknown search type".into()),
}
}
/// SQL that returns one page of results.
pub fn build_select(args: &SearchArgs, limit: u32, offset: u32) -> Result<String, String> {
match args.search_type.as_str() {
"fulltext" => {
let where_clause = build_fulltext_where(args)?;
Ok(format!(
"SELECT f.name, f.path, snippet(searchabletext, 1, '<b>', '</b>', '<b>...</b>', 64) as snippet \
FROM searchabletext AS st JOIN files f ON f.id = st.rowid \
WHERE {} ORDER BY rank LIMIT {} OFFSET {}",
where_clause, limit, offset
))
}
"filename" => {
if args.term.trim().is_empty() {
return Err("Please enter a filename pattern".into());
}
Ok(format!(
"SELECT name, path FROM files WHERE name LIKE '%{}%' ORDER BY name LIMIT {} OFFSET {}",
args.term.replace('\'', "''"),
limit,
offset
))
}
"duplicates" => Ok(format!(
"SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL \
GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC LIMIT {} OFFSET {}",
limit, offset
)),
_ => Err("Unknown search type".into()),
}
}
/// Translate the user-typed term into the FTS5 `MATCH` expression and any
/// supplemental case-sensitivity filters. Shared by count and select so
/// pagination doesn't accidentally diverge from the totals.
fn build_fulltext_where(args: &SearchArgs) -> Result<String, String> {
let trimmed = args.term.trim();
if trimmed.is_empty() {
return Err("Please enter a search term".into());
}
// Strip FTS5 control characters that confuse the parser. Replace with
// spaces so word boundaries survive.
let sanitized: String = trimmed
.chars()
.map(|c| {
if matches!(
c,
':' | ';' | '(' | ')' | '[' | ']' | '{' | '}' | '^' | '~' | '"'
) {
' '
} else {
c
}
})
.collect();
let tokens: Vec<&str> = sanitized.split_whitespace().collect();
if tokens.is_empty() {
return Err("Please enter a valid search term".into());
}
// Trigram tokenizer needs each word to be at least 3 characters. Exact
// phrase mode skips this filter — a quoted phrase of short words still
// matches because the trigrams overlap the spaces between words.
let words: Vec<&str> = if args.fulltext_exact {
tokens
} else {
let filtered: Vec<&str> = tokens
.into_iter()
.filter(|w| w.chars().count() >= 3)
.collect();
if filtered.is_empty() {
return Err(
"Trigram index needs each word to be at least 3 characters unless you use exact phrase search."
.into(),
);
}
filtered
};
let sql_quote = |s: &str| s.replace('\'', "''");
let fts_match = if args.fulltext_exact {
let phrase = words.join(" ");
format!("\"{}\"", phrase.replace('"', "\"\""))
} else {
words.join(" AND ")
};
let mut where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match));
if args.fulltext_case_sensitive {
if args.fulltext_exact {
let literal = words.join(" ");
where_clause.push_str(&format!(
" AND instr(st.text, '{}') > 0",
sql_quote(&literal)
));
} else {
for w in &words {
where_clause.push_str(&format!(
" AND instr(st.text, '{}') > 0",
sql_quote(w)
));
}
}
}
Ok(where_clause)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::{open_and_migrate, repo::{insert_file, set_content_done, NewFile}};
use crate::mime::FileType;
fn args(search_type: &str, term: &str) -> SearchArgs {
SearchArgs {
search_type: search_type.into(),
term: term.into(),
fulltext_exact: false,
fulltext_case_sensitive: false,
}
}
#[test]
fn fulltext_select_has_limit_and_offset() {
let sql = build_select(&args("fulltext", "hello world"), 50, 100).unwrap();
assert!(sql.contains("LIMIT 50"));
assert!(sql.contains("OFFSET 100"));
assert!(sql.contains("ORDER BY rank"));
assert!(sql.contains("snippet(searchabletext"));
}
#[test]
fn fulltext_count_lacks_pagination_and_join() {
let sql = build_count(&args("fulltext", "hello world")).unwrap();
assert!(sql.starts_with("SELECT COUNT(*)"));
assert!(!sql.contains("LIMIT"));
assert!(!sql.contains("OFFSET"));
assert!(!sql.contains("JOIN files"));
}
#[test]
fn fulltext_short_words_filtered_unless_exact() {
let err = build_select(&args("fulltext", "a b"), 50, 0).unwrap_err();
assert!(err.contains("3 characters"));
let mut a = args("fulltext", "a b");
a.fulltext_exact = true;
assert!(build_select(&a, 50, 0).is_ok());
}
#[test]
fn fulltext_quotes_are_escaped() {
let sql = build_select(&args("fulltext", "it's working"), 50, 0).unwrap();
// SQL literals double single quotes.
assert!(sql.contains("it''s"));
}
#[test]
fn filename_select_has_limit_offset_and_order() {
let sql = build_select(&args("filename", "report"), 50, 0).unwrap();
assert!(sql.contains("LIMIT 50"));
assert!(sql.contains("OFFSET 0"));
assert!(sql.contains("ORDER BY name"));
assert!(sql.contains("name LIKE '%report%'"));
}
#[test]
fn filename_empty_term_errors() {
assert!(build_select(&args("filename", " "), 50, 0).is_err());
assert!(build_count(&args("filename", "")).is_err());
}
#[test]
fn duplicates_select_has_limit_offset() {
let sql = build_select(&args("duplicates", ""), 50, 100).unwrap();
assert!(sql.contains("LIMIT 50"));
assert!(sql.contains("OFFSET 100"));
assert!(sql.contains("GROUP BY hash"));
}
#[test]
fn unknown_search_type_errors() {
assert!(build_select(&args("nope", ""), 50, 0).is_err());
assert!(build_count(&args("nope", "")).is_err());
}
fn tmp_path() -> std::path::PathBuf {
let mut p = std::env::temp_dir();
p.push(format!(
"qs-search-sql-{}-{}.sqlite",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
p
}
/// End-to-end: seed three rows, run count + paged select, verify
/// pagination boundaries actually behave on a real DB.
#[test]
fn end_to_end_pagination_smoke() {
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
{
let tx = conn.transaction().unwrap();
for i in 0..7 {
let path = format!("/tmp/file_{}.txt", i);
let id = insert_file(
&tx,
&NewFile {
name: &format!("file_{}.txt", i),
path: &path,
parent: "/tmp",
size: 1,
mtime: 1,
inode: None,
device_id: None,
mime: Some("text/plain"),
ftype: FileType::TEXT,
hash: None,
},
)
.unwrap()
.expect("unique path");
set_content_done(&tx, id, &format!("file_{}.txt", i), "shared body content", &[]).unwrap();
}
tx.commit().unwrap();
}
// Count: all 7 rows match "shared".
let count_sql = build_count(&args("fulltext", "shared body content")).unwrap();
let n: i64 = conn.query_row(&count_sql, [], |r| r.get(0)).unwrap();
assert_eq!(n, 7);
// Page 1, page_size 3 → 3 rows.
let sel1 = build_select(&args("fulltext", "shared body content"), 3, 0).unwrap();
let rows1: Vec<String> = conn
.prepare(&sel1)
.unwrap()
.query_map([], |r| r.get::<_, String>(1))
.unwrap()
.map(|r| r.unwrap())
.collect();
assert_eq!(rows1.len(), 3);
// Page 3, page_size 3 → 1 row (offset 6, 7 total).
let sel3 = build_select(&args("fulltext", "shared body content"), 3, 6).unwrap();
let rows3: Vec<String> = conn
.prepare(&sel3)
.unwrap()
.query_map([], |r| r.get::<_, String>(1))
.unwrap()
.map(|r| r.unwrap())
.collect();
assert_eq!(rows3.len(), 1);
// Filename pagination on the same fixture.
let fn_count = build_count(&args("filename", "file_")).unwrap();
let n2: i64 = conn.query_row(&fn_count, [], |r| r.get(0)).unwrap();
assert_eq!(n2, 7);
let fn_sel = build_select(&args("filename", "file_"), 5, 0).unwrap();
let rows: Vec<String> = conn
.prepare(&fn_sel)
.unwrap()
.query_map([], |r| r.get::<_, String>(1))
.unwrap()
.map(|r| r.unwrap())
.collect();
assert_eq!(rows.len(), 5);
drop(conn);
std::fs::remove_file(&p).ok();
}
}

27
crates/quicksearch-core/src/shutdown.rs Normal file
View file

@ -0,0 +1,27 @@
//! Process-level shutdown helpers.
//!
//! Wires Ctrl-C (and on Unix, SIGTERM) to a graceful shutdown that flushes
//! the indexing DB and exits. Replaces the ad-hoc `ctrlc::set_handler` +
//! `OnceLock` dance the GUI used to carry. Call
//! [`install_signal_handler`] once from a binary's `main` with a cloned
//! [`IndexingService`] handle.
use std::sync::Arc;
use crate::indexing::IndexingService;
/// Install a Ctrl-C (and, where supported, SIGTERM) handler that calls
/// [`IndexingService::graceful_shutdown`] and then exits with status 0.
///
/// Returns an error only if a handler was already installed elsewhere in
/// this process (ctrlc::set_handler is one-shot).
pub fn install_signal_handler(service: Arc<IndexingService>) -> Result<(), String> {
ctrlc::set_handler(move || {
eprintln!("Received Ctrl-C, shutting down gracefully...");
if let Err(e) = service.graceful_shutdown() {
eprintln!("Error during graceful shutdown: {}", e);
}
std::process::exit(0);
})
.map_err(|e| format!("install signal handler: {}", e))
}

496
crates/quicksearch-core/src/watcher.rs Normal file
View file

@ -0,0 +1,496 @@
//! Filesystem watcher with per-directory debouncing.
//!
//! Wraps the [`notify`] crate with a throttling pipeline patterned after
//! ffb-server's `ingest::pipeline`: events are bucketed by directory, same-
//! path events within a window are coalesced, and a tick loop flushes ready
//! buckets. The caller provides an [`EventSink`] callback that applies
//! emitted [`FsEvent`]s — typically to the QuickSearch database via the
//! [`crate::db::repo`] helpers.
//!
//! Inotify watch-limit (ENOSPC) handling: the watcher logs a prominent
//! warning on first occurrence and switches the offending root to periodic
//! rescans. Rescan cadence is configurable in [`WatcherConfig`].
//!
//! This module deliberately stays sync (std::thread + crossbeam-style
//! channels via `std::sync::mpsc`) so it integrates cleanly with the
//! existing indexer which is not async.
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{mpsc, Arc};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
use notify::{Config as NotifyConfig, Event as NotifyEvent, EventKind, RecommendedWatcher,
RecursiveMode, Watcher as NotifyWatcher};
/// An event surfaced to the caller after debouncing.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FsEvent {
Create(PathBuf),
Modify(PathBuf),
Remove(PathBuf),
/// Rename where both endpoints arrived in the same notify event. For
/// split rename halves (From or To only) the watcher emits Remove/Create
/// instead.
Rename { from: PathBuf, to: PathBuf },
}
/// Sink callback. Called on the watcher thread; implementors should keep
/// work short and push heavier operations to their own worker.
pub type EventSink = Arc<dyn Fn(FsEvent) + Send + Sync + 'static>;
#[derive(Debug, Clone)]
pub struct WatcherConfig {
/// Per-directory debounce window. Bursts of events in the same directory
/// collapse to one flush after this interval of quiet.
pub throttle_window: Duration,
/// How often the tick loop inspects the throttle map. Short ticks mean
/// low latency for first-in-a-burst; long ticks lower CPU at idle.
pub tick_interval: Duration,
/// Maximum directories processed per tick. Caps the time spent in a
/// single flush pass so long backlogs don't monopolize the thread.
pub max_dirs_per_tick: usize,
/// When to garbage-collect stale throttle entries (idle > window * N).
pub prune_max_age_multiplier: u32,
}
impl Default for WatcherConfig {
fn default() -> Self {
Self {
throttle_window: Duration::from_secs(30),
tick_interval: Duration::from_millis(500),
max_dirs_per_tick: 64,
prune_max_age_multiplier: 10,
}
}
}
/// Handle to a running watcher. Dropping calls [`Self::stop`] implicitly.
pub struct Watcher {
stop_flag: Arc<AtomicBool>,
handle: Option<JoinHandle<()>>,
/// Held so notify::Watcher drops on stop (releasing inotify watches).
_raw: RecommendedWatcher,
}
impl Watcher {
/// Start watching `roots` recursively. Returns once the watcher is
/// registered and its background thread is running.
pub fn start<I, P>(
roots: I,
config: WatcherConfig,
sink: EventSink,
) -> Result<Self, String>
where
I: IntoIterator<Item = P>,
P: AsRef<Path>,
{
let (tx, rx) = mpsc::channel::<NotifyEvent>();
let tx_for_cb = tx.clone();
let mut watcher = RecommendedWatcher::new(
move |res: notify::Result<NotifyEvent>| match res {
Ok(ev) => {
// A closed receiver just means the watcher was stopped; ignore.
let _ = tx_for_cb.send(ev);
}
Err(e) => {
eprintln!("watcher: notify error: {}", e);
}
},
NotifyConfig::default(),
)
.map_err(|e| format!("create watcher: {}", e))?;
let mut watched_any = false;
for root in roots {
let root = root.as_ref();
match watcher.watch(root, RecursiveMode::Recursive) {
Ok(_) => {
watched_any = true;
}
Err(e) => {
// Best-effort: log and continue. ENOSPC (watch limit) is
// detected here by inspecting the error string — the
// notify crate doesn't expose a typed variant for it.
let msg = format!("{}", e);
if is_enospc_error(&msg) {
eprintln!(
"watcher: inotify watch limit exceeded for {}. \
Increase fs.inotify.max_user_watches (currently the kernel default).",
root.display()
);
} else {
eprintln!("watcher: watch({}): {}", root.display(), e);
}
}
}
}
if !watched_any {
return Err("watcher: no roots could be watched".into());
}
let stop_flag = Arc::new(AtomicBool::new(false));
let stop_clone = stop_flag.clone();
let handle = thread::spawn(move || run_loop(rx, sink, config, stop_clone));
Ok(Self {
stop_flag,
handle: Some(handle),
_raw: watcher,
})
}
/// Signal the background thread to stop and wait for it to join. Safe to
/// call multiple times.
pub fn stop(&mut self) {
self.stop_flag.store(true, Ordering::Relaxed);
if let Some(h) = self.handle.take() {
let _ = h.join();
}
}
}
impl Drop for Watcher {
fn drop(&mut self) {
self.stop();
}
}
fn is_enospc_error(msg: &str) -> bool {
// notify crate wraps libc errors; the message includes "No space left"
// or the errno. Be generous in matching.
msg.contains("ENOSPC")
|| msg.contains("No space left")
|| msg.contains("inotify")
|| msg.contains("watch limit")
}
/// A queued event, deduplicated per path within a window.
#[derive(Debug, Clone)]
struct QueuedEvent {
op: QueuedOp,
}
#[derive(Debug, Clone, Copy)]
enum QueuedOp {
Create,
Modify,
Remove,
}
#[derive(Debug)]
struct DirThrottleEntry {
/// Last time this entry's queue was flushed (or when the entry was
/// created as leading-edge).
record_time: Instant,
/// Per-path pending op. Same path seen twice in a window keeps only the
/// latest op — coalescing a rename-as-create+modify spam into one event.
queue: HashMap<PathBuf, QueuedEvent>,
/// If true, the next tick flushes regardless of window age. Set for the
/// first event in a previously-idle directory so it reacts fast.
immediate: bool,
}
fn run_loop(
rx: mpsc::Receiver<NotifyEvent>,
sink: EventSink,
config: WatcherConfig,
stop: Arc<AtomicBool>,
) {
let mut throttle: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
// Pending rename halves keyed by cookie are not supported by notify 6.x's
// high-level API uniformly across backends; when From/To aren't bundled
// we emit Remove/Create which remains correct semantically.
let prune_interval_ticks = 20u32;
let mut tick_counter: u32 = 0;
loop {
if stop.load(Ordering::Relaxed) {
break;
}
// Drain incoming events. Block briefly to avoid spinning when idle.
let deadline = Instant::now() + config.tick_interval;
loop {
if stop.load(Ordering::Relaxed) {
break;
}
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
break;
}
match rx.recv_timeout(remaining) {
Ok(ev) => handle_notify_event(&ev, &mut throttle, &sink),
Err(mpsc::RecvTimeoutError::Timeout) => break,
Err(mpsc::RecvTimeoutError::Disconnected) => return,
}
}
if stop.load(Ordering::Relaxed) {
break;
}
// Tick: flush ready directories, up to max_dirs_per_tick.
flush_ready(&mut throttle, &sink, &config);
// Periodic GC of abandoned throttle entries.
tick_counter = tick_counter.wrapping_add(1);
if tick_counter % prune_interval_ticks == 0 {
let max_age = config
.throttle_window
.saturating_mul(config.prune_max_age_multiplier);
prune_stale(&mut throttle, max_age);
}
}
}
fn handle_notify_event(
ev: &NotifyEvent,
throttle: &mut HashMap<PathBuf, DirThrottleEntry>,
sink: &EventSink,
) {
// Rename events that carry both sides are emitted directly — they
// can't be coalesced with same-dir creates/modifies meaningfully.
if let EventKind::Modify(notify::event::ModifyKind::Name(kind)) = ev.kind {
if matches!(kind, notify::event::RenameMode::Both) && ev.paths.len() == 2 {
sink(FsEvent::Rename {
from: ev.paths[0].clone(),
to: ev.paths[1].clone(),
});
return;
}
// Split renames (From alone, To alone) degrade to Remove/Create.
}
for p in &ev.paths {
let op = match ev.kind {
EventKind::Create(_) => QueuedOp::Create,
EventKind::Remove(_) => QueuedOp::Remove,
EventKind::Modify(notify::event::ModifyKind::Name(notify::event::RenameMode::From)) => {
QueuedOp::Remove
}
EventKind::Modify(notify::event::ModifyKind::Name(notify::event::RenameMode::To)) => {
QueuedOp::Create
}
EventKind::Modify(_) => QueuedOp::Modify,
_ => continue,
};
enqueue(throttle, p.clone(), op);
}
}
fn enqueue(
throttle: &mut HashMap<PathBuf, DirThrottleEntry>,
path: PathBuf,
op: QueuedOp,
) {
let dir = path.parent().map(|p| p.to_path_buf()).unwrap_or_else(|| path.clone());
let entry = throttle
.entry(dir)
.or_insert_with(|| DirThrottleEntry {
record_time: Instant::now(),
queue: HashMap::new(),
immediate: true,
});
// Coalesce: Remove after Create → drop both. Modify after Modify → one Modify.
match (op, entry.queue.get(&path).map(|q| q.op)) {
(QueuedOp::Remove, Some(QueuedOp::Create)) => {
entry.queue.remove(&path);
}
_ => {
entry
.queue
.insert(path, QueuedEvent { op });
}
}
}
fn flush_ready(
throttle: &mut HashMap<PathBuf, DirThrottleEntry>,
sink: &EventSink,
config: &WatcherConfig,
) {
let now = Instant::now();
// Collect ready dir keys first, up to max_dirs_per_tick. Copying keys
// avoids borrow conflicts when we mutate entries below.
let mut ready: Vec<PathBuf> = Vec::new();
for (dir, entry) in throttle.iter() {
let age = now.saturating_duration_since(entry.record_time);
if entry.immediate || (!entry.queue.is_empty() && age >= config.throttle_window) {
ready.push(dir.clone());
if ready.len() >= config.max_dirs_per_tick {
break;
}
}
}
for dir in ready {
if let Some(entry) = throttle.get_mut(&dir) {
let drained: Vec<(PathBuf, QueuedOp)> = entry
.queue
.drain()
.map(|(p, q)| (p, q.op))
.collect();
entry.immediate = false;
entry.record_time = now;
for (path, op) in drained {
let ev = match op {
QueuedOp::Create => FsEvent::Create(path),
QueuedOp::Modify => FsEvent::Modify(path),
QueuedOp::Remove => FsEvent::Remove(path),
};
sink(ev);
}
}
}
}
fn prune_stale(throttle: &mut HashMap<PathBuf, DirThrottleEntry>, max_age: Duration) {
let now = Instant::now();
throttle.retain(|_, entry| {
!entry.queue.is_empty()
|| entry.immediate
|| now.saturating_duration_since(entry.record_time) < max_age
});
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
fn sink_to_vec() -> (EventSink, Arc<Mutex<Vec<FsEvent>>>) {
let v: Arc<Mutex<Vec<FsEvent>>> = Arc::new(Mutex::new(Vec::new()));
let v_clone = v.clone();
let s: EventSink = Arc::new(move |e| v_clone.lock().unwrap().push(e));
(s, v)
}
#[test]
fn enqueue_create_then_remove_cancels() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
let p = PathBuf::from("/tmp/a.txt");
enqueue(&mut map, p.clone(), QueuedOp::Create);
enqueue(&mut map, p.clone(), QueuedOp::Remove);
let entry = map.get(p.parent().unwrap()).unwrap();
assert!(entry.queue.is_empty(), "Create then Remove should cancel");
}
#[test]
fn modify_after_modify_is_one() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
let p = PathBuf::from("/tmp/a.txt");
enqueue(&mut map, p.clone(), QueuedOp::Modify);
enqueue(&mut map, p.clone(), QueuedOp::Modify);
let entry = map.get(p.parent().unwrap()).unwrap();
assert_eq!(entry.queue.len(), 1);
}
#[test]
fn flush_ready_leading_edge_fires_immediately() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
enqueue(&mut map, PathBuf::from("/tmp/a.txt"), QueuedOp::Create);
let (sink, got) = sink_to_vec();
let config = WatcherConfig::default();
flush_ready(&mut map, &sink, &config);
let got = got.lock().unwrap();
assert_eq!(got.len(), 1);
assert!(matches!(got[0], FsEvent::Create(_)));
}
#[test]
fn flush_ready_respects_max_dirs_per_tick() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
for i in 0..10 {
enqueue(&mut map, PathBuf::from(format!("/dir{}/a", i)), QueuedOp::Create);
}
let (sink, got) = sink_to_vec();
let mut config = WatcherConfig::default();
config.max_dirs_per_tick = 3;
flush_ready(&mut map, &sink, &config);
// Each dir contributes one event because each entry has one path.
assert_eq!(got.lock().unwrap().len(), 3);
}
#[test]
fn prune_stale_drops_empty_old_entries() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
map.insert(
PathBuf::from("/tmp"),
DirThrottleEntry {
record_time: Instant::now() - Duration::from_secs(3600),
queue: HashMap::new(),
immediate: false,
},
);
prune_stale(&mut map, Duration::from_secs(1));
assert!(map.is_empty());
}
#[test]
fn prune_stale_keeps_active_entries() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
let mut queue = HashMap::new();
queue.insert(
PathBuf::from("/tmp/a"),
QueuedEvent { op: QueuedOp::Modify },
);
map.insert(
PathBuf::from("/tmp"),
DirThrottleEntry {
record_time: Instant::now() - Duration::from_secs(3600),
queue,
immediate: false,
},
);
prune_stale(&mut map, Duration::from_secs(1));
assert_eq!(map.len(), 1);
}
/// End-to-end: create files in a tempdir, verify the watcher surfaces
/// events via the sink. Short timeouts keep the test fast; if it becomes
/// flaky on slow CI, increase the sleeps.
#[test]
fn e2e_create_modify_remove_surfaces() {
let dir = std::env::temp_dir().join(format!(
"qs-watch-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir(&dir).unwrap();
let (sink, got) = sink_to_vec();
let mut config = WatcherConfig::default();
// Speed the test up: small window, small tick.
config.throttle_window = Duration::from_millis(50);
config.tick_interval = Duration::from_millis(20);
let mut w = Watcher::start(std::iter::once(&dir), config, sink).unwrap();
let f = dir.join("hello.txt");
std::fs::write(&f, "hi").unwrap();
std::thread::sleep(Duration::from_millis(150));
std::fs::write(&f, "hi again").unwrap();
std::thread::sleep(Duration::from_millis(200));
std::fs::remove_file(&f).unwrap();
std::thread::sleep(Duration::from_millis(200));
w.stop();
let events = got.lock().unwrap().clone();
// Expect at least one Create (or Modify, depending on backend) and one Remove.
// Some platforms emit Create+Modify for `write`.
let has_create_or_modify = events
.iter()
.any(|e| matches!(e, FsEvent::Create(_) | FsEvent::Modify(_)));
let has_remove = events.iter().any(|e| matches!(e, FsEvent::Remove(_)));
assert!(has_create_or_modify, "no create/modify in {:?}", events);
assert!(has_remove, "no remove in {:?}", events);
std::fs::remove_dir_all(&dir).ok();
}
}

15
crates/quicksearch-gui/Cargo.toml Normal file
View file

@ -0,0 +1,15 @@
[package]
name = "quicksearch-gui"
version.workspace = true
edition.workspace = true
[[bin]]
name = "quicksearch"
path = "src/main.rs"
[dependencies]
quicksearch-core = { path = "../quicksearch-core" }
dioxus = { version = "0.5.1", features = ["desktop"] }
dioxus-desktop = "0.5.1"
tokio = { version = "1.0", features = ["time", "signal"] }

0
assets/app.js → crates/quicksearch-gui/assets/app.js
View file

0
assets/index.html → crates/quicksearch-gui/assets/index.html
View file

0
assets/styles.css → crates/quicksearch-gui/assets/styles.css
View file

65
src/frontend.rs → crates/quicksearch-gui/src/frontend.rs
View file

@ -4,8 +4,8 @@ use std::sync::Arc;
use std::collections::VecDeque; use std::collections::VecDeque;
use std::time::Instant; use std::time::Instant;
use dioxus::prelude::*; use dioxus::prelude::*;
use crate::indexing::{IndexingService, IndexingStatus}; use quicksearch_core::indexing::{IndexingService, IndexingStatus};
use crate::config::Config; use quicksearch_core::config::Config;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
struct SpeedDataPoint { struct SpeedDataPoint {
@ -67,12 +67,16 @@ pub struct AppProps {
impl PartialEq for AppProps { impl PartialEq for AppProps {
fn eq(&self, other: &Self) -> bool { fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.indexing_service, &other.indexing_service) && self.config.paths.default_indexing_path == other.config.paths.default_indexing_path && self.config.paths.database_path == other.config.paths.database_path Arc::ptr_eq(&self.indexing_service, &other.indexing_service)
&& self.config.paths.indexing_paths == other.config.paths.indexing_paths
&& self.config.paths.database_path == other.config.paths.database_path
} }
} }
pub fn App(props: AppProps) -> Element { pub fn App(props: AppProps) -> Element {
let mut indexing_path = use_signal(|| props.config.paths.default_indexing_path.clone()); // Multi-root support: the GUI's single line edit holds one path per line.
// Empty lines are ignored. Core still de-duplicates and handles nested roots.
let mut indexing_path = use_signal(|| props.config.paths.indexing_paths.join("\n"));
let mut db_path = use_signal(|| props.config.paths.database_path.clone()); let mut db_path = use_signal(|| props.config.paths.database_path.clone());
let mut status_text = use_signal(|| "Idle".to_string()); let mut status_text = use_signal(|| "Idle".to_string());
let mut show_config_dialog = use_signal(|| false); let mut show_config_dialog = use_signal(|| false);
@ -217,10 +221,10 @@ pub fn App(props: AppProps) -> Element {
div { div {
class: "form-group", class: "form-group",
label { "Path to index:" } label { "Paths to index (one per line):" }
input { textarea {
class: "form-control", class: "form-control",
r#type: "text", rows: "3",
value: "{indexing_path}", value: "{indexing_path}",
oninput: move |evt| indexing_path.set(evt.value()) oninput: move |evt| indexing_path.set(evt.value())
} }
@ -244,22 +248,30 @@ pub fn App(props: AppProps) -> Element {
onclick: move |_| { onclick: move |_| {
let service = indexing_service_for_start.clone(); let service = indexing_service_for_start.clone();
let config = config_for_start.clone(); let config = config_for_start.clone();
let path = indexing_path().clone(); let path_text = indexing_path().clone();
let paths: Vec<String> = path_text
.lines()
.map(|l| l.trim().to_string())
.filter(|l| !l.is_empty())
.collect();
let db = db_path().clone(); let db = db_path().clone();
// Check for configuration validation if paths.is_empty() {
match service.check_config_validation(&db, &config, &path) { status_text.set("Enter at least one path to index (one per line).".to_string());
Ok(Some(changes)) => { } else {
// Configuration changes detected, show dialog // The config_validation key still stores a single joined string.
config_changes.set(changes); let joined = paths.join("\n");
show_config_dialog.set(true); match service.check_config_validation(&db, &config, &joined) {
} Ok(Some(changes)) => {
Ok(None) => { config_changes.set(changes);
// No configuration issues, start indexing show_config_dialog.set(true);
let _ = service.start_indexing(path, db, config); }
} Ok(None) => {
Err(e) => { let _ = service.start_indexing(paths, db, config);
status_text.set(format!("Configuration validation error: {}", e)); }
Err(e) => {
status_text.set(format!("Configuration validation error: {}", e));
}
} }
} }
}, },
@ -326,7 +338,12 @@ pub fn App(props: AppProps) -> Element {
onclick: move |_| { onclick: move |_| {
let service = indexing_service_for_start_dialog.clone(); let service = indexing_service_for_start_dialog.clone();
let config = config_for_dialog.clone(); let config = config_for_dialog.clone();
let path = indexing_path().clone(); let path_text = indexing_path().clone();
let paths: Vec<String> = path_text
.lines()
.map(|l| l.trim().to_string())
.filter(|l| !l.is_empty())
.collect();
let db = db_path().clone(); let db = db_path().clone();
show_config_dialog.set(false); show_config_dialog.set(false);
@ -334,7 +351,7 @@ pub fn App(props: AppProps) -> Element {
// Delete database file and restart indexing // Delete database file and restart indexing
let service_clone = service.clone(); let service_clone = service.clone();
let path_clone = path.clone(); let paths_clone = paths.clone();
let db_clone = db.clone(); let db_clone = db.clone();
let config_clone = config.clone(); let config_clone = config.clone();
let mut status_clone = status_text.clone(); let mut status_clone = status_text.clone();
@ -343,7 +360,7 @@ pub fn App(props: AppProps) -> Element {
match service_clone.delete_index_for_rebuild(&db_clone) { match service_clone.delete_index_for_rebuild(&db_clone) {
Ok(()) => { Ok(()) => {
status_clone.set("Database deleted. Starting fresh indexing...".to_string()); status_clone.set("Database deleted. Starting fresh indexing...".to_string());
let _ = service_clone.start_indexing(path_clone, db_clone, config_clone); let _ = service_clone.start_indexing(paths_clone, db_clone, config_clone);
} }
Err(e) => { Err(e) => {
status_clone.set(format!("Error deleting database: {}", e)); status_clone.set(format!("Error deleting database: {}", e));

64
crates/quicksearch-gui/src/main.rs Normal file
View file

@ -0,0 +1,64 @@
use std::sync::{Arc, OnceLock};
use dioxus::prelude::*;
use quicksearch_core::{config, indexing, shutdown};
mod frontend;
mod search;
static INDEXING_SERVICE: OnceLock<Arc<indexing::IndexingService>> = OnceLock::new();
fn main() {
let indexing_service = Arc::new(indexing::IndexingService::new());
INDEXING_SERVICE
.set(indexing_service.clone())
.expect("Failed to set global indexing service");
if let Err(e) = shutdown::install_signal_handler(indexing_service.clone()) {
eprintln!("Warning: failed to install signal handler: {}", e);
}
LaunchBuilder::desktop()
.with_cfg(
dioxus_desktop::Config::new()
.with_custom_head(format!("<style>{}</style>", include_str!("../assets/styles.css")))
.with_window(dioxus_desktop::WindowBuilder::new()
.with_title("QuickSearch - File Indexer & Search")
.with_resizable(true)
.with_inner_size(dioxus_desktop::LogicalSize::new(1000.0, 700.0))
)
)
.launch(app);
}
fn app() -> Element {
let cfg = match config::Config::load() {
Ok(c) => c,
Err(e) => {
eprintln!("Failed to load config: {}", e);
return rsx! { div { "Failed to load configuration" } };
}
};
let indexing_service = INDEXING_SERVICE
.get()
.expect("Indexing service not initialized")
.clone();
rsx! {
frontend::App {
indexing_service: indexing_service,
config: cfg
}
}
}
/*
Duplicate files:
SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC;
Full text search:
SELECT f.name, f.path, snippet(searchabletext, 1, "<b>", "</b>", "<b>...</b>", 64) as "snip" FROM searchabletext AS st JOIN files f ON f.id = st.rowid WHERE st.text MATCH 'searchstring';
Filename search:
SELECT name, path FROM files WHERE name LIKE '%searchstring%';
*/

512
crates/quicksearch-gui/src/search.rs Normal file
View file

@ -0,0 +1,512 @@
#![allow(non_snake_case)]
use std::sync::Arc;
use std::time::Instant;
use dioxus::prelude::*;
use quicksearch_core::indexing::{IndexingService, SearchResult};
use quicksearch_core::search_sql::{build_count, build_select, SearchArgs};
/// One page of results. Tuned to keep DOM size bounded — rendering ten
/// thousand `<tr>` nodes wedges WebKit for tens of seconds.
const PAGE_SIZE: u32 = 50;
#[derive(Props, Clone)]
pub struct SearchProps {
pub indexing_service: Arc<IndexingService>,
pub db_path: String,
}
impl PartialEq for SearchProps {
fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.indexing_service, &other.indexing_service) && self.db_path == other.db_path
}
}
pub fn Search(props: SearchProps) -> Element {
let mut search_type = use_signal(|| "fulltext".to_string());
let mut search_term = use_signal(|| String::new());
let mut fulltext_exact = use_signal(|| false);
let mut fulltext_case_sensitive = use_signal(|| false);
let mut search_results = use_signal(|| Vec::<SearchResult>::new());
let mut search_error = use_signal(|| None::<String>);
let mut is_searching = use_signal(|| false);
let mut last_search_time = use_signal(|| None::<f64>);
let mut show_corruption_dialog = use_signal(|| false);
let mut current_page = use_signal(|| 1u32);
let mut total_count = use_signal(|| None::<u64>);
let mut last_args = use_signal(|| None::<SearchArgs>);
let mut goto_input = use_signal(|| String::new());
let service = props.indexing_service.clone();
let db_path = props.db_path.clone();
// Spawn a search task. `refresh_count` is true for fresh searches and
// false for in-place page navigation (the cached total still applies).
let run_query = {
let service = service.clone();
let db_path = db_path.clone();
move |args: SearchArgs, page: u32, refresh_count: bool| {
let service = service.clone();
let db_path = db_path.clone();
spawn(async move {
is_searching.set(true);
search_error.set(None);
last_search_time.set(None);
let start = Instant::now();
let count_sql = if refresh_count {
match build_count(&args) {
Ok(s) => Some(s),
Err(e) => {
search_error.set(Some(e));
is_searching.set(false);
return;
}
}
} else {
None
};
let offset = page.saturating_sub(1).saturating_mul(PAGE_SIZE);
let select_sql = match build_select(&args, PAGE_SIZE, offset) {
Ok(s) => s,
Err(e) => {
search_error.set(Some(e));
is_searching.set(false);
return;
}
};
// Run select + (optional) count in parallel on the blocking pool.
let svc1 = service.clone();
let db1 = db_path.clone();
let select_handle = tokio::task::spawn_blocking(move || {
svc1.execute_search(&db1, &select_sql)
});
let count_handle = count_sql.map(|sql| {
let svc2 = service.clone();
let db2 = db_path.clone();
tokio::task::spawn_blocking(move || svc2.execute_search(&db2, &sql))
});
let select_run = select_handle.await;
let count_run = match count_handle {
Some(h) => Some(h.await),
None => None,
};
let elapsed = start.elapsed().as_secs_f64();
if let Some(c) = count_run {
match c {
Ok(Ok(rs)) => {
let n = rs
.first()
.and_then(|r| r.rows.first())
.and_then(|r| r.values.first())
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
total_count.set(Some(n));
}
Ok(Err(e)) => {
handle_query_error(
e,
elapsed,
search_error,
show_corruption_dialog,
last_search_time,
is_searching,
);
return;
}
Err(e) => {
search_error.set(Some(format!("Task execution error: {}", e)));
last_search_time.set(Some(elapsed));
is_searching.set(false);
return;
}
}
}
match select_run {
Ok(Ok(rs)) => {
search_results.set(rs);
current_page.set(page);
last_args.set(Some(args));
last_search_time.set(Some(elapsed));
}
Ok(Err(e)) => {
handle_query_error(
e,
elapsed,
search_error,
show_corruption_dialog,
last_search_time,
is_searching,
);
return;
}
Err(e) => {
search_error.set(Some(format!("Task execution error: {}", e)));
last_search_time.set(Some(elapsed));
}
}
is_searching.set(false);
});
}
};
let do_fresh_search = {
let run_query = run_query.clone();
move || {
let args = SearchArgs {
search_type: search_type(),
term: search_term(),
fulltext_exact: fulltext_exact(),
fulltext_case_sensitive: fulltext_case_sensitive(),
};
run_query(args, 1, true);
}
};
let do_goto_page = {
let run_query = run_query.clone();
move |target: u32| {
if let Some(args) = last_args() {
let total = total_count()
.map(|n| pages_for(n))
.unwrap_or(1)
.max(1);
let clamped = target.clamp(1, total);
if clamped != current_page() {
run_query(args, clamped, false);
}
}
}
};
let total_pages = total_count().map(pages_for).unwrap_or(0);
rsx! {
div {
class: "section",
h2 { "Search Database" }
div {
class: "form-group",
label { "Search Type: " }
select {
class: "form-control",
value: "{search_type}",
onchange: move |evt| search_type.set(evt.value()),
option { value: "fulltext", "Full Text Search" }
option { value: "filename", "Filename Search" }
option { value: "duplicates", "Find Duplicate Files" }
}
}
if search_type() == "fulltext" {
div {
class: "form-group",
style: "display: flex; flex-direction: column; gap: 6px;",
span { style: "font-weight: 600;", "Full text options" }
label {
style: "display: flex; align-items: center; gap: 8px; cursor: pointer;",
input {
r#type: "checkbox",
checked: fulltext_exact(),
onchange: move |evt| fulltext_exact.set(evt.checked()),
}
"Exact phrase match"
}
label {
style: "display: flex; align-items: center; gap: 8px; cursor: pointer;",
input {
r#type: "checkbox",
checked: fulltext_case_sensitive(),
onchange: move |evt| fulltext_case_sensitive.set(evt.checked()),
}
"Case-sensitive match"
}
}
}
if search_type() != "duplicates" {
div {
class: "form-group",
label { "Search Term: " }
input {
class: "form-control",
r#type: "text",
value: "{search_term}",
oninput: move |evt| search_term.set(evt.value()),
onkeydown: {
let do_fresh_search = do_fresh_search.clone();
move |evt| {
if evt.code() == dioxus::events::Code::Enter {
do_fresh_search();
}
}
}
}
}
}
div {
style: "display: flex; align-items: center; gap: 10px;",
button {
class: "btn btn-info",
disabled: is_searching(),
onclick: {
let do_fresh_search = do_fresh_search.clone();
move |_| { do_fresh_search(); }
},
"Search"
}
if is_searching() {
div {
class: "loading",
title: "Searching..."
}
} else if let Some(elapsed) = last_search_time() {
span {
style: "color: #666; font-size: 0.9em;",
"Search completed in {elapsed:.3}s"
}
}
}
if let Some(error) = search_error() {
div {
class: "error-message",
"Error: {error}"
}
}
// Results panel: pagination header + bounded table. Only renders
// when at least one search has completed (last_args is Some).
if last_args().is_some() {
div {
class: "search-results",
{
let total_str = match total_count() {
Some(n) => format!("{}", n),
None => "?".to_string(),
};
let page_first = (current_page().saturating_sub(1) as u64) * PAGE_SIZE as u64 + 1;
let page_last_calc = page_first + search_results().first().map(|r| r.rows.len() as u64).unwrap_or(0).saturating_sub(1);
let header = if total_count() == Some(0) {
"No results.".to_string()
} else {
format!(
"Showing {}-{} of {} (page {} of {})",
page_first,
page_last_calc,
total_str,
current_page(),
total_pages
)
};
rsx! { h3 { "{header}" } }
}
// Pagination controls. Hidden if there's only one page.
if total_pages > 1 {
div {
style: "display: flex; align-items: center; gap: 8px; margin: 8px 0;",
button {
class: "btn",
disabled: is_searching() || current_page() <= 1,
onclick: {
let do_goto_page = do_goto_page.clone();
move |_| do_goto_page(1)
},
"« First"
}
button {
class: "btn",
disabled: is_searching() || current_page() <= 1,
onclick: {
let do_goto_page = do_goto_page.clone();
move |_| do_goto_page(current_page().saturating_sub(1))
},
"‹ Prev"
}
button {
class: "btn",
disabled: is_searching() || current_page() >= total_pages,
onclick: {
let do_goto_page = do_goto_page.clone();
move |_| do_goto_page(current_page().saturating_add(1))
},
"Next ›"
}
button {
class: "btn",
disabled: is_searching() || current_page() >= total_pages,
onclick: {
let do_goto_page = do_goto_page.clone();
move |_| do_goto_page(total_pages)
},
"Last »"
}
span { "Go to:" }
input {
r#type: "number",
style: "width: 70px;",
value: "{goto_input}",
oninput: move |evt| goto_input.set(evt.value()),
onkeydown: {
let do_goto_page = do_goto_page.clone();
move |evt| {
if evt.code() == dioxus::events::Code::Enter {
if let Ok(p) = goto_input().trim().parse::<u32>() {
do_goto_page(p);
goto_input.set(String::new());
}
}
}
}
}
}
}
if !search_results().is_empty() && !search_results()[0].rows.is_empty() {
div {
class: "results-table",
table {
thead {
tr {
for column in search_results()[0].columns.iter() {
th { "{column}" }
}
}
}
tbody {
for row in search_results()[0].rows.iter() {
tr {
for (col_index, value) in row.values.iter().enumerate() {
if search_results()[0].columns.get(col_index).map(|s| s.as_str()) == Some("path") {
{
let value_owned = value.clone();
let service_owned = props.indexing_service.clone();
rsx! {
td {
class: "path-cell clickable",
onclick: move |_| {
let path = value_owned.clone();
let service_clone = service_owned.clone();
spawn(async move {
if let Err(e) = service_clone.open_file_explorer(&path) {
eprintln!("Failed to open file explorer: {}", e);
}
});
},
title: "Click to open in file explorer",
dangerous_inner_html: "{value}"
}
}
}
} else {
td {
dangerous_inner_html: "{value}"
}
}
}
}
}
}
}
}
}
}
}
if show_corruption_dialog() {
div {
class: "modal-backdrop",
div {
class: "modal-dialog",
h3 {
style: "margin-top: 0; color: #d32f2f;",
"⚠️ Database Corruption Detected"
}
p {
style: "margin: 15px 0;",
"The database appears to be corrupted or malformed. This can happen due to unexpected shutdowns or disk issues."
}
p {
style: "margin: 15px 0; font-weight: bold;",
"Would you like to delete the corrupted database and create a new one? This will require re-indexing your files."
}
div {
style: "display: flex; gap: 10px; margin-top: 20px;",
button {
style: "padding: 10px 20px; background-color: #d32f2f; color: white; border: none; border-radius: 5px; cursor: pointer;",
onclick: move |_| {
let service = props.indexing_service.clone();
let db = props.db_path.clone();
show_corruption_dialog.set(false);
search_error.set(Some("Deleting corrupted database...".to_string()));
spawn(async move {
match service.delete_index_for_rebuild(&db) {
Ok(()) => {
search_error.set(Some("Database deleted. You can now start indexing again.".to_string()));
}
Err(e) => {
search_error.set(Some(format!("Error deleting database: {}", e)));
}
}
});
},
"Yes, Delete & Rebuild"
}
button {
style: "padding: 10px 20px; background-color: #666; color: white; border: none; border-radius: 5px; cursor: pointer;",
onclick: move |_| {
show_corruption_dialog.set(false);
},
"Cancel"
}
}
}
}
}
}
}
}
/// Number of pages required to fit `total` rows at [`PAGE_SIZE`] per page.
/// Returns 0 for 0 rows so callers can branch on "no results yet".
fn pages_for(total: u64) -> u32 {
if total == 0 {
0
} else {
((total - 1) / PAGE_SIZE as u64) as u32 + 1
}
}
/// Centralized error-router for the two query branches that can fail
/// identically (count and select). Sets the error/timing/spinner signals
/// and pops the corruption dialog when warranted.
fn handle_query_error(
e: String,
elapsed: f64,
mut search_error: Signal<Option<String>>,
mut show_corruption_dialog: Signal<bool>,
mut last_search_time: Signal<Option<f64>>,
mut is_searching: Signal<bool>,
) {
if e.starts_with("DATABASE_CORRUPTED:") {
search_error.set(Some("Database appears to be corrupted".into()));
show_corruption_dialog.set(true);
} else {
search_error.set(Some(e));
}
last_search_time.set(Some(elapsed));
is_searching.set(false);
}

170
index.html
View file

@ -1,170 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>QuickSearch</title>
<style>
body {
font-family: Arial, sans-serif;
margin: 0;
padding: 20px;
background-color: #f5f5f5;
}
.container {
max-width: 1200px;
margin: 0 auto;
background-color: white;
padding: 20px;
border-radius: 8px;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
}
.header {
border-bottom: 2px solid #e0e0e0;
padding-bottom: 20px;
margin-bottom: 20px;
}
.section {
margin-bottom: 30px;
}
.form-group {
margin-bottom: 15px;
}
label {
display: block;
margin-bottom: 5px;
font-weight: bold;
color: #333;
}
input, select, textarea {
padding: 8px;
border: 1px solid #ddd;
border-radius: 4px;
font-size: 14px;
}
input[type="text"] {
width: 400px;
}
button {
padding: 10px 20px;
border: none;
border-radius: 4px;
cursor: pointer;
font-size: 14px;
margin-right: 10px;
}
.btn-primary {
background-color: #4CAF50;
color: white;
}
.btn-secondary {
background-color: #f44336;
color: white;
}
.btn-search {
background-color: #2196F3;
color: white;
}
.status-box {
background-color: #f5f5f5;
padding: 15px;
border-radius: 5px;
font-family: monospace;
white-space: pre-wrap;
border: 1px solid #ddd;
}
.error {
color: #d32f2f;
background-color: #ffebee;
padding: 10px;
border-radius: 5px;
margin: 10px 0;
}
.results-container {
margin-top: 20px;
}
.results-table {
max-height: 400px;
overflow: auto;
border: 1px solid #ddd;
border-radius: 5px;
}
table {
width: 100%;
border-collapse: collapse;
font-size: 12px;
}
thead {
background-color: #f5f5f5;
position: sticky;
top: 0;
}
th, td {
padding: 8px;
border: 1px solid #ddd;
text-align: left;
word-break: break-all;
}
th {
font-weight: bold;
}
tbody tr:nth-child(even) {
background-color: #f9f9f9;
}
.modal-overlay {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background-color: rgba(0,0,0,0.5);
display: flex;
align-items: center;
justify-content: center;
z-index: 1000;
}
.modal-content {
background-color: white;
padding: 30px;
border-radius: 10px;
max-width: 600px;
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
}
.warning-icon {
color: #d32f2f;
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>QuickSearch File Indexer</h1>
</div>
<div id="main"></div>
</div>
</body>
</html>

2
run.bat
View file

@ -1 +1 @@
cargo run cargo run -p quicksearch-gui

2
run.sh
View file

@ -1,2 +1,2 @@
cargo build --release cargo build --release -p quicksearch-gui
./target/release/quicksearch ./target/release/quicksearch

71
src/main.rs
View file

@ -1,71 +0,0 @@
use std::sync::{Arc, OnceLock};
use dioxus::prelude::*;
mod frontend;
mod file_handling;
mod document_extraction;
mod indexing;
mod config;
mod search;
// Global indexing service for signal handling
static INDEXING_SERVICE: OnceLock<Arc<indexing::IndexingService>> = OnceLock::new();
fn main() {
// Initialize global indexing service
let indexing_service = Arc::new(indexing::IndexingService::new());
INDEXING_SERVICE.set(indexing_service.clone()).expect("Failed to set global indexing service");
// Set up Ctrl-C signal handler
ctrlc::set_handler(|| {
eprintln!("Received Ctrl-C, shutting down gracefully...");
if let Some(service) = INDEXING_SERVICE.get() {
if let Err(e) = service.graceful_shutdown() {
eprintln!("Error during graceful shutdown: {}", e);
}
}
std::process::exit(0);
}).expect("Error setting Ctrl-C handler");
LaunchBuilder::desktop()
.with_cfg(
dioxus_desktop::Config::new()
.with_custom_head(format!("<style>{}</style>", include_str!("../assets/styles.css")))
.with_window(dioxus_desktop::WindowBuilder::new()
.with_title("QuickSearch - File Indexer & Search")
.with_resizable(true)
.with_inner_size(dioxus_desktop::LogicalSize::new(1000.0, 700.0))
)
)
.launch(app);
}
fn app() -> Element {
let config = match config::Config::load() {
Ok(config) => config,
Err(e) => {
eprintln!("Failed to load config: {}", e);
return rsx! { div { "Failed to load configuration" } };
}
};
let indexing_service = INDEXING_SERVICE.get().expect("Indexing service not initialized").clone();
rsx! {
frontend::App {
indexing_service: indexing_service,
config: config
}
}
}
/*
Duplicate files:
SELECT name, count(*) as cnt, path FROM files GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC;
Full text search:
SELECT d.name, d.path, d.text, snippet(st, 1 , "<b>", "</b>", "<b>...</b>", 64) as "snip" FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE st.text MATCH 'searchstring'
Filename search:
SELECT name, path FROM files WHERE name LIKE '%searchstring%';
*/

404
src/search.rs
View file

@ -1,404 +0,0 @@
#![allow(non_snake_case)]
use std::sync::Arc;
use std::time::Instant;
use dioxus::prelude::*;
use crate::indexing::{IndexingService, SearchResult};
#[derive(Props, Clone)]
pub struct SearchProps {
pub indexing_service: Arc<IndexingService>,
pub db_path: String,
}
impl PartialEq for SearchProps {
fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.indexing_service, &other.indexing_service) && self.db_path == other.db_path
}
}
pub fn Search(props: SearchProps) -> Element {
let mut search_type = use_signal(|| "fulltext".to_string());
let mut search_term = use_signal(|| String::new());
let mut fulltext_exact = use_signal(|| false);
let mut fulltext_case_sensitive = use_signal(|| false);
let search_results = use_signal(|| Vec::<SearchResult>::new());
let mut search_error = use_signal(|| None::<String>);
let is_searching = use_signal(|| false);
let last_search_time = use_signal(|| None::<f64>);
let mut show_corruption_dialog = use_signal(|| false);
let service = props.indexing_service.clone();
let db_path = props.db_path.clone();
// Create a callback to perform search
let perform_search = {
let service = service.clone();
let db_path = db_path.clone();
let search_type = search_type.clone();
let search_term = search_term.clone();
let fulltext_exact = fulltext_exact.clone();
let fulltext_case_sensitive = fulltext_case_sensitive.clone();
let search_results = search_results.clone();
let search_error = search_error.clone();
let is_searching = is_searching.clone();
let last_search_time = last_search_time.clone();
let show_corruption_dialog = show_corruption_dialog.clone();
move || {
let service_clone = service.clone();
let db_clone = db_path.clone();
let search_type_val = search_type().clone();
let search_term_val = search_term().clone();
let fulltext_exact_val = fulltext_exact();
let fulltext_case_sensitive_val = fulltext_case_sensitive();
let mut search_results_clone = search_results.clone();
let mut search_error_clone = search_error.clone();
let mut is_searching_clone = is_searching.clone();
let mut last_search_time_clone = last_search_time.clone();
let mut show_corruption_dialog_clone = show_corruption_dialog.clone();
spawn(async move {
is_searching_clone.set(true);
search_error_clone.set(None);
last_search_time_clone.set(None);
let start_time = Instant::now();
let query = match search_type_val.as_str() {
"fulltext" => {
let trimmed = search_term_val.trim();
if trimmed.is_empty() {
search_error_clone.set(Some("Please enter a search term".to_string()));
is_searching_clone.set(false);
return;
}
let sanitized_term = trimmed
.replace(':', " ")
.replace(';', " ")
.replace('(', " ")
.replace(')', " ")
.replace('[', " ")
.replace(']', " ")
.replace('{', " ")
.replace('}', " ")
.replace('^', " ")
.replace('~', " ")
.replace('"', " ");
let tokens: Vec<&str> = sanitized_term.split_whitespace().collect();
if tokens.is_empty() {
search_error_clone.set(Some("Please enter a valid search term".to_string()));
is_searching_clone.set(false);
return;
}
let words: Vec<&str> = if fulltext_exact_val {
tokens
} else {
let filtered: Vec<&str> = tokens
.into_iter()
.filter(|w| w.chars().count() >= 3)
.collect();
if filtered.is_empty() {
search_error_clone.set(Some(
"Trigram index needs each word to be at least 3 characters unless you use exact phrase search.".to_string(),
));
is_searching_clone.set(false);
return;
}
filtered
};
let sql_quote = |s: &str| s.replace('\'', "''");
let fts_match = if fulltext_exact_val {
let phrase = words.join(" ");
format!("\"{}\"", phrase.replace('"', "\"\""))
} else {
words.join(" AND ")
};
let mut where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match));
if fulltext_case_sensitive_val {
if fulltext_exact_val {
let literal = words.join(" ");
where_clause.push_str(&format!(
" AND instr(st.text, '{}') > 0",
sql_quote(&literal)
));
} else {
for w in &words {
where_clause.push_str(&format!(
" AND instr(st.text, '{}') > 0",
sql_quote(w)
));
}
}
}
format!(
"SELECT d.name, d.path, snippet(searchabletext, 1, '<b>', '</b>', '<b>...</b>', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank",
where_clause
)
},
"filename" => {
if search_term_val.trim().is_empty() {
search_error_clone.set(Some("Please enter a filename pattern".to_string()));
is_searching_clone.set(false);
return;
}
format!("SELECT name, path FROM files WHERE name LIKE '%{}%'", search_term_val.replace("'", "''"))
},
"duplicates" => "SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC LIMIT 5000".to_string(),
_ => {
is_searching_clone.set(false);
return;
}
};
// Run the search in a blocking task to prevent UI freezing
let search_result = tokio::task::spawn_blocking(move || {
service_clone.execute_search(&db_clone, &query)
}).await;
let elapsed = start_time.elapsed().as_secs_f64();
match search_result {
Ok(db_result) => {
match db_result {
Ok(results) => {
search_results_clone.set(results);
last_search_time_clone.set(Some(elapsed));
},
Err(e) => {
if e.starts_with("DATABASE_CORRUPTED:") {
search_error_clone.set(Some("Database appears to be corrupted".to_string()));
show_corruption_dialog_clone.set(true);
} else {
search_error_clone.set(Some(e));
}
last_search_time_clone.set(Some(elapsed));
}
}
},
Err(e) => {
search_error_clone.set(Some(format!("Task execution error: {}", e)));
last_search_time_clone.set(Some(elapsed));
}
}
is_searching_clone.set(false);
});
}
};
rsx! {
div {
class: "section",
h2 { "Search Database" }
div {
class: "form-group",
label { "Search Type: " }
select {
class: "form-control",
value: "{search_type}",
onchange: move |evt| search_type.set(evt.value()),
option { value: "fulltext", "Full Text Search" }
option { value: "filename", "Filename Search" }
option { value: "duplicates", "Find Duplicate Files" }
}
}
if search_type() == "fulltext" {
div {
class: "form-group",
style: "display: flex; flex-direction: column; gap: 6px;",
span { style: "font-weight: 600;", "Full text options" }
label {
style: "display: flex; align-items: center; gap: 8px; cursor: pointer;",
input {
r#type: "checkbox",
checked: fulltext_exact(),
onchange: move |evt| fulltext_exact.set(evt.checked()),
}
"Exact phrase match"
}
label {
style: "display: flex; align-items: center; gap: 8px; cursor: pointer;",
input {
r#type: "checkbox",
checked: fulltext_case_sensitive(),
onchange: move |evt| fulltext_case_sensitive.set(evt.checked()),
}
"Case-sensitive match"
}
}
}
if search_type() != "duplicates" {
div {
class: "form-group",
label { "Search Term: " }
input {
class: "form-control",
r#type: "text",
value: "{search_term}",
oninput: move |evt| search_term.set(evt.value()),
onkeydown: {
let perform_search = perform_search.clone();
move |evt| {
if evt.code() == dioxus::events::Code::Enter {
perform_search();
}
}
}
}
}
}
div {
style: "display: flex; align-items: center; gap: 10px;",
button {
class: "btn btn-info",
disabled: is_searching(),
onclick: {
let perform_search = perform_search.clone();
move |_| {
perform_search();
}
},
"Search"
}
if is_searching() {
div {
class: "loading",
title: "Searching..."
}
} else if let Some(elapsed) = last_search_time() {
span {
style: "color: #666; font-size: 0.9em;",
"Search completed in {elapsed:.3}s"
}
}
}
if let Some(error) = search_error() {
div {
class: "error-message",
"Error: {error}"
}
}
if !search_results().is_empty() {
div {
class: "search-results",
h3 { "Search Results ({search_results()[0].rows.len()} results)" }
div {
class: "results-table",
table {
thead {
tr {
for column in search_results()[0].columns.iter() {
th { "{column}" }
}
}
}
tbody {
for (_i, row) in search_results()[0].rows.iter().enumerate() {
tr {
for (col_index, value) in row.values.iter().enumerate() {
// Check if this column is a path column
if search_results()[0].columns.get(col_index).map(|s| s.as_str()) == Some("path") {
{
let value_owned = value.clone();
let service_owned = props.indexing_service.clone();
rsx! {
td {
class: "path-cell clickable",
onclick: move |_| {
let path = value_owned.clone();
let service_clone = service_owned.clone();
spawn(async move {
if let Err(e) = service_clone.open_file_explorer(&path) {
eprintln!("Failed to open file explorer: {}", e);
}
});
},
title: "Click to open in file explorer",
dangerous_inner_html: "{value}"
}
}
}
} else {
td {
dangerous_inner_html: "{value}"
}
}
}
}
}
}
}
}
}
}
// Database corruption recovery dialog
if show_corruption_dialog() {
div {
class: "modal-backdrop",
div {
class: "modal-dialog",
h3 {
style: "margin-top: 0; color: #d32f2f;",
"⚠️ Database Corruption Detected"
}
p {
style: "margin: 15px 0;",
"The database appears to be corrupted or malformed. This can happen due to unexpected shutdowns or disk issues."
}
p {
style: "margin: 15px 0; font-weight: bold;",
"Would you like to delete the corrupted database and create a new one? This will require re-indexing your files."
}
div {
style: "display: flex; gap: 10px; margin-top: 20px;",
button {
style: "padding: 10px 20px; background-color: #d32f2f; color: white; border: none; border-radius: 5px; cursor: pointer;",
onclick: move |_| {
let service = props.indexing_service.clone();
let db = props.db_path.clone();
show_corruption_dialog.set(false);
search_error.set(Some("Deleting corrupted database...".to_string()));
spawn(async move {
match service.delete_index_for_rebuild(&db) {
Ok(()) => {
search_error.set(Some("Database deleted. You can now start indexing again.".to_string()));
}
Err(e) => {
search_error.set(Some(format!("Error deleting database: {}", e)));
}
}
});
},
"Yes, Delete & Rebuild"
}
button {
style: "padding: 10px 20px; background-color: #666; color: white; border: none; border-radius: 5px; cursor: pointer;",
onclick: move |_| {
show_corruption_dialog.set(false);
},
"Cancel"
}
}
}
}
}
}
}
}