diff --git a/config_example.toml b/config_example.toml index 67debd1..df5cf0c 100644 --- a/config_example.toml +++ b/config_example.toml @@ -1,7 +1,7 @@ [paths] # One or more directory roots to index. Walked in order; duplicate and # nested roots are de-duplicated automatically. -indexing_paths = ["C:\\"] +indexing_paths = ["/"] database_path = "QuickSearch.db" [processing] diff --git a/crates/quicksearch-core/src/cli.rs b/crates/quicksearch-core/src/cli.rs index bb3f1f0..4a94f24 100644 --- a/crates/quicksearch-core/src/cli.rs +++ b/crates/quicksearch-core/src/cli.rs @@ -142,6 +142,23 @@ pub fn index_size_breakdown(db_path: &str) -> Result { }) } +/// Count files with `content_state = 0` (pending). Distinct from +/// `files_row_count − searchabletext_row_count`, which over-counts because +/// files whose content doesn't apply (binary formats, too-large files) sit +/// with `content_state = 3` (NA) forever and never become FTS rows. +/// +/// Used by the Baloo compat daemon to report the "Files waiting for content +/// indexing" figure both to balooctl and to the LMDB mirror. +pub fn pending_content_count(db_path: &str) -> Result { + let conn = open_and_migrate(db_path, "trigram")?; + conn.query_row( + "SELECT COUNT(*) FROM files WHERE content_state = ?1", + rusqlite::params![crate::db::repo::STATE_PENDING], + |r| r.get(0), + ) + .map_err(|e| format!("pending_content_count: {}", e)) +} + /// Remove a single file from the index. Returns whether a row was deleted. /// Keeps FTS/documents/properties in sync via the repo helpers. pub fn clear_path(db_path: &str, path: &str) -> Result { diff --git a/crates/quicksearch-core/src/db/migrate.rs b/crates/quicksearch-core/src/db/migrate.rs index bcbe68b..dcab9d9 100644 --- a/crates/quicksearch-core/src/db/migrate.rs +++ b/crates/quicksearch-core/src/db/migrate.rs @@ -14,7 +14,7 @@ use std::path::Path; use rusqlite::{params, Connection, OptionalExtension}; -use super::schema::{fts_create_sql, PRAGMAS_FAST, SCHEMA_CURRENT}; +use super::schema::{effective_tokenizer, fts_create_sql, PRAGMAS_FAST, SCHEMA_CURRENT}; pub const CURRENT_SCHEMA_VERSION: u32 = 2; @@ -34,7 +34,15 @@ pub fn open_and_migrate(db_path: &str, tokenizer: &str) -> Result {} + Some(v) if v == CURRENT_SCHEMA_VERSION => { + // Schema version matches. Before returning, check whether the + // FTS5 tokenizer config has drifted (e.g. someone upgraded to a + // build that switched the default to `trigram remove_diacritics 1`). + // If so, rebuild the FTS table in place — cheaper than a full + // DB wipe since `files` rows stay intact; only content + // extraction (phase 2) re-runs. + maybe_rebuild_fts_for_tokenizer_change(&conn, tokenizer)?; + } Some(v) if v > CURRENT_SCHEMA_VERSION => { return Err(format!( "Database schema version {} is newer than this build ({}). \ @@ -73,6 +81,53 @@ pub fn open_and_migrate(db_path: &str, tokenizer: &str) -> Result Result<(), String> { + let want = effective_tokenizer(tokenizer); + let stored: Option = conn + .query_row( + "SELECT value FROM schema_info WHERE key = 'tokenize'", + [], + |r| r.get(0), + ) + .optional() + .map_err(|e| format!("read schema_info.tokenize: {}", e))?; + + if stored.as_deref() == Some(&*want) { + return Ok(()); + } + + eprintln!( + "QuickSearch: FTS5 tokenizer changed from {:?} to {:?}; rebuilding searchabletext. \ + File metadata is preserved; content extraction will re-run on next indexing pass.", + stored.as_deref().unwrap_or("(none)"), + want + ); + conn.execute("DROP TABLE IF EXISTS searchabletext", []) + .map_err(|e| format!("drop searchabletext: {}", e))?; + let create_sql = fts_create_sql(tokenizer); + conn.execute_batch(&create_sql) + .map_err(|e| format!("recreate searchabletext: {}", e))?; + conn.execute( + "UPDATE files SET content_state = 0 WHERE content_state != 0", + [], + ) + .map_err(|e| format!("reset content_state: {}", e))?; + conn.execute( + "INSERT OR REPLACE INTO schema_info(key, value) VALUES ('tokenize', ?1)", + params![want], + ) + .map_err(|e| format!("update schema_info.tokenize: {}", e))?; + Ok(()) +} + fn wipe_and_reopen( conn: Connection, path_for_rebuild: &std::path::Path, @@ -166,9 +221,13 @@ fn apply_current_schema(conn: &Connection, tokenizer: &str) -> Result<(), String .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); + // Store the *effective* tokenizer string (with any default options we + // auto-applied). On subsequent opens we compare this against what the + // caller asks for and rebuild the FTS table if it changed. + let effective = effective_tokenizer(tokenizer); conn.execute( "INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)", - params![CURRENT_SCHEMA_VERSION.to_string(), now.to_string(), tokenizer], + params![CURRENT_SCHEMA_VERSION.to_string(), now.to_string(), effective], ) .map_err(|e| format!("Failed to seed schema_info: {}", e))?; diff --git a/crates/quicksearch-core/src/db/schema.rs b/crates/quicksearch-core/src/db/schema.rs index df8e134..208e7ee 100644 --- a/crates/quicksearch-core/src/db/schema.rs +++ b/crates/quicksearch-core/src/db/schema.rs @@ -74,11 +74,52 @@ CREATE TABLE config_validation ( /// INSERT/UPDATE/DELETE work with normal SQL semantics. `rowid` is supplied /// by the caller and must equal `files.id`. pub fn fts_create_sql(tokenizer: &str) -> String { + let effective = effective_tokenizer(tokenizer); format!( "CREATE VIRTUAL TABLE searchabletext USING fts5(\ name, text, properties, \ tokenize='{}'\ );", - tokenizer.replace('\'', "''") + effective.replace('\'', "''") ) } + +/// Map a user-facing tokenizer name to the actual FTS5 option string we +/// apply. The default `trigram` gets `remove_diacritics 1` appended so an +/// ASCII query like `cafe` matches indexed `café`, and vice versa. +/// Without this, the default trigram tokenizer would emit disjoint +/// trigram sets for the two spellings and `MATCH` would miss one of them. +/// Users who want precise match semantics can pass the full option string +/// (e.g. `"trigram case_sensitive 1 remove_diacritics 0"`) and we'll use +/// it verbatim. +pub fn effective_tokenizer(tokenizer: &str) -> String { + let trimmed = tokenizer.trim(); + if trimmed.eq_ignore_ascii_case("trigram") { + // Explicit default includes accent stripping. `case_sensitive 0` + // is FTS5's default too; we repeat it here for clarity. + "trigram remove_diacritics 1".to_string() + } else { + trimmed.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plain_trigram_gets_accent_stripping() { + assert_eq!(effective_tokenizer("trigram"), "trigram remove_diacritics 1"); + assert_eq!(effective_tokenizer(" trigram "), "trigram remove_diacritics 1"); + } + + #[test] + fn explicit_tokenizers_pass_through() { + assert_eq!( + effective_tokenizer("trigram remove_diacritics 0"), + "trigram remove_diacritics 0" + ); + assert_eq!(effective_tokenizer("porter"), "porter"); + assert_eq!(effective_tokenizer("unicode61"), "unicode61"); + } +} diff --git a/crates/quicksearch-core/src/file_handling.rs b/crates/quicksearch-core/src/file_handling.rs index 8420c75..63bdb7d 100644 --- a/crates/quicksearch-core/src/file_handling.rs +++ b/crates/quicksearch-core/src/file_handling.rs @@ -687,6 +687,23 @@ pub fn process_text_indexing( callback("Counting files pending text index…"); } + // Files bigger than our text-file cap can never graduate from + // `content_state = 0`, so they'd otherwise sit in the "pending content + // indexing" column forever and peg System Settings' progress below + // 100%. Flip them to `content_state = 3` (not-applicable) now. Doing + // this every run is idempotent and also handles the case where a user + // *lowers* `maximum_text_file_size` between runs — previously-pending + // files that cross the threshold get correctly marked. + { + let conn = conn_mutex.lock().unwrap(); + conn.execute( + "UPDATE files SET content_state = 3 \ + WHERE content_state = 0 AND size > ?1", + [max_size], + ) + .map_err(|e| format!("mark oversize files NA: {}", e))?; + } + let total_files: usize = { let conn = conn_mutex.lock().unwrap(); conn.query_row( diff --git a/setup.sh b/setup.sh index f84ab36..98c20c8 100644 --- a/setup.sh +++ b/setup.sh @@ -1 +1,23 @@ -sudo apt install -y libsoup-3.0-dev libjavascriptcoregtk-4.1-dev libwebkit2gtk-4.1-dev libxdo-dev +#!/usr/bin/env bash +# One-time system setup for building and running the Dioxus-based +# `quicksearch-gui` binary on Debian / Ubuntu. The core library +# (`quicksearch-core`) has no native system deps and doesn't need this. +# +# Usage: +# ./setup.sh # installs everything +# +# Adds `pkg-config` and `build-essential` explicitly because the Dioxus +# build scripts (glib-sys, gdk-sys, gio-sys, javascriptcoregtk-sys, +# webkit2gtk-sys) shell out to `pkg-config` and will refuse to build +# without it. Some minimal Ubuntu flavors and container images don't +# install pkg-config transitively. +set -e + +sudo apt update +sudo apt install -y \ + pkg-config \ + build-essential \ + libsoup-3.0-dev \ + libjavascriptcoregtk-4.1-dev \ + libwebkit2gtk-4.1-dev \ + libxdo-dev