From 2a584fbfc02a9f007bf1f402e4978054aaadffd9 Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 23 Apr 2026 17:46:11 -0400 Subject: [PATCH] Added zstd compression to text contents, flags for contentless mode, and better baloo integration --- Cargo.lock | 23 +- config_example.toml | 12 +- crates/quicksearch-core/Cargo.toml | 1 + crates/quicksearch-core/src/cli.rs | 95 ++++- crates/quicksearch-core/src/config.rs | 16 + crates/quicksearch-core/src/db/migrate.rs | 340 ------------------ crates/quicksearch-core/src/db/mod.rs | 16 +- crates/quicksearch-core/src/db/open.rs | 329 +++++++++++++++++ crates/quicksearch-core/src/db/repo.rs | 62 +++- crates/quicksearch-core/src/db/schema.rs | 44 ++- crates/quicksearch-core/src/file_handling.rs | 11 +- crates/quicksearch-core/src/indexing.rs | 112 +++++- crates/quicksearch-core/src/lib.rs | 1 + .../quicksearch-core/src/query/translator.rs | 9 +- crates/quicksearch-core/src/search_sql.rs | 116 ++++-- crates/quicksearch-core/src/snippet.rs | 336 +++++++++++++++++ crates/quicksearch-core/tests/snippet_perf.rs | 278 ++++++++++++++ crates/quicksearch-gui/src/main.rs | 10 - crates/quicksearch-gui/src/search.rs | 31 +- 19 files changed, 1407 insertions(+), 435 deletions(-) delete mode 100644 crates/quicksearch-core/src/db/migrate.rs create mode 100644 crates/quicksearch-core/src/db/open.rs create mode 100644 crates/quicksearch-core/src/snippet.rs create mode 100644 crates/quicksearch-core/tests/snippet_perf.rs diff --git a/Cargo.lock b/Cargo.lock index 0d7c745..fe2be86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3405,6 +3405,7 @@ dependencies = [ "toml", "walkdir", "zip", + "zstd 0.13.3", ] [[package]] @@ -5412,7 +5413,7 @@ dependencies = [ "pbkdf2", "sha1", "time", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -5421,7 +5422,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe 7.2.4", ] [[package]] @@ -5434,6 +5444,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.16+zstd.1.5.7" diff --git a/config_example.toml b/config_example.toml index df5cf0c..bdcccd4 100644 --- a/config_example.toml +++ b/config_example.toml @@ -24,4 +24,14 @@ follow_symlinks = false include_hidden = false # FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61') # Look here for more information https://www.sqlite.org/fts5.html#tokenizers -tokenize = "trigram" \ No newline at end of file +tokenize = "trigram" +# If true (default), extracted text is stored zstd-compressed in a sidecar +# table so the GUI's search results can render snippet previews with the +# query terms highlighted. If false, the inverted FTS5 index is still +# populated (so queries match the same files) but nothing is stored +# alongside; result rows carry no snippet. Turning this off drops the +# on-disk footprint to roughly what stock Baloo uses, useful for +# apples-to-apples comparisons and for users who never read snippet text. +# Changing this only affects files indexed *after* the change; existing +# sidecar rows are kept until re-indexed. +store_text_for_snippets = true \ No newline at end of file diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index d327f70..f3b0ee2 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -23,3 +23,4 @@ lofty = "0.19" kamadak-exif = "0.5" notify = "6.1" ctrlc = "3.4" +zstd = "0.13" diff --git a/crates/quicksearch-core/src/cli.rs b/crates/quicksearch-core/src/cli.rs index 4a94f24..512b729 100644 --- a/crates/quicksearch-core/src/cli.rs +++ b/crates/quicksearch-core/src/cli.rs @@ -8,7 +8,7 @@ use rusqlite::{params, OptionalExtension}; -use crate::db::open_and_migrate; +use crate::db::open_or_recreate; use crate::db::repo::{STATE_DONE, STATE_FAILED, STATE_NA, STATE_PENDING}; /// Per-file indexing status, mirroring Baloo's multi-state reporting. @@ -55,19 +55,38 @@ pub struct FailedEntry { /// Storage footprint report. "Partitions" correspond to SQL tables for our /// SQLite layout (Baloo's LMDB has named sub-DBs; our equivalent is per-table /// row/size counts). -#[derive(Debug, Clone, PartialEq, Eq)] +/// +/// `documents_text_*` fields cover the zstd-compressed extracted-text +/// sidecar that replaced the regular FTS5 stored text in schema v3. Ratio +/// is `compressed / raw` so a value of ~0.3 means we saved ~70% vs storing +/// the plaintext verbatim. +#[derive(Debug, Clone, PartialEq)] pub struct SizeReport { pub file_size_bytes: u64, pub files_row_count: i64, pub properties_row_count: i64, pub failed_files_row_count: i64, pub searchabletext_row_count: i64, + pub documents_text_row_count: i64, + pub documents_text_raw_bytes: i64, + pub documents_text_compressed_bytes: i64, +} + +impl SizeReport { + /// Compressed:raw ratio for the stored extracted text. `None` when no + /// rows have been written yet (avoids divide-by-zero). + pub fn documents_text_ratio(&self) -> Option { + if self.documents_text_raw_bytes <= 0 { + return None; + } + Some(self.documents_text_compressed_bytes as f64 / self.documents_text_raw_bytes as f64) + } } /// Query the per-file indexing status. Returns `FileStatus` with /// `basic == NotIndexed` if the path isn't in the database. pub fn status_for_path(db_path: &str, path: &str) -> Result { - let conn = open_and_migrate(db_path, "trigram")?; + let conn = open_or_recreate(db_path, "trigram")?; let row: Option<(i64, i64, Option)> = conn .query_row( "SELECT basic_state, content_state, failure_msg FROM files WHERE path = ?1", @@ -94,7 +113,7 @@ pub fn status_for_path(db_path: &str, path: &str) -> Result /// Return every file that failed content extraction, newest first. pub fn list_failed(db_path: &str, limit: Option) -> Result, String> { - let conn = open_and_migrate(db_path, "trigram")?; + let conn = open_or_recreate(db_path, "trigram")?; let limit_sql = match limit { Some(n) => format!(" LIMIT {}", n), None => String::new(), @@ -128,17 +147,29 @@ pub fn index_size_breakdown(db_path: &str) -> Result { let file_size_bytes = std::fs::metadata(db_path) .map(|m| m.len()) .unwrap_or(0); - let conn = open_and_migrate(db_path, "trigram")?; + let conn = open_or_recreate(db_path, "trigram")?; let count = |table: &str| -> Result { conn.query_row(&format!("SELECT COUNT(*) FROM {}", table), [], |r| r.get(0)) .map_err(|e| format!("count {}: {}", table, e)) }; + let dt_row_count: i64 = count("documents_text")?; + let (dt_raw, dt_compressed): (i64, i64) = conn + .query_row( + "SELECT COALESCE(SUM(text_len), 0), COALESCE(SUM(LENGTH(text_zstd)), 0) FROM documents_text", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .map_err(|e| format!("documents_text size sum: {}", e))?; + Ok(SizeReport { file_size_bytes, files_row_count: count("files")?, properties_row_count: count("properties")?, failed_files_row_count: count("failed_files")?, searchabletext_row_count: count("searchabletext")?, + documents_text_row_count: dt_row_count, + documents_text_raw_bytes: dt_raw, + documents_text_compressed_bytes: dt_compressed, }) } @@ -150,7 +181,7 @@ pub fn index_size_breakdown(db_path: &str) -> Result { /// Used by the Baloo compat daemon to report the "Files waiting for content /// indexing" figure both to balooctl and to the LMDB mirror. pub fn pending_content_count(db_path: &str) -> Result { - let conn = open_and_migrate(db_path, "trigram")?; + let conn = open_or_recreate(db_path, "trigram")?; conn.query_row( "SELECT COUNT(*) FROM files WHERE content_state = ?1", rusqlite::params![crate::db::repo::STATE_PENDING], @@ -162,7 +193,7 @@ pub fn pending_content_count(db_path: &str) -> Result { /// Remove a single file from the index. Returns whether a row was deleted. /// Keeps FTS/documents/properties in sync via the repo helpers. pub fn clear_path(db_path: &str, path: &str) -> Result { - let mut conn = open_and_migrate(db_path, "trigram")?; + let mut conn = open_or_recreate(db_path, "trigram")?; let tx = conn .transaction() .map_err(|e| format!("clear_path begin tx: {}", e))?; @@ -192,7 +223,7 @@ mod tests { } fn seed_fixture(db_path: &str) -> (i64, i64) { - let mut conn = open_and_migrate(db_path, "trigram").unwrap(); + let mut conn = open_or_recreate(db_path, "trigram").unwrap(); let (a, b) = { let tx = conn.transaction().unwrap(); let a = insert_file( @@ -212,7 +243,7 @@ mod tests { ) .unwrap() .expect("unique path"); - set_content_done(&tx, a, "a.txt", "hello", &[]).unwrap(); + set_content_done(&tx, a, "a.txt", "hello", &[], true).unwrap(); let b = insert_file( &tx, &NewFile { @@ -281,6 +312,52 @@ mod tests { assert!(r.file_size_bytes > 0); assert_eq!(r.files_row_count, 2); assert_eq!(r.failed_files_row_count, 1); + // File a got content ("hello"); file b failed. Only one documents_text row. + assert_eq!(r.documents_text_row_count, 1); + assert_eq!(r.documents_text_raw_bytes, "hello".len() as i64); + assert!(r.documents_text_compressed_bytes > 0); + + std::fs::remove_file(&p).ok(); + } + + #[test] + fn documents_text_ratio_reports_savings_on_compressible_prose() { + // Feed highly-compressible prose (lots of repeated words) and verify + // the reported ratio reflects real savings. Guards against anyone + // silently swapping the compression step for a pass-through. + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + { + let tx = conn.transaction().unwrap(); + let id = insert_file( + &tx, + &NewFile { + name: "big.txt", + path: "/tmp/big.txt", + parent: "/tmp", + size: 1, + mtime: 1, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + }, + ) + .unwrap() + .expect("unique path"); + let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500); + set_content_done(&tx, id, "big.txt", &prose, &[], true).unwrap(); + tx.commit().unwrap(); + } + drop(conn); + + let r = index_size_breakdown(p.to_str().unwrap()).unwrap(); + let ratio = r.documents_text_ratio().expect("has rows"); + // Repeating a 44-byte sentence 500x → zstd should hit <20% ratio + // trivially. Loose bound protects the test from zstd version churn. + assert!(ratio < 0.3, "ratio too high: {ratio} raw={} comp={}", + r.documents_text_raw_bytes, r.documents_text_compressed_bytes); std::fs::remove_file(&p).ok(); } diff --git a/crates/quicksearch-core/src/config.rs b/crates/quicksearch-core/src/config.rs index e39acd6..35078e1 100644 --- a/crates/quicksearch-core/src/config.rs +++ b/crates/quicksearch-core/src/config.rs @@ -31,6 +31,10 @@ fn default_home_path() -> String { ".".to_string() } +fn default_store_text_for_snippets() -> bool { + true +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ProcessingConfig { pub hash_length: usize, @@ -46,6 +50,17 @@ pub struct ProcessingConfig { pub follow_symlinks: bool, #[serde(default)] pub include_hidden: bool, + /// When `true` (default), extracted text is stored zstd-compressed in + /// `documents_text` so search results can render snippet/highlight + /// previews without re-reading the source file. When `false` the + /// inverted FTS5 index still gets the tokens (so queries return the + /// same hits) but nothing is stored alongside; search results carry + /// no snippet text and rely on filename/path only. This mode drops + /// the on-disk footprint to roughly what stock Baloo uses, at the + /// cost of snippet functionality — useful for apples-to-apples size + /// comparisons and for users who never look at result previews. + #[serde(default = "default_store_text_for_snippets")] + pub store_text_for_snippets: bool, } impl Default for Config { @@ -65,6 +80,7 @@ impl Default for Config { precount_files_for_progress: false, follow_symlinks: false, include_hidden: false, + store_text_for_snippets: true, }, } } diff --git a/crates/quicksearch-core/src/db/migrate.rs b/crates/quicksearch-core/src/db/migrate.rs deleted file mode 100644 index dcab9d9..0000000 --- a/crates/quicksearch-core/src/db/migrate.rs +++ /dev/null @@ -1,340 +0,0 @@ -//! Schema version detection and upgrade. -//! -//! Callers should always enter the DB via [`open_and_migrate`]. It applies -//! pragmas, detects the on-disk schema version, and upgrades or recreates as -//! required. -//! -//! Current policy: Set A introduces schema v1 and is the first versioned -//! release. Any pre-A database (has the legacy `files(name, path, size, -//! moddate, hash)` shape and no `schema_info` table) is wiped and rebuilt — -//! the user will re-index. A prominent log line is printed so the behavior is -//! not silent. Future migrations should prefer ALTER TABLE. - -use std::path::Path; - -use rusqlite::{params, Connection, OptionalExtension}; - -use super::schema::{effective_tokenizer, fts_create_sql, PRAGMAS_FAST, SCHEMA_CURRENT}; - -pub const CURRENT_SCHEMA_VERSION: u32 = 2; - -/// Open the database at `db_path`, apply pragmas, and ensure the schema is at -/// [`CURRENT_SCHEMA_VERSION`]. Recreates the DB if a pre-versioned layout is -/// detected. -/// -/// `tokenizer` is used when (re)creating the FTS5 virtual table. It has no -/// effect on an already-current DB. -pub fn open_and_migrate(db_path: &str, tokenizer: &str) -> Result { - let path_for_rebuild = Path::new(db_path).to_path_buf(); - let mut conn = Connection::open(db_path) - .map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?; - - conn.execute_batch(PRAGMAS_FAST) - .map_err(|e| format!("Failed to apply pragmas: {}", e))?; - - let version = read_schema_version(&conn)?; - match version { - Some(v) if v == CURRENT_SCHEMA_VERSION => { - // Schema version matches. Before returning, check whether the - // FTS5 tokenizer config has drifted (e.g. someone upgraded to a - // build that switched the default to `trigram remove_diacritics 1`). - // If so, rebuild the FTS table in place — cheaper than a full - // DB wipe since `files` rows stay intact; only content - // extraction (phase 2) re-runs. - maybe_rebuild_fts_for_tokenizer_change(&conn, tokenizer)?; - } - Some(v) if v > CURRENT_SCHEMA_VERSION => { - return Err(format!( - "Database schema version {} is newer than this build ({}). \ - Use a newer QuickSearch or move the database aside.", - v, CURRENT_SCHEMA_VERSION - )); - } - Some(v) => { - // Older schema version. Set A's upgrade policy: wipe and rebuild. - // When we start adding ALTER-based migrations this match arm - // will gain a proper stepwise runner. - eprintln!( - "QuickSearch: database at {} is schema v{}; rebuilding to v{}. \ - Existing rows will be re-scanned.", - db_path, v, CURRENT_SCHEMA_VERSION - ); - conn = wipe_and_reopen(conn, &path_for_rebuild)?; - apply_current_schema(&conn, tokenizer)?; - } - None => { - // No schema_info row. Either an empty DB (good — just create) or a - // legacy pre-A layout (detected by presence of the old `files` - // table). Legacy layouts are wiped. - if has_legacy_layout(&conn)? { - eprintln!( - "QuickSearch: legacy database detected at {}; rebuilding index with schema v{}. \ - Existing file rows will be re-scanned.", - db_path, CURRENT_SCHEMA_VERSION - ); - conn = wipe_and_reopen(conn, &path_for_rebuild)?; - } - apply_current_schema(&conn, tokenizer)?; - } - } - - Ok(conn) -} - -/// If the stored `schema_info.tokenize` value doesn't match the effective -/// tokenizer the caller wants now, drop and recreate `searchabletext` with -/// the new tokenizer and reset `files.content_state` so the text-extraction -/// phase re-runs on next indexing pass. Keeps the `files`, `properties`, -/// and `failed_files` rows intact. -fn maybe_rebuild_fts_for_tokenizer_change( - conn: &Connection, - tokenizer: &str, -) -> Result<(), String> { - let want = effective_tokenizer(tokenizer); - let stored: Option = conn - .query_row( - "SELECT value FROM schema_info WHERE key = 'tokenize'", - [], - |r| r.get(0), - ) - .optional() - .map_err(|e| format!("read schema_info.tokenize: {}", e))?; - - if stored.as_deref() == Some(&*want) { - return Ok(()); - } - - eprintln!( - "QuickSearch: FTS5 tokenizer changed from {:?} to {:?}; rebuilding searchabletext. \ - File metadata is preserved; content extraction will re-run on next indexing pass.", - stored.as_deref().unwrap_or("(none)"), - want - ); - conn.execute("DROP TABLE IF EXISTS searchabletext", []) - .map_err(|e| format!("drop searchabletext: {}", e))?; - let create_sql = fts_create_sql(tokenizer); - conn.execute_batch(&create_sql) - .map_err(|e| format!("recreate searchabletext: {}", e))?; - conn.execute( - "UPDATE files SET content_state = 0 WHERE content_state != 0", - [], - ) - .map_err(|e| format!("reset content_state: {}", e))?; - conn.execute( - "INSERT OR REPLACE INTO schema_info(key, value) VALUES ('tokenize', ?1)", - params![want], - ) - .map_err(|e| format!("update schema_info.tokenize: {}", e))?; - Ok(()) -} - -fn wipe_and_reopen( - conn: Connection, - path_for_rebuild: &std::path::Path, -) -> Result { - drop(conn); - std::fs::remove_file(path_for_rebuild) - .map_err(|e| format!("Failed to remove old database: {}", e))?; - // Remove WAL/SHM/journal sidecars defensively even though journal_mode=OFF. - for suffix in ["-wal", "-shm", "-journal"] { - let sidecar = path_for_rebuild.with_file_name(format!( - "{}{}", - path_for_rebuild - .file_name() - .and_then(|s| s.to_str()) - .unwrap_or(""), - suffix - )); - let _ = std::fs::remove_file(sidecar); - } - let conn = Connection::open(path_for_rebuild) - .map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?; - conn.execute_batch(PRAGMAS_FAST) - .map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?; - Ok(conn) -} - -fn read_schema_version(conn: &Connection) -> Result, String> { - let has_info: bool = conn - .query_row( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_info'", - [], - |_| Ok(true), - ) - .optional() - .map_err(|e| format!("sqlite_master schema_info: {}", e))? - .unwrap_or(false); - if !has_info { - return Ok(None); - } - let v: Option = conn - .query_row( - "SELECT value FROM schema_info WHERE key = 'version'", - [], - |r| r.get(0), - ) - .optional() - .map_err(|e| format!("read schema_info.version: {}", e))?; - match v { - Some(s) => s - .parse::() - .map(Some) - .map_err(|e| format!("invalid schema_info.version {:?}: {}", s, e)), - None => Ok(None), - } -} - -fn has_legacy_layout(conn: &Connection) -> Result { - // Old layout has a `files` table without an `id INTEGER PRIMARY KEY`. - let has_files: bool = conn - .query_row( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='files'", - [], - |_| Ok(true), - ) - .optional() - .map_err(|e| format!("sqlite_master files: {}", e))? - .unwrap_or(false); - if !has_files { - return Ok(false); - } - // Check whether the columns match the legacy shape. - let mut stmt = conn - .prepare("PRAGMA table_info(files)") - .map_err(|e| format!("pragma table_info: {}", e))?; - let has_id = stmt - .query_map([], |row| row.get::<_, String>(1)) - .map_err(|e| format!("table_info query: {}", e))? - .filter_map(|r| r.ok()) - .any(|name| name == "id"); - Ok(!has_id) -} - -fn apply_current_schema(conn: &Connection, tokenizer: &str) -> Result<(), String> { - conn.execute_batch(SCHEMA_CURRENT) - .map_err(|e| format!("Failed to create current schema tables: {}", e))?; - let fts = fts_create_sql(tokenizer); - conn.execute_batch(&fts) - .map_err(|e| format!("Failed to create searchabletext: {}", e))?; - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - // Store the *effective* tokenizer string (with any default options we - // auto-applied). On subsequent opens we compare this against what the - // caller asks for and rebuild the FTS table if it changed. - let effective = effective_tokenizer(tokenizer); - conn.execute( - "INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)", - params![CURRENT_SCHEMA_VERSION.to_string(), now.to_string(), effective], - ) - .map_err(|e| format!("Failed to seed schema_info: {}", e))?; - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn tmp_db_path() -> std::path::PathBuf { - let mut p = std::env::temp_dir(); - p.push(format!( - "quicksearch-test-{}-{}.sqlite", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - p - } - - #[test] - fn fresh_db_gets_current_version() { - let p = tmp_db_path(); - let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); - let v: String = conn - .query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0)) - .unwrap(); - assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string()); - drop(conn); - std::fs::remove_file(&p).ok(); - } - - #[test] - fn reopen_is_idempotent() { - let p = tmp_db_path(); - { - let _ = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); - } - let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); - let v: String = conn - .query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0)) - .unwrap(); - assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string()); - drop(conn); - std::fs::remove_file(&p).ok(); - } - - #[test] - fn older_versioned_db_is_wiped_and_recreated() { - // Simulate a DB that was created at a previous schema version. - let p = tmp_db_path(); - { - let conn = Connection::open(&p).unwrap(); - conn.execute("CREATE TABLE schema_info (key TEXT PRIMARY KEY, value TEXT NOT NULL)", []).unwrap(); - conn.execute( - "INSERT INTO schema_info(key,value) VALUES('version','1')", - [], - ) - .unwrap(); - conn.execute("CREATE TABLE files (id INTEGER PRIMARY KEY, name TEXT)", []).unwrap(); - conn.execute("INSERT INTO files(name) VALUES('a.txt')", []).unwrap(); - } - let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); - let v: String = conn - .query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0)) - .unwrap(); - assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string()); - let count: i64 = conn - .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) - .unwrap(); - assert_eq!(count, 0, "old rows should be wiped"); - drop(conn); - std::fs::remove_file(&p).ok(); - } - - #[test] - fn legacy_db_is_wiped_and_recreated() { - let p = tmp_db_path(); - { - // Simulate a pre-A database. - let conn = Connection::open(&p).unwrap(); - conn.execute( - "CREATE TABLE files (name TEXT, path TEXT, size INTEGER, moddate INTEGER, hash BLOB)", - [], - ) - .unwrap(); - conn.execute( - "INSERT INTO files VALUES ('a.txt', '/tmp/a.txt', 1, 2, X'00')", - [], - ) - .unwrap(); - } - let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); - // Old row should be gone. - let count: i64 = conn - .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) - .unwrap(); - assert_eq!(count, 0); - // New columns should exist. - let _ = conn - .query_row("SELECT basic_state, content_state, type, mime FROM files LIMIT 0", [], |_| Ok(())) - .or_else(|e| if matches!(e, rusqlite::Error::QueryReturnedNoRows) { Ok(()) } else { Err(e) }) - .unwrap(); - drop(conn); - std::fs::remove_file(&p).ok(); - } -} diff --git a/crates/quicksearch-core/src/db/mod.rs b/crates/quicksearch-core/src/db/mod.rs index 34f35aa..d0be283 100644 --- a/crates/quicksearch-core/src/db/mod.rs +++ b/crates/quicksearch-core/src/db/mod.rs @@ -1,13 +1,13 @@ -//! SQLite schema, migrations, and row-level repository helpers. +//! SQLite schema, on-disk open/recreate, and row-level repository helpers. //! -//! The only "live" schema is `CURRENT_SCHEMA_VERSION` (see [`schema`]). Older -//! databases are detected in [`migrate::open_and_migrate`] and recreated from -//! scratch — Set A of the QuickSearch → Baloo work is the first schema bump -//! and carries no rows we'd want to preserve. Subsequent migrations should -//! prefer `ALTER TABLE` and versioned steps. +//! Policy: a single [`open::open_or_recreate`] is the only entry point. Any +//! schema mismatch — wrong version, drifted tokenizer, absent `schema_info` +//! — wipes the DB and rebuilds from [`schema::SCHEMA_CURRENT`]. There are +//! no in-place migrations by design; re-indexing is accepted as the cost +//! of avoiding migration-path complexity. -pub mod migrate; +pub mod open; pub mod repo; pub mod schema; -pub use migrate::{open_and_migrate, CURRENT_SCHEMA_VERSION}; +pub use open::{open_or_recreate, CURRENT_SCHEMA_VERSION}; diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs new file mode 100644 index 0000000..b6cd06c --- /dev/null +++ b/crates/quicksearch-core/src/db/open.rs @@ -0,0 +1,329 @@ +//! Open-or-recreate: the sole entry point into the on-disk database. +//! +//! **Policy**: any schema mismatch — wrong `schema_info.version`, wrong +//! stored `tokenize` string, absent `schema_info` table, or any other +//! drift from what this build expects — wipes the database file and +//! recreates it from scratch. There are deliberately **no** in-place +//! migrations. +//! +//! The tradeoff: users pay a re-index cost every time the shipped schema +//! changes. Our indexing is fast (see `bench/`) and schema changes are +//! rare in practice, so the code-complexity cost of maintaining real +//! migration paths wasn't worth it. A single `open_or_recreate` replaces +//! what used to be version detection + tokenizer-drift FTS rebuild + +//! legacy-layout recovery, all of which ultimately wiped anyway. + +use std::path::Path; + +use rusqlite::{params, Connection, OptionalExtension}; + +use super::schema::{effective_tokenizer, fts_create_sql, PRAGMAS_FAST, SCHEMA_CURRENT}; + +/// Bump this whenever [`SCHEMA_CURRENT`] or [`fts_create_sql`] changes in +/// a way that makes an old DB unreadable by new code. Any such bump +/// causes existing indexes to be wiped on next open — there's no +/// migration path by design. +pub const CURRENT_SCHEMA_VERSION: u32 = 3; + +/// Open `db_path`, applying fast-path pragmas, and ensure the on-disk +/// schema matches what this build expects. If it doesn't, delete the +/// file and recreate it empty — callers will need to re-index. +/// +/// `tokenizer` is passed to FTS5's `tokenize=` option when (re)creating +/// `searchabletext`. Changing it against an existing DB counts as a +/// schema mismatch and triggers the wipe-and-recreate path. +pub fn open_or_recreate(db_path: &str, tokenizer: &str) -> Result { + let path = Path::new(db_path).to_path_buf(); + let conn = Connection::open(db_path) + .map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?; + conn.execute_batch(PRAGMAS_FAST) + .map_err(|e| format!("Failed to apply pragmas: {}", e))?; + + if db_matches_current(&conn, tokenizer)? { + return Ok(conn); + } + + // Schema is present but stale, or pre-existing rows belong to an + // older layout, or the tokenizer drifted. Log once so the rebuild + // isn't silent, then wipe + recreate. + eprintln!( + "QuickSearch: database at {} does not match current schema; rebuilding. \ + Existing rows will be re-scanned on next indexing run.", + db_path + ); + let conn = wipe_and_reopen(conn, &path)?; + apply_current_schema(&conn, tokenizer)?; + Ok(conn) +} + +/// True iff the DB has `schema_info` with the current version *and* the +/// effective-tokenizer string this caller asked for. Anything else — +/// missing table, wrong version, different tokenizer — returns false. +fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result { + let has_info: bool = conn + .query_row( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_info'", + [], + |_| Ok(true), + ) + .optional() + .map_err(|e| format!("sqlite_master schema_info: {}", e))? + .unwrap_or(false); + if !has_info { + return Ok(false); + } + + let version: Option = conn + .query_row( + "SELECT value FROM schema_info WHERE key = 'version'", + [], + |r| r.get(0), + ) + .optional() + .map_err(|e| format!("read schema_info.version: {}", e))?; + let version_ok = version.as_deref() == Some(&CURRENT_SCHEMA_VERSION.to_string()); + if !version_ok { + return Ok(false); + } + + let stored_tokenize: Option = conn + .query_row( + "SELECT value FROM schema_info WHERE key = 'tokenize'", + [], + |r| r.get(0), + ) + .optional() + .map_err(|e| format!("read schema_info.tokenize: {}", e))?; + let want_tokenize = effective_tokenizer(tokenizer); + Ok(stored_tokenize.as_deref() == Some(&*want_tokenize)) +} + +/// Drop the current connection, delete the DB file + its WAL/SHM/journal +/// sidecars, reopen a fresh file, re-apply pragmas. +fn wipe_and_reopen(conn: Connection, path: &Path) -> Result { + drop(conn); + // Primary file may already be absent (fresh open that just needed + // the table applied). Ignore NotFound; anything else is an error. + match std::fs::remove_file(path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(format!("Failed to remove old database: {}", e)), + } + // Sidecars are optional — delete best-effort. + for suffix in ["-wal", "-shm", "-journal"] { + let sidecar = path.with_file_name(format!( + "{}{}", + path.file_name().and_then(|s| s.to_str()).unwrap_or(""), + suffix + )); + let _ = std::fs::remove_file(sidecar); + } + let conn = Connection::open(path) + .map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?; + conn.execute_batch(PRAGMAS_FAST) + .map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?; + Ok(conn) +} + +/// Apply [`SCHEMA_CURRENT`] + [`fts_create_sql`] to a blank DB and seed +/// `schema_info` with the matching version/tokenize markers. +fn apply_current_schema(conn: &Connection, tokenizer: &str) -> Result<(), String> { + conn.execute_batch(SCHEMA_CURRENT) + .map_err(|e| format!("Failed to create current schema tables: {}", e))?; + let fts = fts_create_sql(tokenizer); + conn.execute_batch(&fts) + .map_err(|e| format!("Failed to create searchabletext: {}", e))?; + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let effective = effective_tokenizer(tokenizer); + conn.execute( + "INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)", + params![ + CURRENT_SCHEMA_VERSION.to_string(), + now.to_string(), + effective + ], + ) + .map_err(|e| format!("Failed to seed schema_info: {}", e))?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp_db_path() -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "quicksearch-test-{}-{}.sqlite", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + p + } + + #[test] + fn fresh_db_gets_current_version() { + let p = tmp_db_path(); + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let v: String = conn + .query_row( + "SELECT value FROM schema_info WHERE key='version'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string()); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn reopen_is_idempotent() { + let p = tmp_db_path(); + { + let _ = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + } + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let v: String = conn + .query_row( + "SELECT value FROM schema_info WHERE key='version'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string()); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn older_versioned_db_is_wiped_and_recreated() { + // Simulate a DB from a prior schema version. Our policy is to + // wipe without attempting any migration. + let p = tmp_db_path(); + { + let conn = Connection::open(&p).unwrap(); + conn.execute( + "CREATE TABLE schema_info (key TEXT PRIMARY KEY, value TEXT NOT NULL)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO schema_info(key,value) VALUES('version','1')", + [], + ) + .unwrap(); + conn.execute("CREATE TABLE files (id INTEGER PRIMARY KEY, name TEXT)", []) + .unwrap(); + conn.execute("INSERT INTO files(name) VALUES('a.txt')", []) + .unwrap(); + } + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let v: String = conn + .query_row( + "SELECT value FROM schema_info WHERE key='version'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string()); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 0, "old rows should be wiped"); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn legacy_layout_db_is_wiped_and_recreated() { + // Pre-A layout with no `schema_info` at all. Same policy — wipe. + let p = tmp_db_path(); + { + let conn = Connection::open(&p).unwrap(); + conn.execute( + "CREATE TABLE files (name TEXT, path TEXT, size INTEGER, moddate INTEGER, hash BLOB)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO files VALUES ('a.txt', '/tmp/a.txt', 1, 2, X'00')", + [], + ) + .unwrap(); + } + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + // Old row should be gone. + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 0); + // New columns should exist (just prepare the SELECT — an + // unknown column name would parse-error here). + let _ = conn + .query_row( + "SELECT basic_state, content_state, type, mime FROM files LIMIT 0", + [], + |_| Ok(()), + ) + .or_else(|e| { + if matches!(e, rusqlite::Error::QueryReturnedNoRows) { + Ok(()) + } else { + Err(e) + } + }) + .unwrap(); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn tokenizer_drift_wipes_db() { + // Previously this was "rebuild FTS in place and reset + // content_state". New policy: full wipe. + let p = tmp_db_path(); + let first_effective = { + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + conn.execute( + "INSERT INTO files (name, path, parent, size, mtime) \ + VALUES ('x', '/x', '/', 0, 0)", + [], + ) + .unwrap(); + let stored: String = conn + .query_row( + "SELECT value FROM schema_info WHERE key='tokenize'", + [], + |r| r.get(0), + ) + .unwrap(); + drop(conn); + stored + }; + // Second open with a different tokenizer. + let conn = open_or_recreate(p.to_str().unwrap(), "unicode61").unwrap(); + let files_count: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(files_count, 0, "tokenizer drift should wipe rows"); + let new_stored: String = conn + .query_row( + "SELECT value FROM schema_info WHERE key='tokenize'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_ne!(first_effective, new_stored); + drop(conn); + std::fs::remove_file(&p).ok(); + } +} diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs index a643627..88cb2ca 100644 --- a/crates/quicksearch-core/src/db/repo.rs +++ b/crates/quicksearch-core/src/db/repo.rs @@ -121,15 +121,21 @@ pub fn update_file_basic( } /// Mark a file's content indexing as complete and write the extracted text + -/// properties atomically. FTS5 stores the canonical text for this file; -/// `properties` are stored both as a structured side-table (for exact -/// retrieval) and concatenated into the FTS `properties` column (for MATCH). +/// properties atomically. The plaintext is fed to the contentless FTS5 +/// tokenizer (which keeps only the inverted index) and — when `store_text` +/// is `true` — separately stored zstd-compressed in `documents_text` for +/// on-demand snippet rendering. When `store_text=false` the sidecar INSERT +/// is skipped: queries still match the right files but result rows can't +/// render snippets. `properties` are stored both as a structured side- +/// table (for exact retrieval) and concatenated into the FTS `properties` +/// column (for MATCH). pub fn set_content_done( tx: &Transaction<'_>, file_id: i64, name: &str, text: &str, properties: &[(String, String)], + store_text: bool, ) -> Result<(), String> { // Clear any previous extraction (in case of re-run). remove_content_for_id(tx, file_id)?; @@ -142,12 +148,28 @@ pub fn set_content_done( .map_err(|e| format!("insert property {}={}: {}", k, v, e))?; } let props_blob = encode_properties_for_fts(properties); + // Contentless FTS5 still accepts values on INSERT — the tokenizer needs + // them — it simply doesn't persist the raw column values. tx.execute( "INSERT INTO searchabletext(rowid, name, text, properties) VALUES (?1, ?2, ?3, ?4)", params![file_id, name, text, props_blob], ) .map_err(|e| format!("insert FTS row {}: {}", file_id, e))?; + // Skip the compressed sidecar when: the config disables snippet storage + // outright, or there's no body text (e.g. an image whose extractor + // returned only EXIF properties). The second case saves a zstd frame + // on what would otherwise be an empty blob. + if store_text && !text.is_empty() { + let compressed = zstd::encode_all(text.as_bytes(), ZSTD_LEVEL) + .map_err(|e| format!("zstd encode for file {}: {}", file_id, e))?; + tx.execute( + "INSERT INTO documents_text(file_id, text_zstd, text_len) VALUES (?1, ?2, ?3)", + params![file_id, compressed, text.len() as i64], + ) + .map_err(|e| format!("insert documents_text {}: {}", file_id, e))?; + } + tx.execute( "UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2", params![STATE_DONE, file_id], @@ -159,6 +181,13 @@ pub fn set_content_done( Ok(()) } +/// zstd level tuned for extracted-text prose. Level 3 hits ~3-5× on English +/// prose at high throughput (hundreds of MB/s) — level 9+ would shave a few +/// percent more but at 10× the CPU cost during indexing. Wrong knob to +/// tune: readers decompress far faster than writers compress, so keep +/// write-side cost low. +const ZSTD_LEVEL: i32 = 3; + /// Mark a file's content extraction as failed. Keeps the basic row in place. pub fn set_content_failed( tx: &Transaction<'_>, @@ -213,16 +242,22 @@ pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result, file_id: i64) -> Result<(), String> { - // Regular FTS5 supports a plain DELETE by rowid; no need to supply the - // old column values the way a contentless table would require. + // `contentless_delete=1` on the FTS5 table makes this work without + // re-supplying the old column values (it tombstones the rowid). tx.execute( "DELETE FROM searchabletext WHERE rowid = ?1", params![file_id], ) .map_err(|e| format!("FTS delete row {}: {}", file_id, e))?; + tx.execute( + "DELETE FROM documents_text WHERE file_id = ?1", + params![file_id], + ) + .map_err(|e| format!("delete documents_text {}: {}", file_id, e))?; tx.execute("DELETE FROM properties WHERE file_id = ?1", params![file_id]) .map_err(|e| format!("delete properties {}: {}", file_id, e))?; Ok(()) @@ -257,7 +292,7 @@ pub fn checkpoint_and_close(conn: Connection) { #[cfg(test)] mod tests { use super::*; - use crate::db::open_and_migrate; + use crate::db::open_or_recreate; fn tmp_path() -> std::path::PathBuf { let mut p = std::env::temp_dir(); @@ -275,7 +310,7 @@ mod tests { #[test] fn insert_update_delete_round_trip() { let p = tmp_path(); - let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); { let tx = conn.transaction().unwrap(); let id = insert_file( @@ -301,6 +336,7 @@ mod tests { "a.txt", "hello world", &[("title".to_string(), "hi".to_string())], + true, ) .unwrap(); tx.commit().unwrap(); @@ -338,7 +374,7 @@ mod tests { #[test] fn update_resets_content_state() { let p = tmp_path(); - let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let id = { let tx = conn.transaction().unwrap(); let id = insert_file( @@ -358,7 +394,7 @@ mod tests { ) .unwrap() .expect("unique path"); - set_content_done(&tx, id, "a.txt", "old text", &[]).unwrap(); + set_content_done(&tx, id, "a.txt", "old text", &[], true).unwrap(); tx.commit().unwrap(); id }; @@ -409,7 +445,7 @@ mod tests { // (overlapping roots, symlink resolution quirks), the second INSERT // must be a silent no-op, not a run-ending error. let p = tmp_path(); - let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let tx = conn.transaction().unwrap(); let row = NewFile { name: "dup.txt", @@ -447,7 +483,7 @@ mod tests { #[test] fn set_content_failed_writes_failed_table() { let p = tmp_path(); - let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); let id = { let tx = conn.transaction().unwrap(); let id = insert_file( diff --git a/crates/quicksearch-core/src/db/schema.rs b/crates/quicksearch-core/src/db/schema.rs index 208e7ee..dbb6abd 100644 --- a/crates/quicksearch-core/src/db/schema.rs +++ b/crates/quicksearch-core/src/db/schema.rs @@ -12,12 +12,16 @@ pub const PRAGMAS_FAST: &str = " PRAGMA foreign_keys = ON; "; -/// The full current schema. Applied by [`migrate::open_and_migrate`] when -/// the DB is fresh or has been wiped during upgrade. +/// The full current schema. Applied by [`super::open::open_or_recreate`] +/// when the DB is fresh or has just been wiped because it drifted from +/// [`super::open::CURRENT_SCHEMA_VERSION`]. /// -/// FTS5 is a *regular* (non-contentless) virtual table so `snippet()` and -/// `highlight()` can read the stored text. This also means there is no -/// separate `documents` table — FTS5 *is* the text store. +/// FTS5 is *contentless* (see [`fts_create_sql`]): the inverted index is kept +/// but the column values aren't stored. The canonical extracted text lives +/// in a separate `documents_text` table, zstd-compressed. Snippet rendering +/// for search results decompresses on demand and highlights matches in Rust +/// (see `crate::snippet`). This keeps the on-disk footprint close to Baloo's +/// LMDB-only size while still supporting snippet/highlight features. pub const SCHEMA_CURRENT: &str = r#" CREATE TABLE schema_info ( key TEXT PRIMARY KEY, @@ -61,6 +65,16 @@ CREATE TABLE failed_files ( ts INTEGER NOT NULL ); +-- Canonical extracted text for every successfully content-indexed file. +-- Compressed with zstd (see `crate::db::repo::set_content_done`). Only +-- written when the extractor produced text; absent rows mean "no body +-- text" (e.g. an image with only EXIF properties). +CREATE TABLE documents_text ( + file_id INTEGER PRIMARY KEY REFERENCES files(id) ON DELETE CASCADE, + text_zstd BLOB NOT NULL, + text_len INTEGER NOT NULL -- original byte length pre-compression +); + CREATE TABLE config_validation ( key TEXT PRIMARY KEY, value TEXT NOT NULL @@ -69,16 +83,26 @@ CREATE TABLE config_validation ( /// FTS5 virtual table DDL. Separate because the tokenizer is config-driven. /// -/// Regular (not contentless, not external-content) FTS5: the table stores -/// its own text, which enables `snippet()`/`highlight()` and makes row-level -/// INSERT/UPDATE/DELETE work with normal SQL semantics. `rowid` is supplied -/// by the caller and must equal `files.id`. +/// *Contentless* FTS5 (`content=''`): the inverted index is built from the +/// column values supplied on INSERT, but those values are not stored. This +/// is the main lever that pulls our on-disk footprint down toward Baloo's. +/// `contentless_delete=1` (SQLite 3.43+) lets us `DELETE FROM … WHERE +/// rowid=?` without replaying the original row text, at the cost of a +/// modest tombstone bitmap. The tokenizer is config-driven; by default +/// (`trigram`) we append `remove_diacritics 1` so queries and stored text +/// fold the same way. +/// +/// Snippet/highlight aren't available through SQLite's built-in `snippet()` +/// in contentless mode — we render them in Rust from the zstd-compressed +/// `documents_text` sidecar instead. pub fn fts_create_sql(tokenizer: &str) -> String { let effective = effective_tokenizer(tokenizer); format!( "CREATE VIRTUAL TABLE searchabletext USING fts5(\ name, text, properties, \ - tokenize='{}'\ + tokenize='{}', \ + content='', \ + contentless_delete=1\ );", effective.replace('\'', "''") ) diff --git a/crates/quicksearch-core/src/file_handling.rs b/crates/quicksearch-core/src/file_handling.rs index 63bdb7d..b1511f6 100644 --- a/crates/quicksearch-core/src/file_handling.rs +++ b/crates/quicksearch-core/src/file_handling.rs @@ -793,9 +793,14 @@ pub fn process_text_indexing( safe_truncate_string(&content.text, config.processing.maximum_text_size); } let props = content.properties_sorted(); - if let Err(e) = - repo::set_content_done(&tx, *file_id, fname, &content.text, &props) - { + if let Err(e) = repo::set_content_done( + &tx, + *file_id, + fname, + &content.text, + &props, + config.processing.store_text_for_snippets, + ) { eprintln!("Warning: set_content_done for {}: {}", fpath, e); } } diff --git a/crates/quicksearch-core/src/indexing.rs b/crates/quicksearch-core/src/indexing.rs index f63fea1..1f26657 100644 --- a/crates/quicksearch-core/src/indexing.rs +++ b/crates/quicksearch-core/src/indexing.rs @@ -81,6 +81,24 @@ pub struct IndexingService { /// Polling interval for `should_abort` while suspended. const SUSPEND_POLL_MS: u64 = 100; +/// Map a rusqlite/SQLite error string into the tagged form the GUI's +/// search panel recognises (so it can pop the corruption-recovery dialog, +/// special-case FTS5 syntax errors, etc.). Centralized so both generic +/// `execute_search` and the fulltext-specific path classify errors the +/// same way. +fn classify_sql_err(error_msg: &str) -> String { + if error_msg.contains("malformed") + || error_msg.contains("corrupt") + || error_msg.contains("database disk image is malformed") + { + format!("DATABASE_CORRUPTED: {}", error_msg) + } else if error_msg.contains("fts5: syntax error") { + "Search syntax error: The search term contains characters that cannot be processed. Please try a simpler search term.".into() + } else { + format!("Failed to execute query: {}", error_msg) + } +} + /// Combined stop/suspend check used by worker loops. Returns `true` iff the /// caller should abort the operation. While the suspend flag is set and stop /// is not, this parks the thread by sleeping in short increments so a later @@ -237,9 +255,97 @@ impl IndexingService { self.stop_indexing() } + /// Fulltext-specific search path that emits rendered snippets. Runs the + /// SQL produced by [`crate::search_sql::build_select`] for a fulltext + /// `SearchArgs`, which returns `(name, path, file_id, text_zstd)`, then + /// decompresses each text blob and hands it to [`crate::snippet::render`] + /// together with the query terms. Case-sensitive mode is re-applied in + /// Rust since contentless FTS5 no longer stores the raw text. + /// + /// The shape of the returned [`SearchResult`] matches what the GUI's + /// results table expects: columns `(name, path, snippet)`. + pub fn execute_fulltext_search( + &self, + db_path: &str, + args: &crate::search_sql::SearchArgs, + limit: u32, + offset: u32, + ) -> Result, String> { + use crate::search_sql::{build_select, fulltext_terms}; + use crate::snippet; + + let sql = build_select(args, limit, offset)?; + let terms = fulltext_terms(args); + let term_refs: Vec<&str> = terms.iter().map(String::as_str).collect(); + + let conn = db::open_or_recreate(db_path, "trigram").map_err(|e| { + if e.contains("corrupt") || e.contains("malformed") { + format!("DATABASE_CORRUPTED: {}", e) + } else { + e + } + })?; + + let mut stmt = conn + .prepare(&sql) + .map_err(|e| classify_sql_err(&e.to_string()))?; + + let rows_iter = stmt + .query_map([], |row| { + let name: String = row.get(0)?; + let path: String = row.get(1)?; + let _file_id: i64 = row.get(2)?; + let blob: Option> = row.get(3)?; + Ok((name, path, blob)) + }) + .map_err(|e| classify_sql_err(&e.to_string()))?; + + let snippet_opts = snippet::Options::default(); + let mut out_rows: Vec = Vec::new(); + for r in rows_iter { + let (name, path, blob) = + r.map_err(|e| classify_sql_err(&e.to_string()))?; + let body = match blob { + None => String::new(), + Some(bytes) if bytes.is_empty() => String::new(), + Some(bytes) => match zstd::decode_all(bytes.as_slice()) { + Ok(raw) => String::from_utf8_lossy(&raw).into_owned(), + Err(_) => String::new(), + }, + }; + + // Case-sensitive post-filter: every query term must appear with + // the exact case supplied by the user. Applied to the body + // text; filename hits are still matched by the FTS MATCH which + // is case-insensitive (filenames are usually mixed case and the + // user probably doesn't care). Exact-phrase mode treats the + // whole quoted phrase as one token. + if args.fulltext_case_sensitive && !body.is_empty() { + let all_present = terms.iter().all(|t| body.contains(t.as_str())); + if !all_present { + continue; + } + } + + let rendered = if body.is_empty() { + String::new() + } else { + snippet::render(&body, &term_refs, &snippet_opts) + }; + out_rows.push(SearchResultRow { + values: vec![name, path, rendered], + }); + } + + Ok(vec![SearchResult { + columns: vec!["name".into(), "path".into(), "snippet".into()], + rows: out_rows, + }]) + } + /// Execute a search query against the database pub fn execute_search(&self, db_path: &str, query: &str) -> Result, String> { - let conn = db::open_and_migrate(db_path, "trigram") + let conn = db::open_or_recreate(db_path, "trigram") .map_err(|e| { if e.contains("corrupt") || e.contains("malformed") { format!("DATABASE_CORRUPTED: {}", e) @@ -404,7 +510,7 @@ impl IndexingService { /// Check if configuration changes require index recreation pub fn check_config_validation(&self, db_path: &str, config: &Config, indexing_path: &str) -> Result>, String> { - let conn = db::open_and_migrate(db_path, &config.processing.tokenize)?; + let conn = db::open_or_recreate(db_path, &config.processing.tokenize)?; Self::validate_config(&conn, config, indexing_path) } @@ -580,7 +686,7 @@ impl IndexingService { .collect(); // Open and migrate the database to the current schema version. - let conn = db::open_and_migrate(db_path, &config.processing.tokenize)?; + let conn = db::open_or_recreate(db_path, &config.processing.tokenize)?; // Update configuration (for new installations or when no validation issues). // `indexing_path` in the validation table stores the joined list so diff --git a/crates/quicksearch-core/src/lib.rs b/crates/quicksearch-core/src/lib.rs index b1af394..453302b 100644 --- a/crates/quicksearch-core/src/lib.rs +++ b/crates/quicksearch-core/src/lib.rs @@ -9,4 +9,5 @@ pub mod mime; pub mod query; pub mod search_sql; pub mod shutdown; +pub mod snippet; pub mod watcher; diff --git a/crates/quicksearch-core/src/query/translator.rs b/crates/quicksearch-core/src/query/translator.rs index 958ba34..8eb58a9 100644 --- a/crates/quicksearch-core/src/query/translator.rs +++ b/crates/quicksearch-core/src/query/translator.rs @@ -547,7 +547,7 @@ mod tests { #[test] fn end_to_end_combined_filter_executes() { use crate::db::{ - open_and_migrate, + open_or_recreate, repo::{insert_file, set_content_done, NewFile}, }; use crate::mime::FileType; @@ -562,7 +562,7 @@ mod tests { .unwrap() .as_nanos() )); - let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); // Two audio files, one document. Only the "audio with beatles" file // should be returned for `type:Audio beatles`. @@ -591,6 +591,7 @@ mod tests { "beatles-track.mp3", "beatles hey jude", &[("artist".into(), "The Beatles".into())], + true, ) .unwrap(); let b = insert_file( @@ -610,7 +611,7 @@ mod tests { ) .unwrap() .expect("unique path"); - set_content_done(&tx, b, "bach.flac", "bach prelude", &[]).unwrap(); + set_content_done(&tx, b, "bach.flac", "bach prelude", &[], true).unwrap(); let c = insert_file( &tx, &NewFile { @@ -628,7 +629,7 @@ mod tests { ) .unwrap() .expect("unique path"); - set_content_done(&tx, c, "notes.txt", "beatles biography", &[]).unwrap(); + set_content_done(&tx, c, "notes.txt", "beatles biography", &[], true).unwrap(); tx.commit().unwrap(); vec![a, b, c] }; diff --git a/crates/quicksearch-core/src/search_sql.rs b/crates/quicksearch-core/src/search_sql.rs index 6f15677..4087f45 100644 --- a/crates/quicksearch-core/src/search_sql.rs +++ b/crates/quicksearch-core/src/search_sql.rs @@ -47,14 +47,21 @@ pub fn build_count(args: &SearchArgs) -> Result { } } -/// SQL that returns one page of results. +/// SQL that returns one page of results. Columns emitted by the `fulltext` +/// branch are `(name, path, file_id, text_zstd)` — the snippet is rendered +/// in Rust from the zstd-compressed `documents_text` row (FTS5 is +/// contentless, so SQLite's `snippet()` doesn't work on it). The GUI +/// should use [`crate::indexing::IndexingService::execute_fulltext_search`] +/// which stitches the decompress + snippet step on top of this SQL. pub fn build_select(args: &SearchArgs, limit: u32, offset: u32) -> Result { match args.search_type.as_str() { "fulltext" => { let where_clause = build_fulltext_where(args)?; Ok(format!( - "SELECT f.name, f.path, snippet(searchabletext, 1, '', '', '...', 64) as snippet \ - FROM searchabletext AS st JOIN files f ON f.id = st.rowid \ + "SELECT f.name, f.path, f.id, dt.text_zstd \ + FROM searchabletext AS st \ + JOIN files f ON f.id = st.rowid \ + LEFT JOIN documents_text dt ON dt.file_id = f.id \ WHERE {} ORDER BY rank LIMIT {} OFFSET {}", where_clause, limit, offset )) @@ -137,30 +144,62 @@ fn build_fulltext_where(args: &SearchArgs) -> Result { words.join(" AND ") }; - let mut where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match)); - if args.fulltext_case_sensitive { - if args.fulltext_exact { - let literal = words.join(" "); - where_clause.push_str(&format!( - " AND instr(st.text, '{}') > 0", - sql_quote(&literal) - )); - } else { - for w in &words { - where_clause.push_str(&format!( - " AND instr(st.text, '{}') > 0", - sql_quote(w) - )); - } - } - } + // Contentless FTS5 doesn't store column text, so case-sensitive + // filtering can't live in SQL anymore. It's re-applied in + // `IndexingService::execute_fulltext_search` by checking the + // decompressed body text for literal-case matches before returning + // the row. The MATCH itself stays case-insensitive (tokenizer folds), + // which is the correct candidate-set for a post-filter. + let where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match)); Ok(where_clause) } +/// Pull out the raw words the user typed so the snippet renderer and the +/// case-sensitive post-filter can see the same tokens `build_fulltext_where` +/// fed into FTS5. Returns empty when the user's term is empty or contains +/// only too-short words under the non-exact path. +pub fn fulltext_terms(args: &SearchArgs) -> Vec { + let trimmed = args.term.trim(); + if trimmed.is_empty() { + return Vec::new(); + } + let sanitized: String = trimmed + .chars() + .map(|c| { + if matches!( + c, + ':' | ';' | '(' | ')' | '[' | ']' | '{' | '}' | '^' | '~' | '"' + ) { + ' ' + } else { + c + } + }) + .collect(); + let tokens: Vec = sanitized + .split_whitespace() + .map(|s| s.to_string()) + .collect(); + if tokens.is_empty() { + return Vec::new(); + } + if args.fulltext_exact { + // One composite phrase. Snippet rendering wants to highlight the + // whole phrase contiguously; the renderer supports multiple terms + // already so we collapse to the joined form. + vec![tokens.join(" ")] + } else { + tokens + .into_iter() + .filter(|w| w.chars().count() >= 3) + .collect() + } +} + #[cfg(test)] mod tests { use super::*; - use crate::db::{open_and_migrate, repo::{insert_file, set_content_done, NewFile}}; + use crate::db::{open_or_recreate, repo::{insert_file, set_content_done, NewFile}}; use crate::mime::FileType; fn args(search_type: &str, term: &str) -> SearchArgs { @@ -178,7 +217,36 @@ mod tests { assert!(sql.contains("LIMIT 50")); assert!(sql.contains("OFFSET 100")); assert!(sql.contains("ORDER BY rank")); - assert!(sql.contains("snippet(searchabletext")); + // Snippet rendering moved to Rust; SQL returns the compressed blob + // so the post-processor can decompress + highlight. + assert!(sql.contains("dt.text_zstd"), "got {sql}"); + } + + #[test] + fn fulltext_case_sensitive_no_longer_in_sql() { + let mut a = args("fulltext", "Hello"); + a.fulltext_case_sensitive = true; + let sql = build_select(&a, 50, 0).unwrap(); + // Post-filter lives in Rust now — no instr() or st.text reference. + assert!(!sql.contains("instr("), "got {sql}"); + } + + #[test] + fn fulltext_terms_extract_non_exact() { + let a = args("fulltext", "the quick brown"); + let t = fulltext_terms(&a); + // Short words like "the" are dropped (trigram min length 3 applies + // in non-exact mode — matches the SQL build rules). + assert!(t.iter().any(|s| s == "quick")); + assert!(t.iter().any(|s| s == "brown")); + } + + #[test] + fn fulltext_terms_exact_mode_returns_joined_phrase() { + let mut a = args("fulltext", "hello world"); + a.fulltext_exact = true; + let t = fulltext_terms(&a); + assert_eq!(t, vec!["hello world".to_string()]); } #[test] @@ -253,7 +321,7 @@ mod tests { #[test] fn end_to_end_pagination_smoke() { let p = tmp_path(); - let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); { let tx = conn.transaction().unwrap(); for i in 0..7 { @@ -275,7 +343,7 @@ mod tests { ) .unwrap() .expect("unique path"); - set_content_done(&tx, id, &format!("file_{}.txt", i), "shared body content", &[]).unwrap(); + set_content_done(&tx, id, &format!("file_{}.txt", i), "shared body content", &[], true).unwrap(); } tx.commit().unwrap(); } diff --git a/crates/quicksearch-core/src/snippet.rs b/crates/quicksearch-core/src/snippet.rs new file mode 100644 index 0000000..0767267 --- /dev/null +++ b/crates/quicksearch-core/src/snippet.rs @@ -0,0 +1,336 @@ +//! Snippet / highlight rendering for search results. +//! +//! We store extracted text in the `documents_text` sidecar (zstd-compressed) +//! rather than in FTS5, so SQLite's built-in `snippet()` / `highlight()` +//! auxiliary functions aren't available (contentless FTS5 doesn't support +//! them). This module reproduces the parts we actually need in Rust: find +//! a window of context around the first match, bold every query-term +//! occurrence inside that window, trim with ellipsis markers. +//! +//! Matching is ASCII-case-insensitive on the *rendering* side. That aligns +//! with the search path which is already case-insensitive via the trigram +//! tokenizer; exact-case-only snippets aren't a feature users expect here. +//! Unicode accent folding isn't applied at the rendering layer — a query +//! for `cafe` will still *find* a file containing `café` (because the FTS +//! tokenizer strips diacritics) but the snippet won't highlight the +//! accented occurrence. The surrounding text is still returned verbatim. +//! +//! The API is intentionally small: one `render` function plus an `Options` +//! struct. Callers that want different pre/post tags, ellipsis, or window +//! size pass them in; there are sensible defaults for the GUI case. + +/// Options controlling snippet rendering. The defaults mirror the old +/// `snippet(searchabletext, 1, '', '', '...', 64)` call that +/// the GUI used to run directly as SQL. +#[derive(Debug, Clone)] +pub struct Options<'a> { + pub pre: &'a str, + pub post: &'a str, + pub ellipsis: &'a str, + /// Approximate character budget for the returned snippet. Matches + /// expand the window if needed to keep their tags on; the budget is a + /// soft target, not a hard cap. + pub approx_chars: usize, +} + +impl<'a> Default for Options<'a> { + fn default() -> Self { + Options { + pre: "", + post: "", + ellipsis: "...", + approx_chars: 200, + } + } +} + +/// Render a snippet from `text` highlighting every occurrence of any term +/// in `terms`. Returns a string with `pre`/`post` wrapping each match, and +/// `ellipsis` prepended/appended when the returned window doesn't reach +/// the text's edges. +/// +/// If no term matches, returns the first `approx_chars` of `text` (char- +/// aligned), suffixed with `ellipsis` when truncated. +pub fn render(text: &str, terms: &[&str], opts: &Options<'_>) -> String { + // Short-circuit trivial inputs. + if text.is_empty() { + return String::new(); + } + let effective_terms: Vec<&str> = terms + .iter() + .map(|t| t.trim()) + .filter(|t| !t.is_empty()) + .collect(); + if effective_terms.is_empty() { + return truncate_head(text, opts.approx_chars, opts.ellipsis); + } + + // Case-fold once; we do all positioning on the folded buffer and emit + // slices from the original. Both buffers have identical byte layout + // because `to_ascii_lowercase` is a byte-for-byte map that preserves + // multi-byte UTF-8 sequences unchanged (it only touches ASCII letters). + let folded = text.to_ascii_lowercase(); + let folded_bytes = folded.as_bytes(); + + let mut matches: Vec<(usize, usize)> = Vec::new(); + for term in &effective_terms { + let pattern: String = term.to_ascii_lowercase(); + let pbytes = pattern.as_bytes(); + if pbytes.is_empty() { + continue; + } + let mut start = 0; + while start + pbytes.len() <= folded_bytes.len() { + if let Some(rel) = memfind(&folded_bytes[start..], pbytes) { + let at = start + rel; + matches.push((at, at + pbytes.len())); + // Advance past this match to avoid zero-width loops on + // empty patterns (already guarded above) and to allow + // overlapping matches of *different* terms in the next + // outer-loop iteration. + start = at + pbytes.len(); + } else { + break; + } + } + } + + if matches.is_empty() { + return truncate_head(text, opts.approx_chars, opts.ellipsis); + } + + // Dedupe + sort so overlapping matches from different terms (e.g. + // "rust" and "rustc") don't produce nested tags. + matches.sort_by_key(|(a, _)| *a); + matches = coalesce_overlapping(matches); + + // Pick the window. Start a third of the budget before the first match + // so the hit isn't pinned to the left edge. Round both ends to char + // boundaries so we never slice a multi-byte UTF-8 sequence. + let pre_pad = opts.approx_chars / 3; + let first_match_start = matches[0].0; + let mut win_start = first_match_start.saturating_sub(pre_pad); + let mut win_end = win_start + opts.approx_chars; + if win_end > text.len() { + win_end = text.len(); + } + while win_start > 0 && !text.is_char_boundary(win_start) { + win_start -= 1; + } + while win_end < text.len() && !text.is_char_boundary(win_end) { + win_end += 1; + } + + // Expand the window to include the full end of any match that would + // otherwise be cut off mid-tag. Keeps rendering sane when a long term + // sits at the right edge of the budget. + let last_match_in_window = matches.iter().rfind(|(s, _)| *s < win_end); + if let Some((_, end)) = last_match_in_window { + if *end > win_end { + win_end = *end; + while win_end < text.len() && !text.is_char_boundary(win_end) { + win_end += 1; + } + } + } + + // Render: walk matches that fall inside the window, splicing pre/post + // around each. Prepend/append ellipsis when we've chopped off content. + let mut out = String::with_capacity(win_end - win_start + 32); + if win_start > 0 { + out.push_str(opts.ellipsis); + } + let mut cursor = win_start; + for (ms, me) in matches.iter() { + if *me <= win_start || *ms >= win_end { + continue; + } + // Clamp to the window. + let ms = (*ms).max(win_start); + let me = (*me).min(win_end); + if ms > cursor { + out.push_str(&text[cursor..ms]); + } + out.push_str(opts.pre); + out.push_str(&text[ms..me]); + out.push_str(opts.post); + cursor = me; + } + if cursor < win_end { + out.push_str(&text[cursor..win_end]); + } + if win_end < text.len() { + out.push_str(opts.ellipsis); + } + out +} + +/// Return the first `n` characters of `text`, suffixed with `ellipsis` if +/// truncation actually happened. Respects UTF-8 char boundaries. +fn truncate_head(text: &str, n: usize, ellipsis: &str) -> String { + if text.len() <= n { + return text.to_string(); + } + let mut cut = n; + while cut > 0 && !text.is_char_boundary(cut) { + cut -= 1; + } + let mut out = String::with_capacity(cut + ellipsis.len()); + out.push_str(&text[..cut]); + out.push_str(ellipsis); + out +} + +/// Merge adjacent / overlapping (start, end) ranges in place. Input must be +/// sorted by start. +fn coalesce_overlapping(mut v: Vec<(usize, usize)>) -> Vec<(usize, usize)> { + if v.len() < 2 { + return v; + } + let mut out = Vec::with_capacity(v.len()); + let mut cur = v.remove(0); + for next in v { + if next.0 <= cur.1 { + cur.1 = cur.1.max(next.1); + } else { + out.push(cur); + cur = next; + } + } + out.push(cur); + out +} + +/// Locate the first occurrence of `needle` in `hay`. A byte-level search; +/// callers have already lowercased both sides so case is normalized. +fn memfind(hay: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || needle.len() > hay.len() { + return None; + } + let first = needle[0]; + let mut i = 0; + while i + needle.len() <= hay.len() { + if hay[i] == first && &hay[i..i + needle.len()] == needle { + return Some(i); + } + i += 1; + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + fn opts_small() -> Options<'static> { + Options { + pre: "", + post: "", + ellipsis: "…", + approx_chars: 40, + } + } + + #[test] + fn empty_text_returns_empty() { + let s = render("", &["foo"], &Options::default()); + assert_eq!(s, ""); + } + + #[test] + fn no_terms_returns_head_with_ellipsis_when_truncated() { + let long = "abcdefghijklmnop".repeat(10); + let s = render(&long, &[], &opts_small()); + assert!(s.ends_with("…")); + assert!(s.len() < long.len() + 4); + } + + #[test] + fn no_terms_untruncated_has_no_ellipsis() { + let s = render("short text", &[], &opts_small()); + assert_eq!(s, "short text"); + } + + #[test] + fn simple_highlight_wraps_matches() { + let s = render("the quick brown fox", &["quick"], &opts_small()); + assert!(s.contains("quick")); + } + + #[test] + fn case_insensitive_match() { + let s = render("The QUICK brown fox", &["quick"], &opts_small()); + assert!(s.contains("QUICK"), "got {s}"); + } + + #[test] + fn multiple_terms_both_highlighted() { + let s = render( + "the quick brown fox jumps over the lazy dog", + &["quick", "lazy"], + &Options { + approx_chars: 200, + ..Options::default() + }, + ); + assert!(s.contains("quick"), "got {s}"); + assert!(s.contains("lazy"), "got {s}"); + } + + #[test] + fn window_trims_with_ellipsis_on_both_sides() { + let text = + "prefix ".repeat(20) + "MATCH in middle " + &"suffix ".repeat(20); + let s = render(&text, &["MATCH"], &opts_small()); + assert!(s.starts_with("…"), "got {s}"); + assert!(s.ends_with("…"), "got {s}"); + assert!(s.contains("MATCH"), "got {s}"); + } + + #[test] + fn match_at_start_has_no_leading_ellipsis() { + let s = render("MATCH right at the start of this paragraph", &["match"], &opts_small()); + assert!(!s.starts_with("…"), "got {s}"); + } + + #[test] + fn no_match_on_tail_returns_head() { + let text = "alpha beta gamma delta epsilon zeta eta theta iota kappa"; + let s = render(text, &["nomatch"], &opts_small()); + assert!(!s.contains("")); + assert!(s.starts_with("alpha")); + } + + #[test] + fn overlapping_terms_do_not_nest_tags() { + // Two terms matching the same span must coalesce. + let s = render("the RUSTC compiler", &["rust", "rustc"], &opts_small()); + assert!(s.contains("RUSTC"), "got {s}"); + // No nested tags. + assert!(!s.contains(""), "got {s}"); + } + + #[test] + fn utf8_boundary_safe_truncation() { + // Insert multi-byte chars near the window boundary. + let text = "café café café café café café café café café café"; + let s = render(text, &["nope"], &opts_small()); + // Returned string must be valid UTF-8 (push_str guarantees this only + // if we sliced on char boundaries). Assert by round-trip. + assert_eq!(s.as_str(), &s.clone()); + } + + #[test] + fn match_near_right_edge_is_fully_shown() { + let prefix = "x".repeat(30); + let text = format!("{}{}", prefix, "LONGMATCHTERMTEXT"); + let s = render(&text, &["LONGMATCHTERMTEXT"], &opts_small()); + assert!(s.contains("LONGMATCHTERMTEXT"), "got {s}"); + } + + #[test] + fn empty_query_term_ignored() { + let s = render("hello world", &["", "world"], &opts_small()); + assert!(s.contains("world"), "got {s}"); + } +} diff --git a/crates/quicksearch-core/tests/snippet_perf.rs b/crates/quicksearch-core/tests/snippet_perf.rs new file mode 100644 index 0000000..d4ffebe --- /dev/null +++ b/crates/quicksearch-core/tests/snippet_perf.rs @@ -0,0 +1,278 @@ +//! Side-by-side timing for snippet rendering: SQLite's built-in +//! `snippet()` against a regular FTS5 table, vs our zstd-compressed +//! sidecar + Rust renderer. +//! +//! Not a micro-benchmark — we care about end-to-end cost per result page +//! (FTS match + text retrieval + snippet construction) rather than the +//! renderer in isolation. Both paths are run against the same on-disk DB +//! seeded with the same prose, and each query is timed end-to-end from +//! `Connection::prepare` through the final snippet string. +//! +//! Gated by the `QSB_SNIPPET_PERF` env var so the test harness doesn't +//! pay the ~1 s seed cost on every `cargo test`. To run it: +//! +//! ``` +//! QSB_SNIPPET_PERF=1 cargo test --release -p quicksearch-core \ +//! --test snippet_perf -- --nocapture +//! ``` + +use std::path::PathBuf; +use std::time::Instant; + +use quicksearch_core::snippet; +use rusqlite::{params, Connection}; + +const NUM_DOCS: usize = 1000; +const PAGE_SIZE: usize = 50; + +/// Word list we draw text from. Has enough variety that trigram posting +/// lists stay non-trivial (hundreds of terms, not "the" 10000 times). +const WORDS: &[&str] = &[ + "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", + "iota", "kappa", "lambda", "mu", "nu", "xi", "omicron", "pi", "rho", + "sigma", "tau", "upsilon", "phi", "chi", "psi", "omega", + "quick", "brown", "fox", "jumps", "over", "lazy", "dog", + "rust", "cargo", "sqlite", "baloo", "indexer", "tokenizer", "trigram", + "contentless", "posting", "fts5", "snippet", "highlight", + "morning", "afternoon", "evening", "midnight", "yesterday", "today", + "ocean", "forest", "mountain", "river", "valley", "bridge", "tunnel", + "tokyo", "paris", "london", "berlin", "rome", "madrid", "vienna", +]; + +/// Query terms that appear in the seeded corpus, so every query returns +/// real hits (not zero rows, which would skew against both paths equally +/// but wouldn't exercise the snippet renderer at all). +const QUERIES: &[&str] = &[ + "quick", "rust", "baloo", "morning", "paris", + "tokyo", "forest", "indexer", "contentless", "trigram", +]; + +fn seed_text(rng: &mut u64, target_words: usize) -> String { + let mut out = String::with_capacity(target_words * 6); + for _ in 0..target_words { + // xorshift64 — cheap, portable, good enough for text generation. + *rng ^= *rng << 13; + *rng ^= *rng >> 7; + *rng ^= *rng << 17; + let w = WORDS[(*rng as usize) % WORDS.len()]; + out.push_str(w); + out.push(' '); + } + out +} + +fn tmp_path(tag: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "qs-snippet-perf-{}-{}-{}.sqlite", + tag, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + p +} + +#[test] +fn snippet_paths_perf_comparison() { + if std::env::var("QSB_SNIPPET_PERF").is_err() { + eprintln!("skipping: set QSB_SNIPPET_PERF=1 to run"); + return; + } + + let p = tmp_path("both"); + let conn = Connection::open(&p).unwrap(); + conn.execute_batch( + "PRAGMA journal_mode = OFF; + PRAGMA synchronous = 0; + PRAGMA temp_store = MEMORY;", + ) + .unwrap(); + + // Path A: the 'old' shape — regular FTS5 with stored text, snippet() + // built into SQLite. This is what our search SQL used to run before + // schema v3. + conn.execute_batch( + "CREATE VIRTUAL TABLE st_regular USING fts5( + name, text, + tokenize='trigram remove_diacritics 1' + );", + ) + .unwrap(); + + // Path B: the new shape — contentless FTS5 + zstd-compressed sidecar. + // Matches the real schema. We rebuild it in-place here so perf is + // measured against the same DB layout production runs against. + conn.execute_batch( + "CREATE VIRTUAL TABLE st_contentless USING fts5( + name, text, + tokenize='trigram remove_diacritics 1', + content='', + contentless_delete=1 + ); + CREATE TABLE documents_text ( + file_id INTEGER PRIMARY KEY, + text_zstd BLOB NOT NULL, + text_len INTEGER NOT NULL + );", + ) + .unwrap(); + + // Seed both tables with identical content. + let seed_start = Instant::now(); + let mut rng: u64 = 0x1234_5678_9ABC_DEF0; + { + let tx = conn.unchecked_transaction().unwrap(); + { + let mut ins_reg = tx + .prepare( + "INSERT INTO st_regular(rowid, name, text) VALUES (?1, ?2, ?3)", + ) + .unwrap(); + let mut ins_con = tx + .prepare( + "INSERT INTO st_contentless(rowid, name, text) VALUES (?1, ?2, ?3)", + ) + .unwrap(); + let mut ins_blob = tx + .prepare( + "INSERT INTO documents_text(file_id, text_zstd, text_len) VALUES (?1, ?2, ?3)", + ) + .unwrap(); + for i in 1..=NUM_DOCS { + let target = 50 + ((rng as usize) % 400); + let text = seed_text(&mut rng, target); + let name = format!("doc_{:05}.txt", i); + ins_reg.execute(params![i as i64, &name, &text]).unwrap(); + ins_con.execute(params![i as i64, &name, &text]).unwrap(); + let compressed = zstd::encode_all(text.as_bytes(), 3).unwrap(); + ins_blob + .execute(params![i as i64, &compressed, text.len() as i64]) + .unwrap(); + } + } + tx.commit().expect("seed commit"); + } + eprintln!( + "seeded {} docs in both FTS5 shapes in {:.2?}", + NUM_DOCS, + seed_start.elapsed() + ); + + // Warm each table's page cache so the first run doesn't skew. + for q in QUERIES.iter().take(2) { + let mut s = conn + .prepare( + "SELECT rowid FROM st_regular WHERE st_regular MATCH ?1 LIMIT 50", + ) + .unwrap(); + let _ = s.query_map(params![q], |r| r.get::<_, i64>(0)).unwrap().count(); + let mut s = conn + .prepare( + "SELECT rowid FROM st_contentless WHERE st_contentless MATCH ?1 LIMIT 50", + ) + .unwrap(); + let _ = s.query_map(params![q], |r| r.get::<_, i64>(0)).unwrap().count(); + } + + // Path A: SQLite's built-in snippet() on a regular FTS5 table. + let a_reps = 10; + let start_a = Instant::now(); + let mut rows_a_total = 0usize; + for _ in 0..a_reps { + for q in QUERIES { + let mut stmt = conn + .prepare( + "SELECT rowid, name, snippet(st_regular, 1, '', '', '...', 64) \ + FROM st_regular WHERE st_regular MATCH ?1 \ + ORDER BY rank LIMIT ?2", + ) + .unwrap(); + let rows = stmt + .query_map(params![q, PAGE_SIZE as i64], |r| { + Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)) + }) + .unwrap(); + for r in rows { + let _ = r.unwrap(); + rows_a_total += 1; + } + } + } + let dur_a = start_a.elapsed(); + + // Path B: contentless FTS match → pull text_zstd → decompress → render. + let b_reps = 10; + let start_b = Instant::now(); + let mut rows_b_total = 0usize; + let opts = snippet::Options { + pre: "", + post: "", + ellipsis: "...", + approx_chars: 64, + }; + for _ in 0..b_reps { + for q in QUERIES { + // Contentless FTS5 returns NULL for stored columns (that's the + // point of contentless). The real search SQL joins to `files` + // for name/path; here we don't need those fields — we're + // timing the snippet pipeline, not the row projection. + let mut stmt = conn + .prepare( + "SELECT st.rowid, dt.text_zstd \ + FROM st_contentless AS st \ + LEFT JOIN documents_text dt ON dt.file_id = st.rowid \ + WHERE st_contentless MATCH ?1 \ + ORDER BY rank LIMIT ?2", + ) + .unwrap(); + let rows = stmt + .query_map(params![q, PAGE_SIZE as i64], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, Option>>(1)?, + )) + }) + .unwrap(); + for row in rows { + let (_rowid, blob) = row.unwrap(); + let text = match blob { + Some(b) => { + let raw = zstd::decode_all(b.as_slice()).unwrap(); + String::from_utf8(raw).unwrap() + } + None => String::new(), + }; + let _snip = snippet::render(&text, &[q], &opts); + rows_b_total += 1; + } + } + } + let dur_b = start_b.elapsed(); + + // Report — `cargo test -- --nocapture` surfaces this. + let a_per_query = dur_a.as_secs_f64() / (a_reps * QUERIES.len()) as f64 * 1000.0; + let b_per_query = dur_b.as_secs_f64() / (b_reps * QUERIES.len()) as f64 * 1000.0; + let a_per_row = dur_a.as_secs_f64() / rows_a_total as f64 * 1_000_000.0; + let b_per_row = dur_b.as_secs_f64() / rows_b_total as f64 * 1_000_000.0; + eprintln!(); + eprintln!("snippet perf (NUM_DOCS={NUM_DOCS}, PAGE_SIZE={PAGE_SIZE}, QUERIES={}, reps={a_reps}):", + QUERIES.len()); + eprintln!( + " A (SQLite snippet(), regular FTS5): {:.2?} total, {:.2} ms/query, {:.1} µs/row ({} rows)", + dur_a, a_per_query, a_per_row, rows_a_total + ); + eprintln!( + " B (contentless + zstd + Rust snippet): {:.2?} total, {:.2} ms/query, {:.1} µs/row ({} rows)", + dur_b, b_per_query, b_per_row, rows_b_total + ); + eprintln!( + " ratio B/A: {:.2}x (>1 means our path is slower)", + b_per_query / a_per_query + ); + + drop(conn); + let _ = std::fs::remove_file(&p); +} diff --git a/crates/quicksearch-gui/src/main.rs b/crates/quicksearch-gui/src/main.rs index 6756fd1..4a42a9c 100644 --- a/crates/quicksearch-gui/src/main.rs +++ b/crates/quicksearch-gui/src/main.rs @@ -52,13 +52,3 @@ fn app() -> Element { } } -/* -Duplicate files: -SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC; - -Full text search: -SELECT f.name, f.path, snippet(searchabletext, 1, "", "", "...", 64) as "snip" FROM searchabletext AS st JOIN files f ON f.id = st.rowid WHERE st.text MATCH 'searchstring'; - -Filename search: -SELECT name, path FROM files WHERE name LIKE '%searchstring%'; -*/ diff --git a/crates/quicksearch-gui/src/search.rs b/crates/quicksearch-gui/src/search.rs index c2c2ead..7fd6b32 100644 --- a/crates/quicksearch-gui/src/search.rs +++ b/crates/quicksearch-gui/src/search.rs @@ -68,20 +68,35 @@ pub fn Search(props: SearchProps) -> Element { }; let offset = page.saturating_sub(1).saturating_mul(PAGE_SIZE); - let select_sql = match build_select(&args, PAGE_SIZE, offset) { - Ok(s) => s, - Err(e) => { - search_error.set(Some(e)); - is_searching.set(false); - return; + // Validate early for the filename/duplicates branch so we + // surface parse errors before dispatching the blocking task. + let precomputed_select_sql = if args.search_type == "fulltext" { + None + } else { + match build_select(&args, PAGE_SIZE, offset) { + Ok(s) => Some(s), + Err(e) => { + search_error.set(Some(e)); + is_searching.set(false); + return; + } } }; - // Run select + (optional) count in parallel on the blocking pool. + // Fulltext takes the snippet-aware path (decompresses + // documents_text and highlights in Rust); filename + + // duplicates go through the plain SQL executor. let svc1 = service.clone(); let db1 = db_path.clone(); + let args_for_select = args.clone(); let select_handle = tokio::task::spawn_blocking(move || { - svc1.execute_search(&db1, &select_sql) + if args_for_select.search_type == "fulltext" { + svc1.execute_fulltext_search(&db1, &args_for_select, PAGE_SIZE, offset) + } else { + let sql = precomputed_select_sql + .expect("non-fulltext select SQL was prebuilt above"); + svc1.execute_search(&db1, &sql) + } }); let count_handle = count_sql.map(|sql| {