jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

Added zstd compression to text contents, flags for contentless mode, and better baloo integration

Browse source
This commit is contained in:
= 2026-04-23 17:46:11 -04:00
parent a3597dfb31
commit 2a584fbfc0
19 changed files with 1407 additions and 435 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
Cargo.lock generated
View file

@ -3405,6 +3405,7 @@ dependencies = [
"toml",
"walkdir",
"zip",
"zstd 0.13.3",
]
[[package]]
@ -5412,7 +5413,7 @@ dependencies = [
"pbkdf2",
"sha1",
"time",
"zstd",
"zstd 0.11.2+zstd.1.5.2",
]
[[package]]
@ -5421,7 +5422,16 @@ version = "0.11.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
dependencies = [
"zstd-safe",
"zstd-safe 5.0.2+zstd.1.5.2",
]
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe 7.2.4",
]
[[package]]
@ -5434,6 +5444,15 @@ dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-safe"
version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.16+zstd.1.5.7"

10
config_example.toml
View file

@ -25,3 +25,13 @@ include_hidden = false
# FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61')
# Look here for more information https://www.sqlite.org/fts5.html#tokenizers
tokenize = "trigram"
# If true (default), extracted text is stored zstd-compressed in a sidecar
# table so the GUI's search results can render snippet previews with the
# query terms highlighted. If false, the inverted FTS5 index is still
# populated (so queries match the same files) but nothing is stored
# alongside; result rows carry no snippet. Turning this off drops the
# on-disk footprint to roughly what stock Baloo uses, useful for
# apples-to-apples comparisons and for users who never read snippet text.
# Changing this only affects files indexed *after* the change; existing
# sidecar rows are kept until re-indexed.
store_text_for_snippets = true

1
crates/quicksearch-core/Cargo.toml
View file

@ -23,3 +23,4 @@ lofty = "0.19"
kamadak-exif = "0.5"
notify = "6.1"
ctrlc = "3.4"
zstd = "0.13"

95
crates/quicksearch-core/src/cli.rs
View file

@ -8,7 +8,7 @@
use rusqlite::{params, OptionalExtension};
use crate::db::open_and_migrate;
use crate::db::open_or_recreate;
use crate::db::repo::{STATE_DONE, STATE_FAILED, STATE_NA, STATE_PENDING};
/// Per-file indexing status, mirroring Baloo's multi-state reporting.
@ -55,19 +55,38 @@ pub struct FailedEntry {
/// Storage footprint report. "Partitions" correspond to SQL tables for our
/// SQLite layout (Baloo's LMDB has named sub-DBs; our equivalent is per-table
/// row/size counts).
#[derive(Debug, Clone, PartialEq, Eq)]
///
/// `documents_text_*` fields cover the zstd-compressed extracted-text
/// sidecar that replaced the regular FTS5 stored text in schema v3. Ratio
/// is `compressed / raw` so a value of ~0.3 means we saved ~70% vs storing
/// the plaintext verbatim.
#[derive(Debug, Clone, PartialEq)]
pub struct SizeReport {
pub file_size_bytes: u64,
pub files_row_count: i64,
pub properties_row_count: i64,
pub failed_files_row_count: i64,
pub searchabletext_row_count: i64,
pub documents_text_row_count: i64,
pub documents_text_raw_bytes: i64,
pub documents_text_compressed_bytes: i64,
}
impl SizeReport {
/// Compressed:raw ratio for the stored extracted text. `None` when no
/// rows have been written yet (avoids divide-by-zero).
pub fn documents_text_ratio(&self) -> Option<f64> {
if self.documents_text_raw_bytes <= 0 {
return None;
}
Some(self.documents_text_compressed_bytes as f64 / self.documents_text_raw_bytes as f64)
}
}
/// Query the per-file indexing status. Returns `FileStatus` with
/// `basic == NotIndexed` if the path isn't in the database.
pub fn status_for_path(db_path: &str, path: &str) -> Result<FileStatus, String> {
let conn = open_and_migrate(db_path, "trigram")?;
let conn = open_or_recreate(db_path, "trigram")?;
let row: Option<(i64, i64, Option<String>)> = conn
.query_row(
"SELECT basic_state, content_state, failure_msg FROM files WHERE path = ?1",
@ -94,7 +113,7 @@ pub fn status_for_path(db_path: &str, path: &str) -> Result<FileStatus, String>
/// Return every file that failed content extraction, newest first.
pub fn list_failed(db_path: &str, limit: Option<u32>) -> Result<Vec<FailedEntry>, String> {
let conn = open_and_migrate(db_path, "trigram")?;
let conn = open_or_recreate(db_path, "trigram")?;
let limit_sql = match limit {
Some(n) => format!(" LIMIT {}", n),
None => String::new(),
@ -128,17 +147,29 @@ pub fn index_size_breakdown(db_path: &str) -> Result<SizeReport, String> {
let file_size_bytes = std::fs::metadata(db_path)
.map(|m| m.len())
.unwrap_or(0);
let conn = open_and_migrate(db_path, "trigram")?;
let conn = open_or_recreate(db_path, "trigram")?;
let count = |table: &str| -> Result<i64, String> {
conn.query_row(&format!("SELECT COUNT(*) FROM {}", table), [], |r| r.get(0))
.map_err(|e| format!("count {}: {}", table, e))
};
let dt_row_count: i64 = count("documents_text")?;
let (dt_raw, dt_compressed): (i64, i64) = conn
.query_row(
"SELECT COALESCE(SUM(text_len), 0), COALESCE(SUM(LENGTH(text_zstd)), 0) FROM documents_text",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.map_err(|e| format!("documents_text size sum: {}", e))?;
Ok(SizeReport {
file_size_bytes,
files_row_count: count("files")?,
properties_row_count: count("properties")?,
failed_files_row_count: count("failed_files")?,
searchabletext_row_count: count("searchabletext")?,
documents_text_row_count: dt_row_count,
documents_text_raw_bytes: dt_raw,
documents_text_compressed_bytes: dt_compressed,
})
}
@ -150,7 +181,7 @@ pub fn index_size_breakdown(db_path: &str) -> Result<SizeReport, String> {
/// Used by the Baloo compat daemon to report the "Files waiting for content
/// indexing" figure both to balooctl and to the LMDB mirror.
pub fn pending_content_count(db_path: &str) -> Result<i64, String> {
let conn = open_and_migrate(db_path, "trigram")?;
let conn = open_or_recreate(db_path, "trigram")?;
conn.query_row(
"SELECT COUNT(*) FROM files WHERE content_state = ?1",
rusqlite::params![crate::db::repo::STATE_PENDING],
@ -162,7 +193,7 @@ pub fn pending_content_count(db_path: &str) -> Result<i64, String> {
/// Remove a single file from the index. Returns whether a row was deleted.
/// Keeps FTS/documents/properties in sync via the repo helpers.
pub fn clear_path(db_path: &str, path: &str) -> Result<bool, String> {
let mut conn = open_and_migrate(db_path, "trigram")?;
let mut conn = open_or_recreate(db_path, "trigram")?;
let tx = conn
.transaction()
.map_err(|e| format!("clear_path begin tx: {}", e))?;
@ -192,7 +223,7 @@ mod tests {
}
fn seed_fixture(db_path: &str) -> (i64, i64) {
let mut conn = open_and_migrate(db_path, "trigram").unwrap();
let mut conn = open_or_recreate(db_path, "trigram").unwrap();
let (a, b) = {
let tx = conn.transaction().unwrap();
let a = insert_file(
@ -212,7 +243,7 @@ mod tests {
)
.unwrap()
.expect("unique path");
set_content_done(&tx, a, "a.txt", "hello", &[]).unwrap();
set_content_done(&tx, a, "a.txt", "hello", &[], true).unwrap();
let b = insert_file(
&tx,
&NewFile {
@ -281,6 +312,52 @@ mod tests {
assert!(r.file_size_bytes > 0);
assert_eq!(r.files_row_count, 2);
assert_eq!(r.failed_files_row_count, 1);
// File a got content ("hello"); file b failed. Only one documents_text row.
assert_eq!(r.documents_text_row_count, 1);
assert_eq!(r.documents_text_raw_bytes, "hello".len() as i64);
assert!(r.documents_text_compressed_bytes > 0);
std::fs::remove_file(&p).ok();
}
#[test]
fn documents_text_ratio_reports_savings_on_compressible_prose() {
// Feed highly-compressible prose (lots of repeated words) and verify
// the reported ratio reflects real savings. Guards against anyone
// silently swapping the compression step for a pass-through.
let p = tmp_path();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
{
let tx = conn.transaction().unwrap();
let id = insert_file(
&tx,
&NewFile {
name: "big.txt",
path: "/tmp/big.txt",
parent: "/tmp",
size: 1,
mtime: 1,
inode: None,
device_id: None,
mime: Some("text/plain"),
ftype: FileType::TEXT,
hash: None,
},
)
.unwrap()
.expect("unique path");
let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500);
set_content_done(&tx, id, "big.txt", &prose, &[], true).unwrap();
tx.commit().unwrap();
}
drop(conn);
let r = index_size_breakdown(p.to_str().unwrap()).unwrap();
let ratio = r.documents_text_ratio().expect("has rows");
// Repeating a 44-byte sentence 500x → zstd should hit <20% ratio
// trivially. Loose bound protects the test from zstd version churn.
assert!(ratio < 0.3, "ratio too high: {ratio} raw={} comp={}",
r.documents_text_raw_bytes, r.documents_text_compressed_bytes);
std::fs::remove_file(&p).ok();
}

16
crates/quicksearch-core/src/config.rs
View file

@ -31,6 +31,10 @@ fn default_home_path() -> String {
".".to_string()
}
fn default_store_text_for_snippets() -> bool {
true
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ProcessingConfig {
pub hash_length: usize,
@ -46,6 +50,17 @@ pub struct ProcessingConfig {
pub follow_symlinks: bool,
#[serde(default)]
pub include_hidden: bool,
/// When `true` (default), extracted text is stored zstd-compressed in
/// `documents_text` so search results can render snippet/highlight
/// previews without re-reading the source file. When `false` the
/// inverted FTS5 index still gets the tokens (so queries return the
/// same hits) but nothing is stored alongside; search results carry
/// no snippet text and rely on filename/path only. This mode drops
/// the on-disk footprint to roughly what stock Baloo uses, at the
/// cost of snippet functionality — useful for apples-to-apples size
/// comparisons and for users who never look at result previews.
#[serde(default = "default_store_text_for_snippets")]
pub store_text_for_snippets: bool,
}
impl Default for Config {
@ -65,6 +80,7 @@ impl Default for Config {
precount_files_for_progress: false,
follow_symlinks: false,
include_hidden: false,
store_text_for_snippets: true,
},
}
}

340
crates/quicksearch-core/src/db/migrate.rs
View file

@ -1,340 +0,0 @@
//! Schema version detection and upgrade.
//!
//! Callers should always enter the DB via [`open_and_migrate`]. It applies
//! pragmas, detects the on-disk schema version, and upgrades or recreates as
//! required.
//!
//! Current policy: Set A introduces schema v1 and is the first versioned
//! release. Any pre-A database (has the legacy `files(name, path, size,
//! moddate, hash)` shape and no `schema_info` table) is wiped and rebuilt —
//! the user will re-index. A prominent log line is printed so the behavior is
//! not silent. Future migrations should prefer ALTER TABLE.
use std::path::Path;
use rusqlite::{params, Connection, OptionalExtension};
use super::schema::{effective_tokenizer, fts_create_sql, PRAGMAS_FAST, SCHEMA_CURRENT};
pub const CURRENT_SCHEMA_VERSION: u32 = 2;
/// Open the database at `db_path`, apply pragmas, and ensure the schema is at
/// [`CURRENT_SCHEMA_VERSION`]. Recreates the DB if a pre-versioned layout is
/// detected.
///
/// `tokenizer` is used when (re)creating the FTS5 virtual table. It has no
/// effect on an already-current DB.
pub fn open_and_migrate(db_path: &str, tokenizer: &str) -> Result<Connection, String> {
let path_for_rebuild = Path::new(db_path).to_path_buf();
let mut conn = Connection::open(db_path)
.map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?;
conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas: {}", e))?;
let version = read_schema_version(&conn)?;
match version {
Some(v) if v == CURRENT_SCHEMA_VERSION => {
// Schema version matches. Before returning, check whether the
// FTS5 tokenizer config has drifted (e.g. someone upgraded to a
// build that switched the default to `trigram remove_diacritics 1`).
// If so, rebuild the FTS table in place — cheaper than a full
// DB wipe since `files` rows stay intact; only content
// extraction (phase 2) re-runs.
maybe_rebuild_fts_for_tokenizer_change(&conn, tokenizer)?;
}
Some(v) if v > CURRENT_SCHEMA_VERSION => {
return Err(format!(
"Database schema version {} is newer than this build ({}). \
Use a newer QuickSearch or move the database aside.",
v, CURRENT_SCHEMA_VERSION
));
}
Some(v) => {
// Older schema version. Set A's upgrade policy: wipe and rebuild.
// When we start adding ALTER-based migrations this match arm
// will gain a proper stepwise runner.
eprintln!(
"QuickSearch: database at {} is schema v{}; rebuilding to v{}. \
Existing rows will be re-scanned.",
db_path, v, CURRENT_SCHEMA_VERSION
);
conn = wipe_and_reopen(conn, &path_for_rebuild)?;
apply_current_schema(&conn, tokenizer)?;
}
None => {
// No schema_info row. Either an empty DB (good — just create) or a
// legacy pre-A layout (detected by presence of the old `files`
// table). Legacy layouts are wiped.
if has_legacy_layout(&conn)? {
eprintln!(
"QuickSearch: legacy database detected at {}; rebuilding index with schema v{}. \
Existing file rows will be re-scanned.",
db_path, CURRENT_SCHEMA_VERSION
);
conn = wipe_and_reopen(conn, &path_for_rebuild)?;
}
apply_current_schema(&conn, tokenizer)?;
}
}
Ok(conn)
}
/// If the stored `schema_info.tokenize` value doesn't match the effective
/// tokenizer the caller wants now, drop and recreate `searchabletext` with
/// the new tokenizer and reset `files.content_state` so the text-extraction
/// phase re-runs on next indexing pass. Keeps the `files`, `properties`,
/// and `failed_files` rows intact.
fn maybe_rebuild_fts_for_tokenizer_change(
conn: &Connection,
tokenizer: &str,
) -> Result<(), String> {
let want = effective_tokenizer(tokenizer);
let stored: Option<String> = conn
.query_row(
"SELECT value FROM schema_info WHERE key = 'tokenize'",
[],
|r| r.get(0),
)
.optional()
.map_err(|e| format!("read schema_info.tokenize: {}", e))?;
if stored.as_deref() == Some(&*want) {
return Ok(());
}
eprintln!(
"QuickSearch: FTS5 tokenizer changed from {:?} to {:?}; rebuilding searchabletext. \
File metadata is preserved; content extraction will re-run on next indexing pass.",
stored.as_deref().unwrap_or("(none)"),
want
);
conn.execute("DROP TABLE IF EXISTS searchabletext", [])
.map_err(|e| format!("drop searchabletext: {}", e))?;
let create_sql = fts_create_sql(tokenizer);
conn.execute_batch(&create_sql)
.map_err(|e| format!("recreate searchabletext: {}", e))?;
conn.execute(
"UPDATE files SET content_state = 0 WHERE content_state != 0",
[],
)
.map_err(|e| format!("reset content_state: {}", e))?;
conn.execute(
"INSERT OR REPLACE INTO schema_info(key, value) VALUES ('tokenize', ?1)",
params![want],
)
.map_err(|e| format!("update schema_info.tokenize: {}", e))?;
Ok(())
}
fn wipe_and_reopen(
conn: Connection,
path_for_rebuild: &std::path::Path,
) -> Result<Connection, String> {
drop(conn);
std::fs::remove_file(path_for_rebuild)
.map_err(|e| format!("Failed to remove old database: {}", e))?;
// Remove WAL/SHM/journal sidecars defensively even though journal_mode=OFF.
for suffix in ["-wal", "-shm", "-journal"] {
let sidecar = path_for_rebuild.with_file_name(format!(
"{}{}",
path_for_rebuild
.file_name()
.and_then(|s| s.to_str())
.unwrap_or(""),
suffix
));
let _ = std::fs::remove_file(sidecar);
}
let conn = Connection::open(path_for_rebuild)
.map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?;
conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?;
Ok(conn)
}
fn read_schema_version(conn: &Connection) -> Result<Option<u32>, String> {
let has_info: bool = conn
.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_info'",
[],
|_| Ok(true),
)
.optional()
.map_err(|e| format!("sqlite_master schema_info: {}", e))?
.unwrap_or(false);
if !has_info {
return Ok(None);
}
let v: Option<String> = conn
.query_row(
"SELECT value FROM schema_info WHERE key = 'version'",
[],
|r| r.get(0),
)
.optional()
.map_err(|e| format!("read schema_info.version: {}", e))?;
match v {
Some(s) => s
.parse::<u32>()
.map(Some)
.map_err(|e| format!("invalid schema_info.version {:?}: {}", s, e)),
None => Ok(None),
}
}
fn has_legacy_layout(conn: &Connection) -> Result<bool, String> {
// Old layout has a `files` table without an `id INTEGER PRIMARY KEY`.
let has_files: bool = conn
.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='files'",
[],
|_| Ok(true),
)
.optional()
.map_err(|e| format!("sqlite_master files: {}", e))?
.unwrap_or(false);
if !has_files {
return Ok(false);
}
// Check whether the columns match the legacy shape.
let mut stmt = conn
.prepare("PRAGMA table_info(files)")
.map_err(|e| format!("pragma table_info: {}", e))?;
let has_id = stmt
.query_map([], |row| row.get::<_, String>(1))
.map_err(|e| format!("table_info query: {}", e))?
.filter_map(|r| r.ok())
.any(|name| name == "id");
Ok(!has_id)
}
fn apply_current_schema(conn: &Connection, tokenizer: &str) -> Result<(), String> {
conn.execute_batch(SCHEMA_CURRENT)
.map_err(|e| format!("Failed to create current schema tables: {}", e))?;
let fts = fts_create_sql(tokenizer);
conn.execute_batch(&fts)
.map_err(|e| format!("Failed to create searchabletext: {}", e))?;
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
// Store the *effective* tokenizer string (with any default options we
// auto-applied). On subsequent opens we compare this against what the
// caller asks for and rebuild the FTS table if it changed.
let effective = effective_tokenizer(tokenizer);
conn.execute(
"INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)",
params![CURRENT_SCHEMA_VERSION.to_string(), now.to_string(), effective],
)
.map_err(|e| format!("Failed to seed schema_info: {}", e))?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn tmp_db_path() -> std::path::PathBuf {
let mut p = std::env::temp_dir();
p.push(format!(
"quicksearch-test-{}-{}.sqlite",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
p
}
#[test]
fn fresh_db_gets_current_version() {
let p = tmp_db_path();
let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0))
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn reopen_is_idempotent() {
let p = tmp_db_path();
{
let _ = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
}
let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0))
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn older_versioned_db_is_wiped_and_recreated() {
// Simulate a DB that was created at a previous schema version.
let p = tmp_db_path();
{
let conn = Connection::open(&p).unwrap();
conn.execute("CREATE TABLE schema_info (key TEXT PRIMARY KEY, value TEXT NOT NULL)", []).unwrap();
conn.execute(
"INSERT INTO schema_info(key,value) VALUES('version','1')",
[],
)
.unwrap();
conn.execute("CREATE TABLE files (id INTEGER PRIMARY KEY, name TEXT)", []).unwrap();
conn.execute("INSERT INTO files(name) VALUES('a.txt')", []).unwrap();
}
let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row("SELECT value FROM schema_info WHERE key='version'", [], |r| r.get(0))
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0, "old rows should be wiped");
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn legacy_db_is_wiped_and_recreated() {
let p = tmp_db_path();
{
// Simulate a pre-A database.
let conn = Connection::open(&p).unwrap();
conn.execute(
"CREATE TABLE files (name TEXT, path TEXT, size INTEGER, moddate INTEGER, hash BLOB)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO files VALUES ('a.txt', '/tmp/a.txt', 1, 2, X'00')",
[],
)
.unwrap();
}
let conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
// Old row should be gone.
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0);
// New columns should exist.
let _ = conn
.query_row("SELECT basic_state, content_state, type, mime FROM files LIMIT 0", [], |_| Ok(()))
.or_else(|e| if matches!(e, rusqlite::Error::QueryReturnedNoRows) { Ok(()) } else { Err(e) })
.unwrap();
drop(conn);
std::fs::remove_file(&p).ok();
}
}

16
crates/quicksearch-core/src/db/mod.rs
View file

@ -1,13 +1,13 @@
//! SQLite schema, migrations, and row-level repository helpers.
//! SQLite schema, on-disk open/recreate, and row-level repository helpers.
//!
//! The only "live" schema is `CURRENT_SCHEMA_VERSION` (see [`schema`]). Older
//! databases are detected in [`migrate::open_and_migrate`] and recreated from
//! scratch — Set A of the QuickSearch → Baloo work is the first schema bump
//! and carries no rows we'd want to preserve. Subsequent migrations should
//! prefer `ALTER TABLE` and versioned steps.
//! Policy: a single [`open::open_or_recreate`] is the only entry point. Any
//! schema mismatch — wrong version, drifted tokenizer, absent `schema_info`
//! — wipes the DB and rebuilds from [`schema::SCHEMA_CURRENT`]. There are
//! no in-place migrations by design; re-indexing is accepted as the cost
//! of avoiding migration-path complexity.
pub mod migrate;
pub mod open;
pub mod repo;
pub mod schema;
pub use migrate::{open_and_migrate, CURRENT_SCHEMA_VERSION};
pub use open::{open_or_recreate, CURRENT_SCHEMA_VERSION};

329
crates/quicksearch-core/src/db/open.rs Normal file
View file

@ -0,0 +1,329 @@
//! Open-or-recreate: the sole entry point into the on-disk database.
//!
//! **Policy**: any schema mismatch — wrong `schema_info.version`, wrong
//! stored `tokenize` string, absent `schema_info` table, or any other
//! drift from what this build expects — wipes the database file and
//! recreates it from scratch. There are deliberately **no** in-place
//! migrations.
//!
//! The tradeoff: users pay a re-index cost every time the shipped schema
//! changes. Our indexing is fast (see `bench/`) and schema changes are
//! rare in practice, so the code-complexity cost of maintaining real
//! migration paths wasn't worth it. A single `open_or_recreate` replaces
//! what used to be version detection + tokenizer-drift FTS rebuild +
//! legacy-layout recovery, all of which ultimately wiped anyway.
use std::path::Path;
use rusqlite::{params, Connection, OptionalExtension};
use super::schema::{effective_tokenizer, fts_create_sql, PRAGMAS_FAST, SCHEMA_CURRENT};
/// Bump this whenever [`SCHEMA_CURRENT`] or [`fts_create_sql`] changes in
/// a way that makes an old DB unreadable by new code. Any such bump
/// causes existing indexes to be wiped on next open — there's no
/// migration path by design.
pub const CURRENT_SCHEMA_VERSION: u32 = 3;
/// Open `db_path`, applying fast-path pragmas, and ensure the on-disk
/// schema matches what this build expects. If it doesn't, delete the
/// file and recreate it empty — callers will need to re-index.
///
/// `tokenizer` is passed to FTS5's `tokenize=` option when (re)creating
/// `searchabletext`. Changing it against an existing DB counts as a
/// schema mismatch and triggers the wipe-and-recreate path.
pub fn open_or_recreate(db_path: &str, tokenizer: &str) -> Result<Connection, String> {
let path = Path::new(db_path).to_path_buf();
let conn = Connection::open(db_path)
.map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?;
conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas: {}", e))?;
if db_matches_current(&conn, tokenizer)? {
return Ok(conn);
}
// Schema is present but stale, or pre-existing rows belong to an
// older layout, or the tokenizer drifted. Log once so the rebuild
// isn't silent, then wipe + recreate.
eprintln!(
"QuickSearch: database at {} does not match current schema; rebuilding. \
Existing rows will be re-scanned on next indexing run.",
db_path
);
let conn = wipe_and_reopen(conn, &path)?;
apply_current_schema(&conn, tokenizer)?;
Ok(conn)
}
/// True iff the DB has `schema_info` with the current version *and* the
/// effective-tokenizer string this caller asked for. Anything else —
/// missing table, wrong version, different tokenizer — returns false.
fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result<bool, String> {
let has_info: bool = conn
.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_info'",
[],
|_| Ok(true),
)
.optional()
.map_err(|e| format!("sqlite_master schema_info: {}", e))?
.unwrap_or(false);
if !has_info {
return Ok(false);
}
let version: Option<String> = conn
.query_row(
"SELECT value FROM schema_info WHERE key = 'version'",
[],
|r| r.get(0),
)
.optional()
.map_err(|e| format!("read schema_info.version: {}", e))?;
let version_ok = version.as_deref() == Some(&CURRENT_SCHEMA_VERSION.to_string());
if !version_ok {
return Ok(false);
}
let stored_tokenize: Option<String> = conn
.query_row(
"SELECT value FROM schema_info WHERE key = 'tokenize'",
[],
|r| r.get(0),
)
.optional()
.map_err(|e| format!("read schema_info.tokenize: {}", e))?;
let want_tokenize = effective_tokenizer(tokenizer);
Ok(stored_tokenize.as_deref() == Some(&*want_tokenize))
}
/// Drop the current connection, delete the DB file + its WAL/SHM/journal
/// sidecars, reopen a fresh file, re-apply pragmas.
fn wipe_and_reopen(conn: Connection, path: &Path) -> Result<Connection, String> {
drop(conn);
// Primary file may already be absent (fresh open that just needed
// the table applied). Ignore NotFound; anything else is an error.
match std::fs::remove_file(path) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(format!("Failed to remove old database: {}", e)),
}
// Sidecars are optional — delete best-effort.
for suffix in ["-wal", "-shm", "-journal"] {
let sidecar = path.with_file_name(format!(
"{}{}",
path.file_name().and_then(|s| s.to_str()).unwrap_or(""),
suffix
));
let _ = std::fs::remove_file(sidecar);
}
let conn = Connection::open(path)
.map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?;
conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?;
Ok(conn)
}
/// Apply [`SCHEMA_CURRENT`] + [`fts_create_sql`] to a blank DB and seed
/// `schema_info` with the matching version/tokenize markers.
fn apply_current_schema(conn: &Connection, tokenizer: &str) -> Result<(), String> {
conn.execute_batch(SCHEMA_CURRENT)
.map_err(|e| format!("Failed to create current schema tables: {}", e))?;
let fts = fts_create_sql(tokenizer);
conn.execute_batch(&fts)
.map_err(|e| format!("Failed to create searchabletext: {}", e))?;
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
let effective = effective_tokenizer(tokenizer);
conn.execute(
"INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)",
params![
CURRENT_SCHEMA_VERSION.to_string(),
now.to_string(),
effective
],
)
.map_err(|e| format!("Failed to seed schema_info: {}", e))?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn tmp_db_path() -> std::path::PathBuf {
let mut p = std::env::temp_dir();
p.push(format!(
"quicksearch-test-{}-{}.sqlite",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
p
}
#[test]
fn fresh_db_gets_current_version() {
let p = tmp_db_path();
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row(
"SELECT value FROM schema_info WHERE key='version'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn reopen_is_idempotent() {
let p = tmp_db_path();
{
let _ = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
}
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row(
"SELECT value FROM schema_info WHERE key='version'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn older_versioned_db_is_wiped_and_recreated() {
// Simulate a DB from a prior schema version. Our policy is to
// wipe without attempting any migration.
let p = tmp_db_path();
{
let conn = Connection::open(&p).unwrap();
conn.execute(
"CREATE TABLE schema_info (key TEXT PRIMARY KEY, value TEXT NOT NULL)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO schema_info(key,value) VALUES('version','1')",
[],
)
.unwrap();
conn.execute("CREATE TABLE files (id INTEGER PRIMARY KEY, name TEXT)", [])
.unwrap();
conn.execute("INSERT INTO files(name) VALUES('a.txt')", [])
.unwrap();
}
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row(
"SELECT value FROM schema_info WHERE key='version'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0, "old rows should be wiped");
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn legacy_layout_db_is_wiped_and_recreated() {
// Pre-A layout with no `schema_info` at all. Same policy — wipe.
let p = tmp_db_path();
{
let conn = Connection::open(&p).unwrap();
conn.execute(
"CREATE TABLE files (name TEXT, path TEXT, size INTEGER, moddate INTEGER, hash BLOB)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO files VALUES ('a.txt', '/tmp/a.txt', 1, 2, X'00')",
[],
)
.unwrap();
}
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
// Old row should be gone.
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0);
// New columns should exist (just prepare the SELECT — an
// unknown column name would parse-error here).
let _ = conn
.query_row(
"SELECT basic_state, content_state, type, mime FROM files LIMIT 0",
[],
|_| Ok(()),
)
.or_else(|e| {
if matches!(e, rusqlite::Error::QueryReturnedNoRows) {
Ok(())
} else {
Err(e)
}
})
.unwrap();
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn tokenizer_drift_wipes_db() {
// Previously this was "rebuild FTS in place and reset
// content_state". New policy: full wipe.
let p = tmp_db_path();
let first_effective = {
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('x', '/x', '/', 0, 0)",
[],
)
.unwrap();
let stored: String = conn
.query_row(
"SELECT value FROM schema_info WHERE key='tokenize'",
[],
|r| r.get(0),
)
.unwrap();
drop(conn);
stored
};
// Second open with a different tokenizer.
let conn = open_or_recreate(p.to_str().unwrap(), "unicode61").unwrap();
let files_count: i64 = conn
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap();
assert_eq!(files_count, 0, "tokenizer drift should wipe rows");
let new_stored: String = conn
.query_row(
"SELECT value FROM schema_info WHERE key='tokenize'",
[],
|r| r.get(0),
)
.unwrap();
assert_ne!(first_effective, new_stored);
drop(conn);
std::fs::remove_file(&p).ok();
}
}

62
crates/quicksearch-core/src/db/repo.rs
View file

@ -121,15 +121,21 @@ pub fn update_file_basic(
}
/// Mark a file's content indexing as complete and write the extracted text +
/// properties atomically. FTS5 stores the canonical text for this file;
/// `properties` are stored both as a structured side-table (for exact
/// retrieval) and concatenated into the FTS `properties` column (for MATCH).
/// properties atomically. The plaintext is fed to the contentless FTS5
/// tokenizer (which keeps only the inverted index) and — when `store_text`
/// is `true` — separately stored zstd-compressed in `documents_text` for
/// on-demand snippet rendering. When `store_text=false` the sidecar INSERT
/// is skipped: queries still match the right files but result rows can't
/// render snippets. `properties` are stored both as a structured side-
/// table (for exact retrieval) and concatenated into the FTS `properties`
/// column (for MATCH).
pub fn set_content_done(
tx: &Transaction<'_>,
file_id: i64,
name: &str,
text: &str,
properties: &[(String, String)],
store_text: bool,
) -> Result<(), String> {
// Clear any previous extraction (in case of re-run).
remove_content_for_id(tx, file_id)?;
@ -142,12 +148,28 @@ pub fn set_content_done(
.map_err(|e| format!("insert property {}={}: {}", k, v, e))?;
}
let props_blob = encode_properties_for_fts(properties);
// Contentless FTS5 still accepts values on INSERT — the tokenizer needs
// them — it simply doesn't persist the raw column values.
tx.execute(
"INSERT INTO searchabletext(rowid, name, text, properties) VALUES (?1, ?2, ?3, ?4)",
params![file_id, name, text, props_blob],
)
.map_err(|e| format!("insert FTS row {}: {}", file_id, e))?;
// Skip the compressed sidecar when: the config disables snippet storage
// outright, or there's no body text (e.g. an image whose extractor
// returned only EXIF properties). The second case saves a zstd frame
// on what would otherwise be an empty blob.
if store_text && !text.is_empty() {
let compressed = zstd::encode_all(text.as_bytes(), ZSTD_LEVEL)
.map_err(|e| format!("zstd encode for file {}: {}", file_id, e))?;
tx.execute(
"INSERT INTO documents_text(file_id, text_zstd, text_len) VALUES (?1, ?2, ?3)",
params![file_id, compressed, text.len() as i64],
)
.map_err(|e| format!("insert documents_text {}: {}", file_id, e))?;
}
tx.execute(
"UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2",
params![STATE_DONE, file_id],
@ -159,6 +181,13 @@ pub fn set_content_done(
Ok(())
}
/// zstd level tuned for extracted-text prose. Level 3 hits ~3-5× on English
/// prose at high throughput (hundreds of MB/s) — level 9+ would shave a few
/// percent more but at 10× the CPU cost during indexing. Wrong knob to
/// tune: readers decompress far faster than writers compress, so keep
/// write-side cost low.
const ZSTD_LEVEL: i32 = 3;
/// Mark a file's content extraction as failed. Keeps the basic row in place.
pub fn set_content_failed(
tx: &Transaction<'_>,
@ -213,16 +242,22 @@ pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result<bool, Str
Ok(true)
}
/// Remove the FTS row and any `properties` rows for a given file id. Does
/// not touch the `files` row itself. Idempotent — a missing FTS row is fine.
/// Remove the FTS row, compressed text blob, and any `properties` rows for
/// a given file id. Does not touch the `files` row itself. Idempotent — a
/// missing row is fine.
pub fn remove_content_for_id(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
// Regular FTS5 supports a plain DELETE by rowid; no need to supply the
// old column values the way a contentless table would require.
// `contentless_delete=1` on the FTS5 table makes this work without
// re-supplying the old column values (it tombstones the rowid).
tx.execute(
"DELETE FROM searchabletext WHERE rowid = ?1",
params![file_id],
)
.map_err(|e| format!("FTS delete row {}: {}", file_id, e))?;
tx.execute(
"DELETE FROM documents_text WHERE file_id = ?1",
params![file_id],
)
.map_err(|e| format!("delete documents_text {}: {}", file_id, e))?;
tx.execute("DELETE FROM properties WHERE file_id = ?1", params![file_id])
.map_err(|e| format!("delete properties {}: {}", file_id, e))?;
Ok(())
@ -257,7 +292,7 @@ pub fn checkpoint_and_close(conn: Connection) {
#[cfg(test)]
mod tests {
use super::*;
use crate::db::open_and_migrate;
use crate::db::open_or_recreate;
fn tmp_path() -> std::path::PathBuf {
let mut p = std::env::temp_dir();
@ -275,7 +310,7 @@ mod tests {
#[test]
fn insert_update_delete_round_trip() {
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
{
let tx = conn.transaction().unwrap();
let id = insert_file(
@ -301,6 +336,7 @@ mod tests {
"a.txt",
"hello world",
&[("title".to_string(), "hi".to_string())],
true,
)
.unwrap();
tx.commit().unwrap();
@ -338,7 +374,7 @@ mod tests {
#[test]
fn update_resets_content_state() {
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let id = {
let tx = conn.transaction().unwrap();
let id = insert_file(
@ -358,7 +394,7 @@ mod tests {
)
.unwrap()
.expect("unique path");
set_content_done(&tx, id, "a.txt", "old text", &[]).unwrap();
set_content_done(&tx, id, "a.txt", "old text", &[], true).unwrap();
tx.commit().unwrap();
id
};
@ -409,7 +445,7 @@ mod tests {
// (overlapping roots, symlink resolution quirks), the second INSERT
// must be a silent no-op, not a run-ending error.
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let tx = conn.transaction().unwrap();
let row = NewFile {
name: "dup.txt",
@ -447,7 +483,7 @@ mod tests {
#[test]
fn set_content_failed_writes_failed_table() {
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let id = {
let tx = conn.transaction().unwrap();
let id = insert_file(

44
crates/quicksearch-core/src/db/schema.rs
View file

@ -12,12 +12,16 @@ pub const PRAGMAS_FAST: &str = "
PRAGMA foreign_keys = ON;
";
/// The full current schema. Applied by [`migrate::open_and_migrate`] when
/// the DB is fresh or has been wiped during upgrade.
/// The full current schema. Applied by [`super::open::open_or_recreate`]
/// when the DB is fresh or has just been wiped because it drifted from
/// [`super::open::CURRENT_SCHEMA_VERSION`].
///
/// FTS5 is a *regular* (non-contentless) virtual table so `snippet()` and
/// `highlight()` can read the stored text. This also means there is no
/// separate `documents` table — FTS5 *is* the text store.
/// FTS5 is *contentless* (see [`fts_create_sql`]): the inverted index is kept
/// but the column values aren't stored. The canonical extracted text lives
/// in a separate `documents_text` table, zstd-compressed. Snippet rendering
/// for search results decompresses on demand and highlights matches in Rust
/// (see `crate::snippet`). This keeps the on-disk footprint close to Baloo's
/// LMDB-only size while still supporting snippet/highlight features.
pub const SCHEMA_CURRENT: &str = r#"
CREATE TABLE schema_info (
key TEXT PRIMARY KEY,
@ -61,6 +65,16 @@ CREATE TABLE failed_files (
ts INTEGER NOT NULL
);
-- Canonical extracted text for every successfully content-indexed file.
-- Compressed with zstd (see `crate::db::repo::set_content_done`). Only
-- written when the extractor produced text; absent rows mean "no body
-- text" (e.g. an image with only EXIF properties).
CREATE TABLE documents_text (
file_id INTEGER PRIMARY KEY REFERENCES files(id) ON DELETE CASCADE,
text_zstd BLOB NOT NULL,
text_len INTEGER NOT NULL -- original byte length pre-compression
);
CREATE TABLE config_validation (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
@ -69,16 +83,26 @@ CREATE TABLE config_validation (
/// FTS5 virtual table DDL. Separate because the tokenizer is config-driven.
///
/// Regular (not contentless, not external-content) FTS5: the table stores
/// its own text, which enables `snippet()`/`highlight()` and makes row-level
/// INSERT/UPDATE/DELETE work with normal SQL semantics. `rowid` is supplied
/// by the caller and must equal `files.id`.
/// *Contentless* FTS5 (`content=''`): the inverted index is built from the
/// column values supplied on INSERT, but those values are not stored. This
/// is the main lever that pulls our on-disk footprint down toward Baloo's.
/// `contentless_delete=1` (SQLite 3.43+) lets us `DELETE FROM … WHERE
/// rowid=?` without replaying the original row text, at the cost of a
/// modest tombstone bitmap. The tokenizer is config-driven; by default
/// (`trigram`) we append `remove_diacritics 1` so queries and stored text
/// fold the same way.
///
/// Snippet/highlight aren't available through SQLite's built-in `snippet()`
/// in contentless mode — we render them in Rust from the zstd-compressed
/// `documents_text` sidecar instead.
pub fn fts_create_sql(tokenizer: &str) -> String {
let effective = effective_tokenizer(tokenizer);
format!(
"CREATE VIRTUAL TABLE searchabletext USING fts5(\
name, text, properties, \
tokenize='{}'\
tokenize='{}', \
content='', \
contentless_delete=1\
);",
effective.replace('\'', "''")
)

11
crates/quicksearch-core/src/file_handling.rs
View file

@ -793,9 +793,14 @@ pub fn process_text_indexing(
safe_truncate_string(&content.text, config.processing.maximum_text_size);
}
let props = content.properties_sorted();
if let Err(e) =
repo::set_content_done(&tx, *file_id, fname, &content.text, &props)
{
if let Err(e) = repo::set_content_done(
&tx,
*file_id,
fname,
&content.text,
&props,
config.processing.store_text_for_snippets,
) {
eprintln!("Warning: set_content_done for {}: {}", fpath, e);
}
}

112
crates/quicksearch-core/src/indexing.rs
View file

@ -81,6 +81,24 @@ pub struct IndexingService {
/// Polling interval for `should_abort` while suspended.
const SUSPEND_POLL_MS: u64 = 100;
/// Map a rusqlite/SQLite error string into the tagged form the GUI's
/// search panel recognises (so it can pop the corruption-recovery dialog,
/// special-case FTS5 syntax errors, etc.). Centralized so both generic
/// `execute_search` and the fulltext-specific path classify errors the
/// same way.
fn classify_sql_err(error_msg: &str) -> String {
if error_msg.contains("malformed")
|| error_msg.contains("corrupt")
|| error_msg.contains("database disk image is malformed")
{
format!("DATABASE_CORRUPTED: {}", error_msg)
} else if error_msg.contains("fts5: syntax error") {
"Search syntax error: The search term contains characters that cannot be processed. Please try a simpler search term.".into()
} else {
format!("Failed to execute query: {}", error_msg)
}
}
/// Combined stop/suspend check used by worker loops. Returns `true` iff the
/// caller should abort the operation. While the suspend flag is set and stop
/// is not, this parks the thread by sleeping in short increments so a later
@ -237,9 +255,97 @@ impl IndexingService {
self.stop_indexing()
}
/// Fulltext-specific search path that emits rendered snippets. Runs the
/// SQL produced by [`crate::search_sql::build_select`] for a fulltext
/// `SearchArgs`, which returns `(name, path, file_id, text_zstd)`, then
/// decompresses each text blob and hands it to [`crate::snippet::render`]
/// together with the query terms. Case-sensitive mode is re-applied in
/// Rust since contentless FTS5 no longer stores the raw text.
///
/// The shape of the returned [`SearchResult`] matches what the GUI's
/// results table expects: columns `(name, path, snippet)`.
pub fn execute_fulltext_search(
&self,
db_path: &str,
args: &crate::search_sql::SearchArgs,
limit: u32,
offset: u32,
) -> Result<Vec<SearchResult>, String> {
use crate::search_sql::{build_select, fulltext_terms};
use crate::snippet;
let sql = build_select(args, limit, offset)?;
let terms = fulltext_terms(args);
let term_refs: Vec<&str> = terms.iter().map(String::as_str).collect();
let conn = db::open_or_recreate(db_path, "trigram").map_err(|e| {
if e.contains("corrupt") || e.contains("malformed") {
format!("DATABASE_CORRUPTED: {}", e)
} else {
e
}
})?;
let mut stmt = conn
.prepare(&sql)
.map_err(|e| classify_sql_err(&e.to_string()))?;
let rows_iter = stmt
.query_map([], |row| {
let name: String = row.get(0)?;
let path: String = row.get(1)?;
let _file_id: i64 = row.get(2)?;
let blob: Option<Vec<u8>> = row.get(3)?;
Ok((name, path, blob))
})
.map_err(|e| classify_sql_err(&e.to_string()))?;
let snippet_opts = snippet::Options::default();
let mut out_rows: Vec<SearchResultRow> = Vec::new();
for r in rows_iter {
let (name, path, blob) =
r.map_err(|e| classify_sql_err(&e.to_string()))?;
let body = match blob {
None => String::new(),
Some(bytes) if bytes.is_empty() => String::new(),
Some(bytes) => match zstd::decode_all(bytes.as_slice()) {
Ok(raw) => String::from_utf8_lossy(&raw).into_owned(),
Err(_) => String::new(),
},
};
// Case-sensitive post-filter: every query term must appear with
// the exact case supplied by the user. Applied to the body
// text; filename hits are still matched by the FTS MATCH which
// is case-insensitive (filenames are usually mixed case and the
// user probably doesn't care). Exact-phrase mode treats the
// whole quoted phrase as one token.
if args.fulltext_case_sensitive && !body.is_empty() {
let all_present = terms.iter().all(|t| body.contains(t.as_str()));
if !all_present {
continue;
}
}
let rendered = if body.is_empty() {
String::new()
} else {
snippet::render(&body, &term_refs, &snippet_opts)
};
out_rows.push(SearchResultRow {
values: vec![name, path, rendered],
});
}
Ok(vec![SearchResult {
columns: vec!["name".into(), "path".into(), "snippet".into()],
rows: out_rows,
}])
}
/// Execute a search query against the database
pub fn execute_search(&self, db_path: &str, query: &str) -> Result<Vec<SearchResult>, String> {
let conn = db::open_and_migrate(db_path, "trigram")
let conn = db::open_or_recreate(db_path, "trigram")
.map_err(|e| {
if e.contains("corrupt") || e.contains("malformed") {
format!("DATABASE_CORRUPTED: {}", e)
@ -404,7 +510,7 @@ impl IndexingService {
/// Check if configuration changes require index recreation
pub fn check_config_validation(&self, db_path: &str, config: &Config, indexing_path: &str) -> Result<Option<Vec<String>>, String> {
let conn = db::open_and_migrate(db_path, &config.processing.tokenize)?;
let conn = db::open_or_recreate(db_path, &config.processing.tokenize)?;
Self::validate_config(&conn, config, indexing_path)
}
@ -580,7 +686,7 @@ impl IndexingService {
.collect();
// Open and migrate the database to the current schema version.
let conn = db::open_and_migrate(db_path, &config.processing.tokenize)?;
let conn = db::open_or_recreate(db_path, &config.processing.tokenize)?;
// Update configuration (for new installations or when no validation issues).
// `indexing_path` in the validation table stores the joined list so

1
crates/quicksearch-core/src/lib.rs
View file

@ -9,4 +9,5 @@ pub mod mime;
pub mod query;
pub mod search_sql;
pub mod shutdown;
pub mod snippet;
pub mod watcher;

9
crates/quicksearch-core/src/query/translator.rs
View file

@ -547,7 +547,7 @@ mod tests {
#[test]
fn end_to_end_combined_filter_executes() {
use crate::db::{
open_and_migrate,
open_or_recreate,
repo::{insert_file, set_content_done, NewFile},
};
use crate::mime::FileType;
@ -562,7 +562,7 @@ mod tests {
.unwrap()
.as_nanos()
));
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
// Two audio files, one document. Only the "audio with beatles" file
// should be returned for `type:Audio beatles`.
@ -591,6 +591,7 @@ mod tests {
"beatles-track.mp3",
"beatles hey jude",
&[("artist".into(), "The Beatles".into())],
true,
)
.unwrap();
let b = insert_file(
@ -610,7 +611,7 @@ mod tests {
)
.unwrap()
.expect("unique path");
set_content_done(&tx, b, "bach.flac", "bach prelude", &[]).unwrap();
set_content_done(&tx, b, "bach.flac", "bach prelude", &[], true).unwrap();
let c = insert_file(
&tx,
&NewFile {
@ -628,7 +629,7 @@ mod tests {
)
.unwrap()
.expect("unique path");
set_content_done(&tx, c, "notes.txt", "beatles biography", &[]).unwrap();
set_content_done(&tx, c, "notes.txt", "beatles biography", &[], true).unwrap();
tx.commit().unwrap();
vec![a, b, c]
};

116
crates/quicksearch-core/src/search_sql.rs
View file

@ -47,14 +47,21 @@ pub fn build_count(args: &SearchArgs) -> Result<String, String> {
}
}
/// SQL that returns one page of results.
/// SQL that returns one page of results. Columns emitted by the `fulltext`
/// branch are `(name, path, file_id, text_zstd)` — the snippet is rendered
/// in Rust from the zstd-compressed `documents_text` row (FTS5 is
/// contentless, so SQLite's `snippet()` doesn't work on it). The GUI
/// should use [`crate::indexing::IndexingService::execute_fulltext_search`]
/// which stitches the decompress + snippet step on top of this SQL.
pub fn build_select(args: &SearchArgs, limit: u32, offset: u32) -> Result<String, String> {
match args.search_type.as_str() {
"fulltext" => {
let where_clause = build_fulltext_where(args)?;
Ok(format!(
"SELECT f.name, f.path, snippet(searchabletext, 1, '<b>', '</b>', '<b>...</b>', 64) as snippet \
FROM searchabletext AS st JOIN files f ON f.id = st.rowid \
"SELECT f.name, f.path, f.id, dt.text_zstd \
FROM searchabletext AS st \
JOIN files f ON f.id = st.rowid \
LEFT JOIN documents_text dt ON dt.file_id = f.id \
WHERE {} ORDER BY rank LIMIT {} OFFSET {}",
where_clause, limit, offset
))
@ -137,30 +144,62 @@ fn build_fulltext_where(args: &SearchArgs) -> Result<String, String> {
words.join(" AND ")
};
let mut where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match));
if args.fulltext_case_sensitive {
if args.fulltext_exact {
let literal = words.join(" ");
where_clause.push_str(&format!(
" AND instr(st.text, '{}') > 0",
sql_quote(&literal)
));
} else {
for w in &words {
where_clause.push_str(&format!(
" AND instr(st.text, '{}') > 0",
sql_quote(w)
));
}
}
}
// Contentless FTS5 doesn't store column text, so case-sensitive
// filtering can't live in SQL anymore. It's re-applied in
// `IndexingService::execute_fulltext_search` by checking the
// decompressed body text for literal-case matches before returning
// the row. The MATCH itself stays case-insensitive (tokenizer folds),
// which is the correct candidate-set for a post-filter.
let where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match));
Ok(where_clause)
}
/// Pull out the raw words the user typed so the snippet renderer and the
/// case-sensitive post-filter can see the same tokens `build_fulltext_where`
/// fed into FTS5. Returns empty when the user's term is empty or contains
/// only too-short words under the non-exact path.
pub fn fulltext_terms(args: &SearchArgs) -> Vec<String> {
let trimmed = args.term.trim();
if trimmed.is_empty() {
return Vec::new();
}
let sanitized: String = trimmed
.chars()
.map(|c| {
if matches!(
c,
':' | ';' | '(' | ')' | '[' | ']' | '{' | '}' | '^' | '~' | '"'
) {
' '
} else {
c
}
})
.collect();
let tokens: Vec<String> = sanitized
.split_whitespace()
.map(|s| s.to_string())
.collect();
if tokens.is_empty() {
return Vec::new();
}
if args.fulltext_exact {
// One composite phrase. Snippet rendering wants to highlight the
// whole phrase contiguously; the renderer supports multiple terms
// already so we collapse to the joined form.
vec![tokens.join(" ")]
} else {
tokens
.into_iter()
.filter(|w| w.chars().count() >= 3)
.collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::{open_and_migrate, repo::{insert_file, set_content_done, NewFile}};
use crate::db::{open_or_recreate, repo::{insert_file, set_content_done, NewFile}};
use crate::mime::FileType;
fn args(search_type: &str, term: &str) -> SearchArgs {
@ -178,7 +217,36 @@ mod tests {
assert!(sql.contains("LIMIT 50"));
assert!(sql.contains("OFFSET 100"));
assert!(sql.contains("ORDER BY rank"));
assert!(sql.contains("snippet(searchabletext"));
// Snippet rendering moved to Rust; SQL returns the compressed blob
// so the post-processor can decompress + highlight.
assert!(sql.contains("dt.text_zstd"), "got {sql}");
}
#[test]
fn fulltext_case_sensitive_no_longer_in_sql() {
let mut a = args("fulltext", "Hello");
a.fulltext_case_sensitive = true;
let sql = build_select(&a, 50, 0).unwrap();
// Post-filter lives in Rust now — no instr() or st.text reference.
assert!(!sql.contains("instr("), "got {sql}");
}
#[test]
fn fulltext_terms_extract_non_exact() {
let a = args("fulltext", "the quick brown");
let t = fulltext_terms(&a);
// Short words like "the" are dropped (trigram min length 3 applies
// in non-exact mode — matches the SQL build rules).
assert!(t.iter().any(|s| s == "quick"));
assert!(t.iter().any(|s| s == "brown"));
}
#[test]
fn fulltext_terms_exact_mode_returns_joined_phrase() {
let mut a = args("fulltext", "hello world");
a.fulltext_exact = true;
let t = fulltext_terms(&a);
assert_eq!(t, vec!["hello world".to_string()]);
}
#[test]
@ -253,7 +321,7 @@ mod tests {
#[test]
fn end_to_end_pagination_smoke() {
let p = tmp_path();
let mut conn = open_and_migrate(p.to_str().unwrap(), "trigram").unwrap();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
{
let tx = conn.transaction().unwrap();
for i in 0..7 {
@ -275,7 +343,7 @@ mod tests {
)
.unwrap()
.expect("unique path");
set_content_done(&tx, id, &format!("file_{}.txt", i), "shared body content", &[]).unwrap();
set_content_done(&tx, id, &format!("file_{}.txt", i), "shared body content", &[], true).unwrap();
}
tx.commit().unwrap();
}

336
crates/quicksearch-core/src/snippet.rs Normal file
View file

@ -0,0 +1,336 @@
//! Snippet / highlight rendering for search results.
//!
//! We store extracted text in the `documents_text` sidecar (zstd-compressed)
//! rather than in FTS5, so SQLite's built-in `snippet()` / `highlight()`
//! auxiliary functions aren't available (contentless FTS5 doesn't support
//! them). This module reproduces the parts we actually need in Rust: find
//! a window of context around the first match, bold every query-term
//! occurrence inside that window, trim with ellipsis markers.
//!
//! Matching is ASCII-case-insensitive on the *rendering* side. That aligns
//! with the search path which is already case-insensitive via the trigram
//! tokenizer; exact-case-only snippets aren't a feature users expect here.
//! Unicode accent folding isn't applied at the rendering layer — a query
//! for `cafe` will still *find* a file containing `café` (because the FTS
//! tokenizer strips diacritics) but the snippet won't highlight the
//! accented occurrence. The surrounding text is still returned verbatim.
//!
//! The API is intentionally small: one `render` function plus an `Options`
//! struct. Callers that want different pre/post tags, ellipsis, or window
//! size pass them in; there are sensible defaults for the GUI case.
/// Options controlling snippet rendering. The defaults mirror the old
/// `snippet(searchabletext, 1, '<b>', '</b>', '<b>...</b>', 64)` call that
/// the GUI used to run directly as SQL.
#[derive(Debug, Clone)]
pub struct Options<'a> {
pub pre: &'a str,
pub post: &'a str,
pub ellipsis: &'a str,
/// Approximate character budget for the returned snippet. Matches
/// expand the window if needed to keep their tags on; the budget is a
/// soft target, not a hard cap.
pub approx_chars: usize,
}
impl<'a> Default for Options<'a> {
fn default() -> Self {
Options {
pre: "<b>",
post: "</b>",
ellipsis: "<b>...</b>",
approx_chars: 200,
}
}
}
/// Render a snippet from `text` highlighting every occurrence of any term
/// in `terms`. Returns a string with `pre`/`post` wrapping each match, and
/// `ellipsis` prepended/appended when the returned window doesn't reach
/// the text's edges.
///
/// If no term matches, returns the first `approx_chars` of `text` (char-
/// aligned), suffixed with `ellipsis` when truncated.
pub fn render(text: &str, terms: &[&str], opts: &Options<'_>) -> String {
// Short-circuit trivial inputs.
if text.is_empty() {
return String::new();
}
let effective_terms: Vec<&str> = terms
.iter()
.map(|t| t.trim())
.filter(|t| !t.is_empty())
.collect();
if effective_terms.is_empty() {
return truncate_head(text, opts.approx_chars, opts.ellipsis);
}
// Case-fold once; we do all positioning on the folded buffer and emit
// slices from the original. Both buffers have identical byte layout
// because `to_ascii_lowercase` is a byte-for-byte map that preserves
// multi-byte UTF-8 sequences unchanged (it only touches ASCII letters).
let folded = text.to_ascii_lowercase();
let folded_bytes = folded.as_bytes();
let mut matches: Vec<(usize, usize)> = Vec::new();
for term in &effective_terms {
let pattern: String = term.to_ascii_lowercase();
let pbytes = pattern.as_bytes();
if pbytes.is_empty() {
continue;
}
let mut start = 0;
while start + pbytes.len() <= folded_bytes.len() {
if let Some(rel) = memfind(&folded_bytes[start..], pbytes) {
let at = start + rel;
matches.push((at, at + pbytes.len()));
// Advance past this match to avoid zero-width loops on
// empty patterns (already guarded above) and to allow
// overlapping matches of *different* terms in the next
// outer-loop iteration.
start = at + pbytes.len();
} else {
break;
}
}
}
if matches.is_empty() {
return truncate_head(text, opts.approx_chars, opts.ellipsis);
}
// Dedupe + sort so overlapping matches from different terms (e.g.
// "rust" and "rustc") don't produce nested tags.
matches.sort_by_key(|(a, _)| *a);
matches = coalesce_overlapping(matches);
// Pick the window. Start a third of the budget before the first match
// so the hit isn't pinned to the left edge. Round both ends to char
// boundaries so we never slice a multi-byte UTF-8 sequence.
let pre_pad = opts.approx_chars / 3;
let first_match_start = matches[0].0;
let mut win_start = first_match_start.saturating_sub(pre_pad);
let mut win_end = win_start + opts.approx_chars;
if win_end > text.len() {
win_end = text.len();
}
while win_start > 0 && !text.is_char_boundary(win_start) {
win_start -= 1;
}
while win_end < text.len() && !text.is_char_boundary(win_end) {
win_end += 1;
}
// Expand the window to include the full end of any match that would
// otherwise be cut off mid-tag. Keeps rendering sane when a long term
// sits at the right edge of the budget.
let last_match_in_window = matches.iter().rfind(|(s, _)| *s < win_end);
if let Some((_, end)) = last_match_in_window {
if *end > win_end {
win_end = *end;
while win_end < text.len() && !text.is_char_boundary(win_end) {
win_end += 1;
}
}
}
// Render: walk matches that fall inside the window, splicing pre/post
// around each. Prepend/append ellipsis when we've chopped off content.
let mut out = String::with_capacity(win_end - win_start + 32);
if win_start > 0 {
out.push_str(opts.ellipsis);
}
let mut cursor = win_start;
for (ms, me) in matches.iter() {
if *me <= win_start || *ms >= win_end {
continue;
}
// Clamp to the window.
let ms = (*ms).max(win_start);
let me = (*me).min(win_end);
if ms > cursor {
out.push_str(&text[cursor..ms]);
}
out.push_str(opts.pre);
out.push_str(&text[ms..me]);
out.push_str(opts.post);
cursor = me;
}
if cursor < win_end {
out.push_str(&text[cursor..win_end]);
}
if win_end < text.len() {
out.push_str(opts.ellipsis);
}
out
}
/// Return the first `n` characters of `text`, suffixed with `ellipsis` if
/// truncation actually happened. Respects UTF-8 char boundaries.
fn truncate_head(text: &str, n: usize, ellipsis: &str) -> String {
if text.len() <= n {
return text.to_string();
}
let mut cut = n;
while cut > 0 && !text.is_char_boundary(cut) {
cut -= 1;
}
let mut out = String::with_capacity(cut + ellipsis.len());
out.push_str(&text[..cut]);
out.push_str(ellipsis);
out
}
/// Merge adjacent / overlapping (start, end) ranges in place. Input must be
/// sorted by start.
fn coalesce_overlapping(mut v: Vec<(usize, usize)>) -> Vec<(usize, usize)> {
if v.len() < 2 {
return v;
}
let mut out = Vec::with_capacity(v.len());
let mut cur = v.remove(0);
for next in v {
if next.0 <= cur.1 {
cur.1 = cur.1.max(next.1);
} else {
out.push(cur);
cur = next;
}
}
out.push(cur);
out
}
/// Locate the first occurrence of `needle` in `hay`. A byte-level search;
/// callers have already lowercased both sides so case is normalized.
fn memfind(hay: &[u8], needle: &[u8]) -> Option<usize> {
if needle.is_empty() || needle.len() > hay.len() {
return None;
}
let first = needle[0];
let mut i = 0;
while i + needle.len() <= hay.len() {
if hay[i] == first && &hay[i..i + needle.len()] == needle {
return Some(i);
}
i += 1;
}
None
}
#[cfg(test)]
mod tests {
use super::*;
fn opts_small() -> Options<'static> {
Options {
pre: "<b>",
post: "</b>",
ellipsis: "…",
approx_chars: 40,
}
}
#[test]
fn empty_text_returns_empty() {
let s = render("", &["foo"], &Options::default());
assert_eq!(s, "");
}
#[test]
fn no_terms_returns_head_with_ellipsis_when_truncated() {
let long = "abcdefghijklmnop".repeat(10);
let s = render(&long, &[], &opts_small());
assert!(s.ends_with("…"));
assert!(s.len() < long.len() + 4);
}
#[test]
fn no_terms_untruncated_has_no_ellipsis() {
let s = render("short text", &[], &opts_small());
assert_eq!(s, "short text");
}
#[test]
fn simple_highlight_wraps_matches() {
let s = render("the quick brown fox", &["quick"], &opts_small());
assert!(s.contains("<b>quick</b>"));
}
#[test]
fn case_insensitive_match() {
let s = render("The QUICK brown fox", &["quick"], &opts_small());
assert!(s.contains("<b>QUICK</b>"), "got {s}");
}
#[test]
fn multiple_terms_both_highlighted() {
let s = render(
"the quick brown fox jumps over the lazy dog",
&["quick", "lazy"],
&Options {
approx_chars: 200,
..Options::default()
},
);
assert!(s.contains("<b>quick</b>"), "got {s}");
assert!(s.contains("<b>lazy</b>"), "got {s}");
}
#[test]
fn window_trims_with_ellipsis_on_both_sides() {
let text =
"prefix ".repeat(20) + "MATCH in middle " + &"suffix ".repeat(20);
let s = render(&text, &["MATCH"], &opts_small());
assert!(s.starts_with("…"), "got {s}");
assert!(s.ends_with("…"), "got {s}");
assert!(s.contains("<b>MATCH</b>"), "got {s}");
}
#[test]
fn match_at_start_has_no_leading_ellipsis() {
let s = render("MATCH right at the start of this paragraph", &["match"], &opts_small());
assert!(!s.starts_with("…"), "got {s}");
}
#[test]
fn no_match_on_tail_returns_head() {
let text = "alpha beta gamma delta epsilon zeta eta theta iota kappa";
let s = render(text, &["nomatch"], &opts_small());
assert!(!s.contains("<b>"));
assert!(s.starts_with("alpha"));
}
#[test]
fn overlapping_terms_do_not_nest_tags() {
// Two terms matching the same span must coalesce.
let s = render("the RUSTC compiler", &["rust", "rustc"], &opts_small());
assert!(s.contains("<b>RUSTC</b>"), "got {s}");
// No nested <b> tags.
assert!(!s.contains("<b><b>"), "got {s}");
}
#[test]
fn utf8_boundary_safe_truncation() {
// Insert multi-byte chars near the window boundary.
let text = "café café café café café café café café café café";
let s = render(text, &["nope"], &opts_small());
// Returned string must be valid UTF-8 (push_str guarantees this only
// if we sliced on char boundaries). Assert by round-trip.
assert_eq!(s.as_str(), &s.clone());
}
#[test]
fn match_near_right_edge_is_fully_shown() {
let prefix = "x".repeat(30);
let text = format!("{}{}", prefix, "LONGMATCHTERMTEXT");
let s = render(&text, &["LONGMATCHTERMTEXT"], &opts_small());
assert!(s.contains("<b>LONGMATCHTERMTEXT</b>"), "got {s}");
}
#[test]
fn empty_query_term_ignored() {
let s = render("hello world", &["", "world"], &opts_small());
assert!(s.contains("<b>world</b>"), "got {s}");
}
}

278
crates/quicksearch-core/tests/snippet_perf.rs Normal file
View file

@ -0,0 +1,278 @@
//! Side-by-side timing for snippet rendering: SQLite's built-in
//! `snippet()` against a regular FTS5 table, vs our zstd-compressed
//! sidecar + Rust renderer.
//!
//! Not a micro-benchmark — we care about end-to-end cost per result page
//! (FTS match + text retrieval + snippet construction) rather than the
//! renderer in isolation. Both paths are run against the same on-disk DB
//! seeded with the same prose, and each query is timed end-to-end from
//! `Connection::prepare` through the final snippet string.
//!
//! Gated by the `QSB_SNIPPET_PERF` env var so the test harness doesn't
//! pay the ~1 s seed cost on every `cargo test`. To run it:
//!
//! ```
//! QSB_SNIPPET_PERF=1 cargo test --release -p quicksearch-core \
//! --test snippet_perf -- --nocapture
//! ```
use std::path::PathBuf;
use std::time::Instant;
use quicksearch_core::snippet;
use rusqlite::{params, Connection};
const NUM_DOCS: usize = 1000;
const PAGE_SIZE: usize = 50;
/// Word list we draw text from. Has enough variety that trigram posting
/// lists stay non-trivial (hundreds of terms, not "the" 10000 times).
const WORDS: &[&str] = &[
"alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta",
"iota", "kappa", "lambda", "mu", "nu", "xi", "omicron", "pi", "rho",
"sigma", "tau", "upsilon", "phi", "chi", "psi", "omega",
"quick", "brown", "fox", "jumps", "over", "lazy", "dog",
"rust", "cargo", "sqlite", "baloo", "indexer", "tokenizer", "trigram",
"contentless", "posting", "fts5", "snippet", "highlight",
"morning", "afternoon", "evening", "midnight", "yesterday", "today",
"ocean", "forest", "mountain", "river", "valley", "bridge", "tunnel",
"tokyo", "paris", "london", "berlin", "rome", "madrid", "vienna",
];
/// Query terms that appear in the seeded corpus, so every query returns
/// real hits (not zero rows, which would skew against both paths equally
/// but wouldn't exercise the snippet renderer at all).
const QUERIES: &[&str] = &[
"quick", "rust", "baloo", "morning", "paris",
"tokyo", "forest", "indexer", "contentless", "trigram",
];
fn seed_text(rng: &mut u64, target_words: usize) -> String {
let mut out = String::with_capacity(target_words * 6);
for _ in 0..target_words {
// xorshift64 — cheap, portable, good enough for text generation.
*rng ^= *rng << 13;
*rng ^= *rng >> 7;
*rng ^= *rng << 17;
let w = WORDS[(*rng as usize) % WORDS.len()];
out.push_str(w);
out.push(' ');
}
out
}
fn tmp_path(tag: &str) -> PathBuf {
let mut p = std::env::temp_dir();
p.push(format!(
"qs-snippet-perf-{}-{}-{}.sqlite",
tag,
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
p
}
#[test]
fn snippet_paths_perf_comparison() {
if std::env::var("QSB_SNIPPET_PERF").is_err() {
eprintln!("skipping: set QSB_SNIPPET_PERF=1 to run");
return;
}
let p = tmp_path("both");
let conn = Connection::open(&p).unwrap();
conn.execute_batch(
"PRAGMA journal_mode = OFF;
PRAGMA synchronous = 0;
PRAGMA temp_store = MEMORY;",
)
.unwrap();
// Path A: the 'old' shape — regular FTS5 with stored text, snippet()
// built into SQLite. This is what our search SQL used to run before
// schema v3.
conn.execute_batch(
"CREATE VIRTUAL TABLE st_regular USING fts5(
name, text,
tokenize='trigram remove_diacritics 1'
);",
)
.unwrap();
// Path B: the new shape — contentless FTS5 + zstd-compressed sidecar.
// Matches the real schema. We rebuild it in-place here so perf is
// measured against the same DB layout production runs against.
conn.execute_batch(
"CREATE VIRTUAL TABLE st_contentless USING fts5(
name, text,
tokenize='trigram remove_diacritics 1',
content='',
contentless_delete=1
);
CREATE TABLE documents_text (
file_id INTEGER PRIMARY KEY,
text_zstd BLOB NOT NULL,
text_len INTEGER NOT NULL
);",
)
.unwrap();
// Seed both tables with identical content.
let seed_start = Instant::now();
let mut rng: u64 = 0x1234_5678_9ABC_DEF0;
{
let tx = conn.unchecked_transaction().unwrap();
{
let mut ins_reg = tx
.prepare(
"INSERT INTO st_regular(rowid, name, text) VALUES (?1, ?2, ?3)",
)
.unwrap();
let mut ins_con = tx
.prepare(
"INSERT INTO st_contentless(rowid, name, text) VALUES (?1, ?2, ?3)",
)
.unwrap();
let mut ins_blob = tx
.prepare(
"INSERT INTO documents_text(file_id, text_zstd, text_len) VALUES (?1, ?2, ?3)",
)
.unwrap();
for i in 1..=NUM_DOCS {
let target = 50 + ((rng as usize) % 400);
let text = seed_text(&mut rng, target);
let name = format!("doc_{:05}.txt", i);
ins_reg.execute(params![i as i64, &name, &text]).unwrap();
ins_con.execute(params![i as i64, &name, &text]).unwrap();
let compressed = zstd::encode_all(text.as_bytes(), 3).unwrap();
ins_blob
.execute(params![i as i64, &compressed, text.len() as i64])
.unwrap();
}
}
tx.commit().expect("seed commit");
}
eprintln!(
"seeded {} docs in both FTS5 shapes in {:.2?}",
NUM_DOCS,
seed_start.elapsed()
);
// Warm each table's page cache so the first run doesn't skew.
for q in QUERIES.iter().take(2) {
let mut s = conn
.prepare(
"SELECT rowid FROM st_regular WHERE st_regular MATCH ?1 LIMIT 50",
)
.unwrap();
let _ = s.query_map(params![q], |r| r.get::<_, i64>(0)).unwrap().count();
let mut s = conn
.prepare(
"SELECT rowid FROM st_contentless WHERE st_contentless MATCH ?1 LIMIT 50",
)
.unwrap();
let _ = s.query_map(params![q], |r| r.get::<_, i64>(0)).unwrap().count();
}
// Path A: SQLite's built-in snippet() on a regular FTS5 table.
let a_reps = 10;
let start_a = Instant::now();
let mut rows_a_total = 0usize;
for _ in 0..a_reps {
for q in QUERIES {
let mut stmt = conn
.prepare(
"SELECT rowid, name, snippet(st_regular, 1, '<b>', '</b>', '...', 64) \
FROM st_regular WHERE st_regular MATCH ?1 \
ORDER BY rank LIMIT ?2",
)
.unwrap();
let rows = stmt
.query_map(params![q, PAGE_SIZE as i64], |r| {
Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?))
})
.unwrap();
for r in rows {
let _ = r.unwrap();
rows_a_total += 1;
}
}
}
let dur_a = start_a.elapsed();
// Path B: contentless FTS match → pull text_zstd → decompress → render.
let b_reps = 10;
let start_b = Instant::now();
let mut rows_b_total = 0usize;
let opts = snippet::Options {
pre: "<b>",
post: "</b>",
ellipsis: "...",
approx_chars: 64,
};
for _ in 0..b_reps {
for q in QUERIES {
// Contentless FTS5 returns NULL for stored columns (that's the
// point of contentless). The real search SQL joins to `files`
// for name/path; here we don't need those fields — we're
// timing the snippet pipeline, not the row projection.
let mut stmt = conn
.prepare(
"SELECT st.rowid, dt.text_zstd \
FROM st_contentless AS st \
LEFT JOIN documents_text dt ON dt.file_id = st.rowid \
WHERE st_contentless MATCH ?1 \
ORDER BY rank LIMIT ?2",
)
.unwrap();
let rows = stmt
.query_map(params![q, PAGE_SIZE as i64], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, Option<Vec<u8>>>(1)?,
))
})
.unwrap();
for row in rows {
let (_rowid, blob) = row.unwrap();
let text = match blob {
Some(b) => {
let raw = zstd::decode_all(b.as_slice()).unwrap();
String::from_utf8(raw).unwrap()
}
None => String::new(),
};
let _snip = snippet::render(&text, &[q], &opts);
rows_b_total += 1;
}
}
}
let dur_b = start_b.elapsed();
// Report — `cargo test -- --nocapture` surfaces this.
let a_per_query = dur_a.as_secs_f64() / (a_reps * QUERIES.len()) as f64 * 1000.0;
let b_per_query = dur_b.as_secs_f64() / (b_reps * QUERIES.len()) as f64 * 1000.0;
let a_per_row = dur_a.as_secs_f64() / rows_a_total as f64 * 1_000_000.0;
let b_per_row = dur_b.as_secs_f64() / rows_b_total as f64 * 1_000_000.0;
eprintln!();
eprintln!("snippet perf (NUM_DOCS={NUM_DOCS}, PAGE_SIZE={PAGE_SIZE}, QUERIES={}, reps={a_reps}):",
QUERIES.len());
eprintln!(
" A (SQLite snippet(), regular FTS5): {:.2?} total, {:.2} ms/query, {:.1} µs/row ({} rows)",
dur_a, a_per_query, a_per_row, rows_a_total
);
eprintln!(
" B (contentless + zstd + Rust snippet): {:.2?} total, {:.2} ms/query, {:.1} µs/row ({} rows)",
dur_b, b_per_query, b_per_row, rows_b_total
);
eprintln!(
" ratio B/A: {:.2}x (>1 means our path is slower)",
b_per_query / a_per_query
);
drop(conn);
let _ = std::fs::remove_file(&p);
}

10
crates/quicksearch-gui/src/main.rs
View file

@ -52,13 +52,3 @@ fn app() -> Element {
}
}
/*
Duplicate files:
SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC;
Full text search:
SELECT f.name, f.path, snippet(searchabletext, 1, "<b>", "</b>", "<b>...</b>", 64) as "snip" FROM searchabletext AS st JOIN files f ON f.id = st.rowid WHERE st.text MATCH 'searchstring';
Filename search:
SELECT name, path FROM files WHERE name LIKE '%searchstring%';
*/

31
crates/quicksearch-gui/src/search.rs
View file

@ -68,20 +68,35 @@ pub fn Search(props: SearchProps) -> Element {
};
let offset = page.saturating_sub(1).saturating_mul(PAGE_SIZE);
let select_sql = match build_select(&args, PAGE_SIZE, offset) {
Ok(s) => s,
Err(e) => {
search_error.set(Some(e));
is_searching.set(false);
return;
// Validate early for the filename/duplicates branch so we
// surface parse errors before dispatching the blocking task.
let precomputed_select_sql = if args.search_type == "fulltext" {
None
} else {
match build_select(&args, PAGE_SIZE, offset) {
Ok(s) => Some(s),
Err(e) => {
search_error.set(Some(e));
is_searching.set(false);
return;
}
}
};
// Run select + (optional) count in parallel on the blocking pool.
// Fulltext takes the snippet-aware path (decompresses
// documents_text and highlights in Rust); filename +
// duplicates go through the plain SQL executor.
let svc1 = service.clone();
let db1 = db_path.clone();
let args_for_select = args.clone();
let select_handle = tokio::task::spawn_blocking(move || {
svc1.execute_search(&db1, &select_sql)
if args_for_select.search_type == "fulltext" {
svc1.execute_fulltext_search(&db1, &args_for_select, PAGE_SIZE, offset)
} else {
let sql = precomputed_select_sql
.expect("non-fulltext select SQL was prebuilt above");
svc1.execute_search(&db1, &sql)
}
});
let count_handle = count_sql.map(|sql| {