From 5c039206c6c600fe7b3ec4f352e33c88e253a9e7 Mon Sep 17 00:00:00 2001 From: Jeremy Karst Date: Tue, 21 Apr 2026 02:34:40 -0400 Subject: [PATCH] Sped up initial file count and reduced memory use, added cleaning step to database on repeat indexing. --- config_example.toml | 13 +- run.bat | 1 + run.sh | 2 + setup.sh | 1 + src/config.rs | 19 +- src/file_handling.rs | 719 +++++++++++++++++++++++++++++++------------ src/frontend.rs | 16 + src/indexing.rs | 420 ++++++++++++++++--------- src/main.rs | 2 +- src/search.rs | 123 ++++++-- 10 files changed, 941 insertions(+), 375 deletions(-) create mode 100644 run.bat create mode 100644 run.sh create mode 100644 setup.sh diff --git a/config_example.toml b/config_example.toml index cca9e9a..56c7db4 100644 --- a/config_example.toml +++ b/config_example.toml @@ -6,11 +6,18 @@ database_path = "QuickSearch.db" # Amount of data in bytes read from start/end of files used to calculate hash hash_length = 8192 # Maximum text content to index per file (bytes) -maximum_text_size = 524288 +maximum_text_size = 262144 # Maximum file size to process for text extraction (bytes) -maximum_file_size = 52428800 -# Number of files to process in each batch +maximum_text_file_size = 2097152 +# Number of files to process in each batch (directory walk / inserts / text extraction batches) batch_size = 200 +# Files per transaction for incremental UPDATE files + DELETE from searchabletext (FTS); larger = fewer commits, more RAM per chunk +fts_update_batch_size = 1000 +# If true, run a fast shell-backed tree count before Phase 1 (enables % progress; Linux uses GNU find -printf '\n' | wc -l when available). +# If false, Phase 1 shows file counts without a percentage. +precount_files_for_progress = false +# If true, follow symbolic links during directory walks (indexing only; shell precount unchanged). +follow_symlinks = false # FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61') # Look here for more information https://www.sqlite.org/fts5.html#tokenizers tokenize = "trigram" \ No newline at end of file diff --git a/run.bat b/run.bat new file mode 100644 index 0000000..0f4c3cf --- /dev/null +++ b/run.bat @@ -0,0 +1 @@ +cargo run \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..1c9c1eb --- /dev/null +++ b/run.sh @@ -0,0 +1,2 @@ +cargo build --release +./target/release/quicksearch \ No newline at end of file diff --git a/setup.sh b/setup.sh new file mode 100644 index 0000000..f84ab36 --- /dev/null +++ b/setup.sh @@ -0,0 +1 @@ +sudo apt install -y libsoup-3.0-dev libjavascriptcoregtk-4.1-dev libwebkit2gtk-4.1-dev libxdo-dev diff --git a/src/config.rs b/src/config.rs index 4cb9771..448a936 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,13 +14,23 @@ pub struct PathConfig { pub database_path: String, } +fn default_fts_update_batch_size() -> usize { + 1000 +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ProcessingConfig { pub hash_length: usize, pub maximum_text_size: usize, - pub maximum_file_size: u64, + pub maximum_text_file_size: u64, pub batch_size: usize, + #[serde(default = "default_fts_update_batch_size")] + pub fts_update_batch_size: usize, pub tokenize: String, + #[serde(default)] + pub precount_files_for_progress: bool, + #[serde(default)] + pub follow_symlinks: bool, } impl Default for Config { @@ -32,10 +42,13 @@ impl Default for Config { }, processing: ProcessingConfig { hash_length: 1024 * 8, - maximum_text_size: 1024 * 512, - maximum_file_size: 1024 * 1024 * 50, + maximum_text_size: 1024 * 256, + maximum_text_file_size: 1024 * 1024 * 2, batch_size: 200, + fts_update_batch_size: 1000, tokenize: "trigram".to_string(), + precount_files_for_progress: false, + follow_symlinks: false, }, } } diff --git a/src/file_handling.rs b/src/file_handling.rs index fb54444..e0aa080 100644 --- a/src/file_handling.rs +++ b/src/file_handling.rs @@ -2,46 +2,54 @@ use std::sync::{Mutex, Arc}; use std::ffi::OsString; use std::fs::{File,read_to_string}; use std::io::{Read, Seek, SeekFrom}; +use std::process::{Command, Stdio}; use std::time::UNIX_EPOCH; use std::collections::HashMap; use sha2::{Sha256, Digest}; -use walkdir::DirEntry; +use walkdir::{DirEntry, WalkDir}; use rusqlite::{params, Connection}; use crate::document_extraction::extract_document_text; use crate::config::Config; -#[derive(Debug, Clone)] -pub struct FileMetadata { - pub path: String, - pub size: u64, +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExistingFileEntry { pub moddate: u64, - pub hash: Vec, } -#[derive(Debug)] -pub struct BatchUpdate { - pub files_to_update: Vec<(DirEntry, FileMetadata)>, - pub files_to_insert: Vec, -} - -pub const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 86] = - ["","txt","rtf","log", // Text Documents - "csv", // Spreadsheet - "sh","bat","cmd","bash","ps1","psm1","psd1","pssc","psrc", // Scripts - "c","cpp","i","cs","csx","caki", // C# - "cpp","cc","cxx","c++","hpp","hh","hxx","h","ii", // C++ - "tex","bib","bbx","cbx", // LaTeX - "css","xml","md","json","yaml","yml", // Markup Languages and others - "html","htm","shtml","xhtml","xht","mdoc","jsp","asp","aspx","jshtm", // HTML - "js","cjs","mjs","es6","es","jsx","ts","tsx", // Javascript and TypeScript - "cfg","conf","ini","gitattributes","gitignore", // Config and related files - "java","jav", // Java - "pl","pm","pod","t","psgi", // Perl - "php","php4","php5","phtml","ctp", // PHP - "py","rpy","pyw","cpy","gyp","gypi","pyi","ipy","pyt","ipynb", // Python - "wasm","wat", // Web Assembly +pub const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 84] = + ["c","cs","csx", // C + "cpp","cc","cxx","hpp","hh","hxx","h", // C++ + "cfg","conf","ini","gitattributes","gitignore", // Config (General) + "toml","env","tf","tfvars", // Config (Infrastructure) + "scss","sass","less", // CSS Preprocessors + "dart", // Dart + "diff","patch", // Diffs + "go", // Go + "graphql","gql", // GraphQL + "html","htm","xhtml","xht","jsp","asp","aspx", // HTML + "java", // Java + "js","cjs","mjs","jsx","ts","tsx", // Javascript and TypeScript + "vue","svelte", // JS Frameworks + "kt","kts", // Kotlin + "tex","bib", // LaTeX + "css","xml","md","json","yaml","yml", // Markup + "m", // Objective-C + "pl","pm","t", // Perl + "php","phtml", // PHP + "proto", // Protocol Buffers + "py","pyw","pyi","ipynb", // Python + "r", // R + "rb", // Ruby + "rs", // Rust + "sh","bat","cmd","bash","ps1","psm1","psd1", // Scripts + "sql", // SQL + "csv", // Spreadsheet + "svg", // SVG + "swift", // Swift + "","txt","rtf","log", // Text Documents + "wasm", // Web Assembly ]; pub const SUPPORTED_DOCUMENT_EXTENSIONS_LIST: [&'static str; 9] = @@ -49,76 +57,183 @@ pub const SUPPORTED_DOCUMENT_EXTENSIONS_LIST: [&'static str; 9] = "ppt", "pptx", "odp", // Presentation "xls", "xlsx", "ods"]; // Spreadsheet -/// Load existing file metadata from database indexed by path -pub fn load_existing_files(conn: &Connection) -> Result, rusqlite::Error> { +/// Load path and moddate per row for incremental classification (hash/size loaded only when updating a file). +pub fn load_existing_files(conn: &Connection) -> Result, rusqlite::Error> { let mut existing_files = HashMap::new(); - let mut stmt = conn.prepare("SELECT path, size, moddate, hash FROM files")?; + let mut stmt = conn.prepare("SELECT path, moddate FROM files")?; let rows = stmt.query_map([], |row| { - Ok(FileMetadata { - path: row.get(0)?, - size: row.get(1)?, - moddate: row.get(2)?, - hash: row.get(3)?, - }) + Ok(( + row.get::<_, String>(0)?, + ExistingFileEntry { + moddate: row.get(1)?, + }, + )) })?; for row in rows { - let metadata = row?; - existing_files.insert(metadata.path.clone(), metadata); + let (path, entry) = row?; + existing_files.insert(path, entry); } - + Ok(existing_files) } -/// Analyze files and determine which need updates vs inserts -pub fn analyze_files_for_batch_update( - entries: &[DirEntry], - existing_files: &HashMap -) -> BatchUpdate { - let mut files_to_update = Vec::new(); - let mut files_to_insert = Vec::new(); +pub fn indexed_walk_file_entries(path: &str, follow_symlinks: bool) -> impl Iterator { + WalkDir::new(path) + .follow_links(follow_symlinks) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|entry| !entry.metadata().map(|m| m.is_dir()).unwrap_or(true)) +} - for entry in entries { - let meta = match entry.metadata() { - Ok(m) if !m.is_dir() => m, - _ => continue, - }; +fn parse_wc_l_stdout(bytes: &[u8]) -> Result { + let s = String::from_utf8_lossy(bytes); + let token = s + .trim() + .split_whitespace() + .next() + .ok_or_else(|| "wc: empty output".to_string())?; + token + .parse() + .map_err(|e| format!("wc: invalid count {:?}: {}", token, e)) +} - let fpath = match entry.path().canonicalize() { - Ok(fp) => { - let path_str = fp.to_string_lossy().to_string(); - // Remove Windows UNC prefix \\?\ - if path_str.starts_with("\\\\?\\") { - path_str[4..].to_string() - } else { - path_str - } - }, - Err(_) => continue, - }; - - let fmodified = match meta.modified() - .ok() - .and_then(|m| m.duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs())) { - Some(time) => time, - None => continue, - }; - - if let Some(existing_metadata) = existing_files.get(&fpath) { - // File exists in database, check if modification date changed - if existing_metadata.moddate != fmodified { - files_to_update.push((entry.clone(), existing_metadata.clone())); - } - // If moddate is same, skip processing this file entirely - } else { - // New file, needs to be inserted - files_to_insert.push(entry.clone()); - } +#[cfg(unix)] +fn count_find_pipe_wc(path: &str) -> Result { + let mut find = Command::new("find") + .arg(path) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| format!("find: {}", e))?; + let find_stdout = find.stdout.take().ok_or("find: stdout")?; + let wc = Command::new("wc") + .arg("-l") + .stdin(find_stdout) + .stdout(Stdio::piped()) + .output() + .map_err(|e| format!("wc: {}", e))?; + find.wait().map_err(|e| format!("find wait: {}", e))?; + if !wc.status.success() { + return Err(format!("wc exited with {}", wc.status)); } + parse_wc_l_stdout(&wc.stdout) +} - BatchUpdate { - files_to_update, - files_to_insert, +#[cfg(target_os = "linux")] +fn count_find_printf_wc(path: &str) -> Result { + let mut find = Command::new("find") + .arg(path) + .arg("-printf") + .arg("\n") + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| format!("find: {}", e))?; + let find_stdout = find.stdout.take().ok_or("find: stdout")?; + let wc = Command::new("wc") + .arg("-l") + .stdin(find_stdout) + .stdout(Stdio::piped()) + .output() + .map_err(|e| format!("wc: {}", e))?; + find.wait().map_err(|e| format!("find wait: {}", e))?; + if !wc.status.success() { + return Err(format!("wc exited with {}", wc.status)); + } + parse_wc_l_stdout(&wc.stdout) +} + +#[cfg(windows)] +fn count_tree_entries_windows(path: &str) -> Result { + let lit = path.replace('\'', "''"); + let ps = format!( + "(Get-ChildItem -LiteralPath '{}' -Recurse -Force -ErrorAction SilentlyContinue | Measure-Object).Count", + lit + ); + let out = Command::new("powershell.exe") + .args(["-NoProfile", "-NonInteractive", "-Command", &ps]) + .output() + .map_err(|e| format!("powershell: {}", e))?; + if !out.status.success() { + return Err(format!( + "powershell exited with {}: {}", + out.status, + String::from_utf8_lossy(&out.stderr) + )); + } + String::from_utf8_lossy(&out.stdout) + .trim() + .parse() + .map_err(|e| format!("invalid count output: {}", e)) +} + +/// Rough tree entry count for progress totals (Linux: `find DIR -printf '\n' | wc -l` when GNU find is available, else `find DIR | wc -l`; macOS/other Unix: `find DIR | wc -l`; Windows: PowerShell `Get-ChildItem -Recurse`). Scope is not identical to the indexer’s classified file count. +pub fn count_tree_entries_fast(path: &str) -> Result { + #[cfg(windows)] + { + return count_tree_entries_windows(path); + } + #[cfg(all(unix, target_os = "linux"))] + { + return count_find_printf_wc(path).or_else(|_| count_find_pipe_wc(path)); + } + #[cfg(all(unix, not(target_os = "linux")))] + { + return count_find_pipe_wc(path); + } + #[cfg(not(any(windows, unix)))] + { + Err("tree entry count is not supported on this target".to_string()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileIndexAction { + Skip, + Update, + Insert, +} + +/// Classify a walkdir entry for Phase 1 file indexing. Returns `None` if the path is not indexable. +pub fn classify_dir_entry_for_indexing( + entry: &DirEntry, + existing_files: &HashMap, +) -> Option { + let fpath = match entry.path().canonicalize() { + Ok(fp) => { + let path_str = fp.to_string_lossy().to_string(); + if path_str.starts_with("\\\\?\\") { + path_str[4..].to_string() + } else { + path_str + } + } + Err(_) => return None, + }; + + let meta = match std::fs::metadata(&fpath) { + Ok(m) if m.is_file() => m, + _ => return None, + }; + + let fmodified = match meta + .modified() + .ok() + .and_then(|m| m.duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs())) + { + Some(time) => time, + None => return None, + }; + + if let Some(existing) = existing_files.get(&fpath) { + if existing.moddate != fmodified { + Some(FileIndexAction::Update) + } else { + Some(FileIndexAction::Skip) + } + } else { + Some(FileIndexAction::Insert) } } @@ -159,127 +274,217 @@ fn get_file_hash(size: u64, path: OsString, hash_length: usize) -> Result) -> String { + match progress_display_total { + Some(t) => format!("{}/{}", visit_index, t), + None => format!("{}", visit_index), + } +} + +const FTS_SQL_AUTOMERGE_8: &str = + "INSERT INTO searchabletext(searchabletext, rank) VALUES('automerge', 8)"; +const FTS_SQL_REBUILD: &str = "INSERT INTO searchabletext(searchabletext) VALUES('rebuild')"; + +pub fn fts_finalize_after_text_indexing(conn: &Connection) -> Result<(), String> { + conn.execute(FTS_SQL_AUTOMERGE_8, []) + .map_err(|e| format!("FTS automerge(8): {}", e))?; + conn.execute(FTS_SQL_REBUILD, []) + .map_err(|e| format!("FTS rebuild: {}", e))?; + Ok(()) +} + +fn fts_remove_document_for_path( + tx: &rusqlite::Transaction<'_>, + path: &str, +) -> Result<(), String> { + let id_opt: Option = match tx.query_row( + "SELECT id FROM documents WHERE path = ?1", + params![path], + |r| r.get(0), + ) { + Ok(id) => Some(id), + Err(rusqlite::Error::QueryReturnedNoRows) => None, + Err(e) => return Err(format!("documents id lookup: {}", e)), + }; + if let Some(doc_id) = id_opt { + tx.execute( + "INSERT INTO searchabletext(searchabletext, rowid) VALUES('delete', ?1)", + params![doc_id], + ) + .map_err(|e| format!("FTS delete doc {}: {}", doc_id, e))?; + tx.execute("DELETE FROM documents WHERE id = ?1", params![doc_id]) + .map_err(|e| format!("delete documents row: {}", e))?; + } + Ok(()) +} + +struct PreparedFileUpdate { + path_db: String, + fsize: u64, + fmodified: u64, + fhash: Vec, + filename: String, + visit_index: usize, +} + /// Process updated files in batch with transaction - files table only (no text extraction) pub fn process_batch_updates_files_only( conn_mutex: &Arc>, - files_to_update: &[(DirEntry, FileMetadata)], + files_to_update: &[(DirEntry, usize)], stop_flag: &Arc>, status_callback: Option>, progress_callback: Option>, - config: &Config + config: &Config, + progress_display_total: Option, ) -> Result<(), String> { if files_to_update.is_empty() { return Ok(()); } - let batch_size = config.processing.batch_size; - let total_files = files_to_update.len(); + let fts_batch = config.processing.fts_update_batch_size.max(1); - // Process files in batches of batch_size - for (batch_idx, batch) in files_to_update.chunks(batch_size).enumerate() { - // Check stop flag at the start of each batch + for batch in files_to_update.chunks(fts_batch) { if *stop_flag.lock().unwrap() { return Ok(()); } - let conn = conn_mutex.lock().unwrap(); - let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?; + let mut prepared: Vec = Vec::new(); - for (i, (entry, _old_metadata)) in batch.iter().enumerate() { - let global_index = batch_idx * batch_size + i + 1; - // Check stop flag + for (entry, visit_index) in batch.iter() { if *stop_flag.lock().unwrap() { - drop(tx); - drop(conn); return Ok(()); } - // Update status with current file + let filename = entry + .path() + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("unknown") + .to_string(); + if let Some(ref callback) = status_callback { - let filename = entry.path().file_name() - .and_then(|n| n.to_str()) - .unwrap_or("unknown"); - callback(&format!("Updating file metadata {}/{}: {}", global_index, total_files, filename)); - } - - // Update progress counter - if let Some(ref progress_cb) = progress_callback { - progress_cb(global_index); + let pair = format_progress_pair(*visit_index, progress_display_total); + callback(&format!("Hashing changed files {}: {}", pair, filename)); } - let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?; - if meta.is_dir() { - continue; + if let Some(ref progress_cb) = progress_callback { + progress_cb(*visit_index); } let fpath = match entry.path().canonicalize() { Ok(fp) => { let path_str = fp.to_string_lossy().to_string(); - // Remove Windows UNC prefix \\?\ if path_str.starts_with("\\\\?\\") { std::ffi::OsString::from(&path_str[4..]) } else { fp.into_os_string() } - }, + } Err(_) => continue, }; + let meta = match std::fs::metadata(&fpath) { + Ok(m) if m.is_file() => m, + _ => continue, + }; + let fsize = meta.len(); - let fmodified = meta.modified() + let fmodified = meta + .modified() .map_err(|e| format!("Failed to get modified time: {}", e))? .duration_since(UNIX_EPOCH) .map_err(|e| format!("Failed to calculate duration: {}", e))? .as_secs(); - let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length) - .map_err(|e| format!("Failed to calculate hash: {}", e))?; + let fhash = match get_file_hash(fsize, fpath.clone(), config.processing.hash_length) { + Ok(h) => h, + Err(e) => { + eprintln!( + "Warning: Skipping file (cannot hash) {}: {}", + fpath.to_string_lossy(), + e + ); + continue; + } + }; - // Check stop flag after hash calculation + if *stop_flag.lock().unwrap() { + return Ok(()); + } + + prepared.push(PreparedFileUpdate { + path_db: fpath.to_string_lossy().into_owned(), + fsize, + fmodified, + fhash, + filename, + visit_index: *visit_index, + }); + } + + if prepared.is_empty() { + continue; + } + + let conn = conn_mutex.lock().unwrap(); + let tx = conn + .unchecked_transaction() + .map_err(|e| format!("Failed to begin transaction: {}", e))?; + + for row in &prepared { if *stop_flag.lock().unwrap() { drop(tx); drop(conn); return Ok(()); } - // Update files table + if let Some(ref callback) = status_callback { + let pair = format_progress_pair(row.visit_index, progress_display_total); + callback(&format!( + "Applying index updates {}: {}", + pair, row.filename + )); + } + tx.execute( "UPDATE files SET size = ?1, moddate = ?2, hash = ?3 WHERE path = ?4", - params![fsize, fmodified, fhash, fpath.to_string_lossy()] - ).map_err(|e| format!("Failed to update file record: {}", e))?; + params![row.fsize, row.fmodified, row.fhash, row.path_db], + ) + .map_err(|e| format!("Failed to update file record: {}", e))?; - // Delete old searchable text entry for this file (will be re-added in text indexing phase) - tx.execute( - "DELETE FROM searchabletext WHERE path = ?1", - params![fpath.to_string_lossy()] - ).map_err(|e| format!("Failed to delete old searchable text: {}", e))?; + fts_remove_document_for_path(&tx, &row.path_db).map_err(|e| { + format!( + "Failed to remove old document / FTS entry for {}: {}", + row.path_db, e + ) + })?; } - tx.commit().map_err(|e| format!("Failed to commit transaction: {}", e))?; + tx.commit() + .map_err(|e| format!("Failed to commit transaction: {}", e))?; } - + Ok(()) } /// Process new files in batch with transaction - files table only (no text extraction) pub fn process_batch_inserts_files_only( conn_mutex: &Arc>, - files_to_insert: &[DirEntry], + files_to_insert: &[(DirEntry, usize)], stop_flag: &Arc>, status_callback: Option>, progress_callback: Option>, - config: &Config + config: &Config, + progress_display_total: Option, ) -> Result<(), String> { if files_to_insert.is_empty() { return Ok(()); } let batch_size = config.processing.batch_size; - let total_files = files_to_insert.len(); // Process files in batches of batch_size - for (batch_idx, batch) in files_to_insert.chunks(batch_size).enumerate() { - // Check stop flag at the start of each batch + for batch in files_to_insert.chunks(batch_size) { if *stop_flag.lock().unwrap() { return Ok(()); } @@ -287,9 +492,8 @@ pub fn process_batch_inserts_files_only( let conn = conn_mutex.lock().unwrap(); let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?; - for (i, entry) in batch.iter().enumerate() { - let global_index = batch_idx * batch_size + i + 1; - // Check stop flag + for (entry, visit_index) in batch.iter() { + // Check stop flag for early termination if *stop_flag.lock().unwrap() { drop(tx); drop(conn); @@ -301,17 +505,13 @@ pub fn process_batch_inserts_files_only( let filename = entry.path().file_name() .and_then(|n| n.to_str()) .unwrap_or("unknown"); - callback(&format!("Indexing file metadata {}/{}: {}", global_index, total_files, filename)); + let pair = format_progress_pair(*visit_index, progress_display_total); + callback(&format!("Indexing file metadata {}: {}", pair, filename)); } // Update progress counter if let Some(ref progress_cb) = progress_callback { - progress_cb(global_index); - } - - let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?; - if meta.is_dir() { - continue; + progress_cb(*visit_index); } let fpath = match entry.path().canonicalize() { @@ -327,6 +527,11 @@ pub fn process_batch_inserts_files_only( Err(_) => continue, }; + let meta = match std::fs::metadata(&fpath) { + Ok(m) if m.is_file() => m, + _ => continue, + }; + let fsize = meta.len(); let fmodified = meta.modified() .map_err(|e| format!("Failed to get modified time: {}", e))? @@ -334,8 +539,17 @@ pub fn process_batch_inserts_files_only( .map_err(|e| format!("Failed to calculate duration: {}", e))? .as_secs(); - let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length) - .map_err(|e| format!("Failed to calculate hash: {}", e))?; + let fhash = match get_file_hash(fsize, fpath.clone(), config.processing.hash_length) { + Ok(h) => h, + Err(e) => { + eprintln!( + "Warning: Skipping file (cannot hash) {}: {}", + fpath.to_string_lossy(), + e + ); + continue; + } + }; let fname = entry.path().file_name().unwrap().to_os_string(); @@ -346,13 +560,69 @@ pub fn process_batch_inserts_files_only( ).map_err(|e| format!("Failed to insert file record: {}", e))?; } + // Update status with current file + if let Some(ref callback) = status_callback { + callback("Committing file updates to database…"); + } + tx.commit().map_err(|e| format!("Failed to commit transaction: {}", e))?; } Ok(()) } -/// Process text indexing for files - adds entries to searchabletext table +pub fn cleanup_stale_index_entries( + conn_mutex: &Arc>, + stale_paths: &[String], + stop_flag: &Arc>, + status_callback: Option>, +) -> Result { + if stale_paths.is_empty() { + return Ok(0); + } + + let conn = conn_mutex.lock().unwrap(); + let tx = conn + .unchecked_transaction() + .map_err(|e| format!("Failed to begin stale cleanup transaction: {}", e))?; + + let mut deleted_count = 0usize; + for path in stale_paths { + if *stop_flag.lock().unwrap() { + let _ = tx.commit(); + drop(conn); + return Ok(deleted_count); + } + + if let Some(ref callback) = status_callback { + callback(&format!("Removing stale index entry: {}", path)); + } + + fts_remove_document_for_path(&tx, path).map_err(|e| { + format!( + "Failed to remove stale document / FTS entry for {}: {}", + path, e + ) + })?; + tx.execute("DELETE FROM files WHERE path = ?1", params![path]) + .map_err(|e| format!("Failed to delete stale file record {}: {}", path, e))?; + deleted_count += 1; + } + + tx.commit() + .map_err(|e| format!("Failed to commit stale cleanup transaction: {}", e))?; + + if deleted_count > 0 && !*stop_flag.lock().unwrap() { + if let Some(ref callback) = status_callback { + callback("Rebuilding FTS index after stale cleanup..."); + } + fts_finalize_after_text_indexing(&conn)?; + } + + Ok(deleted_count) +} + +/// Process text indexing for files - writes `documents`; FTS rebuilt in `fts_finalize_after_text_indexing`. pub fn process_text_indexing( conn_mutex: &Arc>, stop_flag: &Arc>, @@ -360,93 +630,131 @@ pub fn process_text_indexing( progress_callback: Option>, config: &Config ) -> Result<(), String> { - let conn = conn_mutex.lock().unwrap(); - - // Get all files from the files table that don't have corresponding searchabletext entries - let mut stmt = conn.prepare( - "SELECT f.name, f.path, f.size FROM files f - LEFT JOIN searchabletext s ON f.path = s.path - WHERE s.path IS NULL AND f.size <= ?1" - ).map_err(|e| format!("Failed to prepare statement: {}", e))?; - - let file_rows = stmt.query_map([config.processing.maximum_file_size], |row| { - Ok(( - row.get::<_, String>(0)?, // name - row.get::<_, String>(1)?, // path - row.get::<_, u64>(2)? // size - )) - }).map_err(|e| format!("Failed to query files: {}", e))?; - - let files_to_process: Vec<_> = file_rows.collect::, _>>() - .map_err(|e| format!("Failed to collect files: {}", e))?; - - drop(stmt); - drop(conn); - - let total_files = files_to_process.len(); + let max_size = config.processing.maximum_text_file_size; let batch_size = config.processing.batch_size; + let batch_limit = batch_size as i64; - // Process files in batches - for (batch_idx, batch) in files_to_process.chunks(batch_size).enumerate() { - // Check stop flag at the start of each batch + if let Some(ref callback) = status_callback { + callback("Counting files pending text index…"); + } + + let total_files: usize = { + let conn = conn_mutex.lock().unwrap(); + conn.query_row( + "SELECT COUNT(*) FROM files f + LEFT JOIN documents d ON f.path = d.path + WHERE d.path IS NULL AND f.size <= ?1", + [max_size], + |row| row.get(0), + ) + .map_err(|e| format!("Failed to count pending text files: {}", e))? + }; + + let mut cursor_path = String::new(); + let mut global_index: usize = 0; + + loop { if *stop_flag.lock().unwrap() { return Ok(()); } - let conn = conn_mutex.lock().unwrap(); - let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?; + let batch: Vec<(String, String, u64)> = { + let conn = conn_mutex.lock().unwrap(); + let mut stmt = conn + .prepare( + "SELECT f.name, f.path, f.size FROM files f + LEFT JOIN documents d ON f.path = d.path + WHERE d.path IS NULL AND f.size <= ?1 + AND (?2 = '' OR f.path > ?2) + ORDER BY f.path + LIMIT ?3", + ) + .map_err(|e| format!("Failed to prepare text indexing query: {}", e))?; + let rows = stmt + .query_map( + rusqlite::params![max_size, cursor_path.as_str(), batch_limit], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, u64>(2)?, + )) + }, + ) + .map_err(|e| format!("Failed to query files for text indexing: {}", e))?; + rows.collect::, _>>() + .map_err(|e| format!("Failed to read file row: {}", e))? + }; - for (i, (fname, fpath, _fsize)) in batch.iter().enumerate() { - let global_index = batch_idx * batch_size + i + 1; - - // Check stop flag + if batch.is_empty() { + break; + } + + let last_path = batch.last().unwrap().1.clone(); + cursor_path = last_path; + + let conn = conn_mutex.lock().unwrap(); + let tx = conn + .unchecked_transaction() + .map_err(|e| format!("Failed to begin transaction: {}", e))?; + + for (fname, fpath, _fsize) in batch.iter() { if *stop_flag.lock().unwrap() { - // Commit current transaction before stopping let _ = tx.commit(); drop(conn); return Ok(()); } - // Update status + global_index += 1; + if let Some(ref callback) = status_callback { - callback(&format!("Extracting text for search indexing {}/{}: {}", global_index, total_files, fname)); + callback(&format!( + "Extracting text for search indexing {}/{}: {}", + global_index, total_files, fname + )); } - - // Update progress counter + if let Some(ref progress_cb) = progress_callback { progress_cb(global_index); } - let path = std::path::Path::new(fpath); + let path = std::path::Path::new(fpath.as_str()); let default_ext = OsString::new(); - let file_extension = path.extension().unwrap_or(&default_ext) - .to_ascii_lowercase().to_str().unwrap_or("").to_string(); + let file_extension = path + .extension() + .unwrap_or(&default_ext) + .to_ascii_lowercase() + .to_str() + .unwrap_or("") + .to_string(); let ext_str = file_extension.as_str(); - - // Process text content with error handling for individual files + let text_result = if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) { match read_to_string(fpath) { Ok(file_string) => { - let trimmed_file_string = safe_truncate_string(&file_string, config.processing.maximum_text_size); + let trimmed_file_string = + safe_truncate_string(&file_string, config.processing.maximum_text_size); Some(trimmed_file_string) } - Err(_e) => { - // eprintln!("Warning: Failed to read plaintext file {}: {}", fpath, e); - None - } + Err(_e) => None, } } else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) { match extract_document_text(&std::ffi::OsString::from(fpath), ext_str) { Ok(extracted_text) => { if !extracted_text.trim().is_empty() { - let trimmed_file_string = safe_truncate_string(&extracted_text, config.processing.maximum_text_size); - Some(trimmed_file_string) + Some(safe_truncate_string( + &extracted_text, + config.processing.maximum_text_size, + )) } else { None } } Err(e) => { - eprintln!("Warning: Failed to extract text from document {}: {}", fpath, e); + eprintln!( + "Warning: Failed to extract text from document {}: {}", + fpath, e + ); None } } @@ -454,19 +762,24 @@ pub fn process_text_indexing( None }; - // Insert the text content if we successfully extracted it if let Some(text_content) = text_result { if let Err(e) = tx.execute( - "INSERT INTO searchabletext VALUES (?1, ?2, ?3)", - params![fname, fpath, text_content] + "INSERT OR REPLACE INTO documents(name, path, text) VALUES (?1, ?2, ?3)", + params![fname, fpath, text_content], ) { - eprintln!("Warning: Failed to insert searchable text for {}: {}", fpath, e); + eprintln!("Warning: Failed to insert document row for {}: {}", fpath, e); } } } - tx.commit().map_err(|e| format!("Failed to commit transaction: {}", e))?; + tx.commit() + .map_err(|e| format!("Failed to commit transaction: {}", e))?; } - + + if total_files > 0 && !*stop_flag.lock().unwrap() { + let conn = conn_mutex.lock().unwrap(); + fts_finalize_after_text_indexing(&conn)?; + } + Ok(()) } diff --git a/src/frontend.rs b/src/frontend.rs index 780c800..e0fdb00 100644 --- a/src/frontend.rs +++ b/src/frontend.rs @@ -106,6 +106,22 @@ pub fn App(props: AppProps) -> Element { speed_tracker_clone.set(SpeedTracker::new()); "Idle".to_string() }, + IndexingStatus::CountingFiles { + current_file, + start_time, + .. + } => { + let elapsed = start_time.elapsed(); + let current_file_display = current_file + .as_ref() + .map(|f| format!("{}", f)) + .unwrap_or_else(|| "...".to_string()); + format!( + "Phase 0 - Counting paths (shell) - {:.1}s elapsed\n{}", + elapsed.as_secs_f64(), + current_file_display + ) + } IndexingStatus::RunningFileIndex { files_processed, total_files, current_file, start_time } => { // Add data point to speed tracker speed_tracker_clone.with_mut(|tracker| { diff --git a/src/indexing.rs b/src/indexing.rs index cb99fc2..8ae0393 100644 --- a/src/indexing.rs +++ b/src/indexing.rs @@ -2,10 +2,21 @@ use std::sync::{Arc, Mutex, mpsc}; use std::thread; use std::time::Instant; use std::process::Command; -use walkdir::WalkDir; -use rusqlite::{Connection, params}; +use std::collections::HashSet; +use rusqlite::{Connection, OptionalExtension, params}; +use walkdir::DirEntry; -use crate::file_handling::{load_existing_files, analyze_files_for_batch_update, process_batch_updates_files_only, process_batch_inserts_files_only, process_text_indexing}; +use crate::file_handling::{ + classify_dir_entry_for_indexing, + cleanup_stale_index_entries, + count_tree_entries_fast, + indexed_walk_file_entries, + load_existing_files, + process_batch_inserts_files_only, + process_batch_updates_files_only, + process_text_indexing, + FileIndexAction, +}; use crate::config::Config; #[derive(Debug, Clone)] @@ -22,6 +33,12 @@ pub struct SearchResult { #[derive(Debug, Clone)] pub enum IndexingStatus { Idle, + CountingFiles { + _entries_scanned: usize, + _indexable_files_counted: usize, + current_file: Option, + start_time: Instant, + }, RunningFileIndex { files_processed: usize, total_files: Option, @@ -288,6 +305,7 @@ impl IndexingService { } /// Clean up UNC prefixes from existing database entries + #[allow(dead_code)] pub fn clean_unc_prefixes(&self, db_path: &str) -> Result<(), String> { let conn = Connection::open(db_path) .map_err(|e| format!("Failed to open database: {}", e))?; @@ -298,16 +316,26 @@ impl IndexingService { (), ).map_err(|e| format!("Failed to update files table: {}", e))?; - // Clean UNC prefixes from searchabletext table - conn.execute( - "UPDATE searchabletext SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'", - (), - ).map_err(|e| format!("Failed to update searchabletext table: {}", e))?; + let doc_table: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='documents'", + [], + |r| r.get(0), + ) + .unwrap_or(0); + if doc_table > 0 { + conn.execute( + "UPDATE documents SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'", + (), + ) + .map_err(|e| format!("Failed to update documents table: {}", e))?; + } Ok(()) } /// Check if the database is corrupted or malformed + #[allow(dead_code)] pub fn check_database_health(&self, db_path: &str) -> Result { match Connection::open(db_path) { Ok(conn) => { @@ -356,7 +384,10 @@ impl IndexingService { while attempts < 50 { // Wait up to 5 seconds match self.get_status() { IndexingStatus::Idle => break, - IndexingStatus::Stopping | IndexingStatus::RunningFileIndex { .. } | IndexingStatus::RunningTextIndex { .. } => { + IndexingStatus::Stopping + | IndexingStatus::CountingFiles { .. } + | IndexingStatus::RunningFileIndex { .. } + | IndexingStatus::RunningTextIndex { .. } => { std::thread::sleep(std::time::Duration::from_millis(100)); attempts += 1; } @@ -384,7 +415,12 @@ impl IndexingService { while let Ok(command) = command_rx.recv() { match command { IndexingCommand::Start { path, db_path, config } => { - if matches!(*status.lock().unwrap(), IndexingStatus::RunningFileIndex { .. } | IndexingStatus::RunningTextIndex { .. }) { + if matches!( + *status.lock().unwrap(), + IndexingStatus::CountingFiles { .. } + | IndexingStatus::RunningFileIndex { .. } + | IndexingStatus::RunningTextIndex { .. } + ) { continue; // Already running } @@ -394,11 +430,20 @@ impl IndexingService { } *stop_flag.lock().unwrap() = false; - *status.lock().unwrap() = IndexingStatus::RunningFileIndex { - files_processed: 0, - total_files: None, - current_file: None, - start_time: Instant::now(), + *status.lock().unwrap() = if config.processing.precount_files_for_progress { + IndexingStatus::CountingFiles { + _entries_scanned: 0, + _indexable_files_counted: 0, + current_file: Some("Preparing database...".to_string()), + start_time: Instant::now(), + } + } else { + IndexingStatus::RunningFileIndex { + files_processed: 0, + total_files: None, + current_file: None, + start_time: Instant::now(), + } }; // Run indexing in a separate thread @@ -426,7 +471,12 @@ impl IndexingService { })); } IndexingCommand::Stop => { - if matches!(*status.lock().unwrap(), IndexingStatus::RunningFileIndex { .. } | IndexingStatus::RunningTextIndex { .. }) { + if matches!( + *status.lock().unwrap(), + IndexingStatus::CountingFiles { .. } + | IndexingStatus::RunningFileIndex { .. } + | IndexingStatus::RunningTextIndex { .. } + ) { *status.lock().unwrap() = IndexingStatus::Stopping; *stop_flag.lock().unwrap() = true; } @@ -440,6 +490,20 @@ impl IndexingService { } } + fn file_index_status_callback( + status: &Arc>, + ) -> Box { + let st = status.clone(); + Box::new(move |file_status: &str| { + if let Ok(mut status_guard) = st.lock() { + if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard + { + *current_file = Some(file_status.to_string()); + } + } + }) + } + fn run_indexing( status: &Arc>, path: &str, @@ -452,7 +516,7 @@ impl IndexingService { let conn = Connection::open(db_path) .map_err(|e| format!("Failed to open database: {}", e))?; - conn.execute_batch( + conn.execute_batch( // Default SQLITE page size is 4kB, and our memory cache is in units of page count "PRAGMA journal_mode = OFF; PRAGMA synchronous = 0; PRAGMA cache_size = 10000; @@ -471,13 +535,36 @@ impl IndexingService { ) .map_err(|e| format!("Failed to create files table: {}", e))?; + conn.execute( + "CREATE TABLE IF NOT EXISTS documents ( + id INTEGER PRIMARY KEY, + name TEXT, + path TEXT NOT NULL UNIQUE, + text TEXT NOT NULL)", + (), + ) + .map_err(|e| format!("Failed to create documents table: {}", e))?; + + let fts_external = Self::searchabletext_is_external_content(&conn)?; + if !fts_external { + conn.execute("DROP TABLE IF EXISTS searchabletext", ()) + .map_err(|e| format!("Failed to drop legacy searchabletext: {}", e))?; + conn.execute("DROP TABLE IF EXISTS searchabletext_doc", ()) + .map_err(|e| format!("Failed to drop legacy searchabletext_doc: {}", e))?; + } + let create_fts_sql = format!( - "CREATE VIRTUAL TABLE IF NOT EXISTS searchabletext USING fts5 (name, path, text, tokenize = '{}');", + "CREATE VIRTUAL TABLE IF NOT EXISTS searchabletext USING fts5(name, text, content='documents', content_rowid='id', tokenize='{}');", config.processing.tokenize ); conn.execute(&create_fts_sql, ()) .map_err(|e| format!("Failed to create searchabletext table: {}", e))?; + if !fts_external { + conn.execute("INSERT INTO searchabletext(searchabletext) VALUES('rebuild')", ()) + .map_err(|e| format!("Failed to rebuild searchabletext: {}", e))?; + } + conn.execute( "CREATE TABLE IF NOT EXISTS config_validation ( key TEXT PRIMARY KEY, @@ -503,134 +590,166 @@ impl IndexingService { *db_opt = Some(conn_mutex.clone()); } - // Collect all file entries - let walker = WalkDir::new(path).into_iter(); - let entries: Vec<_> = walker - .filter_map(|entry| entry.ok()) - .filter(|entry| !entry.metadata().map(|m| m.is_dir()).unwrap_or(true)) + let progress_display_total: Option = + if config.processing.precount_files_for_progress { + if *stop_flag.lock().unwrap() { + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::Idle; + } + return Ok(()); + } + if let Ok(mut g) = status.lock() { + if let IndexingStatus::CountingFiles { ref mut current_file, .. } = *g { + *current_file = Some("Counting paths (shell)...".to_string()); + } + } + let n = count_tree_entries_fast(path).map_err(|e| format!("Precount: {}", e))?; + if *stop_flag.lock().unwrap() { + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::Idle; + } + return Ok(()); + } + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::RunningFileIndex { + files_processed: 0, + total_files: Some(n), + current_file: None, + start_time: Instant::now(), + }; + } + Some(n) + } else { + None + }; + + let batch_size = config.processing.batch_size; + let mut pending_updates: Vec<(DirEntry, usize)> = Vec::new(); + let mut pending_inserts: Vec<(DirEntry, usize)> = Vec::new(); + let mut seen_existing_paths: HashSet = HashSet::new(); + let mut visit: usize = 0; + let mut had_incremental_work = false; + let flush_updates = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> { + if buf.is_empty() { + return Ok(()); + } + process_batch_updates_files_only( + &conn_mutex, + buf.as_slice(), + stop_flag, + Some(Self::file_index_status_callback(status)), + None, + config, + progress_display_total, + )?; + buf.clear(); + Ok(()) + }; + + let flush_inserts = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> { + if buf.is_empty() { + return Ok(()); + } + process_batch_inserts_files_only( + &conn_mutex, + buf.as_slice(), + stop_flag, + Some(Self::file_index_status_callback(status)), + None, + config, + progress_display_total, + )?; + buf.clear(); + Ok(()) + }; + + for entry in indexed_walk_file_entries(path, config.processing.follow_symlinks) { + if *stop_flag.lock().unwrap() { + flush_updates(&mut pending_updates)?; + flush_inserts(&mut pending_inserts)?; + if let Ok(mut status_guard) = status.lock() { + *status_guard = IndexingStatus::Idle; + } + return Ok(()); + } + + let action = classify_dir_entry_for_indexing(&entry, &existing_files); + let Some(action) = action else { + continue; + }; + let current_path = entry + .path() + .canonicalize() + .ok() + .map(|fp| { + let path_str = fp.to_string_lossy().to_string(); + if path_str.starts_with("\\\\?\\") { + path_str[4..].to_string() + } else { + path_str + } + }); + + visit += 1; + if let Ok(mut g) = status.lock() { + if let IndexingStatus::RunningFileIndex { + ref mut files_processed, + .. + } = *g + { + *files_processed = visit; + } + } + + match action { + FileIndexAction::Skip => { + if let Some(path) = current_path { + seen_existing_paths.insert(path); + } + } + FileIndexAction::Update => { + if let Some(path) = current_path { + seen_existing_paths.insert(path); + } + had_incremental_work = true; + pending_updates.push((entry, visit)); + if pending_updates.len() >= batch_size { + flush_updates(&mut pending_updates)?; + } + } + FileIndexAction::Insert => { + had_incremental_work = true; + pending_inserts.push((entry, visit)); + if pending_inserts.len() >= batch_size { + flush_inserts(&mut pending_inserts)?; + } + } + } + } + + flush_updates(&mut pending_updates)?; + flush_inserts(&mut pending_inserts)?; + + let stale_paths: Vec = existing_files + .keys() + .filter(|p| !seen_existing_paths.contains(*p)) + .cloned() .collect(); - - let total_file_count = entries.len(); - - // Update status with total file count - if let Ok(mut status_guard) = status.lock() { - if let IndexingStatus::RunningFileIndex { ref mut total_files, .. } = *status_guard { - *total_files = Some(total_file_count); - } + let stale_deleted = cleanup_stale_index_entries( + &conn_mutex, + stale_paths.as_slice(), + stop_flag, + Some(Self::file_index_status_callback(status)), + )?; + if stale_deleted > 0 { + had_incremental_work = true; } - // Analyze which files need updates vs inserts - let batch_update = analyze_files_for_batch_update(&entries, &existing_files); - - let total_work = batch_update.files_to_update.len() + batch_update.files_to_insert.len(); - - // Update status to show actual work needed - if let Ok(mut status_guard) = status.lock() { - if let IndexingStatus::RunningFileIndex { ref mut total_files, .. } = *status_guard { - *total_files = Some(total_work); - } - } - - let mut work_completed = 0; - - // Process updated files in batches - if !batch_update.files_to_update.is_empty() { + if !had_incremental_work { if let Ok(mut status_guard) = status.lock() { - if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard { - *current_file = Some(format!("Updating {} modified files...", batch_update.files_to_update.len())); - } - } - - // Create status callback to update current file - let status_clone_1 = status.clone(); - let status_callback = Box::new(move |file_status: &str| { - if let Ok(mut status_guard) = status_clone_1.lock() { - if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard { - *current_file = Some(file_status.to_string()); - } - } - }); - - // Create progress callback to update files_processed - let status_clone_2 = status.clone(); - let base_work_completed = work_completed; - let progress_callback = Box::new(move |current_index: usize| { - if let Ok(mut status_guard) = status_clone_2.lock() { - if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard { - *files_processed = base_work_completed + current_index; - } - } - }); - - if let Err(e) = process_batch_updates_files_only(&conn_mutex, &batch_update.files_to_update, &stop_flag, Some(status_callback), Some(progress_callback), config) { - return Err(format!("Failed to process batch updates: {}", e)); - } - - work_completed += batch_update.files_to_update.len(); - - if let Ok(mut status_guard) = status.lock() { - if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard { - *files_processed = work_completed; - } - } - } - - // Check for stop signal - if *stop_flag.lock().unwrap() { - if let Ok(mut status_guard) = status.lock() { - *status_guard = IndexingStatus::Idle; - } - return Ok(()); - } - - // Process new files in batches - if !batch_update.files_to_insert.is_empty() { - if let Ok(mut status_guard) = status.lock() { - if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard { - *current_file = Some(format!("Indexing {} new files...", batch_update.files_to_insert.len())); - } - } - - // Create status callback for inserts - let status_clone_3 = status.clone(); - let status_callback = Box::new(move |file_status: &str| { - if let Ok(mut status_guard) = status_clone_3.lock() { - if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard { - *current_file = Some(file_status.to_string()); - } - } - }); - - // Create progress callback for inserts - let status_clone_4 = status.clone(); - let base_work_completed = work_completed; - let progress_callback = Box::new(move |current_index: usize| { - if let Ok(mut status_guard) = status_clone_4.lock() { - if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard { - *files_processed = base_work_completed + current_index; - } - } - }); - - if let Err(e) = process_batch_inserts_files_only(&conn_mutex, &batch_update.files_to_insert, &stop_flag, Some(status_callback), Some(progress_callback), config) { - return Err(format!("Failed to process batch inserts: {}", e)); - } - - work_completed += batch_update.files_to_insert.len(); - - if let Ok(mut status_guard) = status.lock() { - if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard { - *files_processed = work_completed; - } - } - } - - // If no incremental work was needed, show completion status for file indexing phase - if total_work == 0 { - if let Ok(mut status_guard) = status.lock() { - if let IndexingStatus::RunningFileIndex { ref mut current_file, ref mut files_processed, .. } = *status_guard { + if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard + { *current_file = Some("File index is up to date".to_string()); - *files_processed = total_file_count; } } } @@ -688,6 +807,25 @@ impl IndexingService { Ok(()) } + fn searchabletext_is_external_content(conn: &Connection) -> Result { + let sql: Option = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE name='searchabletext'", + [], + |r| r.get(0), + ) + .optional() + .map_err(|e| format!("sqlite_master searchabletext: {}", e))?; + Ok(sql + .as_deref() + .map(|s| { + s.contains("content='documents'") + || s.contains("content=\"documents\"") + || s.contains("content=documents") + }) + .unwrap_or(false)) + } + /// Validates configuration against stored values and returns validation results. /// Critical configuration changes that require index recreation: /// - hash_length: affects file hash computation, invalidates existing file metadata diff --git a/src/main.rs b/src/main.rs index 7fac029..5bf65fa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -64,7 +64,7 @@ Duplicate files: SELECT name, count(*) as cnt, path FROM files GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC; Full text search: -SELECT name, path, text, snippet(searchabletext, 2 , "", "", "...", 64) as "snip" FROM searchabletext WHERE text MATCH 'searchstring' +SELECT d.name, d.path, d.text, snippet(st, 1 , "", "", "...", 64) as "snip" FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE st.text MATCH 'searchstring' Filename search: SELECT name, path FROM files WHERE name LIKE '%searchstring%'; diff --git a/src/search.rs b/src/search.rs index cf972c9..5697ed7 100644 --- a/src/search.rs +++ b/src/search.rs @@ -20,6 +20,8 @@ impl PartialEq for SearchProps { pub fn Search(props: SearchProps) -> Element { let mut search_type = use_signal(|| "fulltext".to_string()); let mut search_term = use_signal(|| String::new()); + let mut fulltext_exact = use_signal(|| false); + let mut fulltext_case_sensitive = use_signal(|| false); let search_results = use_signal(|| Vec::::new()); let mut search_error = use_signal(|| None::); let is_searching = use_signal(|| false); @@ -35,6 +37,8 @@ pub fn Search(props: SearchProps) -> Element { let db_path = db_path.clone(); let search_type = search_type.clone(); let search_term = search_term.clone(); + let fulltext_exact = fulltext_exact.clone(); + let fulltext_case_sensitive = fulltext_case_sensitive.clone(); let search_results = search_results.clone(); let search_error = search_error.clone(); let is_searching = is_searching.clone(); @@ -46,6 +50,8 @@ pub fn Search(props: SearchProps) -> Element { let db_clone = db_path.clone(); let search_type_val = search_type().clone(); let search_term_val = search_term().clone(); + let fulltext_exact_val = fulltext_exact(); + let fulltext_case_sensitive_val = fulltext_case_sensitive(); let mut search_results_clone = search_results.clone(); let mut search_error_clone = search_error.clone(); @@ -61,38 +67,81 @@ pub fn Search(props: SearchProps) -> Element { let query = match search_type_val.as_str() { "fulltext" => { - if search_term_val.trim().is_empty() { + let trimmed = search_term_val.trim(); + if trimmed.is_empty() { search_error_clone.set(Some("Please enter a search term".to_string())); is_searching_clone.set(false); return; } - - // Sanitize search term for FTS5 by removing problematic characters - let sanitized_term = search_term_val - .replace("'", "''") // Escape single quotes for SQL - .replace(":", " ") // Replace colons with spaces (common in file paths, times, etc.) - .replace(";", " ") // Replace semicolons with spaces - .replace("(", " ") // Replace parentheses with spaces - .replace(")", " ") - .replace("[", " ") // Replace brackets with spaces - .replace("]", " ") - .replace("{", " ") // Replace braces with spaces - .replace("}", " ") - .replace("^", " ") // Replace carets with spaces - .replace("~", " ") // Replace tildes with spaces - .replace("\"", " "); // Replace quotes with spaces to avoid nesting issues - - // Split into words and rejoin to handle multiple spaces and create a proper FTS5 query - let words: Vec<&str> = sanitized_term.split_whitespace().collect(); - if words.is_empty() { + + let sanitized_term = trimmed + .replace(':', " ") + .replace(';', " ") + .replace('(', " ") + .replace(')', " ") + .replace('[', " ") + .replace(']', " ") + .replace('{', " ") + .replace('}', " ") + .replace('^', " ") + .replace('~', " ") + .replace('"', " "); + + let tokens: Vec<&str> = sanitized_term.split_whitespace().collect(); + if tokens.is_empty() { search_error_clone.set(Some("Please enter a valid search term".to_string())); is_searching_clone.set(false); return; } - - // Join words with AND for better matching - let fts_query = words.join(" AND "); - format!("SELECT name, path, snippet(searchabletext, 2, '', '', '...', 64) as snippet FROM searchabletext WHERE text MATCH '{}'", fts_query) + + let words: Vec<&str> = if fulltext_exact_val { + tokens + } else { + let filtered: Vec<&str> = tokens + .into_iter() + .filter(|w| w.chars().count() >= 3) + .collect(); + if filtered.is_empty() { + search_error_clone.set(Some( + "Trigram index needs each word to be at least 3 characters unless you use exact phrase search.".to_string(), + )); + is_searching_clone.set(false); + return; + } + filtered + }; + + let sql_quote = |s: &str| s.replace('\'', "''"); + + let fts_match = if fulltext_exact_val { + let phrase = words.join(" "); + format!("\"{}\"", phrase.replace('"', "\"\"")) + } else { + words.join(" AND ") + }; + + let mut where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match)); + if fulltext_case_sensitive_val { + if fulltext_exact_val { + let literal = words.join(" "); + where_clause.push_str(&format!( + " AND instr(st.text, '{}') > 0", + sql_quote(&literal) + )); + } else { + for w in &words { + where_clause.push_str(&format!( + " AND instr(st.text, '{}') > 0", + sql_quote(w) + )); + } + } + } + + format!( + "SELECT d.name, d.path, snippet(st, 1, '', '', '...', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank", + where_clause + ) }, "filename" => { if search_term_val.trim().is_empty() { @@ -160,6 +209,32 @@ pub fn Search(props: SearchProps) -> Element { option { value: "duplicates", "Find Duplicate Files" } } } + + if search_type() == "fulltext" { + div { + class: "form-group", + style: "display: flex; flex-direction: column; gap: 6px;", + span { style: "font-weight: 600;", "Full text options" } + label { + style: "display: flex; align-items: center; gap: 8px; cursor: pointer;", + input { + r#type: "checkbox", + checked: fulltext_exact(), + onchange: move |evt| fulltext_exact.set(evt.checked()), + } + "Exact phrase match" + } + label { + style: "display: flex; align-items: center; gap: 8px; cursor: pointer;", + input { + r#type: "checkbox", + checked: fulltext_case_sensitive(), + onchange: move |evt| fulltext_case_sensitive.set(evt.checked()), + } + "Case-sensitive match" + } + } + } if search_type() != "duplicates" { div {