jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

Sped up initial file count and reduced memory use, added cleaning step to database on repeat indexing.

Browse source
This commit is contained in:
Jeremy Karst 2026-04-21 02:34:40 -04:00
parent 2c87a4847e
commit 5c039206c6
10 changed files with 941 additions and 375 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
config_example.toml
View file

@ -6,11 +6,18 @@ database_path = "QuickSearch.db"
# Amount of data in bytes read from start/end of files used to calculate hash
hash_length = 8192
# Maximum text content to index per file (bytes)
maximum_text_size = 524288
maximum_text_size = 262144
# Maximum file size to process for text extraction (bytes)
maximum_file_size = 52428800
# Number of files to process in each batch
maximum_text_file_size = 2097152
# Number of files to process in each batch (directory walk / inserts / text extraction batches)
batch_size = 200
# Files per transaction for incremental UPDATE files + DELETE from searchabletext (FTS); larger = fewer commits, more RAM per chunk
fts_update_batch_size = 1000
# If true, run a fast shell-backed tree count before Phase 1 (enables % progress; Linux uses GNU find -printf '\n' | wc -l when available).
# If false, Phase 1 shows file counts without a percentage.
precount_files_for_progress = false
# If true, follow symbolic links during directory walks (indexing only; shell precount unchanged).
follow_symlinks = false
# FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61')
# Look here for more information https://www.sqlite.org/fts5.html#tokenizers
tokenize = "trigram"

1
run.bat Normal file
View file

@ -0,0 +1 @@
cargo run

2
run.sh Normal file
View file

@ -0,0 +1,2 @@
cargo build --release
./target/release/quicksearch

1
setup.sh Normal file
View file

@ -0,0 +1 @@
sudo apt install -y libsoup-3.0-dev libjavascriptcoregtk-4.1-dev libwebkit2gtk-4.1-dev libxdo-dev

19
src/config.rs
View file

@ -14,13 +14,23 @@ pub struct PathConfig {
pub database_path: String,
}
fn default_fts_update_batch_size() -> usize {
1000
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ProcessingConfig {
pub hash_length: usize,
pub maximum_text_size: usize,
pub maximum_file_size: u64,
pub maximum_text_file_size: u64,
pub batch_size: usize,
#[serde(default = "default_fts_update_batch_size")]
pub fts_update_batch_size: usize,
pub tokenize: String,
#[serde(default)]
pub precount_files_for_progress: bool,
#[serde(default)]
pub follow_symlinks: bool,
}
impl Default for Config {
@ -32,10 +42,13 @@ impl Default for Config {
},
processing: ProcessingConfig {
hash_length: 1024 * 8,
maximum_text_size: 1024 * 512,
maximum_file_size: 1024 * 1024 * 50,
maximum_text_size: 1024 * 256,
maximum_text_file_size: 1024 * 1024 * 2,
batch_size: 200,
fts_update_batch_size: 1000,
tokenize: "trigram".to_string(),
precount_files_for_progress: false,
follow_symlinks: false,
},
}
}

705
src/file_handling.rs
View file

@ -2,46 +2,54 @@ use std::sync::{Mutex, Arc};
use std::ffi::OsString;
use std::fs::{File,read_to_string};
use std::io::{Read, Seek, SeekFrom};
use std::process::{Command, Stdio};
use std::time::UNIX_EPOCH;
use std::collections::HashMap;
use sha2::{Sha256, Digest};
use walkdir::DirEntry;
use walkdir::{DirEntry, WalkDir};
use rusqlite::{params, Connection};
use crate::document_extraction::extract_document_text;
use crate::config::Config;
#[derive(Debug, Clone)]
pub struct FileMetadata {
pub path: String,
pub size: u64,
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ExistingFileEntry {
pub moddate: u64,
pub hash: Vec<u8>,
}
#[derive(Debug)]
pub struct BatchUpdate {
pub files_to_update: Vec<(DirEntry, FileMetadata)>,
pub files_to_insert: Vec<DirEntry>,
}
pub const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 86] =
["","txt","rtf","log", // Text Documents
"csv", // Spreadsheet
"sh","bat","cmd","bash","ps1","psm1","psd1","pssc","psrc", // Scripts
"c","cpp","i","cs","csx","caki", // C#
"cpp","cc","cxx","c++","hpp","hh","hxx","h","ii", // C++
"tex","bib","bbx","cbx", // LaTeX
"css","xml","md","json","yaml","yml", // Markup Languages and others
"html","htm","shtml","xhtml","xht","mdoc","jsp","asp","aspx","jshtm", // HTML
"js","cjs","mjs","es6","es","jsx","ts","tsx", // Javascript and TypeScript
"cfg","conf","ini","gitattributes","gitignore", // Config and related files
"java","jav", // Java
"pl","pm","pod","t","psgi", // Perl
"php","php4","php5","phtml","ctp", // PHP
"py","rpy","pyw","cpy","gyp","gypi","pyi","ipy","pyt","ipynb", // Python
"wasm","wat", // Web Assembly
pub const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 84] =
["c","cs","csx", // C
"cpp","cc","cxx","hpp","hh","hxx","h", // C++
"cfg","conf","ini","gitattributes","gitignore", // Config (General)
"toml","env","tf","tfvars", // Config (Infrastructure)
"scss","sass","less", // CSS Preprocessors
"dart", // Dart
"diff","patch", // Diffs
"go", // Go
"graphql","gql", // GraphQL
"html","htm","xhtml","xht","jsp","asp","aspx", // HTML
"java", // Java
"js","cjs","mjs","jsx","ts","tsx", // Javascript and TypeScript
"vue","svelte", // JS Frameworks
"kt","kts", // Kotlin
"tex","bib", // LaTeX
"css","xml","md","json","yaml","yml", // Markup
"m", // Objective-C
"pl","pm","t", // Perl
"php","phtml", // PHP
"proto", // Protocol Buffers
"py","pyw","pyi","ipynb", // Python
"r", // R
"rb", // Ruby
"rs", // Rust
"sh","bat","cmd","bash","ps1","psm1","psd1", // Scripts
"sql", // SQL
"csv", // Spreadsheet
"svg", // SVG
"swift", // Swift
"","txt","rtf","log", // Text Documents
"wasm", // Web Assembly
];
pub const SUPPORTED_DOCUMENT_EXTENSIONS_LIST: [&'static str; 9] =
@ -49,76 +57,183 @@ pub const SUPPORTED_DOCUMENT_EXTENSIONS_LIST: [&'static str; 9] =
"ppt", "pptx", "odp", // Presentation
"xls", "xlsx", "ods"]; // Spreadsheet
/// Load existing file metadata from database indexed by path
pub fn load_existing_files(conn: &Connection) -> Result<HashMap<String, FileMetadata>, rusqlite::Error> {
/// Load path and moddate per row for incremental classification (hash/size loaded only when updating a file).
pub fn load_existing_files(conn: &Connection) -> Result<HashMap<String, ExistingFileEntry>, rusqlite::Error> {
let mut existing_files = HashMap::new();
let mut stmt = conn.prepare("SELECT path, size, moddate, hash FROM files")?;
let mut stmt = conn.prepare("SELECT path, moddate FROM files")?;
let rows = stmt.query_map([], |row| {
Ok(FileMetadata {
path: row.get(0)?,
size: row.get(1)?,
moddate: row.get(2)?,
hash: row.get(3)?,
})
Ok((
row.get::<_, String>(0)?,
ExistingFileEntry {
moddate: row.get(1)?,
},
))
})?;
for row in rows {
let metadata = row?;
existing_files.insert(metadata.path.clone(), metadata);
let (path, entry) = row?;
existing_files.insert(path, entry);
}
Ok(existing_files)
}
/// Analyze files and determine which need updates vs inserts
pub fn analyze_files_for_batch_update(
entries: &[DirEntry],
existing_files: &HashMap<String, FileMetadata>
) -> BatchUpdate {
let mut files_to_update = Vec::new();
let mut files_to_insert = Vec::new();
pub fn indexed_walk_file_entries(path: &str, follow_symlinks: bool) -> impl Iterator<Item = DirEntry> {
WalkDir::new(path)
.follow_links(follow_symlinks)
.into_iter()
.filter_map(|e| e.ok())
.filter(|entry| !entry.metadata().map(|m| m.is_dir()).unwrap_or(true))
}
for entry in entries {
let meta = match entry.metadata() {
Ok(m) if !m.is_dir() => m,
_ => continue,
};
fn parse_wc_l_stdout(bytes: &[u8]) -> Result<usize, String> {
let s = String::from_utf8_lossy(bytes);
let token = s
.trim()
.split_whitespace()
.next()
.ok_or_else(|| "wc: empty output".to_string())?;
token
.parse()
.map_err(|e| format!("wc: invalid count {:?}: {}", token, e))
}
let fpath = match entry.path().canonicalize() {
Ok(fp) => {
let path_str = fp.to_string_lossy().to_string();
// Remove Windows UNC prefix \\?\
if path_str.starts_with("\\\\?\\") {
path_str[4..].to_string()
} else {
path_str
}
},
Err(_) => continue,
};
let fmodified = match meta.modified()
.ok()
.and_then(|m| m.duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs())) {
Some(time) => time,
None => continue,
};
if let Some(existing_metadata) = existing_files.get(&fpath) {
// File exists in database, check if modification date changed
if existing_metadata.moddate != fmodified {
files_to_update.push((entry.clone(), existing_metadata.clone()));
}
// If moddate is same, skip processing this file entirely
} else {
// New file, needs to be inserted
files_to_insert.push(entry.clone());
}
#[cfg(unix)]
fn count_find_pipe_wc(path: &str) -> Result<usize, String> {
let mut find = Command::new("find")
.arg(path)
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.map_err(|e| format!("find: {}", e))?;
let find_stdout = find.stdout.take().ok_or("find: stdout")?;
let wc = Command::new("wc")
.arg("-l")
.stdin(find_stdout)
.stdout(Stdio::piped())
.output()
.map_err(|e| format!("wc: {}", e))?;
find.wait().map_err(|e| format!("find wait: {}", e))?;
if !wc.status.success() {
return Err(format!("wc exited with {}", wc.status));
}
parse_wc_l_stdout(&wc.stdout)
}
BatchUpdate {
files_to_update,
files_to_insert,
#[cfg(target_os = "linux")]
fn count_find_printf_wc(path: &str) -> Result<usize, String> {
let mut find = Command::new("find")
.arg(path)
.arg("-printf")
.arg("\n")
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.map_err(|e| format!("find: {}", e))?;
let find_stdout = find.stdout.take().ok_or("find: stdout")?;
let wc = Command::new("wc")
.arg("-l")
.stdin(find_stdout)
.stdout(Stdio::piped())
.output()
.map_err(|e| format!("wc: {}", e))?;
find.wait().map_err(|e| format!("find wait: {}", e))?;
if !wc.status.success() {
return Err(format!("wc exited with {}", wc.status));
}
parse_wc_l_stdout(&wc.stdout)
}
#[cfg(windows)]
fn count_tree_entries_windows(path: &str) -> Result<usize, String> {
let lit = path.replace('\'', "''");
let ps = format!(
"(Get-ChildItem -LiteralPath '{}' -Recurse -Force -ErrorAction SilentlyContinue | Measure-Object).Count",
lit
);
let out = Command::new("powershell.exe")
.args(["-NoProfile", "-NonInteractive", "-Command", &ps])
.output()
.map_err(|e| format!("powershell: {}", e))?;
if !out.status.success() {
return Err(format!(
"powershell exited with {}: {}",
out.status,
String::from_utf8_lossy(&out.stderr)
));
}
String::from_utf8_lossy(&out.stdout)
.trim()
.parse()
.map_err(|e| format!("invalid count output: {}", e))
}
/// Rough tree entry count for progress totals (Linux: `find DIR -printf '\n' | wc -l` when GNU find is available, else `find DIR | wc -l`; macOS/other Unix: `find DIR | wc -l`; Windows: PowerShell `Get-ChildItem -Recurse`). Scope is not identical to the indexer’s classified file count.
pub fn count_tree_entries_fast(path: &str) -> Result<usize, String> {
#[cfg(windows)]
{
return count_tree_entries_windows(path);
}
#[cfg(all(unix, target_os = "linux"))]
{
return count_find_printf_wc(path).or_else(|_| count_find_pipe_wc(path));
}
#[cfg(all(unix, not(target_os = "linux")))]
{
return count_find_pipe_wc(path);
}
#[cfg(not(any(windows, unix)))]
{
Err("tree entry count is not supported on this target".to_string())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FileIndexAction {
Skip,
Update,
Insert,
}
/// Classify a walkdir entry for Phase 1 file indexing. Returns `None` if the path is not indexable.
pub fn classify_dir_entry_for_indexing(
entry: &DirEntry,
existing_files: &HashMap<String, ExistingFileEntry>,
) -> Option<FileIndexAction> {
let fpath = match entry.path().canonicalize() {
Ok(fp) => {
let path_str = fp.to_string_lossy().to_string();
if path_str.starts_with("\\\\?\\") {
path_str[4..].to_string()
} else {
path_str
}
}
Err(_) => return None,
};
let meta = match std::fs::metadata(&fpath) {
Ok(m) if m.is_file() => m,
_ => return None,
};
let fmodified = match meta
.modified()
.ok()
.and_then(|m| m.duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()))
{
Some(time) => time,
None => return None,
};
if let Some(existing) = existing_files.get(&fpath) {
if existing.moddate != fmodified {
Some(FileIndexAction::Update)
} else {
Some(FileIndexAction::Skip)
}
} else {
Some(FileIndexAction::Insert)
}
}
@ -159,103 +274,194 @@ fn get_file_hash(size: u64, path: OsString, hash_length: usize) -> Result<Vec<u8
Ok(hasher.finalize().to_vec())
}
fn format_progress_pair(visit_index: usize, progress_display_total: Option<usize>) -> String {
match progress_display_total {
Some(t) => format!("{}/{}", visit_index, t),
None => format!("{}", visit_index),
}
}
const FTS_SQL_AUTOMERGE_8: &str =
"INSERT INTO searchabletext(searchabletext, rank) VALUES('automerge', 8)";
const FTS_SQL_REBUILD: &str = "INSERT INTO searchabletext(searchabletext) VALUES('rebuild')";
pub fn fts_finalize_after_text_indexing(conn: &Connection) -> Result<(), String> {
conn.execute(FTS_SQL_AUTOMERGE_8, [])
.map_err(|e| format!("FTS automerge(8): {}", e))?;
conn.execute(FTS_SQL_REBUILD, [])
.map_err(|e| format!("FTS rebuild: {}", e))?;
Ok(())
}
fn fts_remove_document_for_path(
tx: &rusqlite::Transaction<'_>,
path: &str,
) -> Result<(), String> {
let id_opt: Option<i64> = match tx.query_row(
"SELECT id FROM documents WHERE path = ?1",
params![path],
|r| r.get(0),
) {
Ok(id) => Some(id),
Err(rusqlite::Error::QueryReturnedNoRows) => None,
Err(e) => return Err(format!("documents id lookup: {}", e)),
};
if let Some(doc_id) = id_opt {
tx.execute(
"INSERT INTO searchabletext(searchabletext, rowid) VALUES('delete', ?1)",
params![doc_id],
)
.map_err(|e| format!("FTS delete doc {}: {}", doc_id, e))?;
tx.execute("DELETE FROM documents WHERE id = ?1", params![doc_id])
.map_err(|e| format!("delete documents row: {}", e))?;
}
Ok(())
}
struct PreparedFileUpdate {
path_db: String,
fsize: u64,
fmodified: u64,
fhash: Vec<u8>,
filename: String,
visit_index: usize,
}
/// Process updated files in batch with transaction - files table only (no text extraction)
pub fn process_batch_updates_files_only(
conn_mutex: &Arc<Mutex<Connection>>,
files_to_update: &[(DirEntry, FileMetadata)],
files_to_update: &[(DirEntry, usize)],
stop_flag: &Arc<Mutex<bool>>,
status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>,
config: &Config
config: &Config,
progress_display_total: Option<usize>,
) -> Result<(), String> {
if files_to_update.is_empty() {
return Ok(());
}
let batch_size = config.processing.batch_size;
let total_files = files_to_update.len();
let fts_batch = config.processing.fts_update_batch_size.max(1);
// Process files in batches of batch_size
for (batch_idx, batch) in files_to_update.chunks(batch_size).enumerate() {
// Check stop flag at the start of each batch
for batch in files_to_update.chunks(fts_batch) {
if *stop_flag.lock().unwrap() {
return Ok(());
}
let conn = conn_mutex.lock().unwrap();
let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?;
let mut prepared: Vec<PreparedFileUpdate> = Vec::new();
for (i, (entry, _old_metadata)) in batch.iter().enumerate() {
let global_index = batch_idx * batch_size + i + 1;
// Check stop flag
for (entry, visit_index) in batch.iter() {
if *stop_flag.lock().unwrap() {
drop(tx);
drop(conn);
return Ok(());
}
// Update status with current file
let filename = entry
.path()
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("unknown")
.to_string();
if let Some(ref callback) = status_callback {
let filename = entry.path().file_name()
.and_then(|n| n.to_str())
.unwrap_or("unknown");
callback(&format!("Updating file metadata {}/{}: {}", global_index, total_files, filename));
let pair = format_progress_pair(*visit_index, progress_display_total);
callback(&format!("Hashing changed files {}: {}", pair, filename));
}
// Update progress counter
if let Some(ref progress_cb) = progress_callback {
progress_cb(global_index);
}
let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?;
if meta.is_dir() {
continue;
progress_cb(*visit_index);
}
let fpath = match entry.path().canonicalize() {
Ok(fp) => {
let path_str = fp.to_string_lossy().to_string();
// Remove Windows UNC prefix \\?\
if path_str.starts_with("\\\\?\\") {
std::ffi::OsString::from(&path_str[4..])
} else {
fp.into_os_string()
}
},
}
Err(_) => continue,
};
let meta = match std::fs::metadata(&fpath) {
Ok(m) if m.is_file() => m,
_ => continue,
};
let fsize = meta.len();
let fmodified = meta.modified()
let fmodified = meta
.modified()
.map_err(|e| format!("Failed to get modified time: {}", e))?
.duration_since(UNIX_EPOCH)
.map_err(|e| format!("Failed to calculate duration: {}", e))?
.as_secs();
let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length)
.map_err(|e| format!("Failed to calculate hash: {}", e))?;
let fhash = match get_file_hash(fsize, fpath.clone(), config.processing.hash_length) {
Ok(h) => h,
Err(e) => {
eprintln!(
"Warning: Skipping file (cannot hash) {}: {}",
fpath.to_string_lossy(),
e
);
continue;
}
};
// Check stop flag after hash calculation
if *stop_flag.lock().unwrap() {
return Ok(());
}
prepared.push(PreparedFileUpdate {
path_db: fpath.to_string_lossy().into_owned(),
fsize,
fmodified,
fhash,
filename,
visit_index: *visit_index,
});
}
if prepared.is_empty() {
continue;
}
let conn = conn_mutex.lock().unwrap();
let tx = conn
.unchecked_transaction()
.map_err(|e| format!("Failed to begin transaction: {}", e))?;
for row in &prepared {
if *stop_flag.lock().unwrap() {
drop(tx);
drop(conn);
return Ok(());
}
// Update files table
if let Some(ref callback) = status_callback {
let pair = format_progress_pair(row.visit_index, progress_display_total);
callback(&format!(
"Applying index updates {}: {}",
pair, row.filename
));
}
tx.execute(
"UPDATE files SET size = ?1, moddate = ?2, hash = ?3 WHERE path = ?4",
params![fsize, fmodified, fhash, fpath.to_string_lossy()]
).map_err(|e| format!("Failed to update file record: {}", e))?;
params![row.fsize, row.fmodified, row.fhash, row.path_db],
)
.map_err(|e| format!("Failed to update file record: {}", e))?;
// Delete old searchable text entry for this file (will be re-added in text indexing phase)
tx.execute(
"DELETE FROM searchabletext WHERE path = ?1",
params![fpath.to_string_lossy()]
).map_err(|e| format!("Failed to delete old searchable text: {}", e))?;
fts_remove_document_for_path(&tx, &row.path_db).map_err(|e| {
format!(
"Failed to remove old document / FTS entry for {}: {}",
row.path_db, e
)
})?;
}
tx.commit().map_err(|e| format!("Failed to commit transaction: {}", e))?;
tx.commit()
.map_err(|e| format!("Failed to commit transaction: {}", e))?;
}
Ok(())
@ -264,22 +470,21 @@ pub fn process_batch_updates_files_only(
/// Process new files in batch with transaction - files table only (no text extraction)
pub fn process_batch_inserts_files_only(
conn_mutex: &Arc<Mutex<Connection>>,
files_to_insert: &[DirEntry],
files_to_insert: &[(DirEntry, usize)],
stop_flag: &Arc<Mutex<bool>>,
status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>,
config: &Config
config: &Config,
progress_display_total: Option<usize>,
) -> Result<(), String> {
if files_to_insert.is_empty() {
return Ok(());
}
let batch_size = config.processing.batch_size;
let total_files = files_to_insert.len();
// Process files in batches of batch_size
for (batch_idx, batch) in files_to_insert.chunks(batch_size).enumerate() {
// Check stop flag at the start of each batch
for batch in files_to_insert.chunks(batch_size) {
if *stop_flag.lock().unwrap() {
return Ok(());
}
@ -287,9 +492,8 @@ pub fn process_batch_inserts_files_only(
let conn = conn_mutex.lock().unwrap();
let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?;
for (i, entry) in batch.iter().enumerate() {
let global_index = batch_idx * batch_size + i + 1;
// Check stop flag
for (entry, visit_index) in batch.iter() {
// Check stop flag for early termination
if *stop_flag.lock().unwrap() {
drop(tx);
drop(conn);
@ -301,17 +505,13 @@ pub fn process_batch_inserts_files_only(
let filename = entry.path().file_name()
.and_then(|n| n.to_str())
.unwrap_or("unknown");
callback(&format!("Indexing file metadata {}/{}: {}", global_index, total_files, filename));
let pair = format_progress_pair(*visit_index, progress_display_total);
callback(&format!("Indexing file metadata {}: {}", pair, filename));
}
// Update progress counter
if let Some(ref progress_cb) = progress_callback {
progress_cb(global_index);
}
let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?;
if meta.is_dir() {
continue;
progress_cb(*visit_index);
}
let fpath = match entry.path().canonicalize() {
@ -327,6 +527,11 @@ pub fn process_batch_inserts_files_only(
Err(_) => continue,
};
let meta = match std::fs::metadata(&fpath) {
Ok(m) if m.is_file() => m,
_ => continue,
};
let fsize = meta.len();
let fmodified = meta.modified()
.map_err(|e| format!("Failed to get modified time: {}", e))?
@ -334,8 +539,17 @@ pub fn process_batch_inserts_files_only(
.map_err(|e| format!("Failed to calculate duration: {}", e))?
.as_secs();
let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length)
.map_err(|e| format!("Failed to calculate hash: {}", e))?;
let fhash = match get_file_hash(fsize, fpath.clone(), config.processing.hash_length) {
Ok(h) => h,
Err(e) => {
eprintln!(
"Warning: Skipping file (cannot hash) {}: {}",
fpath.to_string_lossy(),
e
);
continue;
}
};
let fname = entry.path().file_name().unwrap().to_os_string();
@ -346,13 +560,69 @@ pub fn process_batch_inserts_files_only(
).map_err(|e| format!("Failed to insert file record: {}", e))?;
}
// Update status with current file
if let Some(ref callback) = status_callback {
callback("Committing file updates to database…");
}
tx.commit().map_err(|e| format!("Failed to commit transaction: {}", e))?;
}
Ok(())
}
/// Process text indexing for files - adds entries to searchabletext table
pub fn cleanup_stale_index_entries(
conn_mutex: &Arc<Mutex<Connection>>,
stale_paths: &[String],
stop_flag: &Arc<Mutex<bool>>,
status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
) -> Result<usize, String> {
if stale_paths.is_empty() {
return Ok(0);
}
let conn = conn_mutex.lock().unwrap();
let tx = conn
.unchecked_transaction()
.map_err(|e| format!("Failed to begin stale cleanup transaction: {}", e))?;
let mut deleted_count = 0usize;
for path in stale_paths {
if *stop_flag.lock().unwrap() {
let _ = tx.commit();
drop(conn);
return Ok(deleted_count);
}
if let Some(ref callback) = status_callback {
callback(&format!("Removing stale index entry: {}", path));
}
fts_remove_document_for_path(&tx, path).map_err(|e| {
format!(
"Failed to remove stale document / FTS entry for {}: {}",
path, e
)
})?;
tx.execute("DELETE FROM files WHERE path = ?1", params![path])
.map_err(|e| format!("Failed to delete stale file record {}: {}", path, e))?;
deleted_count += 1;
}
tx.commit()
.map_err(|e| format!("Failed to commit stale cleanup transaction: {}", e))?;
if deleted_count > 0 && !*stop_flag.lock().unwrap() {
if let Some(ref callback) = status_callback {
callback("Rebuilding FTS index after stale cleanup...");
}
fts_finalize_after_text_indexing(&conn)?;
}
Ok(deleted_count)
}
/// Process text indexing for files - writes `documents`; FTS rebuilt in `fts_finalize_after_text_indexing`.
pub fn process_text_indexing(
conn_mutex: &Arc<Mutex<Connection>>,
stop_flag: &Arc<Mutex<bool>>,
@ -360,93 +630,131 @@ pub fn process_text_indexing(
progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>,
config: &Config
) -> Result<(), String> {
let conn = conn_mutex.lock().unwrap();
// Get all files from the files table that don't have corresponding searchabletext entries
let mut stmt = conn.prepare(
"SELECT f.name, f.path, f.size FROM files f
LEFT JOIN searchabletext s ON f.path = s.path
WHERE s.path IS NULL AND f.size <= ?1"
).map_err(|e| format!("Failed to prepare statement: {}", e))?;
let file_rows = stmt.query_map([config.processing.maximum_file_size], |row| {
Ok((
row.get::<_, String>(0)?, // name
row.get::<_, String>(1)?, // path
row.get::<_, u64>(2)? // size
))
}).map_err(|e| format!("Failed to query files: {}", e))?;
let files_to_process: Vec<_> = file_rows.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("Failed to collect files: {}", e))?;
drop(stmt);
drop(conn);
let total_files = files_to_process.len();
let max_size = config.processing.maximum_text_file_size;
let batch_size = config.processing.batch_size;
let batch_limit = batch_size as i64;
// Process files in batches
for (batch_idx, batch) in files_to_process.chunks(batch_size).enumerate() {
// Check stop flag at the start of each batch
if let Some(ref callback) = status_callback {
callback("Counting files pending text index…");
}
let total_files: usize = {
let conn = conn_mutex.lock().unwrap();
conn.query_row(
"SELECT COUNT(*) FROM files f
LEFT JOIN documents d ON f.path = d.path
WHERE d.path IS NULL AND f.size <= ?1",
[max_size],
|row| row.get(0),
)
.map_err(|e| format!("Failed to count pending text files: {}", e))?
};
let mut cursor_path = String::new();
let mut global_index: usize = 0;
loop {
if *stop_flag.lock().unwrap() {
return Ok(());
}
let batch: Vec<(String, String, u64)> = {
let conn = conn_mutex.lock().unwrap();
let mut stmt = conn
.prepare(
"SELECT f.name, f.path, f.size FROM files f
LEFT JOIN documents d ON f.path = d.path
WHERE d.path IS NULL AND f.size <= ?1
AND (?2 = '' OR f.path > ?2)
ORDER BY f.path
LIMIT ?3",
)
.map_err(|e| format!("Failed to prepare text indexing query: {}", e))?;
let rows = stmt
.query_map(
rusqlite::params![max_size, cursor_path.as_str(), batch_limit],
|row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, u64>(2)?,
))
},
)
.map_err(|e| format!("Failed to query files for text indexing: {}", e))?;
rows.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("Failed to read file row: {}", e))?
};
if batch.is_empty() {
break;
}
let last_path = batch.last().unwrap().1.clone();
cursor_path = last_path;
let conn = conn_mutex.lock().unwrap();
let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?;
let tx = conn
.unchecked_transaction()
.map_err(|e| format!("Failed to begin transaction: {}", e))?;
for (i, (fname, fpath, _fsize)) in batch.iter().enumerate() {
let global_index = batch_idx * batch_size + i + 1;
// Check stop flag
for (fname, fpath, _fsize) in batch.iter() {
if *stop_flag.lock().unwrap() {
// Commit current transaction before stopping
let _ = tx.commit();
drop(conn);
return Ok(());
}
// Update status
global_index += 1;
if let Some(ref callback) = status_callback {
callback(&format!("Extracting text for search indexing {}/{}: {}", global_index, total_files, fname));
callback(&format!(
"Extracting text for search indexing {}/{}: {}",
global_index, total_files, fname
));
}
// Update progress counter
if let Some(ref progress_cb) = progress_callback {
progress_cb(global_index);
}
let path = std::path::Path::new(fpath);
let path = std::path::Path::new(fpath.as_str());
let default_ext = OsString::new();
let file_extension = path.extension().unwrap_or(&default_ext)
.to_ascii_lowercase().to_str().unwrap_or("").to_string();
let file_extension = path
.extension()
.unwrap_or(&default_ext)
.to_ascii_lowercase()
.to_str()
.unwrap_or("")
.to_string();
let ext_str = file_extension.as_str();
// Process text content with error handling for individual files
let text_result = if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) {
match read_to_string(fpath) {
Ok(file_string) => {
let trimmed_file_string = safe_truncate_string(&file_string, config.processing.maximum_text_size);
let trimmed_file_string =
safe_truncate_string(&file_string, config.processing.maximum_text_size);
Some(trimmed_file_string)
}
Err(_e) => {
// eprintln!("Warning: Failed to read plaintext file {}: {}", fpath, e);
None
}
Err(_e) => None,
}
} else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) {
match extract_document_text(&std::ffi::OsString::from(fpath), ext_str) {
Ok(extracted_text) => {
if !extracted_text.trim().is_empty() {
let trimmed_file_string = safe_truncate_string(&extracted_text, config.processing.maximum_text_size);
Some(trimmed_file_string)
Some(safe_truncate_string(
&extracted_text,
config.processing.maximum_text_size,
))
} else {
None
}
}
Err(e) => {
eprintln!("Warning: Failed to extract text from document {}: {}", fpath, e);
eprintln!(
"Warning: Failed to extract text from document {}: {}",
fpath, e
);
None
}
}
@ -454,18 +762,23 @@ pub fn process_text_indexing(
None
};
// Insert the text content if we successfully extracted it
if let Some(text_content) = text_result {
if let Err(e) = tx.execute(
"INSERT INTO searchabletext VALUES (?1, ?2, ?3)",
params![fname, fpath, text_content]
"INSERT OR REPLACE INTO documents(name, path, text) VALUES (?1, ?2, ?3)",
params![fname, fpath, text_content],
) {
eprintln!("Warning: Failed to insert searchable text for {}: {}", fpath, e);
eprintln!("Warning: Failed to insert document row for {}: {}", fpath, e);
}
}
}
tx.commit().map_err(|e| format!("Failed to commit transaction: {}", e))?;
tx.commit()
.map_err(|e| format!("Failed to commit transaction: {}", e))?;
}
if total_files > 0 && !*stop_flag.lock().unwrap() {
let conn = conn_mutex.lock().unwrap();
fts_finalize_after_text_indexing(&conn)?;
}
Ok(())

16
src/frontend.rs
View file

@ -106,6 +106,22 @@ pub fn App(props: AppProps) -> Element {
speed_tracker_clone.set(SpeedTracker::new());
"Idle".to_string()
},
IndexingStatus::CountingFiles {
current_file,
start_time,
..
} => {
let elapsed = start_time.elapsed();
let current_file_display = current_file
.as_ref()
.map(|f| format!("{}", f))
.unwrap_or_else(|| "...".to_string());
format!(
"Phase 0 - Counting paths (shell) - {:.1}s elapsed\n{}",
elapsed.as_secs_f64(),
current_file_display
)
}
IndexingStatus::RunningFileIndex { files_processed, total_files, current_file, start_time } => {
// Add data point to speed tracker
speed_tracker_clone.with_mut(|tracker| {

420
src/indexing.rs
View file

@ -2,10 +2,21 @@ use std::sync::{Arc, Mutex, mpsc};
use std::thread;
use std::time::Instant;
use std::process::Command;
use walkdir::WalkDir;
use rusqlite::{Connection, params};
use std::collections::HashSet;
use rusqlite::{Connection, OptionalExtension, params};
use walkdir::DirEntry;
use crate::file_handling::{load_existing_files, analyze_files_for_batch_update, process_batch_updates_files_only, process_batch_inserts_files_only, process_text_indexing};
use crate::file_handling::{
classify_dir_entry_for_indexing,
cleanup_stale_index_entries,
count_tree_entries_fast,
indexed_walk_file_entries,
load_existing_files,
process_batch_inserts_files_only,
process_batch_updates_files_only,
process_text_indexing,
FileIndexAction,
};
use crate::config::Config;
#[derive(Debug, Clone)]
@ -22,6 +33,12 @@ pub struct SearchResult {
#[derive(Debug, Clone)]
pub enum IndexingStatus {
Idle,
CountingFiles {
_entries_scanned: usize,
_indexable_files_counted: usize,
current_file: Option<String>,
start_time: Instant,
},
RunningFileIndex {
files_processed: usize,
total_files: Option<usize>,
@ -288,6 +305,7 @@ impl IndexingService {
}
/// Clean up UNC prefixes from existing database entries
#[allow(dead_code)]
pub fn clean_unc_prefixes(&self, db_path: &str) -> Result<(), String> {
let conn = Connection::open(db_path)
.map_err(|e| format!("Failed to open database: {}", e))?;
@ -298,16 +316,26 @@ impl IndexingService {
(),
).map_err(|e| format!("Failed to update files table: {}", e))?;
// Clean UNC prefixes from searchabletext table
conn.execute(
"UPDATE searchabletext SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'",
(),
).map_err(|e| format!("Failed to update searchabletext table: {}", e))?;
let doc_table: i64 = conn
.query_row(
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='documents'",
[],
|r| r.get(0),
)
.unwrap_or(0);
if doc_table > 0 {
conn.execute(
"UPDATE documents SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'",
(),
)
.map_err(|e| format!("Failed to update documents table: {}", e))?;
}
Ok(())
}
/// Check if the database is corrupted or malformed
#[allow(dead_code)]
pub fn check_database_health(&self, db_path: &str) -> Result<bool, String> {
match Connection::open(db_path) {
Ok(conn) => {
@ -356,7 +384,10 @@ impl IndexingService {
while attempts < 50 { // Wait up to 5 seconds
match self.get_status() {
IndexingStatus::Idle => break,
IndexingStatus::Stopping | IndexingStatus::RunningFileIndex { .. } | IndexingStatus::RunningTextIndex { .. } => {
IndexingStatus::Stopping
| IndexingStatus::CountingFiles { .. }
| IndexingStatus::RunningFileIndex { .. }
| IndexingStatus::RunningTextIndex { .. } => {
std::thread::sleep(std::time::Duration::from_millis(100));
attempts += 1;
}
@ -384,7 +415,12 @@ impl IndexingService {
while let Ok(command) = command_rx.recv() {
match command {
IndexingCommand::Start { path, db_path, config } => {
if matches!(*status.lock().unwrap(), IndexingStatus::RunningFileIndex { .. } | IndexingStatus::RunningTextIndex { .. }) {
if matches!(
*status.lock().unwrap(),
IndexingStatus::CountingFiles { .. }
| IndexingStatus::RunningFileIndex { .. }
| IndexingStatus::RunningTextIndex { .. }
) {
continue; // Already running
}
@ -394,11 +430,20 @@ impl IndexingService {
}
*stop_flag.lock().unwrap() = false;
*status.lock().unwrap() = IndexingStatus::RunningFileIndex {
files_processed: 0,
total_files: None,
current_file: None,
start_time: Instant::now(),
*status.lock().unwrap() = if config.processing.precount_files_for_progress {
IndexingStatus::CountingFiles {
_entries_scanned: 0,
_indexable_files_counted: 0,
current_file: Some("Preparing database...".to_string()),
start_time: Instant::now(),
}
} else {
IndexingStatus::RunningFileIndex {
files_processed: 0,
total_files: None,
current_file: None,
start_time: Instant::now(),
}
};
// Run indexing in a separate thread
@ -426,7 +471,12 @@ impl IndexingService {
}));
}
IndexingCommand::Stop => {
if matches!(*status.lock().unwrap(), IndexingStatus::RunningFileIndex { .. } | IndexingStatus::RunningTextIndex { .. }) {
if matches!(
*status.lock().unwrap(),
IndexingStatus::CountingFiles { .. }
| IndexingStatus::RunningFileIndex { .. }
| IndexingStatus::RunningTextIndex { .. }
) {
*status.lock().unwrap() = IndexingStatus::Stopping;
*stop_flag.lock().unwrap() = true;
}
@ -440,6 +490,20 @@ impl IndexingService {
}
}
fn file_index_status_callback(
status: &Arc<Mutex<IndexingStatus>>,
) -> Box<dyn Fn(&str) + Send + Sync> {
let st = status.clone();
Box::new(move |file_status: &str| {
if let Ok(mut status_guard) = st.lock() {
if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard
{
*current_file = Some(file_status.to_string());
}
}
})
}
fn run_indexing(
status: &Arc<Mutex<IndexingStatus>>,
path: &str,
@ -452,7 +516,7 @@ impl IndexingService {
let conn = Connection::open(db_path)
.map_err(|e| format!("Failed to open database: {}", e))?;
conn.execute_batch(
conn.execute_batch( // Default SQLITE page size is 4kB, and our memory cache is in units of page count
"PRAGMA journal_mode = OFF;
PRAGMA synchronous = 0;
PRAGMA cache_size = 10000;
@ -471,13 +535,36 @@ impl IndexingService {
)
.map_err(|e| format!("Failed to create files table: {}", e))?;
conn.execute(
"CREATE TABLE IF NOT EXISTS documents (
id INTEGER PRIMARY KEY,
name TEXT,
path TEXT NOT NULL UNIQUE,
text TEXT NOT NULL)",
(),
)
.map_err(|e| format!("Failed to create documents table: {}", e))?;
let fts_external = Self::searchabletext_is_external_content(&conn)?;
if !fts_external {
conn.execute("DROP TABLE IF EXISTS searchabletext", ())
.map_err(|e| format!("Failed to drop legacy searchabletext: {}", e))?;
conn.execute("DROP TABLE IF EXISTS searchabletext_doc", ())
.map_err(|e| format!("Failed to drop legacy searchabletext_doc: {}", e))?;
}
let create_fts_sql = format!(
"CREATE VIRTUAL TABLE IF NOT EXISTS searchabletext USING fts5 (name, path, text, tokenize = '{}');",
"CREATE VIRTUAL TABLE IF NOT EXISTS searchabletext USING fts5(name, text, content='documents', content_rowid='id', tokenize='{}');",
config.processing.tokenize
);
conn.execute(&create_fts_sql, ())
.map_err(|e| format!("Failed to create searchabletext table: {}", e))?;
if !fts_external {
conn.execute("INSERT INTO searchabletext(searchabletext) VALUES('rebuild')", ())
.map_err(|e| format!("Failed to rebuild searchabletext: {}", e))?;
}
conn.execute(
"CREATE TABLE IF NOT EXISTS config_validation (
key TEXT PRIMARY KEY,
@ -503,134 +590,166 @@ impl IndexingService {
*db_opt = Some(conn_mutex.clone());
}
// Collect all file entries
let walker = WalkDir::new(path).into_iter();
let entries: Vec<_> = walker
.filter_map(|entry| entry.ok())
.filter(|entry| !entry.metadata().map(|m| m.is_dir()).unwrap_or(true))
let progress_display_total: Option<usize> =
if config.processing.precount_files_for_progress {
if *stop_flag.lock().unwrap() {
if let Ok(mut status_guard) = status.lock() {
*status_guard = IndexingStatus::Idle;
}
return Ok(());
}
if let Ok(mut g) = status.lock() {
if let IndexingStatus::CountingFiles { ref mut current_file, .. } = *g {
*current_file = Some("Counting paths (shell)...".to_string());
}
}
let n = count_tree_entries_fast(path).map_err(|e| format!("Precount: {}", e))?;
if *stop_flag.lock().unwrap() {
if let Ok(mut status_guard) = status.lock() {
*status_guard = IndexingStatus::Idle;
}
return Ok(());
}
if let Ok(mut status_guard) = status.lock() {
*status_guard = IndexingStatus::RunningFileIndex {
files_processed: 0,
total_files: Some(n),
current_file: None,
start_time: Instant::now(),
};
}
Some(n)
} else {
None
};
let batch_size = config.processing.batch_size;
let mut pending_updates: Vec<(DirEntry, usize)> = Vec::new();
let mut pending_inserts: Vec<(DirEntry, usize)> = Vec::new();
let mut seen_existing_paths: HashSet<String> = HashSet::new();
let mut visit: usize = 0;
let mut had_incremental_work = false;
let flush_updates = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> {
if buf.is_empty() {
return Ok(());
}
process_batch_updates_files_only(
&conn_mutex,
buf.as_slice(),
stop_flag,
Some(Self::file_index_status_callback(status)),
None,
config,
progress_display_total,
)?;
buf.clear();
Ok(())
};
let flush_inserts = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> {
if buf.is_empty() {
return Ok(());
}
process_batch_inserts_files_only(
&conn_mutex,
buf.as_slice(),
stop_flag,
Some(Self::file_index_status_callback(status)),
None,
config,
progress_display_total,
)?;
buf.clear();
Ok(())
};
for entry in indexed_walk_file_entries(path, config.processing.follow_symlinks) {
if *stop_flag.lock().unwrap() {
flush_updates(&mut pending_updates)?;
flush_inserts(&mut pending_inserts)?;
if let Ok(mut status_guard) = status.lock() {
*status_guard = IndexingStatus::Idle;
}
return Ok(());
}
let action = classify_dir_entry_for_indexing(&entry, &existing_files);
let Some(action) = action else {
continue;
};
let current_path = entry
.path()
.canonicalize()
.ok()
.map(|fp| {
let path_str = fp.to_string_lossy().to_string();
if path_str.starts_with("\\\\?\\") {
path_str[4..].to_string()
} else {
path_str
}
});
visit += 1;
if let Ok(mut g) = status.lock() {
if let IndexingStatus::RunningFileIndex {
ref mut files_processed,
..
} = *g
{
*files_processed = visit;
}
}
match action {
FileIndexAction::Skip => {
if let Some(path) = current_path {
seen_existing_paths.insert(path);
}
}
FileIndexAction::Update => {
if let Some(path) = current_path {
seen_existing_paths.insert(path);
}
had_incremental_work = true;
pending_updates.push((entry, visit));
if pending_updates.len() >= batch_size {
flush_updates(&mut pending_updates)?;
}
}
FileIndexAction::Insert => {
had_incremental_work = true;
pending_inserts.push((entry, visit));
if pending_inserts.len() >= batch_size {
flush_inserts(&mut pending_inserts)?;
}
}
}
}
flush_updates(&mut pending_updates)?;
flush_inserts(&mut pending_inserts)?;
let stale_paths: Vec<String> = existing_files
.keys()
.filter(|p| !seen_existing_paths.contains(*p))
.cloned()
.collect();
let total_file_count = entries.len();
// Update status with total file count
if let Ok(mut status_guard) = status.lock() {
if let IndexingStatus::RunningFileIndex { ref mut total_files, .. } = *status_guard {
*total_files = Some(total_file_count);
}
let stale_deleted = cleanup_stale_index_entries(
&conn_mutex,
stale_paths.as_slice(),
stop_flag,
Some(Self::file_index_status_callback(status)),
)?;
if stale_deleted > 0 {
had_incremental_work = true;
}
// Analyze which files need updates vs inserts
let batch_update = analyze_files_for_batch_update(&entries, &existing_files);
let total_work = batch_update.files_to_update.len() + batch_update.files_to_insert.len();
// Update status to show actual work needed
if let Ok(mut status_guard) = status.lock() {
if let IndexingStatus::RunningFileIndex { ref mut total_files, .. } = *status_guard {
*total_files = Some(total_work);
}
}
let mut work_completed = 0;
// Process updated files in batches
if !batch_update.files_to_update.is_empty() {
if !had_incremental_work {
if let Ok(mut status_guard) = status.lock() {
if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard {
*current_file = Some(format!("Updating {} modified files...", batch_update.files_to_update.len()));
}
}
// Create status callback to update current file
let status_clone_1 = status.clone();
let status_callback = Box::new(move |file_status: &str| {
if let Ok(mut status_guard) = status_clone_1.lock() {
if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard {
*current_file = Some(file_status.to_string());
}
}
});
// Create progress callback to update files_processed
let status_clone_2 = status.clone();
let base_work_completed = work_completed;
let progress_callback = Box::new(move |current_index: usize| {
if let Ok(mut status_guard) = status_clone_2.lock() {
if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard {
*files_processed = base_work_completed + current_index;
}
}
});
if let Err(e) = process_batch_updates_files_only(&conn_mutex, &batch_update.files_to_update, &stop_flag, Some(status_callback), Some(progress_callback), config) {
return Err(format!("Failed to process batch updates: {}", e));
}
work_completed += batch_update.files_to_update.len();
if let Ok(mut status_guard) = status.lock() {
if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard {
*files_processed = work_completed;
}
}
}
// Check for stop signal
if *stop_flag.lock().unwrap() {
if let Ok(mut status_guard) = status.lock() {
*status_guard = IndexingStatus::Idle;
}
return Ok(());
}
// Process new files in batches
if !batch_update.files_to_insert.is_empty() {
if let Ok(mut status_guard) = status.lock() {
if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard {
*current_file = Some(format!("Indexing {} new files...", batch_update.files_to_insert.len()));
}
}
// Create status callback for inserts
let status_clone_3 = status.clone();
let status_callback = Box::new(move |file_status: &str| {
if let Ok(mut status_guard) = status_clone_3.lock() {
if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard {
*current_file = Some(file_status.to_string());
}
}
});
// Create progress callback for inserts
let status_clone_4 = status.clone();
let base_work_completed = work_completed;
let progress_callback = Box::new(move |current_index: usize| {
if let Ok(mut status_guard) = status_clone_4.lock() {
if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard {
*files_processed = base_work_completed + current_index;
}
}
});
if let Err(e) = process_batch_inserts_files_only(&conn_mutex, &batch_update.files_to_insert, &stop_flag, Some(status_callback), Some(progress_callback), config) {
return Err(format!("Failed to process batch inserts: {}", e));
}
work_completed += batch_update.files_to_insert.len();
if let Ok(mut status_guard) = status.lock() {
if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard {
*files_processed = work_completed;
}
}
}
// If no incremental work was needed, show completion status for file indexing phase
if total_work == 0 {
if let Ok(mut status_guard) = status.lock() {
if let IndexingStatus::RunningFileIndex { ref mut current_file, ref mut files_processed, .. } = *status_guard {
if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard
{
*current_file = Some("File index is up to date".to_string());
*files_processed = total_file_count;
}
}
}
@ -688,6 +807,25 @@ impl IndexingService {
Ok(())
}
fn searchabletext_is_external_content(conn: &Connection) -> Result<bool, String> {
let sql: Option<String> = conn
.query_row(
"SELECT sql FROM sqlite_master WHERE name='searchabletext'",
[],
|r| r.get(0),
)
.optional()
.map_err(|e| format!("sqlite_master searchabletext: {}", e))?;
Ok(sql
.as_deref()
.map(|s| {
s.contains("content='documents'")
|| s.contains("content=\"documents\"")
|| s.contains("content=documents")
})
.unwrap_or(false))
}
/// Validates configuration against stored values and returns validation results.
/// Critical configuration changes that require index recreation:
/// - hash_length: affects file hash computation, invalidates existing file metadata

2
src/main.rs
View file

@ -64,7 +64,7 @@ Duplicate files:
SELECT name, count(*) as cnt, path FROM files GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC;
Full text search:
SELECT name, path, text, snippet(searchabletext, 2 , "<b>", "</b>", "<b>...</b>", 64) as "snip" FROM searchabletext WHERE text MATCH 'searchstring'
SELECT d.name, d.path, d.text, snippet(st, 1 , "<b>", "</b>", "<b>...</b>", 64) as "snip" FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE st.text MATCH 'searchstring'
Filename search:
SELECT name, path FROM files WHERE name LIKE '%searchstring%';

117
src/search.rs
View file

@ -20,6 +20,8 @@ impl PartialEq for SearchProps {
pub fn Search(props: SearchProps) -> Element {
let mut search_type = use_signal(|| "fulltext".to_string());
let mut search_term = use_signal(|| String::new());
let mut fulltext_exact = use_signal(|| false);
let mut fulltext_case_sensitive = use_signal(|| false);
let search_results = use_signal(|| Vec::<SearchResult>::new());
let mut search_error = use_signal(|| None::<String>);
let is_searching = use_signal(|| false);
@ -35,6 +37,8 @@ pub fn Search(props: SearchProps) -> Element {
let db_path = db_path.clone();
let search_type = search_type.clone();
let search_term = search_term.clone();
let fulltext_exact = fulltext_exact.clone();
let fulltext_case_sensitive = fulltext_case_sensitive.clone();
let search_results = search_results.clone();
let search_error = search_error.clone();
let is_searching = is_searching.clone();
@ -46,6 +50,8 @@ pub fn Search(props: SearchProps) -> Element {
let db_clone = db_path.clone();
let search_type_val = search_type().clone();
let search_term_val = search_term().clone();
let fulltext_exact_val = fulltext_exact();
let fulltext_case_sensitive_val = fulltext_case_sensitive();
let mut search_results_clone = search_results.clone();
let mut search_error_clone = search_error.clone();
@ -61,38 +67,81 @@ pub fn Search(props: SearchProps) -> Element {
let query = match search_type_val.as_str() {
"fulltext" => {
if search_term_val.trim().is_empty() {
let trimmed = search_term_val.trim();
if trimmed.is_empty() {
search_error_clone.set(Some("Please enter a search term".to_string()));
is_searching_clone.set(false);
return;
}
// Sanitize search term for FTS5 by removing problematic characters
let sanitized_term = search_term_val
.replace("'", "''") // Escape single quotes for SQL
.replace(":", " ") // Replace colons with spaces (common in file paths, times, etc.)
.replace(";", " ") // Replace semicolons with spaces
.replace("(", " ") // Replace parentheses with spaces
.replace(")", " ")
.replace("[", " ") // Replace brackets with spaces
.replace("]", " ")
.replace("{", " ") // Replace braces with spaces
.replace("}", " ")
.replace("^", " ") // Replace carets with spaces
.replace("~", " ") // Replace tildes with spaces
.replace("\"", " "); // Replace quotes with spaces to avoid nesting issues
let sanitized_term = trimmed
.replace(':', " ")
.replace(';', " ")
.replace('(', " ")
.replace(')', " ")
.replace('[', " ")
.replace(']', " ")
.replace('{', " ")
.replace('}', " ")
.replace('^', " ")
.replace('~', " ")
.replace('"', " ");
// Split into words and rejoin to handle multiple spaces and create a proper FTS5 query
let words: Vec<&str> = sanitized_term.split_whitespace().collect();
if words.is_empty() {
let tokens: Vec<&str> = sanitized_term.split_whitespace().collect();
if tokens.is_empty() {
search_error_clone.set(Some("Please enter a valid search term".to_string()));
is_searching_clone.set(false);
return;
}
// Join words with AND for better matching
let fts_query = words.join(" AND ");
format!("SELECT name, path, snippet(searchabletext, 2, '<b>', '</b>', '<b>...</b>', 64) as snippet FROM searchabletext WHERE text MATCH '{}'", fts_query)
let words: Vec<&str> = if fulltext_exact_val {
tokens
} else {
let filtered: Vec<&str> = tokens
.into_iter()
.filter(|w| w.chars().count() >= 3)
.collect();
if filtered.is_empty() {
search_error_clone.set(Some(
"Trigram index needs each word to be at least 3 characters unless you use exact phrase search.".to_string(),
));
is_searching_clone.set(false);
return;
}
filtered
};
let sql_quote = |s: &str| s.replace('\'', "''");
let fts_match = if fulltext_exact_val {
let phrase = words.join(" ");
format!("\"{}\"", phrase.replace('"', "\"\""))
} else {
words.join(" AND ")
};
let mut where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match));
if fulltext_case_sensitive_val {
if fulltext_exact_val {
let literal = words.join(" ");
where_clause.push_str(&format!(
" AND instr(st.text, '{}') > 0",
sql_quote(&literal)
));
} else {
for w in &words {
where_clause.push_str(&format!(
" AND instr(st.text, '{}') > 0",
sql_quote(w)
));
}
}
}
format!(
"SELECT d.name, d.path, snippet(st, 1, '<b>', '</b>', '<b>...</b>', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank",
where_clause
)
},
"filename" => {
if search_term_val.trim().is_empty() {
@ -161,6 +210,32 @@ pub fn Search(props: SearchProps) -> Element {
}
}
if search_type() == "fulltext" {
div {
class: "form-group",
style: "display: flex; flex-direction: column; gap: 6px;",
span { style: "font-weight: 600;", "Full text options" }
label {
style: "display: flex; align-items: center; gap: 8px; cursor: pointer;",
input {
r#type: "checkbox",
checked: fulltext_exact(),
onchange: move |evt| fulltext_exact.set(evt.checked()),
}
"Exact phrase match"
}
label {
style: "display: flex; align-items: center; gap: 8px; cursor: pointer;",
input {
r#type: "checkbox",
checked: fulltext_case_sensitive(),
onchange: move |evt| fulltext_case_sensitive.set(evt.checked()),
}
"Case-sensitive match"
}
}
}
if search_type() != "duplicates" {
div {
class: "form-group",