diff --git a/config_example.toml b/config_example.toml index 3ed9698..9e4c337 100644 --- a/config_example.toml +++ b/config_example.toml @@ -1,3 +1,13 @@ [paths] default_indexing_path = "C:\\" database_path = "QuickSearch.db" + +[processing] +# Size of hash blocks read from start/end of files (bytes) +hash_length = 4096 +# Maximum text content to index per file (bytes) +maximum_text_size = 524288 +# Maximum file size to process for text extraction (bytes) +maximum_file_size = 52428800 +# Number of files to process in each batch +batch_size = 200 \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 94a42d8..a7774cb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,6 +5,7 @@ use std::path::Path; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct Config { pub paths: PathConfig, + pub processing: ProcessingConfig, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -13,6 +14,14 @@ pub struct PathConfig { pub database_path: String, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ProcessingConfig { + pub hash_length: usize, + pub maximum_text_size: usize, + pub maximum_file_size: u64, + pub batch_size: usize, +} + impl Default for Config { fn default() -> Self { Config { @@ -20,6 +29,12 @@ impl Default for Config { default_indexing_path: "C:\\".to_string(), database_path: "QuickSearch.db".to_string(), }, + processing: ProcessingConfig { + hash_length: 1024 * 8, + maximum_text_size: 1024 * 512, + maximum_file_size: 1024 * 1024 * 50, + batch_size: 200, + }, } } } diff --git a/src/file_handling.rs b/src/file_handling.rs index ca9edac..9d93c19 100644 --- a/src/file_handling.rs +++ b/src/file_handling.rs @@ -10,6 +10,7 @@ use walkdir::DirEntry; use rusqlite::{params, Connection}; use crate::document_extraction::extract_document_text; +use crate::config::Config; #[derive(Debug, Clone)] pub struct FileMetadata { @@ -25,9 +26,6 @@ pub struct BatchUpdate { pub files_to_insert: Vec, } -const HASHLEN:usize = 1024 * 8; -const MAXIMUM_TEXT_SIZE:usize = 1024 * 512; -const MAXIMUM_FILE_SIZE:u64 = 1024 * 1024 * 50; const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 86] = ["","txt","rtf","log", // Text Documents "csv", // Spreadsheet @@ -116,19 +114,19 @@ pub fn analyze_files_for_batch_update( } } -/// Get a hash of a file by reading the first and last HASHLEN bytes of the file -fn get_file_hash(size: u64, path: OsString) -> Result, std::io::Error> { +/// Get a hash of a file by reading the first and last hash_length bytes of the file +fn get_file_hash(size: u64, path: OsString, hash_length: usize) -> Result, std::io::Error> { let mut hasher = Sha256::new(); let mut f: File = File::open(path)?; hasher.update(&size.to_le_bytes()); - if size > HASHLEN as u64 { - let mut file_start_block = [0u8; HASHLEN]; + if size > hash_length as u64 { + let mut file_start_block = vec![0u8; hash_length]; f.read_exact(&mut file_start_block)?; - hasher.update(file_start_block); - f.seek(SeekFrom::End(0 - HASHLEN as i64))?; - let mut file_end_block = [0u8; HASHLEN]; + hasher.update(&file_start_block); + f.seek(SeekFrom::End(0 - hash_length as i64))?; + let mut file_end_block = vec![0u8; hash_length]; f.read_exact(&mut file_end_block)?; - hasher.update(file_end_block); + hasher.update(&file_end_block); } else if size > 0 { let mut file_block = Vec::new(); f.read_to_end(&mut file_block)?; @@ -143,17 +141,19 @@ pub fn process_batch_updates( conn_mutex: &Arc>, files_to_update: &[(DirEntry, FileMetadata)], stop_flag: &Arc>, - status_callback: Option> + status_callback: Option>, + progress_callback: Option>, + config: &Config ) -> Result<(), String> { if files_to_update.is_empty() { return Ok(()); } - const BATCH_SIZE: usize = 1000; + let batch_size = config.processing.batch_size; let total_files = files_to_update.len(); - // Process files in batches of BATCH_SIZE - for (batch_idx, batch) in files_to_update.chunks(BATCH_SIZE).enumerate() { + // Process files in batches of batch_size + for (batch_idx, batch) in files_to_update.chunks(batch_size).enumerate() { // Check stop flag at the start of each batch if *stop_flag.lock().unwrap() { return Ok(()); @@ -163,7 +163,7 @@ pub fn process_batch_updates( let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?; for (i, (entry, _old_metadata)) in batch.iter().enumerate() { - let global_index = batch_idx * BATCH_SIZE + i + 1; + let global_index = batch_idx * batch_size + i + 1; // Check stop flag if *stop_flag.lock().unwrap() { drop(tx); @@ -178,6 +178,11 @@ pub fn process_batch_updates( .unwrap_or("unknown"); callback(&format!("Updating file {}/{}: {}", global_index, total_files, filename)); } + + // Update progress counter + if let Some(ref progress_cb) = progress_callback { + progress_cb(global_index); + } let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?; if meta.is_dir() { continue; @@ -195,7 +200,7 @@ pub fn process_batch_updates( .map_err(|e| format!("Failed to calculate duration: {}", e))? .as_secs(); - let fhash = get_file_hash(fsize, fpath.clone()) + let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length) .map_err(|e| format!("Failed to calculate hash: {}", e))?; // Check stop flag after hash calculation @@ -218,7 +223,7 @@ pub fn process_batch_updates( ).map_err(|e| format!("Failed to delete old searchable text: {}", e))?; // Insert new searchable text if applicable - if fsize <= MAXIMUM_FILE_SIZE { + if fsize <= config.processing.maximum_file_size { let default_ext = OsString::new(); let file_extension = entry.path().extension().unwrap_or(&default_ext) .to_ascii_lowercase().to_str().unwrap_or("").to_string(); @@ -226,8 +231,8 @@ pub fn process_batch_updates( if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) { if let Ok(file_string) = read_to_string(&fpath) { - let trimmed_file_string = if file_string.len() > MAXIMUM_TEXT_SIZE { - file_string[..MAXIMUM_TEXT_SIZE].to_string() + let trimmed_file_string = if file_string.len() > config.processing.maximum_text_size { + file_string[..config.processing.maximum_text_size].to_string() } else { file_string }; @@ -241,8 +246,8 @@ pub fn process_batch_updates( } else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) { if let Ok(extracted_text) = extract_document_text(&fpath, ext_str) { if !extracted_text.trim().is_empty() { - let trimmed_file_string = if extracted_text.len() > MAXIMUM_TEXT_SIZE { - extracted_text[..MAXIMUM_TEXT_SIZE].to_string() + let trimmed_file_string = if extracted_text.len() > config.processing.maximum_text_size { + extracted_text[..config.processing.maximum_text_size].to_string() } else { extracted_text }; @@ -269,17 +274,19 @@ pub fn process_batch_inserts( conn_mutex: &Arc>, files_to_insert: &[DirEntry], stop_flag: &Arc>, - status_callback: Option> + status_callback: Option>, + progress_callback: Option>, + config: &Config ) -> Result<(), String> { if files_to_insert.is_empty() { return Ok(()); } - const BATCH_SIZE: usize = 1000; + let batch_size = config.processing.batch_size; let total_files = files_to_insert.len(); - // Process files in batches of BATCH_SIZE - for (batch_idx, batch) in files_to_insert.chunks(BATCH_SIZE).enumerate() { + // Process files in batches of batch_size + for (batch_idx, batch) in files_to_insert.chunks(batch_size).enumerate() { // Check stop flag at the start of each batch if *stop_flag.lock().unwrap() { return Ok(()); @@ -289,7 +296,7 @@ pub fn process_batch_inserts( let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?; for (i, entry) in batch.iter().enumerate() { - let global_index = batch_idx * BATCH_SIZE + i + 1; + let global_index = batch_idx * batch_size + i + 1; // Check stop flag if *stop_flag.lock().unwrap() { drop(tx); @@ -299,10 +306,14 @@ pub fn process_batch_inserts( // Update status with current file if let Some(ref callback) = status_callback { - let filename = entry.path().file_name() - .and_then(|n| n.to_str()) + let file_path = entry.path().to_str() .unwrap_or("unknown"); - callback(&format!("Indexing file {}/{}: {}", global_index, total_files, filename)); + callback(&format!("Indexing file: {}", file_path)); + } + + // Update progress counter + if let Some(ref progress_cb) = progress_callback { + progress_cb(global_index); } let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?; if meta.is_dir() { @@ -321,7 +332,7 @@ pub fn process_batch_inserts( .map_err(|e| format!("Failed to calculate duration: {}", e))? .as_secs(); - let fhash = get_file_hash(fsize, fpath.clone()) + let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length) .map_err(|e| format!("Failed to calculate hash: {}", e))?; let fname = entry.path().file_name().unwrap().to_os_string(); @@ -333,7 +344,7 @@ pub fn process_batch_inserts( ).map_err(|e| format!("Failed to insert file record: {}", e))?; // Insert searchable text if applicable - if fsize <= MAXIMUM_FILE_SIZE { + if fsize <= config.processing.maximum_file_size { let default_ext = OsString::new(); let file_extension = entry.path().extension().unwrap_or(&default_ext) .to_ascii_lowercase().to_str().unwrap_or("").to_string(); @@ -341,8 +352,8 @@ pub fn process_batch_inserts( if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) { if let Ok(file_string) = read_to_string(&fpath) { - let trimmed_file_string = if file_string.len() > MAXIMUM_TEXT_SIZE { - file_string[..MAXIMUM_TEXT_SIZE].to_string() + let trimmed_file_string = if file_string.len() > config.processing.maximum_text_size { + file_string[..config.processing.maximum_text_size].to_string() } else { file_string }; @@ -355,8 +366,8 @@ pub fn process_batch_inserts( } else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) { if let Ok(extracted_text) = extract_document_text(&fpath, ext_str) { if !extracted_text.trim().is_empty() { - let trimmed_file_string = if extracted_text.len() > MAXIMUM_TEXT_SIZE { - extracted_text[..MAXIMUM_TEXT_SIZE].to_string() + let trimmed_file_string = if extracted_text.len() > config.processing.maximum_text_size { + extracted_text[..config.processing.maximum_text_size].to_string() } else { extracted_text }; @@ -376,122 +387,3 @@ pub fn process_batch_inserts( Ok(()) } - -pub fn process_entry(conn_mutex: &Arc>, entry: DirEntry) { - let meta = entry.metadata().unwrap(); - if !meta.is_dir() { - // let fpath = entry.path().canonicalize()?.into_os_string(); - let fpath_result = entry.path().canonicalize(); - let fpath = match fpath_result { - Ok(fp) => fp.into_os_string(), - Err(error) => { - println!("Error converting fpath: {:?}", error); - return; - }, - }; - - // Get basic file properties - let fsize = meta.len(); - let fmodified = meta.modified().unwrap().duration_since(UNIX_EPOCH).unwrap().as_secs(); - - // Get file hash - let fhash_result = get_file_hash(fsize, fpath.clone()); - let fhash = match fhash_result { - Ok(fh) => fh, - Err(error) => { - println!("Error digesting hash: {:?}", error); - return; - }, - }; - // let fhash = b""; - - // Insert results into database - let query = "INSERT INTO files VALUES (?1,?2,?3,?4,?5)"; - let conn = conn_mutex.lock().unwrap(); - let mut stmt = conn.prepare_cached(query).unwrap(); - let fname = entry.path().file_name().unwrap().to_os_string(); - - let stmt_result = stmt.execute(params![fname.to_str(), fpath.to_string_lossy(), fsize, fmodified, fhash]); - match stmt_result { - Ok(us) => us, - Err(error) => { - println!("Error with sqlite transaction: {:?}", error); - return; - }, - }; - std::mem::drop(stmt); // Free the mutex lock so that other threads can access the database - std::mem::drop(conn); - - // Generate searchable plain text for file if applicable - if fsize <= MAXIMUM_FILE_SIZE { - let default_ext = OsString::new(); - let file_extension = entry.path().extension().unwrap_or(&default_ext).to_ascii_lowercase().to_str().unwrap().to_string(); - let ext_str = file_extension.as_str(); - if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) { - let file_contents_result = read_to_string(fpath.clone()); - let file_string = match file_contents_result { - Ok(fs) => fs, - Err(_error) => { - // println!("Error reading file to string: {:?}", error); - return; - }, - }; - - let trimmed_file_string; - // Trim file contents if too large - if file_string.len() > MAXIMUM_TEXT_SIZE { - trimmed_file_string = file_string[..MAXIMUM_TEXT_SIZE].to_string(); - } else { - trimmed_file_string = file_string; - } - - // Insert file contents into database - let query2 = "INSERT INTO searchabletext VALUES (?1,?2,?3)"; - let conn2 = conn_mutex.lock().unwrap(); - let mut stmt2 = conn2.prepare_cached(query2).unwrap(); - - let stmt_result2 = stmt2.execute(params![fname.to_str(), fpath.to_string_lossy(), trimmed_file_string]); - match stmt_result2 { - Ok(us) => us, - Err(error) => { - println!("Error with sqlite transaction 2: {:?}", error); - return; - }, - }; - } - else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) { - // Extract text from office documents - match extract_document_text(&fpath, ext_str) { - Ok(extracted_text) => { - if !extracted_text.trim().is_empty() { - let trimmed_file_string; - // Trim file contents if too large - if extracted_text.len() > MAXIMUM_TEXT_SIZE { - trimmed_file_string = extracted_text[..MAXIMUM_TEXT_SIZE].to_string(); - } else { - trimmed_file_string = extracted_text; - } - - // Insert file contents into database - let query2 = "INSERT INTO searchabletext VALUES (?1,?2,?3)"; - let conn2 = conn_mutex.lock().unwrap(); - let mut stmt2 = conn2.prepare_cached(query2).unwrap(); - - let stmt_result2 = stmt2.execute(params![fname.to_str(), fpath.to_string_lossy(), trimmed_file_string]); - match stmt_result2 { - Ok(_) => {}, - Err(error) => { - println!("Error with sqlite transaction for document: {:?}", error); - }, - }; - } - } - Err(error) => { - println!("Error extracting text from document {}: {:?}", fpath.to_string_lossy(), error); - } - } - } - } - - } -} \ No newline at end of file diff --git a/src/frontend.rs b/src/frontend.rs index 1a89214..626a335 100644 --- a/src/frontend.rs +++ b/src/frontend.rs @@ -21,50 +21,16 @@ pub fn App(props: AppProps) -> Element { let mut indexing_path = use_signal(|| props.config.paths.default_indexing_path.clone()); let mut db_path = use_signal(|| props.config.paths.database_path.clone()); let mut status_text = use_signal(|| "Idle".to_string()); + let mut show_config_dialog = use_signal(|| false); + let mut config_changes = use_signal(|| Vec::::new()); let indexing_service_for_start = props.indexing_service.clone(); + let indexing_service_for_start_dialog = props.indexing_service.clone(); let indexing_service_for_stop = props.indexing_service.clone(); - let indexing_service_for_refresh = props.indexing_service.clone(); let indexing_service_for_timer = props.indexing_service.clone(); + let config_for_start = props.config.clone(); + let config_for_dialog = props.config.clone(); - // Manual status refresh function - let refresh_status = move |_| { - let status = indexing_service_for_refresh.get_status(); - let status_str = match status { - IndexingStatus::Idle => "Idle".to_string(), - IndexingStatus::Running { files_processed, total_files, current_file, start_time } => { - let elapsed = start_time.elapsed(); - let current_file_display = current_file - .as_ref() - .map(|f| format!("Current: {}", f.split('\\').last().unwrap_or(f))) - .unwrap_or_default(); - - if let Some(total) = total_files { - let percentage = if total > 0 { - (files_processed as f64 / total as f64 * 100.0) as u32 - } else { 0 }; - format!( - "Running: {}/{} files ({}%) - {:.1}s elapsed\n{}", - files_processed, - total, - percentage, - elapsed.as_secs_f64(), - current_file_display - ) - } else { - format!( - "Running: {} files processed - {:.1}s elapsed\n{}", - files_processed, - elapsed.as_secs_f64(), - current_file_display - ) - } - } - IndexingStatus::Stopping => "Indexing Stopped".to_string(), - IndexingStatus::Error(ref e) => format!("Error: {}", e), - }; - status_text.set(status_str); - }; // Automatic status updates every second { @@ -74,7 +40,7 @@ pub fn App(props: AppProps) -> Element { let service = service_clone.clone(); async move { loop { - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; let status = service.get_status(); let status_str = match status { @@ -83,7 +49,7 @@ pub fn App(props: AppProps) -> Element { let elapsed = start_time.elapsed(); let current_file_display = current_file .as_ref() - .map(|f| format!("Current: {}", f.split('\\').last().unwrap_or(f))) + .map(|f| format!("Current: {}", f)) .unwrap_or_default(); if let Some(total) = total_files { @@ -159,10 +125,26 @@ pub fn App(props: AppProps) -> Element { button { style: "margin-right: 10px; padding: 10px 20px; background-color: #4CAF50; color: white; border: none; cursor: pointer;", onclick: move |_| { - let _ = indexing_service_for_start.start_indexing( - indexing_path().clone(), - db_path().clone() - ); + let service = indexing_service_for_start.clone(); + let config = config_for_start.clone(); + let path = indexing_path().clone(); + let db = db_path().clone(); + + // Check for configuration validation + match service.check_config_validation(&db, &config, &path) { + Ok(Some(changes)) => { + // Configuration changes detected, show dialog + config_changes.set(changes); + show_config_dialog.set(true); + } + Ok(None) => { + // No configuration issues, start indexing + let _ = service.start_indexing(path, db, config); + } + Err(e) => { + status_text.set(format!("Configuration validation error: {}", e)); + } + } }, "Start Indexing" } @@ -173,11 +155,6 @@ pub fn App(props: AppProps) -> Element { }, "Stop Indexing" } - button { - style: "margin-left: 10px; padding: 10px 20px; background-color: #2196F3; color: white; border: none; cursor: pointer;", - onclick: refresh_status, - "Refresh Status" - } } } @@ -189,5 +166,78 @@ pub fn App(props: AppProps) -> Element { } } } + + // Configuration validation dialog + if show_config_dialog() { + div { + style: "position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: rgba(0,0,0,0.5); display: flex; align-items: center; justify-content: center; z-index: 1000;", + div { + style: "background-color: white; padding: 30px; border-radius: 10px; max-width: 600px; box-shadow: 0 4px 6px rgba(0,0,0,0.1);", + h3 { + style: "margin-top: 0; color: #d32f2f;", + "⚠️ Configuration Changes Detected" + } + p { + style: "margin: 15px 0;", + "The following configuration changes require deleting and rebuilding the search index:" + } + ul { + style: "margin: 15px 0; padding-left: 20px;", + for change in config_changes().iter() { + li { + style: "margin: 5px 0; font-family: monospace; background-color: #f5f5f5; padding: 5px; border-radius: 3px;", + "{change}" + } + } + } + p { + style: "margin: 15px 0; font-weight: bold;", + "This will delete the existing index and rebuild it from scratch." + } + div { + style: "display: flex; gap: 10px; margin-top: 20px;", + button { + style: "padding: 10px 20px; background-color: #d32f2f; color: white; border: none; border-radius: 5px; cursor: pointer;", + onclick: move |_| { + let service = indexing_service_for_start_dialog.clone(); + let config = config_for_dialog.clone(); + let path = indexing_path().clone(); + let db = db_path().clone(); + + show_config_dialog.set(false); + status_text.set("Stopping indexing and deleting database...".to_string()); + + // Delete database file and restart indexing + let service_clone = service.clone(); + let path_clone = path.clone(); + let db_clone = db.clone(); + let config_clone = config.clone(); + let mut status_clone = status_text.clone(); + + spawn(async move { + match service_clone.delete_index_for_rebuild(&db_clone) { + Ok(()) => { + status_clone.set("Database deleted. Starting fresh indexing...".to_string()); + let _ = service_clone.start_indexing(path_clone, db_clone, config_clone); + } + Err(e) => { + status_clone.set(format!("Error deleting database: {}", e)); + } + } + }); + }, + "Yes, Rebuild Index" + } + button { + style: "padding: 10px 20px; background-color: #666; color: white; border: none; border-radius: 5px; cursor: pointer;", + onclick: move |_| { + show_config_dialog.set(false); + }, + "Cancel" + } + } + } + } + } } } \ No newline at end of file diff --git a/src/indexing.rs b/src/indexing.rs index cabac80..47feb7d 100644 --- a/src/indexing.rs +++ b/src/indexing.rs @@ -2,9 +2,10 @@ use std::sync::{Arc, Mutex, mpsc}; use std::thread; use std::time::Instant; use walkdir::WalkDir; -use rusqlite::Connection; +use rusqlite::{Connection, params}; use crate::file_handling::{load_existing_files, analyze_files_for_batch_update, process_batch_updates, process_batch_inserts}; +use crate::config::Config; #[derive(Debug, Clone)] pub enum IndexingStatus { @@ -24,6 +25,7 @@ pub enum IndexingCommand { Start { path: String, db_path: String, + config: Config, }, Stop, } @@ -34,6 +36,34 @@ pub struct IndexingService { _handle: thread::JoinHandle<()>, } +/// Set process priority for background operation +// fn set_background_priority() { +// #[cfg(windows)] +// { +// use std::os::windows::raw::HANDLE; + +// // Windows implementation +// extern "system" { +// fn GetCurrentProcess() -> HANDLE; +// fn SetPriorityClass(hprocess: HANDLE, dwpriorityclass: u32) -> i32; +// } + +// const BELOW_NORMAL_PRIORITY_CLASS: u32 = 0x00004000; +// unsafe { +// SetPriorityClass(GetCurrentProcess(), BELOW_NORMAL_PRIORITY_CLASS); +// } +// } + +// #[cfg(unix)] +// { +// // Unix implementation +// use std::os::unix::process::CommandExt; +// unsafe { +// libc::nice(10); // Lower priority +// } +// } +// } + impl IndexingService { pub fn new() -> Self { let status = Arc::new(Mutex::new(IndexingStatus::Idle)); @@ -51,9 +81,9 @@ impl IndexingService { } } - pub fn start_indexing(&self, path: String, db_path: String) -> Result<(), String> { + pub fn start_indexing(&self, path: String, db_path: String, config: Config) -> Result<(), String> { self.command_tx - .send(IndexingCommand::Start { path, db_path }) + .send(IndexingCommand::Start { path, db_path, config }) .map_err(|e| format!("Failed to send start command: {}", e)) } @@ -67,13 +97,57 @@ impl IndexingService { self.status.lock().unwrap().clone() } + /// Check if configuration changes require index recreation + pub fn check_config_validation(&self, db_path: &str, config: &Config, indexing_path: &str) -> Result>, String> { + let conn = Connection::open(db_path) + .map_err(|e| format!("Failed to open database: {}", e))?; + + // Create config validation table if it doesn't exist + conn.execute( + "CREATE TABLE IF NOT EXISTS config_validation ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL);", + (), + ).map_err(|e| format!("Failed to create config_validation table: {}", e))?; + + Self::validate_config(&conn, config, indexing_path) + } + + /// Stop indexing and delete the database file for a clean rebuild + pub fn delete_index_for_rebuild(&self, db_path: &str) -> Result<(), String> { + // Stop any running indexing first + self.stop_indexing() + .map_err(|e| format!("Failed to stop indexing: {}", e))?; + + // Wait for indexing to actually stop + let mut attempts = 0; + while attempts < 50 { // Wait up to 5 seconds + match self.get_status() { + IndexingStatus::Idle => break, + IndexingStatus::Stopping | IndexingStatus::Running { .. } => { + std::thread::sleep(std::time::Duration::from_millis(100)); + attempts += 1; + } + IndexingStatus::Error(_) => break, // Consider error state as stopped + } + } + + // Delete the database file + if std::path::Path::new(db_path).exists() { + std::fs::remove_file(db_path) + .map_err(|e| format!("Failed to delete database file: {}", e))?; + } + + Ok(()) + } + fn indexing_thread(status: Arc>, command_rx: mpsc::Receiver) { let stop_flag = Arc::new(Mutex::new(false)); let mut indexing_handle: Option> = None; while let Ok(command) = command_rx.recv() { match command { - IndexingCommand::Start { path, db_path } => { + IndexingCommand::Start { path, db_path, config } => { if matches!(*status.lock().unwrap(), IndexingStatus::Running { .. }) { continue; // Already running } @@ -96,9 +170,10 @@ impl IndexingService { let stop_flag_clone = stop_flag.clone(); let path_owned = path.clone(); let db_path_owned = db_path.clone(); + let config_owned = config.clone(); indexing_handle = Some(thread::spawn(move || { - if let Err(e) = Self::run_indexing(&status_clone, &path_owned, &db_path_owned, &stop_flag_clone) { + if let Err(e) = Self::run_indexing(&status_clone, &path_owned, &db_path_owned, &stop_flag_clone, &config_owned) { *status_clone.lock().unwrap() = IndexingStatus::Error(e); } else { // Only set to Idle if we weren't stopped @@ -128,6 +203,7 @@ impl IndexingService { path: &str, db_path: &str, stop_flag: &Arc>, + config: &Config, ) -> Result<(), String> { // Set up database let conn = Connection::open(db_path) @@ -158,6 +234,17 @@ impl IndexingService { ) .map_err(|e| format!("Failed to create searchabletext table: {}", e))?; + conn.execute( + "CREATE TABLE IF NOT EXISTS config_validation ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL);", + (), + ) + .map_err(|e| format!("Failed to create config_validation table: {}", e))?; + + // Update configuration (for new installations or when no validation issues) + Self::update_config(&conn, config, path)?; + // Load existing files from database for incremental indexing let existing_files = { let conn_ref = &conn; @@ -205,27 +292,28 @@ impl IndexingService { } } - // Create status callback to update current file and progress - let status_clone = status.clone(); + // Create status callback to update current file + let status_clone_1 = status.clone(); let status_callback = Box::new(move |file_status: &str| { - if let Ok(mut status_guard) = status_clone.lock() { - if let IndexingStatus::Running { ref mut current_file, ref mut files_processed, .. } = *status_guard { + if let Ok(mut status_guard) = status_clone_1.lock() { + if let IndexingStatus::Running { ref mut current_file, .. } = *status_guard { *current_file = Some(file_status.to_string()); - // Extract the current count from the status string - if let Some(slash_pos) = file_status.find('/') { - if let Some(space_pos) = file_status.rfind(' ') { - if space_pos + 1 < slash_pos { - if let Ok(current) = file_status[space_pos + 1..slash_pos].parse::() { - *files_processed = work_completed + current; - } - } - } - } + } + } + }); + + // Create progress callback to update files_processed + let status_clone_2 = status.clone(); + let base_work_completed = work_completed; + let progress_callback = Box::new(move |current_index: usize| { + if let Ok(mut status_guard) = status_clone_2.lock() { + if let IndexingStatus::Running { ref mut files_processed, .. } = *status_guard { + *files_processed = base_work_completed + current_index; } } }); - if let Err(e) = process_batch_updates(&conn_mutex, &batch_update.files_to_update, &stop_flag, Some(status_callback)) { + if let Err(e) = process_batch_updates(&conn_mutex, &batch_update.files_to_update, &stop_flag, Some(status_callback), Some(progress_callback), config) { return Err(format!("Failed to process batch updates: {}", e)); } @@ -254,28 +342,28 @@ impl IndexingService { } } - // Create status callback to update current file and progress - let status_clone = status.clone(); - let base_work_completed = work_completed; + // Create status callback for inserts + let status_clone_3 = status.clone(); let status_callback = Box::new(move |file_status: &str| { - if let Ok(mut status_guard) = status_clone.lock() { - if let IndexingStatus::Running { ref mut current_file, ref mut files_processed, .. } = *status_guard { + if let Ok(mut status_guard) = status_clone_3.lock() { + if let IndexingStatus::Running { ref mut current_file, .. } = *status_guard { *current_file = Some(file_status.to_string()); - // Extract the current count from the status string - if let Some(slash_pos) = file_status.find('/') { - if let Some(space_pos) = file_status.rfind(' ') { - if space_pos + 1 < slash_pos { - if let Ok(current) = file_status[space_pos + 1..slash_pos].parse::() { - *files_processed = base_work_completed + current; - } - } - } - } + } + } + }); + + // Create progress callback for inserts + let status_clone_4 = status.clone(); + let base_work_completed = work_completed; + let progress_callback = Box::new(move |current_index: usize| { + if let Ok(mut status_guard) = status_clone_4.lock() { + if let IndexingStatus::Running { ref mut files_processed, .. } = *status_guard { + *files_processed = base_work_completed + current_index; } } }); - if let Err(e) = process_batch_inserts(&conn_mutex, &batch_update.files_to_insert, &stop_flag, Some(status_callback)) { + if let Err(e) = process_batch_inserts(&conn_mutex, &batch_update.files_to_insert, &stop_flag, Some(status_callback), Some(progress_callback), config) { return Err(format!("Failed to process batch inserts: {}", e)); } @@ -300,6 +388,85 @@ impl IndexingService { Ok(()) } + + /// Validates configuration against stored values and returns validation results. + /// Critical configuration changes that require index recreation: + /// - hash_length: affects file hash computation, invalidates existing file metadata + /// - indexing_path: changes the scope of indexed files + fn validate_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result>, String> { + // Critical configuration values that require index recreation + let hash_length = config.processing.hash_length.to_string(); + let normalized_path = std::path::Path::new(indexing_path) + .canonicalize() + .unwrap_or_else(|_| std::path::PathBuf::from(indexing_path)) + .to_string_lossy() + .to_string(); + + // Check stored configuration values + let mut stored_hash_length: Option = None; + let mut stored_indexing_path: Option = None; + + if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path')") { + if let Ok(rows) = stmt.query_map([], |row| { + let key: String = row.get(0)?; + let value: String = row.get(1)?; + Ok((key, value)) + }) { + for row in rows.flatten() { + match row.0.as_str() { + "hash_length" => stored_hash_length = Some(row.1), + "indexing_path" => stored_indexing_path = Some(row.1), + _ => {} + } + } + } + } + + // Check if configuration is invalid + let hash_length_changed = stored_hash_length.as_ref().map_or(false, |stored| stored != &hash_length); + let indexing_path_changed = stored_indexing_path.as_ref().map_or(false, |stored| stored != &normalized_path); + + if hash_length_changed || indexing_path_changed { + let mut changes = Vec::new(); + if hash_length_changed { + changes.push(format!("hash_length: {} -> {}", + stored_hash_length.unwrap_or_else(|| "unknown".to_string()), hash_length)); + } + if indexing_path_changed { + changes.push(format!("indexing_path: {} -> {}", + stored_indexing_path.unwrap_or_else(|| "unknown".to_string()), normalized_path)); + } + + return Ok(Some(changes)); + } + + // No configuration changes detected + Ok(None) + } + + + /// Updates stored configuration values without clearing the index + fn update_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result<(), String> { + let hash_length = config.processing.hash_length.to_string(); + let normalized_path = std::path::Path::new(indexing_path) + .canonicalize() + .unwrap_or_else(|_| std::path::PathBuf::from(indexing_path)) + .to_string_lossy() + .to_string(); + + // Update stored configuration values + conn.execute( + "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('hash_length', ?1)", + params![hash_length], + ).map_err(|e| format!("Failed to store hash_length config: {}", e))?; + + conn.execute( + "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('indexing_path', ?1)", + params![normalized_path], + ).map_err(|e| format!("Failed to store indexing_path config: {}", e))?; + + Ok(()) + } } impl Default for IndexingService {