jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

Moved more things into configuration options, added index invalidation checks for some config items.

Browse source
This commit is contained in:
Jeremy Karst 2025-09-11 16:32:21 -04:00
parent a85d2261dd
commit 86c52ae65b
5 changed files with 375 additions and 241 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
config_example.toml
View file

@ -1,3 +1,13 @@
[paths] [paths]
default_indexing_path = "C:\\" default_indexing_path = "C:\\"
database_path = "QuickSearch.db" database_path = "QuickSearch.db"
[processing]
# Size of hash blocks read from start/end of files (bytes)
hash_length = 4096
# Maximum text content to index per file (bytes)
maximum_text_size = 524288
# Maximum file size to process for text extraction (bytes)
maximum_file_size = 52428800
# Number of files to process in each batch
batch_size = 200

15
src/config.rs
View file

@ -5,6 +5,7 @@ use std::path::Path;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Config { pub struct Config {
pub paths: PathConfig, pub paths: PathConfig,
pub processing: ProcessingConfig,
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
@ -13,6 +14,14 @@ pub struct PathConfig {
pub database_path: String, pub database_path: String,
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ProcessingConfig {
pub hash_length: usize,
pub maximum_text_size: usize,
pub maximum_file_size: u64,
pub batch_size: usize,
}
impl Default for Config { impl Default for Config {
fn default() -> Self { fn default() -> Self {
Config { Config {
@ -20,6 +29,12 @@ impl Default for Config {
default_indexing_path: "C:\\".to_string(), default_indexing_path: "C:\\".to_string(),
database_path: "QuickSearch.db".to_string(), database_path: "QuickSearch.db".to_string(),
}, },
processing: ProcessingConfig {
hash_length: 1024 * 8,
maximum_text_size: 1024 * 512,
maximum_file_size: 1024 * 1024 * 50,
batch_size: 200,
},
} }
} }
} }

202
src/file_handling.rs
View file

@ -10,6 +10,7 @@ use walkdir::DirEntry;
use rusqlite::{params, Connection}; use rusqlite::{params, Connection};
use crate::document_extraction::extract_document_text; use crate::document_extraction::extract_document_text;
use crate::config::Config;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct FileMetadata { pub struct FileMetadata {
@ -25,9 +26,6 @@ pub struct BatchUpdate {
pub files_to_insert: Vec<DirEntry>, pub files_to_insert: Vec<DirEntry>,
} }
const HASHLEN:usize = 1024 * 8;
const MAXIMUM_TEXT_SIZE:usize = 1024 * 512;
const MAXIMUM_FILE_SIZE:u64 = 1024 * 1024 * 50;
const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 86] = const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 86] =
["","txt","rtf","log", // Text Documents ["","txt","rtf","log", // Text Documents
"csv", // Spreadsheet "csv", // Spreadsheet
@ -116,19 +114,19 @@ pub fn analyze_files_for_batch_update(
} }
} }
/// Get a hash of a file by reading the first and last HASHLEN bytes of the file /// Get a hash of a file by reading the first and last hash_length bytes of the file
fn get_file_hash(size: u64, path: OsString) -> Result<Vec<u8>, std::io::Error> { fn get_file_hash(size: u64, path: OsString, hash_length: usize) -> Result<Vec<u8>, std::io::Error> {
let mut hasher = Sha256::new(); let mut hasher = Sha256::new();
let mut f: File = File::open(path)?; let mut f: File = File::open(path)?;
hasher.update(&size.to_le_bytes()); hasher.update(&size.to_le_bytes());
if size > HASHLEN as u64 { if size > hash_length as u64 {
let mut file_start_block = [0u8; HASHLEN]; let mut file_start_block = vec![0u8; hash_length];
f.read_exact(&mut file_start_block)?; f.read_exact(&mut file_start_block)?;
hasher.update(file_start_block); hasher.update(&file_start_block);
f.seek(SeekFrom::End(0 - HASHLEN as i64))?; f.seek(SeekFrom::End(0 - hash_length as i64))?;
let mut file_end_block = [0u8; HASHLEN]; let mut file_end_block = vec![0u8; hash_length];
f.read_exact(&mut file_end_block)?; f.read_exact(&mut file_end_block)?;
hasher.update(file_end_block); hasher.update(&file_end_block);
} else if size > 0 { } else if size > 0 {
let mut file_block = Vec::new(); let mut file_block = Vec::new();
f.read_to_end(&mut file_block)?; f.read_to_end(&mut file_block)?;
@ -143,17 +141,19 @@ pub fn process_batch_updates(
conn_mutex: &Arc<Mutex<Connection>>, conn_mutex: &Arc<Mutex<Connection>>,
files_to_update: &[(DirEntry, FileMetadata)], files_to_update: &[(DirEntry, FileMetadata)],
stop_flag: &Arc<Mutex<bool>>, stop_flag: &Arc<Mutex<bool>>,
status_callback: Option<Box<dyn Fn(&str) + Send + Sync>> status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>,
config: &Config
) -> Result<(), String> { ) -> Result<(), String> {
if files_to_update.is_empty() { if files_to_update.is_empty() {
return Ok(()); return Ok(());
} }
const BATCH_SIZE: usize = 1000; let batch_size = config.processing.batch_size;
let total_files = files_to_update.len(); let total_files = files_to_update.len();
// Process files in batches of BATCH_SIZE // Process files in batches of batch_size
for (batch_idx, batch) in files_to_update.chunks(BATCH_SIZE).enumerate() { for (batch_idx, batch) in files_to_update.chunks(batch_size).enumerate() {
// Check stop flag at the start of each batch // Check stop flag at the start of each batch
if *stop_flag.lock().unwrap() { if *stop_flag.lock().unwrap() {
return Ok(()); return Ok(());
@ -163,7 +163,7 @@ pub fn process_batch_updates(
let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?; let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?;
for (i, (entry, _old_metadata)) in batch.iter().enumerate() { for (i, (entry, _old_metadata)) in batch.iter().enumerate() {
let global_index = batch_idx * BATCH_SIZE + i + 1; let global_index = batch_idx * batch_size + i + 1;
// Check stop flag // Check stop flag
if *stop_flag.lock().unwrap() { if *stop_flag.lock().unwrap() {
drop(tx); drop(tx);
@ -178,6 +178,11 @@ pub fn process_batch_updates(
.unwrap_or("unknown"); .unwrap_or("unknown");
callback(&format!("Updating file {}/{}: {}", global_index, total_files, filename)); callback(&format!("Updating file {}/{}: {}", global_index, total_files, filename));
} }
// Update progress counter
if let Some(ref progress_cb) = progress_callback {
progress_cb(global_index);
}
let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?; let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?;
if meta.is_dir() { if meta.is_dir() {
continue; continue;
@ -195,7 +200,7 @@ pub fn process_batch_updates(
.map_err(|e| format!("Failed to calculate duration: {}", e))? .map_err(|e| format!("Failed to calculate duration: {}", e))?
.as_secs(); .as_secs();
let fhash = get_file_hash(fsize, fpath.clone()) let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length)
.map_err(|e| format!("Failed to calculate hash: {}", e))?; .map_err(|e| format!("Failed to calculate hash: {}", e))?;
// Check stop flag after hash calculation // Check stop flag after hash calculation
@ -218,7 +223,7 @@ pub fn process_batch_updates(
).map_err(|e| format!("Failed to delete old searchable text: {}", e))?; ).map_err(|e| format!("Failed to delete old searchable text: {}", e))?;
// Insert new searchable text if applicable // Insert new searchable text if applicable
if fsize <= MAXIMUM_FILE_SIZE { if fsize <= config.processing.maximum_file_size {
let default_ext = OsString::new(); let default_ext = OsString::new();
let file_extension = entry.path().extension().unwrap_or(&default_ext) let file_extension = entry.path().extension().unwrap_or(&default_ext)
.to_ascii_lowercase().to_str().unwrap_or("").to_string(); .to_ascii_lowercase().to_str().unwrap_or("").to_string();
@ -226,8 +231,8 @@ pub fn process_batch_updates(
if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) { if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) {
if let Ok(file_string) = read_to_string(&fpath) { if let Ok(file_string) = read_to_string(&fpath) {
let trimmed_file_string = if file_string.len() > MAXIMUM_TEXT_SIZE { let trimmed_file_string = if file_string.len() > config.processing.maximum_text_size {
file_string[..MAXIMUM_TEXT_SIZE].to_string() file_string[..config.processing.maximum_text_size].to_string()
} else { } else {
file_string file_string
}; };
@ -241,8 +246,8 @@ pub fn process_batch_updates(
} else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) { } else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) {
if let Ok(extracted_text) = extract_document_text(&fpath, ext_str) { if let Ok(extracted_text) = extract_document_text(&fpath, ext_str) {
if !extracted_text.trim().is_empty() { if !extracted_text.trim().is_empty() {
let trimmed_file_string = if extracted_text.len() > MAXIMUM_TEXT_SIZE { let trimmed_file_string = if extracted_text.len() > config.processing.maximum_text_size {
extracted_text[..MAXIMUM_TEXT_SIZE].to_string() extracted_text[..config.processing.maximum_text_size].to_string()
} else { } else {
extracted_text extracted_text
}; };
@ -269,17 +274,19 @@ pub fn process_batch_inserts(
conn_mutex: &Arc<Mutex<Connection>>, conn_mutex: &Arc<Mutex<Connection>>,
files_to_insert: &[DirEntry], files_to_insert: &[DirEntry],
stop_flag: &Arc<Mutex<bool>>, stop_flag: &Arc<Mutex<bool>>,
status_callback: Option<Box<dyn Fn(&str) + Send + Sync>> status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>,
config: &Config
) -> Result<(), String> { ) -> Result<(), String> {
if files_to_insert.is_empty() { if files_to_insert.is_empty() {
return Ok(()); return Ok(());
} }
const BATCH_SIZE: usize = 1000; let batch_size = config.processing.batch_size;
let total_files = files_to_insert.len(); let total_files = files_to_insert.len();
// Process files in batches of BATCH_SIZE // Process files in batches of batch_size
for (batch_idx, batch) in files_to_insert.chunks(BATCH_SIZE).enumerate() { for (batch_idx, batch) in files_to_insert.chunks(batch_size).enumerate() {
// Check stop flag at the start of each batch // Check stop flag at the start of each batch
if *stop_flag.lock().unwrap() { if *stop_flag.lock().unwrap() {
return Ok(()); return Ok(());
@ -289,7 +296,7 @@ pub fn process_batch_inserts(
let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?; let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?;
for (i, entry) in batch.iter().enumerate() { for (i, entry) in batch.iter().enumerate() {
let global_index = batch_idx * BATCH_SIZE + i + 1; let global_index = batch_idx * batch_size + i + 1;
// Check stop flag // Check stop flag
if *stop_flag.lock().unwrap() { if *stop_flag.lock().unwrap() {
drop(tx); drop(tx);
@ -299,10 +306,14 @@ pub fn process_batch_inserts(
// Update status with current file // Update status with current file
if let Some(ref callback) = status_callback { if let Some(ref callback) = status_callback {
let filename = entry.path().file_name() let file_path = entry.path().to_str()
.and_then(|n| n.to_str())
.unwrap_or("unknown"); .unwrap_or("unknown");
callback(&format!("Indexing file {}/{}: {}", global_index, total_files, filename)); callback(&format!("Indexing file: {}", file_path));
}
// Update progress counter
if let Some(ref progress_cb) = progress_callback {
progress_cb(global_index);
} }
let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?; let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?;
if meta.is_dir() { if meta.is_dir() {
@ -321,7 +332,7 @@ pub fn process_batch_inserts(
.map_err(|e| format!("Failed to calculate duration: {}", e))? .map_err(|e| format!("Failed to calculate duration: {}", e))?
.as_secs(); .as_secs();
let fhash = get_file_hash(fsize, fpath.clone()) let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length)
.map_err(|e| format!("Failed to calculate hash: {}", e))?; .map_err(|e| format!("Failed to calculate hash: {}", e))?;
let fname = entry.path().file_name().unwrap().to_os_string(); let fname = entry.path().file_name().unwrap().to_os_string();
@ -333,7 +344,7 @@ pub fn process_batch_inserts(
).map_err(|e| format!("Failed to insert file record: {}", e))?; ).map_err(|e| format!("Failed to insert file record: {}", e))?;
// Insert searchable text if applicable // Insert searchable text if applicable
if fsize <= MAXIMUM_FILE_SIZE { if fsize <= config.processing.maximum_file_size {
let default_ext = OsString::new(); let default_ext = OsString::new();
let file_extension = entry.path().extension().unwrap_or(&default_ext) let file_extension = entry.path().extension().unwrap_or(&default_ext)
.to_ascii_lowercase().to_str().unwrap_or("").to_string(); .to_ascii_lowercase().to_str().unwrap_or("").to_string();
@ -341,8 +352,8 @@ pub fn process_batch_inserts(
if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) { if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) {
if let Ok(file_string) = read_to_string(&fpath) { if let Ok(file_string) = read_to_string(&fpath) {
let trimmed_file_string = if file_string.len() > MAXIMUM_TEXT_SIZE { let trimmed_file_string = if file_string.len() > config.processing.maximum_text_size {
file_string[..MAXIMUM_TEXT_SIZE].to_string() file_string[..config.processing.maximum_text_size].to_string()
} else { } else {
file_string file_string
}; };
@ -355,8 +366,8 @@ pub fn process_batch_inserts(
} else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) { } else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) {
if let Ok(extracted_text) = extract_document_text(&fpath, ext_str) { if let Ok(extracted_text) = extract_document_text(&fpath, ext_str) {
if !extracted_text.trim().is_empty() { if !extracted_text.trim().is_empty() {
let trimmed_file_string = if extracted_text.len() > MAXIMUM_TEXT_SIZE { let trimmed_file_string = if extracted_text.len() > config.processing.maximum_text_size {
extracted_text[..MAXIMUM_TEXT_SIZE].to_string() extracted_text[..config.processing.maximum_text_size].to_string()
} else { } else {
extracted_text extracted_text
}; };
@ -376,122 +387,3 @@ pub fn process_batch_inserts(
Ok(()) Ok(())
} }
pub fn process_entry(conn_mutex: &Arc<Mutex<Connection>>, entry: DirEntry) {
let meta = entry.metadata().unwrap();
if !meta.is_dir() {
// let fpath = entry.path().canonicalize()?.into_os_string();
let fpath_result = entry.path().canonicalize();
let fpath = match fpath_result {
Ok(fp) => fp.into_os_string(),
Err(error) => {
println!("Error converting fpath: {:?}", error);
return;
},
};
// Get basic file properties
let fsize = meta.len();
let fmodified = meta.modified().unwrap().duration_since(UNIX_EPOCH).unwrap().as_secs();
// Get file hash
let fhash_result = get_file_hash(fsize, fpath.clone());
let fhash = match fhash_result {
Ok(fh) => fh,
Err(error) => {
println!("Error digesting hash: {:?}", error);
return;
},
};
// let fhash = b"";
// Insert results into database
let query = "INSERT INTO files VALUES (?1,?2,?3,?4,?5)";
let conn = conn_mutex.lock().unwrap();
let mut stmt = conn.prepare_cached(query).unwrap();
let fname = entry.path().file_name().unwrap().to_os_string();
let stmt_result = stmt.execute(params![fname.to_str(), fpath.to_string_lossy(), fsize, fmodified, fhash]);
match stmt_result {
Ok(us) => us,
Err(error) => {
println!("Error with sqlite transaction: {:?}", error);
return;
},
};
std::mem::drop(stmt); // Free the mutex lock so that other threads can access the database
std::mem::drop(conn);
// Generate searchable plain text for file if applicable
if fsize <= MAXIMUM_FILE_SIZE {
let default_ext = OsString::new();
let file_extension = entry.path().extension().unwrap_or(&default_ext).to_ascii_lowercase().to_str().unwrap().to_string();
let ext_str = file_extension.as_str();
if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) {
let file_contents_result = read_to_string(fpath.clone());
let file_string = match file_contents_result {
Ok(fs) => fs,
Err(_error) => {
// println!("Error reading file to string: {:?}", error);
return;
},
};
let trimmed_file_string;
// Trim file contents if too large
if file_string.len() > MAXIMUM_TEXT_SIZE {
trimmed_file_string = file_string[..MAXIMUM_TEXT_SIZE].to_string();
} else {
trimmed_file_string = file_string;
}
// Insert file contents into database
let query2 = "INSERT INTO searchabletext VALUES (?1,?2,?3)";
let conn2 = conn_mutex.lock().unwrap();
let mut stmt2 = conn2.prepare_cached(query2).unwrap();
let stmt_result2 = stmt2.execute(params![fname.to_str(), fpath.to_string_lossy(), trimmed_file_string]);
match stmt_result2 {
Ok(us) => us,
Err(error) => {
println!("Error with sqlite transaction 2: {:?}", error);
return;
},
};
}
else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) {
// Extract text from office documents
match extract_document_text(&fpath, ext_str) {
Ok(extracted_text) => {
if !extracted_text.trim().is_empty() {
let trimmed_file_string;
// Trim file contents if too large
if extracted_text.len() > MAXIMUM_TEXT_SIZE {
trimmed_file_string = extracted_text[..MAXIMUM_TEXT_SIZE].to_string();
} else {
trimmed_file_string = extracted_text;
}
// Insert file contents into database
let query2 = "INSERT INTO searchabletext VALUES (?1,?2,?3)";
let conn2 = conn_mutex.lock().unwrap();
let mut stmt2 = conn2.prepare_cached(query2).unwrap();
let stmt_result2 = stmt2.execute(params![fname.to_str(), fpath.to_string_lossy(), trimmed_file_string]);
match stmt_result2 {
Ok(_) => {},
Err(error) => {
println!("Error with sqlite transaction for document: {:?}", error);
},
};
}
}
Err(error) => {
println!("Error extracting text from document {}: {:?}", fpath.to_string_lossy(), error);
}
}
}
}
}
}

150
src/frontend.rs
View file

@ -21,50 +21,16 @@ pub fn App(props: AppProps) -> Element {
let mut indexing_path = use_signal(|| props.config.paths.default_indexing_path.clone()); let mut indexing_path = use_signal(|| props.config.paths.default_indexing_path.clone());
let mut db_path = use_signal(|| props.config.paths.database_path.clone()); let mut db_path = use_signal(|| props.config.paths.database_path.clone());
let mut status_text = use_signal(|| "Idle".to_string()); let mut status_text = use_signal(|| "Idle".to_string());
let mut show_config_dialog = use_signal(|| false);
let mut config_changes = use_signal(|| Vec::<String>::new());
let indexing_service_for_start = props.indexing_service.clone(); let indexing_service_for_start = props.indexing_service.clone();
let indexing_service_for_start_dialog = props.indexing_service.clone();
let indexing_service_for_stop = props.indexing_service.clone(); let indexing_service_for_stop = props.indexing_service.clone();
let indexing_service_for_refresh = props.indexing_service.clone();
let indexing_service_for_timer = props.indexing_service.clone(); let indexing_service_for_timer = props.indexing_service.clone();
let config_for_start = props.config.clone();
let config_for_dialog = props.config.clone();
// Manual status refresh function
let refresh_status = move |_| {
let status = indexing_service_for_refresh.get_status();
let status_str = match status {
IndexingStatus::Idle => "Idle".to_string(),
IndexingStatus::Running { files_processed, total_files, current_file, start_time } => {
let elapsed = start_time.elapsed();
let current_file_display = current_file
.as_ref()
.map(|f| format!("Current: {}", f.split('\\').last().unwrap_or(f)))
.unwrap_or_default();
if let Some(total) = total_files {
let percentage = if total > 0 {
(files_processed as f64 / total as f64 * 100.0) as u32
} else { 0 };
format!(
"Running: {}/{} files ({}%) - {:.1}s elapsed\n{}",
files_processed,
total,
percentage,
elapsed.as_secs_f64(),
current_file_display
)
} else {
format!(
"Running: {} files processed - {:.1}s elapsed\n{}",
files_processed,
elapsed.as_secs_f64(),
current_file_display
)
}
}
IndexingStatus::Stopping => "Indexing Stopped".to_string(),
IndexingStatus::Error(ref e) => format!("Error: {}", e),
};
status_text.set(status_str);
};
// Automatic status updates every second // Automatic status updates every second
{ {
@ -74,7 +40,7 @@ pub fn App(props: AppProps) -> Element {
let service = service_clone.clone(); let service = service_clone.clone();
async move { async move {
loop { loop {
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
let status = service.get_status(); let status = service.get_status();
let status_str = match status { let status_str = match status {
@ -83,7 +49,7 @@ pub fn App(props: AppProps) -> Element {
let elapsed = start_time.elapsed(); let elapsed = start_time.elapsed();
let current_file_display = current_file let current_file_display = current_file
.as_ref() .as_ref()
.map(|f| format!("Current: {}", f.split('\\').last().unwrap_or(f))) .map(|f| format!("Current: {}", f))
.unwrap_or_default(); .unwrap_or_default();
if let Some(total) = total_files { if let Some(total) = total_files {
@ -159,10 +125,26 @@ pub fn App(props: AppProps) -> Element {
button { button {
style: "margin-right: 10px; padding: 10px 20px; background-color: #4CAF50; color: white; border: none; cursor: pointer;", style: "margin-right: 10px; padding: 10px 20px; background-color: #4CAF50; color: white; border: none; cursor: pointer;",
onclick: move |_| { onclick: move |_| {
let _ = indexing_service_for_start.start_indexing( let service = indexing_service_for_start.clone();
indexing_path().clone(), let config = config_for_start.clone();
db_path().clone() let path = indexing_path().clone();
); let db = db_path().clone();
// Check for configuration validation
match service.check_config_validation(&db, &config, &path) {
Ok(Some(changes)) => {
// Configuration changes detected, show dialog
config_changes.set(changes);
show_config_dialog.set(true);
}
Ok(None) => {
// No configuration issues, start indexing
let _ = service.start_indexing(path, db, config);
}
Err(e) => {
status_text.set(format!("Configuration validation error: {}", e));
}
}
}, },
"Start Indexing" "Start Indexing"
} }
@ -173,11 +155,6 @@ pub fn App(props: AppProps) -> Element {
}, },
"Stop Indexing" "Stop Indexing"
} }
button {
style: "margin-left: 10px; padding: 10px 20px; background-color: #2196F3; color: white; border: none; cursor: pointer;",
onclick: refresh_status,
"Refresh Status"
}
} }
} }
@ -189,5 +166,78 @@ pub fn App(props: AppProps) -> Element {
} }
} }
} }
// Configuration validation dialog
if show_config_dialog() {
div {
style: "position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: rgba(0,0,0,0.5); display: flex; align-items: center; justify-content: center; z-index: 1000;",
div {
style: "background-color: white; padding: 30px; border-radius: 10px; max-width: 600px; box-shadow: 0 4px 6px rgba(0,0,0,0.1);",
h3 {
style: "margin-top: 0; color: #d32f2f;",
"⚠️ Configuration Changes Detected"
}
p {
style: "margin: 15px 0;",
"The following configuration changes require deleting and rebuilding the search index:"
}
ul {
style: "margin: 15px 0; padding-left: 20px;",
for change in config_changes().iter() {
li {
style: "margin: 5px 0; font-family: monospace; background-color: #f5f5f5; padding: 5px; border-radius: 3px;",
"{change}"
}
}
}
p {
style: "margin: 15px 0; font-weight: bold;",
"This will delete the existing index and rebuild it from scratch."
}
div {
style: "display: flex; gap: 10px; margin-top: 20px;",
button {
style: "padding: 10px 20px; background-color: #d32f2f; color: white; border: none; border-radius: 5px; cursor: pointer;",
onclick: move |_| {
let service = indexing_service_for_start_dialog.clone();
let config = config_for_dialog.clone();
let path = indexing_path().clone();
let db = db_path().clone();
show_config_dialog.set(false);
status_text.set("Stopping indexing and deleting database...".to_string());
// Delete database file and restart indexing
let service_clone = service.clone();
let path_clone = path.clone();
let db_clone = db.clone();
let config_clone = config.clone();
let mut status_clone = status_text.clone();
spawn(async move {
match service_clone.delete_index_for_rebuild(&db_clone) {
Ok(()) => {
status_clone.set("Database deleted. Starting fresh indexing...".to_string());
let _ = service_clone.start_indexing(path_clone, db_clone, config_clone);
}
Err(e) => {
status_clone.set(format!("Error deleting database: {}", e));
}
}
});
},
"Yes, Rebuild Index"
}
button {
style: "padding: 10px 20px; background-color: #666; color: white; border: none; border-radius: 5px; cursor: pointer;",
onclick: move |_| {
show_config_dialog.set(false);
},
"Cancel"
}
}
}
}
}
} }
} }

239
src/indexing.rs
View file

@ -2,9 +2,10 @@ use std::sync::{Arc, Mutex, mpsc};
use std::thread; use std::thread;
use std::time::Instant; use std::time::Instant;
use walkdir::WalkDir; use walkdir::WalkDir;
use rusqlite::Connection; use rusqlite::{Connection, params};
use crate::file_handling::{load_existing_files, analyze_files_for_batch_update, process_batch_updates, process_batch_inserts}; use crate::file_handling::{load_existing_files, analyze_files_for_batch_update, process_batch_updates, process_batch_inserts};
use crate::config::Config;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum IndexingStatus { pub enum IndexingStatus {
@ -24,6 +25,7 @@ pub enum IndexingCommand {
Start { Start {
path: String, path: String,
db_path: String, db_path: String,
config: Config,
}, },
Stop, Stop,
} }
@ -34,6 +36,34 @@ pub struct IndexingService {
_handle: thread::JoinHandle<()>, _handle: thread::JoinHandle<()>,
} }
/// Set process priority for background operation
// fn set_background_priority() {
// #[cfg(windows)]
// {
// use std::os::windows::raw::HANDLE;
// // Windows implementation
// extern "system" {
// fn GetCurrentProcess() -> HANDLE;
// fn SetPriorityClass(hprocess: HANDLE, dwpriorityclass: u32) -> i32;
// }
// const BELOW_NORMAL_PRIORITY_CLASS: u32 = 0x00004000;
// unsafe {
// SetPriorityClass(GetCurrentProcess(), BELOW_NORMAL_PRIORITY_CLASS);
// }
// }
// #[cfg(unix)]
// {
// // Unix implementation
// use std::os::unix::process::CommandExt;
// unsafe {
// libc::nice(10); // Lower priority
// }
// }
// }
impl IndexingService { impl IndexingService {
pub fn new() -> Self { pub fn new() -> Self {
let status = Arc::new(Mutex::new(IndexingStatus::Idle)); let status = Arc::new(Mutex::new(IndexingStatus::Idle));
@ -51,9 +81,9 @@ impl IndexingService {
} }
} }
pub fn start_indexing(&self, path: String, db_path: String) -> Result<(), String> { pub fn start_indexing(&self, path: String, db_path: String, config: Config) -> Result<(), String> {
self.command_tx self.command_tx
.send(IndexingCommand::Start { path, db_path }) .send(IndexingCommand::Start { path, db_path, config })
.map_err(|e| format!("Failed to send start command: {}", e)) .map_err(|e| format!("Failed to send start command: {}", e))
} }
@ -67,13 +97,57 @@ impl IndexingService {
self.status.lock().unwrap().clone() self.status.lock().unwrap().clone()
} }
/// Check if configuration changes require index recreation
pub fn check_config_validation(&self, db_path: &str, config: &Config, indexing_path: &str) -> Result<Option<Vec<String>>, String> {
let conn = Connection::open(db_path)
.map_err(|e| format!("Failed to open database: {}", e))?;
// Create config validation table if it doesn't exist
conn.execute(
"CREATE TABLE IF NOT EXISTS config_validation (
key TEXT PRIMARY KEY,
value TEXT NOT NULL);",
(),
).map_err(|e| format!("Failed to create config_validation table: {}", e))?;
Self::validate_config(&conn, config, indexing_path)
}
/// Stop indexing and delete the database file for a clean rebuild
pub fn delete_index_for_rebuild(&self, db_path: &str) -> Result<(), String> {
// Stop any running indexing first
self.stop_indexing()
.map_err(|e| format!("Failed to stop indexing: {}", e))?;
// Wait for indexing to actually stop
let mut attempts = 0;
while attempts < 50 { // Wait up to 5 seconds
match self.get_status() {
IndexingStatus::Idle => break,
IndexingStatus::Stopping | IndexingStatus::Running { .. } => {
std::thread::sleep(std::time::Duration::from_millis(100));
attempts += 1;
}
IndexingStatus::Error(_) => break, // Consider error state as stopped
}
}
// Delete the database file
if std::path::Path::new(db_path).exists() {
std::fs::remove_file(db_path)
.map_err(|e| format!("Failed to delete database file: {}", e))?;
}
Ok(())
}
fn indexing_thread(status: Arc<Mutex<IndexingStatus>>, command_rx: mpsc::Receiver<IndexingCommand>) { fn indexing_thread(status: Arc<Mutex<IndexingStatus>>, command_rx: mpsc::Receiver<IndexingCommand>) {
let stop_flag = Arc::new(Mutex::new(false)); let stop_flag = Arc::new(Mutex::new(false));
let mut indexing_handle: Option<thread::JoinHandle<()>> = None; let mut indexing_handle: Option<thread::JoinHandle<()>> = None;
while let Ok(command) = command_rx.recv() { while let Ok(command) = command_rx.recv() {
match command { match command {
IndexingCommand::Start { path, db_path } => { IndexingCommand::Start { path, db_path, config } => {
if matches!(*status.lock().unwrap(), IndexingStatus::Running { .. }) { if matches!(*status.lock().unwrap(), IndexingStatus::Running { .. }) {
continue; // Already running continue; // Already running
} }
@ -96,9 +170,10 @@ impl IndexingService {
let stop_flag_clone = stop_flag.clone(); let stop_flag_clone = stop_flag.clone();
let path_owned = path.clone(); let path_owned = path.clone();
let db_path_owned = db_path.clone(); let db_path_owned = db_path.clone();
let config_owned = config.clone();
indexing_handle = Some(thread::spawn(move || { indexing_handle = Some(thread::spawn(move || {
if let Err(e) = Self::run_indexing(&status_clone, &path_owned, &db_path_owned, &stop_flag_clone) { if let Err(e) = Self::run_indexing(&status_clone, &path_owned, &db_path_owned, &stop_flag_clone, &config_owned) {
*status_clone.lock().unwrap() = IndexingStatus::Error(e); *status_clone.lock().unwrap() = IndexingStatus::Error(e);
} else { } else {
// Only set to Idle if we weren't stopped // Only set to Idle if we weren't stopped
@ -128,6 +203,7 @@ impl IndexingService {
path: &str, path: &str,
db_path: &str, db_path: &str,
stop_flag: &Arc<Mutex<bool>>, stop_flag: &Arc<Mutex<bool>>,
config: &Config,
) -> Result<(), String> { ) -> Result<(), String> {
// Set up database // Set up database
let conn = Connection::open(db_path) let conn = Connection::open(db_path)
@ -158,6 +234,17 @@ impl IndexingService {
) )
.map_err(|e| format!("Failed to create searchabletext table: {}", e))?; .map_err(|e| format!("Failed to create searchabletext table: {}", e))?;
conn.execute(
"CREATE TABLE IF NOT EXISTS config_validation (
key TEXT PRIMARY KEY,
value TEXT NOT NULL);",
(),
)
.map_err(|e| format!("Failed to create config_validation table: {}", e))?;
// Update configuration (for new installations or when no validation issues)
Self::update_config(&conn, config, path)?;
// Load existing files from database for incremental indexing // Load existing files from database for incremental indexing
let existing_files = { let existing_files = {
let conn_ref = &conn; let conn_ref = &conn;
@ -205,27 +292,28 @@ impl IndexingService {
} }
} }
// Create status callback to update current file and progress // Create status callback to update current file
let status_clone = status.clone(); let status_clone_1 = status.clone();
let status_callback = Box::new(move |file_status: &str| { let status_callback = Box::new(move |file_status: &str| {
if let Ok(mut status_guard) = status_clone.lock() { if let Ok(mut status_guard) = status_clone_1.lock() {
if let IndexingStatus::Running { ref mut current_file, ref mut files_processed, .. } = *status_guard { if let IndexingStatus::Running { ref mut current_file, .. } = *status_guard {
*current_file = Some(file_status.to_string()); *current_file = Some(file_status.to_string());
// Extract the current count from the status string
if let Some(slash_pos) = file_status.find('/') {
if let Some(space_pos) = file_status.rfind(' ') {
if space_pos + 1 < slash_pos {
if let Ok(current) = file_status[space_pos + 1..slash_pos].parse::<usize>() {
*files_processed = work_completed + current;
}
}
}
}
} }
} }
}); });
if let Err(e) = process_batch_updates(&conn_mutex, &batch_update.files_to_update, &stop_flag, Some(status_callback)) { // Create progress callback to update files_processed
let status_clone_2 = status.clone();
let base_work_completed = work_completed;
let progress_callback = Box::new(move |current_index: usize| {
if let Ok(mut status_guard) = status_clone_2.lock() {
if let IndexingStatus::Running { ref mut files_processed, .. } = *status_guard {
*files_processed = base_work_completed + current_index;
}
}
});
if let Err(e) = process_batch_updates(&conn_mutex, &batch_update.files_to_update, &stop_flag, Some(status_callback), Some(progress_callback), config) {
return Err(format!("Failed to process batch updates: {}", e)); return Err(format!("Failed to process batch updates: {}", e));
} }
@ -254,28 +342,28 @@ impl IndexingService {
} }
} }
// Create status callback to update current file and progress // Create status callback for inserts
let status_clone = status.clone(); let status_clone_3 = status.clone();
let base_work_completed = work_completed;
let status_callback = Box::new(move |file_status: &str| { let status_callback = Box::new(move |file_status: &str| {
if let Ok(mut status_guard) = status_clone.lock() { if let Ok(mut status_guard) = status_clone_3.lock() {
if let IndexingStatus::Running { ref mut current_file, ref mut files_processed, .. } = *status_guard { if let IndexingStatus::Running { ref mut current_file, .. } = *status_guard {
*current_file = Some(file_status.to_string()); *current_file = Some(file_status.to_string());
// Extract the current count from the status string
if let Some(slash_pos) = file_status.find('/') {
if let Some(space_pos) = file_status.rfind(' ') {
if space_pos + 1 < slash_pos {
if let Ok(current) = file_status[space_pos + 1..slash_pos].parse::<usize>() {
*files_processed = base_work_completed + current;
}
}
}
}
} }
} }
}); });
if let Err(e) = process_batch_inserts(&conn_mutex, &batch_update.files_to_insert, &stop_flag, Some(status_callback)) { // Create progress callback for inserts
let status_clone_4 = status.clone();
let base_work_completed = work_completed;
let progress_callback = Box::new(move |current_index: usize| {
if let Ok(mut status_guard) = status_clone_4.lock() {
if let IndexingStatus::Running { ref mut files_processed, .. } = *status_guard {
*files_processed = base_work_completed + current_index;
}
}
});
if let Err(e) = process_batch_inserts(&conn_mutex, &batch_update.files_to_insert, &stop_flag, Some(status_callback), Some(progress_callback), config) {
return Err(format!("Failed to process batch inserts: {}", e)); return Err(format!("Failed to process batch inserts: {}", e));
} }
@ -300,6 +388,85 @@ impl IndexingService {
Ok(()) Ok(())
} }
/// Validates configuration against stored values and returns validation results.
/// Critical configuration changes that require index recreation:
/// - hash_length: affects file hash computation, invalidates existing file metadata
/// - indexing_path: changes the scope of indexed files
fn validate_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result<Option<Vec<String>>, String> {
// Critical configuration values that require index recreation
let hash_length = config.processing.hash_length.to_string();
let normalized_path = std::path::Path::new(indexing_path)
.canonicalize()
.unwrap_or_else(|_| std::path::PathBuf::from(indexing_path))
.to_string_lossy()
.to_string();
// Check stored configuration values
let mut stored_hash_length: Option<String> = None;
let mut stored_indexing_path: Option<String> = None;
if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path')") {
if let Ok(rows) = stmt.query_map([], |row| {
let key: String = row.get(0)?;
let value: String = row.get(1)?;
Ok((key, value))
}) {
for row in rows.flatten() {
match row.0.as_str() {
"hash_length" => stored_hash_length = Some(row.1),
"indexing_path" => stored_indexing_path = Some(row.1),
_ => {}
}
}
}
}
// Check if configuration is invalid
let hash_length_changed = stored_hash_length.as_ref().map_or(false, |stored| stored != &hash_length);
let indexing_path_changed = stored_indexing_path.as_ref().map_or(false, |stored| stored != &normalized_path);
if hash_length_changed || indexing_path_changed {
let mut changes = Vec::new();
if hash_length_changed {
changes.push(format!("hash_length: {} -> {}",
stored_hash_length.unwrap_or_else(|| "unknown".to_string()), hash_length));
}
if indexing_path_changed {
changes.push(format!("indexing_path: {} -> {}",
stored_indexing_path.unwrap_or_else(|| "unknown".to_string()), normalized_path));
}
return Ok(Some(changes));
}
// No configuration changes detected
Ok(None)
}
/// Updates stored configuration values without clearing the index
fn update_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result<(), String> {
let hash_length = config.processing.hash_length.to_string();
let normalized_path = std::path::Path::new(indexing_path)
.canonicalize()
.unwrap_or_else(|_| std::path::PathBuf::from(indexing_path))
.to_string_lossy()
.to_string();
// Update stored configuration values
conn.execute(
"INSERT OR REPLACE INTO config_validation (key, value) VALUES ('hash_length', ?1)",
params![hash_length],
).map_err(|e| format!("Failed to store hash_length config: {}", e))?;
conn.execute(
"INSERT OR REPLACE INTO config_validation (key, value) VALUES ('indexing_path', ?1)",
params![normalized_path],
).map_err(|e| format!("Failed to store indexing_path config: {}", e))?;
Ok(())
}
} }
impl Default for IndexingService { impl Default for IndexingService {