From f0edf3963b0b3835f16f52fac75abd1896ed24dc Mon Sep 17 00:00:00 2001 From: Jeremy Karst Date: Tue, 21 Apr 2026 03:43:28 -0400 Subject: [PATCH] Added skipping of hidden files and fixed some bugs. --- config_example.toml | 2 ++ src/config.rs | 3 +++ src/file_handling.rs | 19 +++++++++++++- src/frontend.rs | 32 ++++++----------------- src/indexing.rs | 62 +++++++++++++++++++++++++++++++++----------- src/search.rs | 4 +-- 6 files changed, 80 insertions(+), 42 deletions(-) diff --git a/config_example.toml b/config_example.toml index 56c7db4..0d8ffc6 100644 --- a/config_example.toml +++ b/config_example.toml @@ -18,6 +18,8 @@ fts_update_batch_size = 1000 precount_files_for_progress = false # If true, follow symbolic links during directory walks (indexing only; shell precount unchanged). follow_symlinks = false +# If true, hidden files and directories will be indexed. +include_hidden = false # FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61') # Look here for more information https://www.sqlite.org/fts5.html#tokenizers tokenize = "trigram" \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 448a936..caaed75 100644 --- a/src/config.rs +++ b/src/config.rs @@ -31,6 +31,8 @@ pub struct ProcessingConfig { pub precount_files_for_progress: bool, #[serde(default)] pub follow_symlinks: bool, + #[serde(default)] + pub include_hidden: bool, } impl Default for Config { @@ -49,6 +51,7 @@ impl Default for Config { tokenize: "trigram".to_string(), precount_files_for_progress: false, follow_symlinks: false, + include_hidden: false, }, } } diff --git a/src/file_handling.rs b/src/file_handling.rs index e0aa080..ddc1dc6 100644 --- a/src/file_handling.rs +++ b/src/file_handling.rs @@ -2,6 +2,7 @@ use std::sync::{Mutex, Arc}; use std::ffi::OsString; use std::fs::{File,read_to_string}; use std::io::{Read, Seek, SeekFrom}; +use std::path::Component; use std::process::{Command, Stdio}; use std::time::UNIX_EPOCH; use std::collections::HashMap; @@ -78,7 +79,10 @@ pub fn load_existing_files(conn: &Connection) -> Result impl Iterator { +pub fn indexed_walk_file_entries( + path: &str, + follow_symlinks: bool, +) -> impl Iterator { WalkDir::new(path) .follow_links(follow_symlinks) .into_iter() @@ -86,6 +90,15 @@ pub fn indexed_walk_file_entries(path: &str, follow_symlinks: bool) -> impl Iter .filter(|entry| !entry.metadata().map(|m| m.is_dir()).unwrap_or(true)) } +pub fn path_has_hidden_component(path: &std::path::Path) -> bool { + path.components().any(|c| { + matches!( + c, + Component::Normal(name) if name.to_string_lossy().starts_with('.') + ) + }) +} + fn parse_wc_l_stdout(bytes: &[u8]) -> Result { let s = String::from_utf8_lossy(bytes); let token = s @@ -772,6 +785,10 @@ pub fn process_text_indexing( } } + if let Some(ref callback) = status_callback { + callback("Rebuilding FTS index after text addition..."); + } + tx.commit() .map_err(|e| format!("Failed to commit transaction: {}", e))?; } diff --git a/src/frontend.rs b/src/frontend.rs index e0fdb00..c258fa4 100644 --- a/src/frontend.rs +++ b/src/frontend.rs @@ -164,7 +164,7 @@ pub fn App(props: AppProps) -> Element { ) } } - IndexingStatus::RunningTextIndex { files_processed, total_files, current_file, start_time } => { + IndexingStatus::RunningTextIndex { files_processed, current_file, start_time } => { // Add data point to speed tracker speed_tracker_clone.with_mut(|tracker| { tracker.add_data_point(files_processed); @@ -182,29 +182,13 @@ pub fn App(props: AppProps) -> Element { .map(|fps| format!(" - {:.1} files/sec", fps)) .unwrap_or_default() }); - - if let Some(total) = total_files { - let percentage = if total > 0 { - (files_processed as f64 / total as f64 * 100.0) as u32 - } else { 0 }; - format!( - "Phase 2 - Text Index: {}/{} files ({}%) - {:.1}s elapsed{}\n{}", - files_processed, - total, - percentage, - elapsed.as_secs_f64(), - speed_display, - current_file_display - ) - } else { - format!( - "Phase 2 - Text Index: {} files processed - {:.1}s elapsed{}\n{}", - files_processed, - elapsed.as_secs_f64(), - speed_display, - current_file_display - ) - } + format!( + "Phase 2 - Text Index: {} files processed - {:.1}s elapsed{}\n{}", + files_processed, + elapsed.as_secs_f64(), + speed_display, + current_file_display + ) } IndexingStatus::Stopping => "Indexing Stopped".to_string(), IndexingStatus::Error(ref e) => format!("Error: {}", e), diff --git a/src/indexing.rs b/src/indexing.rs index 8ae0393..4bc81f2 100644 --- a/src/indexing.rs +++ b/src/indexing.rs @@ -12,6 +12,7 @@ use crate::file_handling::{ count_tree_entries_fast, indexed_walk_file_entries, load_existing_files, + path_has_hidden_component, process_batch_inserts_files_only, process_batch_updates_files_only, process_text_indexing, @@ -47,7 +48,6 @@ pub enum IndexingStatus { }, RunningTextIndex { files_processed: usize, - total_files: Option, current_file: Option, start_time: Instant, }, @@ -188,6 +188,12 @@ impl IndexingService { format!("Failed to open database: {}", e) } })?; + + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);", + (), + ) + .map_err(|e| format!("Failed to prepare duplicate-search index: {}", e))?; let mut stmt = conn.prepare(query) .map_err(|e| { @@ -535,6 +541,12 @@ impl IndexingService { ) .map_err(|e| format!("Failed to create files table: {}", e))?; + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);", + (), + ) + .map_err(|e| format!("Failed to create files hash index: {}", e))?; + conn.execute( "CREATE TABLE IF NOT EXISTS documents ( id INTEGER PRIMARY KEY, @@ -673,6 +685,21 @@ impl IndexingService { return Ok(()); } + visit += 1; + if let Ok(mut g) = status.lock() { + if let IndexingStatus::RunningFileIndex { + ref mut files_processed, + .. + } = *g + { + *files_processed = visit; + } + } + + if !config.processing.include_hidden && path_has_hidden_component(entry.path()) { + continue; + } + let action = classify_dir_entry_for_indexing(&entry, &existing_files); let Some(action) = action else { continue; @@ -690,17 +717,6 @@ impl IndexingService { } }); - visit += 1; - if let Ok(mut g) = status.lock() { - if let IndexingStatus::RunningFileIndex { - ref mut files_processed, - .. - } = *g - { - *files_processed = visit; - } - } - match action { FileIndexAction::Skip => { if let Some(path) = current_path { @@ -766,7 +782,6 @@ impl IndexingService { if let Ok(mut status_guard) = status.lock() { *status_guard = IndexingStatus::RunningTextIndex { files_processed: 0, - total_files: None, current_file: Some("Starting text indexing...".to_string()), start_time: Instant::now(), }; @@ -835,6 +850,7 @@ impl IndexingService { // Critical configuration values that require index recreation let hash_length = config.processing.hash_length.to_string(); let tokenize = config.processing.tokenize.clone(); + let include_hidden = config.processing.include_hidden.to_string(); let normalized_path = { let path = std::path::Path::new(indexing_path) .canonicalize() @@ -853,8 +869,9 @@ impl IndexingService { let mut stored_hash_length: Option = None; let mut stored_indexing_path: Option = None; let mut stored_tokenize: Option = None; + let mut stored_include_hidden: Option = None; - if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path', 'tokenize')") { + if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path', 'tokenize', 'include_hidden')") { if let Ok(rows) = stmt.query_map([], |row| { let key: String = row.get(0)?; let value: String = row.get(1)?; @@ -865,6 +882,7 @@ impl IndexingService { "hash_length" => stored_hash_length = Some(row.1), "indexing_path" => stored_indexing_path = Some(row.1), "tokenize" => stored_tokenize = Some(row.1), + "include_hidden" => stored_include_hidden = Some(row.1), _ => {} } } @@ -875,8 +893,9 @@ impl IndexingService { let hash_length_changed = stored_hash_length.as_ref().map_or(false, |stored| stored != &hash_length); let indexing_path_changed = stored_indexing_path.as_ref().map_or(false, |stored| stored != &normalized_path); let tokenize_changed = stored_tokenize.as_ref().map_or(false, |stored| stored != &tokenize); + let include_hidden_changed = stored_include_hidden.as_ref().map_or(false, |stored| stored != &include_hidden); - if hash_length_changed || indexing_path_changed || tokenize_changed { + if hash_length_changed || indexing_path_changed || tokenize_changed || include_hidden_changed { let mut changes = Vec::new(); if hash_length_changed { changes.push(format!("hash_length: {} -> {}", @@ -890,6 +909,13 @@ impl IndexingService { changes.push(format!("tokenize: {} -> {}", stored_tokenize.unwrap_or_else(|| "unknown".to_string()), tokenize)); } + if include_hidden_changed { + changes.push(format!( + "include_hidden: {} -> {}", + stored_include_hidden.unwrap_or_else(|| "unknown".to_string()), + include_hidden + )); + } return Ok(Some(changes)); } @@ -903,6 +929,7 @@ impl IndexingService { fn update_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result<(), String> { let hash_length = config.processing.hash_length.to_string(); let tokenize = config.processing.tokenize.clone(); + let include_hidden = config.processing.include_hidden.to_string(); let normalized_path = { let path = std::path::Path::new(indexing_path) .canonicalize() @@ -932,6 +959,11 @@ impl IndexingService { "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('tokenize', ?1)", params![tokenize], ).map_err(|e| format!("Failed to store tokenize config: {}", e))?; + + conn.execute( + "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('include_hidden', ?1)", + params![include_hidden], + ).map_err(|e| format!("Failed to store include_hidden config: {}", e))?; Ok(()) } diff --git a/src/search.rs b/src/search.rs index 5697ed7..ab321c9 100644 --- a/src/search.rs +++ b/src/search.rs @@ -139,7 +139,7 @@ pub fn Search(props: SearchProps) -> Element { } format!( - "SELECT d.name, d.path, snippet(st, 1, '', '', '...', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank", + "SELECT d.name, d.path, snippet(searchabletext, 1, '', '', '...', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank", where_clause ) }, @@ -151,7 +151,7 @@ pub fn Search(props: SearchProps) -> Element { } format!("SELECT name, path FROM files WHERE name LIKE '%{}%'", search_term_val.replace("'", "''")) }, - "duplicates" => "SELECT name, count(*) as cnt, path FROM files GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC".to_string(), + "duplicates" => "SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC LIMIT 5000".to_string(), _ => { is_searching_clone.set(false); return;