Added skipping of hidden files and fixed some bugs.

config_example.toml

@@ -18,6 +18,8 @@ fts_update_batch_size = 1000
 precount_files_for_progress = false
 # If true, follow symbolic links during directory walks (indexing only; shell precount unchanged).
 follow_symlinks = false
+# If true, hidden files and directories will be indexed.
+include_hidden = false
 # FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61')
 # Look here for more information https://www.sqlite.org/fts5.html#tokenizers
 tokenize = "trigram"

src/config.rs

@@ -31,6 +31,8 @@ pub struct ProcessingConfig {
     pub precount_files_for_progress: bool,
     #[serde(default)]
     pub follow_symlinks: bool,
+    #[serde(default)]
+    pub include_hidden: bool,
 }
 
 impl Default for Config {
@@ -49,6 +51,7 @@ impl Default for Config {
                 tokenize: "trigram".to_string(),
                 precount_files_for_progress: false,
                 follow_symlinks: false,
+                include_hidden: false,
             },
         }
     }

src/file_handling.rs

@@ -2,6 +2,7 @@ use std::sync::{Mutex, Arc};
 use std::ffi::OsString;
 use std::fs::{File,read_to_string};
 use std::io::{Read, Seek, SeekFrom};
+use std::path::Component;
 use std::process::{Command, Stdio};
 use std::time::UNIX_EPOCH;
 use std::collections::HashMap;
@@ -78,7 +79,10 @@ pub fn load_existing_files(conn: &Connection) -> Result<HashMap<String, Existing
     Ok(existing_files)
 }
 
-pub fn indexed_walk_file_entries(path: &str, follow_symlinks: bool) -> impl Iterator<Item = DirEntry> {
+pub fn indexed_walk_file_entries(
+    path: &str,
+    follow_symlinks: bool,
+) -> impl Iterator<Item = DirEntry> {
     WalkDir::new(path)
         .follow_links(follow_symlinks)
         .into_iter()
@@ -86,6 +90,15 @@ pub fn indexed_walk_file_entries(path: &str, follow_symlinks: bool) -> impl Iter
         .filter(|entry| !entry.metadata().map(|m| m.is_dir()).unwrap_or(true))
 }
 
+pub fn path_has_hidden_component(path: &std::path::Path) -> bool {
+    path.components().any(|c| {
+        matches!(
+            c,
+            Component::Normal(name) if name.to_string_lossy().starts_with('.')
+        )
+    })
+}
+
 fn parse_wc_l_stdout(bytes: &[u8]) -> Result<usize, String> {
     let s = String::from_utf8_lossy(bytes);
     let token = s
@@ -772,6 +785,10 @@ pub fn process_text_indexing(
             }
         }
 
+        if let Some(ref callback) = status_callback {
+            callback("Rebuilding FTS index after text addition...");
+        }
+
         tx.commit()
             .map_err(|e| format!("Failed to commit transaction: {}", e))?;
     }

src/frontend.rs

@@ -164,7 +164,7 @@ pub fn App(props: AppProps) -> Element {
                                 )
                             }
                         }
-                        IndexingStatus::RunningTextIndex { files_processed, total_files, current_file, start_time } => {
+                        IndexingStatus::RunningTextIndex { files_processed, current_file, start_time } => {
                             // Add data point to speed tracker
                             speed_tracker_clone.with_mut(|tracker| {
                                 tracker.add_data_point(files_processed);
@@ -182,29 +182,13 @@ pub fn App(props: AppProps) -> Element {
                                     .map(|fps| format!(" - {:.1} files/sec", fps))
                                     .unwrap_or_default()
                             });
-                            
+                            format!(
-                            if let Some(total) = total_files {
+                                "Phase 2 - Text Index: {} files processed - {:.1}s elapsed{}\n{}",
-                                let percentage = if total > 0 { 
+                                files_processed,
-                                    (files_processed as f64 / total as f64 * 100.0) as u32 
+                                elapsed.as_secs_f64(),
-                                } else { 0 };
+                                speed_display,
-                                format!(
+                                current_file_display
-                                    "Phase 2 - Text Index: {}/{} files ({}%) - {:.1}s elapsed{}\n{}",
+                            )
-                                    files_processed,
-                                    total,
-                                    percentage,
-                                    elapsed.as_secs_f64(),
-                                    speed_display,
-                                    current_file_display
-                                )
-                            } else {
-                                format!(
-                                    "Phase 2 - Text Index: {} files processed - {:.1}s elapsed{}\n{}",
-                                    files_processed,
-                                    elapsed.as_secs_f64(),
-                                    speed_display,
-                                    current_file_display
-                                )
-                            }
                         }
                         IndexingStatus::Stopping => "Indexing Stopped".to_string(),
                         IndexingStatus::Error(ref e) => format!("Error: {}", e),

src/indexing.rs

@@ -12,6 +12,7 @@ use crate::file_handling::{
     count_tree_entries_fast,
     indexed_walk_file_entries,
     load_existing_files,
+    path_has_hidden_component,
     process_batch_inserts_files_only,
     process_batch_updates_files_only,
     process_text_indexing,
@@ -47,7 +48,6 @@ pub enum IndexingStatus {
     },
     RunningTextIndex {
         files_processed: usize,
-        total_files: Option<usize>,
         current_file: Option<String>,
         start_time: Instant,
     },
@@ -189,6 +189,12 @@ impl IndexingService {
                 }
             })?;
 
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);",
+            (),
+        )
+        .map_err(|e| format!("Failed to prepare duplicate-search index: {}", e))?;
+        
         let mut stmt = conn.prepare(query)
             .map_err(|e| {
                 let error_msg = e.to_string();
@@ -535,6 +541,12 @@ impl IndexingService {
         )
         .map_err(|e| format!("Failed to create files table: {}", e))?;
 
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);",
+            (),
+        )
+        .map_err(|e| format!("Failed to create files hash index: {}", e))?;
+
         conn.execute(
             "CREATE TABLE IF NOT EXISTS documents (
                 id      INTEGER PRIMARY KEY,
@@ -673,6 +685,21 @@ impl IndexingService {
                 return Ok(());
             }
 
+            visit += 1;
+            if let Ok(mut g) = status.lock() {
+                if let IndexingStatus::RunningFileIndex {
+                    ref mut files_processed,
+                    ..
+                } = *g
+                {
+                    *files_processed = visit;
+                }
+            }
+
+            if !config.processing.include_hidden && path_has_hidden_component(entry.path()) {
+                continue;
+            }
+
             let action = classify_dir_entry_for_indexing(&entry, &existing_files);
             let Some(action) = action else {
                 continue;
@@ -690,17 +717,6 @@ impl IndexingService {
                     }
                 });
 
-            visit += 1;
-            if let Ok(mut g) = status.lock() {
-                if let IndexingStatus::RunningFileIndex {
-                    ref mut files_processed,
-                    ..
-                } = *g
-                {
-                    *files_processed = visit;
-                }
-            }
-
             match action {
                 FileIndexAction::Skip => {
                     if let Some(path) = current_path {
@@ -766,7 +782,6 @@ impl IndexingService {
         if let Ok(mut status_guard) = status.lock() {
             *status_guard = IndexingStatus::RunningTextIndex {
                 files_processed: 0,
-                total_files: None,
                 current_file: Some("Starting text indexing...".to_string()),
                 start_time: Instant::now(),
             };
@@ -835,6 +850,7 @@ impl IndexingService {
         // Critical configuration values that require index recreation
         let hash_length = config.processing.hash_length.to_string();
         let tokenize = config.processing.tokenize.clone();
+        let include_hidden = config.processing.include_hidden.to_string();
         let normalized_path = {
             let path = std::path::Path::new(indexing_path)
                 .canonicalize()
@@ -853,8 +869,9 @@ impl IndexingService {
         let mut stored_hash_length: Option<String> = None;
         let mut stored_indexing_path: Option<String> = None;
         let mut stored_tokenize: Option<String> = None;
+        let mut stored_include_hidden: Option<String> = None;
         
-        if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path', 'tokenize')") {
+        if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path', 'tokenize', 'include_hidden')") {
             if let Ok(rows) = stmt.query_map([], |row| {
                 let key: String = row.get(0)?;
                 let value: String = row.get(1)?;
@@ -865,6 +882,7 @@ impl IndexingService {
                         "hash_length" => stored_hash_length = Some(row.1),
                         "indexing_path" => stored_indexing_path = Some(row.1),
                         "tokenize" => stored_tokenize = Some(row.1),
+                        "include_hidden" => stored_include_hidden = Some(row.1),
                         _ => {}
                     }
                 }
@@ -875,8 +893,9 @@ impl IndexingService {
         let hash_length_changed = stored_hash_length.as_ref().map_or(false, |stored| stored != &hash_length);
         let indexing_path_changed = stored_indexing_path.as_ref().map_or(false, |stored| stored != &normalized_path);
         let tokenize_changed = stored_tokenize.as_ref().map_or(false, |stored| stored != &tokenize);
+        let include_hidden_changed = stored_include_hidden.as_ref().map_or(false, |stored| stored != &include_hidden);
         
-        if hash_length_changed || indexing_path_changed || tokenize_changed {
+        if hash_length_changed || indexing_path_changed || tokenize_changed || include_hidden_changed {
             let mut changes = Vec::new();
             if hash_length_changed {
                 changes.push(format!("hash_length: {} -> {}", 
@@ -890,6 +909,13 @@ impl IndexingService {
                 changes.push(format!("tokenize: {} -> {}", 
                     stored_tokenize.unwrap_or_else(|| "unknown".to_string()), tokenize));
             }
+            if include_hidden_changed {
+                changes.push(format!(
+                    "include_hidden: {} -> {}",
+                    stored_include_hidden.unwrap_or_else(|| "unknown".to_string()),
+                    include_hidden
+                ));
+            }
             
             return Ok(Some(changes));
         }
@@ -903,6 +929,7 @@ impl IndexingService {
     fn update_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result<(), String> {
         let hash_length = config.processing.hash_length.to_string();
         let tokenize = config.processing.tokenize.clone();
+        let include_hidden = config.processing.include_hidden.to_string();
         let normalized_path = {
             let path = std::path::Path::new(indexing_path)
                 .canonicalize()
@@ -933,6 +960,11 @@ impl IndexingService {
             params![tokenize],
         ).map_err(|e| format!("Failed to store tokenize config: {}", e))?;
         
+        conn.execute(
+            "INSERT OR REPLACE INTO config_validation (key, value) VALUES ('include_hidden', ?1)",
+            params![include_hidden],
+        ).map_err(|e| format!("Failed to store include_hidden config: {}", e))?;
+
         Ok(())
     }
 }

src/search.rs

@@ -139,7 +139,7 @@ pub fn Search(props: SearchProps) -> Element {
                         }
 
                         format!(
-                            "SELECT d.name, d.path, snippet(st, 1, '<b>', '</b>', '<b>...</b>', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank",
+                            "SELECT d.name, d.path, snippet(searchabletext, 1, '<b>', '</b>', '<b>...</b>', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank",
                             where_clause
                         )
                     },
@@ -151,7 +151,7 @@ pub fn Search(props: SearchProps) -> Element {
                         }
                         format!("SELECT name, path FROM files WHERE name LIKE '%{}%'", search_term_val.replace("'", "''"))
                     },
-                    "duplicates" => "SELECT name, count(*) as cnt, path FROM files GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC".to_string(),
+                    "duplicates" => "SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC LIMIT 5000".to_string(),
                     _ => {
                         is_searching_clone.set(false);
                         return;