Sped up initial file count and reduced memory use, added cleaning step to database on repeat indexing.

config_example.toml

@@ -6,11 +6,18 @@ database_path = "QuickSearch.db"
 # Amount of data in bytes read from start/end of files used to calculate hash
 hash_length = 8192
 # Maximum text content to index per file (bytes) 
-maximum_text_size = 524288
+maximum_text_size = 262144
 # Maximum file size to process for text extraction (bytes)
-maximum_file_size = 52428800
+maximum_text_file_size = 2097152
-# Number of files to process in each batch
+# Number of files to process in each batch (directory walk / inserts / text extraction batches)
 batch_size = 200
+# Files per transaction for incremental UPDATE files + DELETE from searchabletext (FTS); larger = fewer commits, more RAM per chunk
+fts_update_batch_size = 1000
+# If true, run a fast shell-backed tree count before Phase 1 (enables % progress; Linux uses GNU find -printf '\n' | wc -l when available).
+# If false, Phase 1 shows file counts without a percentage.
+precount_files_for_progress = false
+# If true, follow symbolic links during directory walks (indexing only; shell precount unchanged).
+follow_symlinks = false
 # FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61')
 # Look here for more information https://www.sqlite.org/fts5.html#tokenizers
 tokenize = "trigram"

run.bat

@@ -0,0 +1 @@
+cargo run

run.sh

@@ -0,0 +1,2 @@
+cargo build --release
+./target/release/quicksearch

setup.sh

@@ -0,0 +1 @@
+sudo apt install -y libsoup-3.0-dev libjavascriptcoregtk-4.1-dev libwebkit2gtk-4.1-dev libxdo-dev

src/config.rs

@@ -14,13 +14,23 @@ pub struct PathConfig {
     pub database_path: String,
 }
 
+fn default_fts_update_batch_size() -> usize {
+    1000
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct ProcessingConfig {
     pub hash_length: usize,
     pub maximum_text_size: usize,
-    pub maximum_file_size: u64,
+    pub maximum_text_file_size: u64,
     pub batch_size: usize,
+    #[serde(default = "default_fts_update_batch_size")]
+    pub fts_update_batch_size: usize,
     pub tokenize: String,
+    #[serde(default)]
+    pub precount_files_for_progress: bool,
+    #[serde(default)]
+    pub follow_symlinks: bool,
 }
 
 impl Default for Config {
@@ -32,10 +42,13 @@ impl Default for Config {
             },
             processing: ProcessingConfig {
                 hash_length: 1024 * 8,
-                maximum_text_size: 1024 * 512,
+                maximum_text_size: 1024 * 256,
-                maximum_file_size: 1024 * 1024 * 50,
+                maximum_text_file_size: 1024 * 1024 * 2,
                 batch_size: 200,
+                fts_update_batch_size: 1000,
                 tokenize: "trigram".to_string(),
+                precount_files_for_progress: false,
+                follow_symlinks: false,
             },
         }
     }

src/file_handling.rs

@@ -2,46 +2,54 @@ use std::sync::{Mutex, Arc};
 use std::ffi::OsString;
 use std::fs::{File,read_to_string};
 use std::io::{Read, Seek, SeekFrom};
+use std::process::{Command, Stdio};
 use std::time::UNIX_EPOCH;
 use std::collections::HashMap;
 
 use sha2::{Sha256, Digest};
-use walkdir::DirEntry;
+use walkdir::{DirEntry, WalkDir};
 use rusqlite::{params, Connection};
 
 use crate::document_extraction::extract_document_text;
 use crate::config::Config;
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct FileMetadata {
+pub struct ExistingFileEntry {
-    pub path: String,
-    pub size: u64,
     pub moddate: u64,
-    pub hash: Vec<u8>,
 }
 
-#[derive(Debug)]
+pub const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 84] = 
-pub struct BatchUpdate {
+    ["c","cs","csx",                                // C
-    pub files_to_update: Vec<(DirEntry, FileMetadata)>,
+    "cpp","cc","cxx","hpp","hh","hxx","h",          // C++
-    pub files_to_insert: Vec<DirEntry>,
+    "cfg","conf","ini","gitattributes","gitignore", // Config (General)
-}
+    "toml","env","tf","tfvars",                     // Config (Infrastructure)
-
+    "scss","sass","less",                           // CSS Preprocessors
-pub const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 86] = 
+    "dart",                                         // Dart
-    ["","txt","rtf","log", // Text Documents
+    "diff","patch",                                 // Diffs
-    "csv", // Spreadsheet
+    "go",                                           // Go
-    "sh","bat","cmd","bash","ps1","psm1","psd1","pssc","psrc", // Scripts
+    "graphql","gql",                                // GraphQL
-    "c","cpp","i","cs","csx","caki", // C#
+    "html","htm","xhtml","xht","jsp","asp","aspx",  // HTML
-    "cpp","cc","cxx","c++","hpp","hh","hxx","h","ii", // C++
+    "java",                                         // Java
-    "tex","bib","bbx","cbx", // LaTeX
+    "js","cjs","mjs","jsx","ts","tsx",              // Javascript and TypeScript
-    "css","xml","md","json","yaml","yml", // Markup Languages and others
+    "vue","svelte",                                 // JS Frameworks
-    "html","htm","shtml","xhtml","xht","mdoc","jsp","asp","aspx","jshtm", // HTML
+    "kt","kts",                                     // Kotlin
-    "js","cjs","mjs","es6","es","jsx","ts","tsx", // Javascript and TypeScript
+    "tex","bib",                                    // LaTeX
-    "cfg","conf","ini","gitattributes","gitignore", // Config and related files
+    "css","xml","md","json","yaml","yml",           // Markup
-    "java","jav", // Java
+    "m",                                            // Objective-C
-    "pl","pm","pod","t","psgi", // Perl
+    "pl","pm","t",                                  // Perl
-    "php","php4","php5","phtml","ctp", // PHP
+    "php","phtml",                                  // PHP
-    "py","rpy","pyw","cpy","gyp","gypi","pyi","ipy","pyt","ipynb", // Python
+    "proto",                                        // Protocol Buffers
-    "wasm","wat", // Web Assembly
+    "py","pyw","pyi","ipynb",                       // Python
+    "r",                                            // R
+    "rb",                                           // Ruby
+    "rs",                                           // Rust
+    "sh","bat","cmd","bash","ps1","psm1","psd1",    // Scripts
+    "sql",                                          // SQL
+    "csv",                                          // Spreadsheet
+    "svg",                                          // SVG
+    "swift",                                        // Swift
+    "","txt","rtf","log",                           // Text Documents
+    "wasm",                                         // Web Assembly
     ];
 
 pub const SUPPORTED_DOCUMENT_EXTENSIONS_LIST: [&'static str; 9] = 
@@ -49,76 +57,183 @@ pub const SUPPORTED_DOCUMENT_EXTENSIONS_LIST: [&'static str; 9] =
     "ppt", "pptx", "odp", // Presentation
     "xls", "xlsx", "ods"]; // Spreadsheet
 
-/// Load existing file metadata from database indexed by path
+/// Load path and moddate per row for incremental classification (hash/size loaded only when updating a file).
-pub fn load_existing_files(conn: &Connection) -> Result<HashMap<String, FileMetadata>, rusqlite::Error> {
+pub fn load_existing_files(conn: &Connection) -> Result<HashMap<String, ExistingFileEntry>, rusqlite::Error> {
     let mut existing_files = HashMap::new();
-    let mut stmt = conn.prepare("SELECT path, size, moddate, hash FROM files")?;
+    let mut stmt = conn.prepare("SELECT path, moddate FROM files")?;
     let rows = stmt.query_map([], |row| {
-        Ok(FileMetadata {
+        Ok((
-            path: row.get(0)?,
+            row.get::<_, String>(0)?,
-            size: row.get(1)?,
+            ExistingFileEntry {
-            moddate: row.get(2)?,
+                moddate: row.get(1)?,
-            hash: row.get(3)?,
+            },
-        })
+        ))
     })?;
 
     for row in rows {
-        let metadata = row?;
+        let (path, entry) = row?;
-        existing_files.insert(metadata.path.clone(), metadata);
+        existing_files.insert(path, entry);
     }
 
     Ok(existing_files)
 }
 
-/// Analyze files and determine which need updates vs inserts
+pub fn indexed_walk_file_entries(path: &str, follow_symlinks: bool) -> impl Iterator<Item = DirEntry> {
-pub fn analyze_files_for_batch_update(
+    WalkDir::new(path)
-    entries: &[DirEntry], 
+        .follow_links(follow_symlinks)
-    existing_files: &HashMap<String, FileMetadata>
+        .into_iter()
-) -> BatchUpdate {
+        .filter_map(|e| e.ok())
-    let mut files_to_update = Vec::new();
+        .filter(|entry| !entry.metadata().map(|m| m.is_dir()).unwrap_or(true))
-    let mut files_to_insert = Vec::new();
+}
 
-    for entry in entries {
+fn parse_wc_l_stdout(bytes: &[u8]) -> Result<usize, String> {
-        let meta = match entry.metadata() {
+    let s = String::from_utf8_lossy(bytes);
-            Ok(m) if !m.is_dir() => m,
+    let token = s
-            _ => continue,
+        .trim()
-        };
+        .split_whitespace()
+        .next()
+        .ok_or_else(|| "wc: empty output".to_string())?;
+    token
+        .parse()
+        .map_err(|e| format!("wc: invalid count {:?}: {}", token, e))
+}
 
-        let fpath = match entry.path().canonicalize() {
+#[cfg(unix)]
-            Ok(fp) => {
+fn count_find_pipe_wc(path: &str) -> Result<usize, String> {
-                let path_str = fp.to_string_lossy().to_string();
+    let mut find = Command::new("find")
-                // Remove Windows UNC prefix \\?\
+        .arg(path)
-                if path_str.starts_with("\\\\?\\") {
+        .stdout(Stdio::piped())
-                    path_str[4..].to_string()
+        .stderr(Stdio::null())
-                } else {
+        .spawn()
-                    path_str
+        .map_err(|e| format!("find: {}", e))?;
-                }
+    let find_stdout = find.stdout.take().ok_or("find: stdout")?;
-            },
+    let wc = Command::new("wc")
-            Err(_) => continue,
+        .arg("-l")
-        };
+        .stdin(find_stdout)
-
+        .stdout(Stdio::piped())
-        let fmodified = match meta.modified()
+        .output()
-            .ok()
+        .map_err(|e| format!("wc: {}", e))?;
-            .and_then(|m| m.duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs())) {
+    find.wait().map_err(|e| format!("find wait: {}", e))?;
-            Some(time) => time,
+    if !wc.status.success() {
-            None => continue,
+        return Err(format!("wc exited with {}", wc.status));
-        };
-
-        if let Some(existing_metadata) = existing_files.get(&fpath) {
-            // File exists in database, check if modification date changed
-            if existing_metadata.moddate != fmodified {
-                files_to_update.push((entry.clone(), existing_metadata.clone()));
-            }
-            // If moddate is same, skip processing this file entirely
-        } else {
-            // New file, needs to be inserted
-            files_to_insert.push(entry.clone());
-        }
     }
+    parse_wc_l_stdout(&wc.stdout)
+}
 
-    BatchUpdate {
+#[cfg(target_os = "linux")]
-        files_to_update,
+fn count_find_printf_wc(path: &str) -> Result<usize, String> {
-        files_to_insert,
+    let mut find = Command::new("find")
+        .arg(path)
+        .arg("-printf")
+        .arg("\n")
+        .stdout(Stdio::piped())
+        .stderr(Stdio::null())
+        .spawn()
+        .map_err(|e| format!("find: {}", e))?;
+    let find_stdout = find.stdout.take().ok_or("find: stdout")?;
+    let wc = Command::new("wc")
+        .arg("-l")
+        .stdin(find_stdout)
+        .stdout(Stdio::piped())
+        .output()
+        .map_err(|e| format!("wc: {}", e))?;
+    find.wait().map_err(|e| format!("find wait: {}", e))?;
+    if !wc.status.success() {
+        return Err(format!("wc exited with {}", wc.status));
+    }
+    parse_wc_l_stdout(&wc.stdout)
+}
+
+#[cfg(windows)]
+fn count_tree_entries_windows(path: &str) -> Result<usize, String> {
+    let lit = path.replace('\'', "''");
+    let ps = format!(
+        "(Get-ChildItem -LiteralPath '{}' -Recurse -Force -ErrorAction SilentlyContinue | Measure-Object).Count",
+        lit
+    );
+    let out = Command::new("powershell.exe")
+        .args(["-NoProfile", "-NonInteractive", "-Command", &ps])
+        .output()
+        .map_err(|e| format!("powershell: {}", e))?;
+    if !out.status.success() {
+        return Err(format!(
+            "powershell exited with {}: {}",
+            out.status,
+            String::from_utf8_lossy(&out.stderr)
+        ));
+    }
+    String::from_utf8_lossy(&out.stdout)
+        .trim()
+        .parse()
+        .map_err(|e| format!("invalid count output: {}", e))
+}
+
+/// Rough tree entry count for progress totals (Linux: `find DIR -printf '\n' | wc -l` when GNU find is available, else `find DIR | wc -l`; macOS/other Unix: `find DIR | wc -l`; Windows: PowerShell `Get-ChildItem -Recurse`). Scope is not identical to the indexer’s classified file count.
+pub fn count_tree_entries_fast(path: &str) -> Result<usize, String> {
+    #[cfg(windows)]
+    {
+        return count_tree_entries_windows(path);
+    }
+    #[cfg(all(unix, target_os = "linux"))]
+    {
+        return count_find_printf_wc(path).or_else(|_| count_find_pipe_wc(path));
+    }
+    #[cfg(all(unix, not(target_os = "linux")))]
+    {
+        return count_find_pipe_wc(path);
+    }
+    #[cfg(not(any(windows, unix)))]
+    {
+        Err("tree entry count is not supported on this target".to_string())
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FileIndexAction {
+    Skip,
+    Update,
+    Insert,
+}
+
+/// Classify a walkdir entry for Phase 1 file indexing. Returns `None` if the path is not indexable.
+pub fn classify_dir_entry_for_indexing(
+    entry: &DirEntry,
+    existing_files: &HashMap<String, ExistingFileEntry>,
+) -> Option<FileIndexAction> {
+    let fpath = match entry.path().canonicalize() {
+        Ok(fp) => {
+            let path_str = fp.to_string_lossy().to_string();
+            if path_str.starts_with("\\\\?\\") {
+                path_str[4..].to_string()
+            } else {
+                path_str
+            }
+        }
+        Err(_) => return None,
+    };
+
+    let meta = match std::fs::metadata(&fpath) {
+        Ok(m) if m.is_file() => m,
+        _ => return None,
+    };
+
+    let fmodified = match meta
+        .modified()
+        .ok()
+        .and_then(|m| m.duration_since(UNIX_EPOCH).ok().map(|d| d.as_secs()))
+    {
+        Some(time) => time,
+        None => return None,
+    };
+
+    if let Some(existing) = existing_files.get(&fpath) {
+        if existing.moddate != fmodified {
+            Some(FileIndexAction::Update)
+        } else {
+            Some(FileIndexAction::Skip)
+        }
+    } else {
+        Some(FileIndexAction::Insert)
     }
 }
 
@@ -159,103 +274,194 @@ fn get_file_hash(size: u64, path: OsString, hash_length: usize) -> Result<Vec<u8
     Ok(hasher.finalize().to_vec()) 
 }
 
+fn format_progress_pair(visit_index: usize, progress_display_total: Option<usize>) -> String {
+    match progress_display_total {
+        Some(t) => format!("{}/{}", visit_index, t),
+        None => format!("{}", visit_index),
+    }
+}
+
+const FTS_SQL_AUTOMERGE_8: &str =
+    "INSERT INTO searchabletext(searchabletext, rank) VALUES('automerge', 8)";
+const FTS_SQL_REBUILD: &str = "INSERT INTO searchabletext(searchabletext) VALUES('rebuild')";
+
+pub fn fts_finalize_after_text_indexing(conn: &Connection) -> Result<(), String> {
+    conn.execute(FTS_SQL_AUTOMERGE_8, [])
+        .map_err(|e| format!("FTS automerge(8): {}", e))?;
+    conn.execute(FTS_SQL_REBUILD, [])
+        .map_err(|e| format!("FTS rebuild: {}", e))?;
+    Ok(())
+}
+
+fn fts_remove_document_for_path(
+    tx: &rusqlite::Transaction<'_>,
+    path: &str,
+) -> Result<(), String> {
+    let id_opt: Option<i64> = match tx.query_row(
+        "SELECT id FROM documents WHERE path = ?1",
+        params![path],
+        |r| r.get(0),
+    ) {
+        Ok(id) => Some(id),
+        Err(rusqlite::Error::QueryReturnedNoRows) => None,
+        Err(e) => return Err(format!("documents id lookup: {}", e)),
+    };
+    if let Some(doc_id) = id_opt {
+        tx.execute(
+            "INSERT INTO searchabletext(searchabletext, rowid) VALUES('delete', ?1)",
+            params![doc_id],
+        )
+        .map_err(|e| format!("FTS delete doc {}: {}", doc_id, e))?;
+        tx.execute("DELETE FROM documents WHERE id = ?1", params![doc_id])
+            .map_err(|e| format!("delete documents row: {}", e))?;
+    }
+    Ok(())
+}
+
+struct PreparedFileUpdate {
+    path_db: String,
+    fsize: u64,
+    fmodified: u64,
+    fhash: Vec<u8>,
+    filename: String,
+    visit_index: usize,
+}
+
 /// Process updated files in batch with transaction - files table only (no text extraction)
 pub fn process_batch_updates_files_only(
     conn_mutex: &Arc<Mutex<Connection>>,
-    files_to_update: &[(DirEntry, FileMetadata)],
+    files_to_update: &[(DirEntry, usize)],
     stop_flag: &Arc<Mutex<bool>>,
     status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
     progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>,
-    config: &Config
+    config: &Config,
+    progress_display_total: Option<usize>,
 ) -> Result<(), String> {
     if files_to_update.is_empty() {
         return Ok(());
     }
 
-    let batch_size = config.processing.batch_size;
+    let fts_batch = config.processing.fts_update_batch_size.max(1);
-    let total_files = files_to_update.len();
 
-    // Process files in batches of batch_size
+    for batch in files_to_update.chunks(fts_batch) {
-    for (batch_idx, batch) in files_to_update.chunks(batch_size).enumerate() {
-        // Check stop flag at the start of each batch
         if *stop_flag.lock().unwrap() {
             return Ok(());
         }
 
-        let conn = conn_mutex.lock().unwrap();
+        let mut prepared: Vec<PreparedFileUpdate> = Vec::new();
-        let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?;
 
-        for (i, (entry, _old_metadata)) in batch.iter().enumerate() {
+        for (entry, visit_index) in batch.iter() {
-            let global_index = batch_idx * batch_size + i + 1;
-            // Check stop flag
             if *stop_flag.lock().unwrap() {
-                drop(tx);
-                drop(conn);
                 return Ok(());
             }
 
-            // Update status with current file
+            let filename = entry
+                .path()
+                .file_name()
+                .and_then(|n| n.to_str())
+                .unwrap_or("unknown")
+                .to_string();
+
             if let Some(ref callback) = status_callback {
-                let filename = entry.path().file_name()
+                let pair = format_progress_pair(*visit_index, progress_display_total);
-                    .and_then(|n| n.to_str())
+                callback(&format!("Hashing changed files {}: {}", pair, filename));
-                    .unwrap_or("unknown");
-                callback(&format!("Updating file metadata {}/{}: {}", global_index, total_files, filename));
             }
 
-            // Update progress counter
             if let Some(ref progress_cb) = progress_callback {
-                progress_cb(global_index);
+                progress_cb(*visit_index);
-            }
-
-            let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?;
-            if meta.is_dir() {
-                continue;
             }
 
             let fpath = match entry.path().canonicalize() {
                 Ok(fp) => {
                     let path_str = fp.to_string_lossy().to_string();
-                    // Remove Windows UNC prefix \\?\
                     if path_str.starts_with("\\\\?\\") {
                         std::ffi::OsString::from(&path_str[4..])
                     } else {
                         fp.into_os_string()
                     }
-                },
+                }
                 Err(_) => continue,
             };
 
+            let meta = match std::fs::metadata(&fpath) {
+                Ok(m) if m.is_file() => m,
+                _ => continue,
+            };
+
             let fsize = meta.len();
-            let fmodified = meta.modified()
+            let fmodified = meta
+                .modified()
                 .map_err(|e| format!("Failed to get modified time: {}", e))?
                 .duration_since(UNIX_EPOCH)
                 .map_err(|e| format!("Failed to calculate duration: {}", e))?
                 .as_secs();
 
-            let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length)
+            let fhash = match get_file_hash(fsize, fpath.clone(), config.processing.hash_length) {
-                .map_err(|e| format!("Failed to calculate hash: {}", e))?;
+                Ok(h) => h,
+                Err(e) => {
+                    eprintln!(
+                        "Warning: Skipping file (cannot hash) {}: {}",
+                        fpath.to_string_lossy(),
+                        e
+                    );
+                    continue;
+                }
+            };
 
-            // Check stop flag after hash calculation
+            if *stop_flag.lock().unwrap() {
+                return Ok(());
+            }
+
+            prepared.push(PreparedFileUpdate {
+                path_db: fpath.to_string_lossy().into_owned(),
+                fsize,
+                fmodified,
+                fhash,
+                filename,
+                visit_index: *visit_index,
+            });
+        }
+
+        if prepared.is_empty() {
+            continue;
+        }
+
+        let conn = conn_mutex.lock().unwrap();
+        let tx = conn
+            .unchecked_transaction()
+            .map_err(|e| format!("Failed to begin transaction: {}", e))?;
+
+        for row in &prepared {
             if *stop_flag.lock().unwrap() {
                 drop(tx);
                 drop(conn);
                 return Ok(());
             }
 
-            // Update files table
+            if let Some(ref callback) = status_callback {
+                let pair = format_progress_pair(row.visit_index, progress_display_total);
+                callback(&format!(
+                    "Applying index updates {}: {}",
+                    pair, row.filename
+                ));
+            }
+
             tx.execute(
                 "UPDATE files SET size = ?1, moddate = ?2, hash = ?3 WHERE path = ?4",
-                params![fsize, fmodified, fhash, fpath.to_string_lossy()]
+                params![row.fsize, row.fmodified, row.fhash, row.path_db],
-            ).map_err(|e| format!("Failed to update file record: {}", e))?;
+            )
+            .map_err(|e| format!("Failed to update file record: {}", e))?;
 
-            // Delete old searchable text entry for this file (will be re-added in text indexing phase)
+            fts_remove_document_for_path(&tx, &row.path_db).map_err(|e| {
-            tx.execute(
+                format!(
-                "DELETE FROM searchabletext WHERE path = ?1",
+                    "Failed to remove old document / FTS entry for {}: {}",
-                params![fpath.to_string_lossy()]
+                    row.path_db, e
-            ).map_err(|e| format!("Failed to delete old searchable text: {}", e))?;
+                )
+            })?;
         }
 
-        tx.commit().map_err(|e| format!("Failed to commit transaction: {}", e))?;
+        tx.commit()
+            .map_err(|e| format!("Failed to commit transaction: {}", e))?;
     }
 
     Ok(())
@@ -264,22 +470,21 @@ pub fn process_batch_updates_files_only(
 /// Process new files in batch with transaction - files table only (no text extraction)
 pub fn process_batch_inserts_files_only(
     conn_mutex: &Arc<Mutex<Connection>>,
-    files_to_insert: &[DirEntry],
+    files_to_insert: &[(DirEntry, usize)],
     stop_flag: &Arc<Mutex<bool>>,
     status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
     progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>,
-    config: &Config
+    config: &Config,
+    progress_display_total: Option<usize>,
 ) -> Result<(), String> {
     if files_to_insert.is_empty() {
         return Ok(());
     }
 
     let batch_size = config.processing.batch_size;
-    let total_files = files_to_insert.len();
 
     // Process files in batches of batch_size
-    for (batch_idx, batch) in files_to_insert.chunks(batch_size).enumerate() {
+    for batch in files_to_insert.chunks(batch_size) {
-        // Check stop flag at the start of each batch
         if *stop_flag.lock().unwrap() {
             return Ok(());
         }
@@ -287,9 +492,8 @@ pub fn process_batch_inserts_files_only(
         let conn = conn_mutex.lock().unwrap();
         let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?;
 
-        for (i, entry) in batch.iter().enumerate() {
+        for (entry, visit_index) in batch.iter() {
-            let global_index = batch_idx * batch_size + i + 1;
+            // Check stop flag for early termination
-            // Check stop flag
             if *stop_flag.lock().unwrap() {
                 drop(tx);
                 drop(conn);
@@ -301,17 +505,13 @@ pub fn process_batch_inserts_files_only(
                 let filename = entry.path().file_name()
                     .and_then(|n| n.to_str())
                     .unwrap_or("unknown");
-                callback(&format!("Indexing file metadata {}/{}: {}", global_index, total_files, filename));
+                let pair = format_progress_pair(*visit_index, progress_display_total);
+                callback(&format!("Indexing file metadata {}: {}", pair, filename));
             }
             
             // Update progress counter
             if let Some(ref progress_cb) = progress_callback {
-                progress_cb(global_index);
+                progress_cb(*visit_index);
-            }
-
-            let meta = entry.metadata().map_err(|e| format!("Failed to get metadata: {}", e))?;
-            if meta.is_dir() {
-                continue;
             }
 
             let fpath = match entry.path().canonicalize() {
@@ -327,6 +527,11 @@ pub fn process_batch_inserts_files_only(
                 Err(_) => continue,
             };
 
+            let meta = match std::fs::metadata(&fpath) {
+                Ok(m) if m.is_file() => m,
+                _ => continue,
+            };
+
             let fsize = meta.len();
             let fmodified = meta.modified()
                 .map_err(|e| format!("Failed to get modified time: {}", e))?
@@ -334,8 +539,17 @@ pub fn process_batch_inserts_files_only(
                 .map_err(|e| format!("Failed to calculate duration: {}", e))?
                 .as_secs();
 
-            let fhash = get_file_hash(fsize, fpath.clone(), config.processing.hash_length)
+            let fhash = match get_file_hash(fsize, fpath.clone(), config.processing.hash_length) {
-                .map_err(|e| format!("Failed to calculate hash: {}", e))?;
+                Ok(h) => h,
+                Err(e) => {
+                    eprintln!(
+                        "Warning: Skipping file (cannot hash) {}: {}",
+                        fpath.to_string_lossy(),
+                        e
+                    );
+                    continue;
+                }
+            };
 
             let fname = entry.path().file_name().unwrap().to_os_string();
 
@@ -346,13 +560,69 @@ pub fn process_batch_inserts_files_only(
             ).map_err(|e| format!("Failed to insert file record: {}", e))?;
         }
 
+        // Update status with current file
+        if let Some(ref callback) = status_callback {
+            callback("Committing file updates to database…");
+        }   
+
         tx.commit().map_err(|e| format!("Failed to commit transaction: {}", e))?;
     }
     
     Ok(())
 }
 
-/// Process text indexing for files - adds entries to searchabletext table
+pub fn cleanup_stale_index_entries(
+    conn_mutex: &Arc<Mutex<Connection>>,
+    stale_paths: &[String],
+    stop_flag: &Arc<Mutex<bool>>,
+    status_callback: Option<Box<dyn Fn(&str) + Send + Sync>>,
+) -> Result<usize, String> {
+    if stale_paths.is_empty() {
+        return Ok(0);
+    }
+
+    let conn = conn_mutex.lock().unwrap();
+    let tx = conn
+        .unchecked_transaction()
+        .map_err(|e| format!("Failed to begin stale cleanup transaction: {}", e))?;
+
+    let mut deleted_count = 0usize;
+    for path in stale_paths {
+        if *stop_flag.lock().unwrap() {
+            let _ = tx.commit();
+            drop(conn);
+            return Ok(deleted_count);
+        }
+
+        if let Some(ref callback) = status_callback {
+            callback(&format!("Removing stale index entry: {}", path));
+        }
+
+        fts_remove_document_for_path(&tx, path).map_err(|e| {
+            format!(
+                "Failed to remove stale document / FTS entry for {}: {}",
+                path, e
+            )
+        })?;
+        tx.execute("DELETE FROM files WHERE path = ?1", params![path])
+            .map_err(|e| format!("Failed to delete stale file record {}: {}", path, e))?;
+        deleted_count += 1;
+    }
+
+    tx.commit()
+        .map_err(|e| format!("Failed to commit stale cleanup transaction: {}", e))?;
+
+    if deleted_count > 0 && !*stop_flag.lock().unwrap() {
+        if let Some(ref callback) = status_callback {
+            callback("Rebuilding FTS index after stale cleanup...");
+        }
+        fts_finalize_after_text_indexing(&conn)?;
+    }
+
+    Ok(deleted_count)
+}
+
+/// Process text indexing for files - writes `documents`; FTS rebuilt in `fts_finalize_after_text_indexing`.
 pub fn process_text_indexing(
     conn_mutex: &Arc<Mutex<Connection>>,
     stop_flag: &Arc<Mutex<bool>>,
@@ -360,93 +630,131 @@ pub fn process_text_indexing(
     progress_callback: Option<Box<dyn Fn(usize) + Send + Sync>>,
     config: &Config
 ) -> Result<(), String> {
-    let conn = conn_mutex.lock().unwrap();
+    let max_size = config.processing.maximum_text_file_size;
-    
-    // Get all files from the files table that don't have corresponding searchabletext entries
-    let mut stmt = conn.prepare(
-        "SELECT f.name, f.path, f.size FROM files f 
-         LEFT JOIN searchabletext s ON f.path = s.path 
-         WHERE s.path IS NULL AND f.size <= ?1"
-    ).map_err(|e| format!("Failed to prepare statement: {}", e))?;
-    
-    let file_rows = stmt.query_map([config.processing.maximum_file_size], |row| {
-        Ok((
-            row.get::<_, String>(0)?, // name
-            row.get::<_, String>(1)?, // path
-            row.get::<_, u64>(2)?     // size
-        ))
-    }).map_err(|e| format!("Failed to query files: {}", e))?;
-
-    let files_to_process: Vec<_> = file_rows.collect::<Result<Vec<_>, _>>()
-        .map_err(|e| format!("Failed to collect files: {}", e))?;
-    
-    drop(stmt);
-    drop(conn);
-
-    let total_files = files_to_process.len();
     let batch_size = config.processing.batch_size;
+    let batch_limit = batch_size as i64;
 
-    // Process files in batches
+    if let Some(ref callback) = status_callback {
-    for (batch_idx, batch) in files_to_process.chunks(batch_size).enumerate() {
+        callback("Counting files pending text index…");
-        // Check stop flag at the start of each batch
+    }
+
+    let total_files: usize = {
+        let conn = conn_mutex.lock().unwrap();
+        conn.query_row(
+            "SELECT COUNT(*) FROM files f
+             LEFT JOIN documents d ON f.path = d.path
+             WHERE d.path IS NULL AND f.size <= ?1",
+            [max_size],
+            |row| row.get(0),
+        )
+        .map_err(|e| format!("Failed to count pending text files: {}", e))?
+    };
+
+    let mut cursor_path = String::new();
+    let mut global_index: usize = 0;
+
+    loop {
         if *stop_flag.lock().unwrap() {
             return Ok(());
         }
 
+        let batch: Vec<(String, String, u64)> = {
+            let conn = conn_mutex.lock().unwrap();
+            let mut stmt = conn
+                .prepare(
+                    "SELECT f.name, f.path, f.size FROM files f
+                     LEFT JOIN documents d ON f.path = d.path
+                     WHERE d.path IS NULL AND f.size <= ?1
+                       AND (?2 = '' OR f.path > ?2)
+                     ORDER BY f.path
+                     LIMIT ?3",
+                )
+                .map_err(|e| format!("Failed to prepare text indexing query: {}", e))?;
+            let rows = stmt
+                .query_map(
+                    rusqlite::params![max_size, cursor_path.as_str(), batch_limit],
+                    |row| {
+                        Ok((
+                            row.get::<_, String>(0)?,
+                            row.get::<_, String>(1)?,
+                            row.get::<_, u64>(2)?,
+                        ))
+                    },
+                )
+                .map_err(|e| format!("Failed to query files for text indexing: {}", e))?;
+            rows.collect::<Result<Vec<_>, _>>()
+                .map_err(|e| format!("Failed to read file row: {}", e))?
+        };
+
+        if batch.is_empty() {
+            break;
+        }
+
+        let last_path = batch.last().unwrap().1.clone();
+        cursor_path = last_path;
+
         let conn = conn_mutex.lock().unwrap();
-        let tx = conn.unchecked_transaction().map_err(|e| format!("Failed to begin transaction: {}", e))?;
+        let tx = conn
+            .unchecked_transaction()
+            .map_err(|e| format!("Failed to begin transaction: {}", e))?;
 
-        for (i, (fname, fpath, _fsize)) in batch.iter().enumerate() {
+        for (fname, fpath, _fsize) in batch.iter() {
-            let global_index = batch_idx * batch_size + i + 1;
-            
-            // Check stop flag
             if *stop_flag.lock().unwrap() {
-                // Commit current transaction before stopping
                 let _ = tx.commit();
                 drop(conn);
                 return Ok(());
             }
 
-            // Update status
+            global_index += 1;
+
             if let Some(ref callback) = status_callback {
-                callback(&format!("Extracting text for search indexing {}/{}: {}", global_index, total_files, fname));
+                callback(&format!(
+                    "Extracting text for search indexing {}/{}: {}",
+                    global_index, total_files, fname
+                ));
             }
 
-            // Update progress counter
             if let Some(ref progress_cb) = progress_callback {
                 progress_cb(global_index);
             }
 
-            let path = std::path::Path::new(fpath);
+            let path = std::path::Path::new(fpath.as_str());
             let default_ext = OsString::new();
-            let file_extension = path.extension().unwrap_or(&default_ext)
+            let file_extension = path
-                .to_ascii_lowercase().to_str().unwrap_or("").to_string();
+                .extension()
+                .unwrap_or(&default_ext)
+                .to_ascii_lowercase()
+                .to_str()
+                .unwrap_or("")
+                .to_string();
             let ext_str = file_extension.as_str();
 
-            // Process text content with error handling for individual files
             let text_result = if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) {
                 match read_to_string(fpath) {
                     Ok(file_string) => {
-                        let trimmed_file_string = safe_truncate_string(&file_string, config.processing.maximum_text_size);
+                        let trimmed_file_string =
+                            safe_truncate_string(&file_string, config.processing.maximum_text_size);
                         Some(trimmed_file_string)
                     }
-                    Err(_e) => {
+                    Err(_e) => None,
-                        // eprintln!("Warning: Failed to read plaintext file {}: {}", fpath, e);
-                        None
-                    }
                 }
             } else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) {
                 match extract_document_text(&std::ffi::OsString::from(fpath), ext_str) {
                     Ok(extracted_text) => {
                         if !extracted_text.trim().is_empty() {
-                            let trimmed_file_string = safe_truncate_string(&extracted_text, config.processing.maximum_text_size);
+                            Some(safe_truncate_string(
-                            Some(trimmed_file_string)
+                                &extracted_text,
+                                config.processing.maximum_text_size,
+                            ))
                         } else {
                             None
                         }
                     }
                     Err(e) => {
-                        eprintln!("Warning: Failed to extract text from document {}: {}", fpath, e);
+                        eprintln!(
+                            "Warning: Failed to extract text from document {}: {}",
+                            fpath, e
+                        );
                         None
                     }
                 }
@@ -454,18 +762,23 @@ pub fn process_text_indexing(
                 None
             };
 
-            // Insert the text content if we successfully extracted it
             if let Some(text_content) = text_result {
                 if let Err(e) = tx.execute(
-                    "INSERT INTO searchabletext VALUES (?1, ?2, ?3)",
+                    "INSERT OR REPLACE INTO documents(name, path, text) VALUES (?1, ?2, ?3)",
-                    params![fname, fpath, text_content]
+                    params![fname, fpath, text_content],
                 ) {
-                    eprintln!("Warning: Failed to insert searchable text for {}: {}", fpath, e);
+                    eprintln!("Warning: Failed to insert document row for {}: {}", fpath, e);
                 }
             }
         }
 
-        tx.commit().map_err(|e| format!("Failed to commit transaction: {}", e))?;
+        tx.commit()
+            .map_err(|e| format!("Failed to commit transaction: {}", e))?;
+    }
+
+    if total_files > 0 && !*stop_flag.lock().unwrap() {
+        let conn = conn_mutex.lock().unwrap();
+        fts_finalize_after_text_indexing(&conn)?;
     }
 
     Ok(())

src/frontend.rs

@@ -106,6 +106,22 @@ pub fn App(props: AppProps) -> Element {
                             speed_tracker_clone.set(SpeedTracker::new());
                             "Idle".to_string()
                         },
+                        IndexingStatus::CountingFiles {
+                            current_file,
+                            start_time,
+                            ..
+                        } => {
+                            let elapsed = start_time.elapsed();
+                            let current_file_display = current_file
+                                .as_ref()
+                                .map(|f| format!("{}", f))
+                                .unwrap_or_else(|| "...".to_string());
+                            format!(
+                                "Phase 0 - Counting paths (shell) - {:.1}s elapsed\n{}",
+                                elapsed.as_secs_f64(),
+                                current_file_display
+                            )
+                        }
                         IndexingStatus::RunningFileIndex { files_processed, total_files, current_file, start_time } => {
                             // Add data point to speed tracker
                             speed_tracker_clone.with_mut(|tracker| {

src/indexing.rs

@@ -2,10 +2,21 @@ use std::sync::{Arc, Mutex, mpsc};
 use std::thread;
 use std::time::Instant;
 use std::process::Command;
-use walkdir::WalkDir;
+use std::collections::HashSet;
-use rusqlite::{Connection, params};
+use rusqlite::{Connection, OptionalExtension, params};
+use walkdir::DirEntry;
 
-use crate::file_handling::{load_existing_files, analyze_files_for_batch_update, process_batch_updates_files_only, process_batch_inserts_files_only, process_text_indexing};
+use crate::file_handling::{
+    classify_dir_entry_for_indexing,
+    cleanup_stale_index_entries,
+    count_tree_entries_fast,
+    indexed_walk_file_entries,
+    load_existing_files,
+    process_batch_inserts_files_only,
+    process_batch_updates_files_only,
+    process_text_indexing,
+    FileIndexAction,
+};
 use crate::config::Config;
 
 #[derive(Debug, Clone)]
@@ -22,6 +33,12 @@ pub struct SearchResult {
 #[derive(Debug, Clone)]
 pub enum IndexingStatus {
     Idle,
+    CountingFiles {
+        _entries_scanned: usize,
+        _indexable_files_counted: usize,
+        current_file: Option<String>,
+        start_time: Instant,
+    },
     RunningFileIndex {
         files_processed: usize,
         total_files: Option<usize>,
@@ -288,6 +305,7 @@ impl IndexingService {
     }
 
     /// Clean up UNC prefixes from existing database entries
+    #[allow(dead_code)]
     pub fn clean_unc_prefixes(&self, db_path: &str) -> Result<(), String> {
         let conn = Connection::open(db_path)
             .map_err(|e| format!("Failed to open database: {}", e))?;
@@ -298,16 +316,26 @@ impl IndexingService {
             (),
         ).map_err(|e| format!("Failed to update files table: {}", e))?;
 
-        // Clean UNC prefixes from searchabletext table  
+        let doc_table: i64 = conn
-        conn.execute(
+            .query_row(
-            "UPDATE searchabletext SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'",
+                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='documents'",
-            (),
+                [],
-        ).map_err(|e| format!("Failed to update searchabletext table: {}", e))?;
+                |r| r.get(0),
+            )
+            .unwrap_or(0);
+        if doc_table > 0 {
+            conn.execute(
+                "UPDATE documents SET path = SUBSTR(path, 5) WHERE path LIKE '\\\\?\\%'",
+                (),
+            )
+            .map_err(|e| format!("Failed to update documents table: {}", e))?;
+        }
 
         Ok(())
     }
 
     /// Check if the database is corrupted or malformed
+    #[allow(dead_code)]
     pub fn check_database_health(&self, db_path: &str) -> Result<bool, String> {
         match Connection::open(db_path) {
             Ok(conn) => {
@@ -356,7 +384,10 @@ impl IndexingService {
         while attempts < 50 { // Wait up to 5 seconds
             match self.get_status() {
                 IndexingStatus::Idle => break,
-                IndexingStatus::Stopping | IndexingStatus::RunningFileIndex { .. } | IndexingStatus::RunningTextIndex { .. } => {
+                IndexingStatus::Stopping
+                | IndexingStatus::CountingFiles { .. }
+                | IndexingStatus::RunningFileIndex { .. }
+                | IndexingStatus::RunningTextIndex { .. } => {
                     std::thread::sleep(std::time::Duration::from_millis(100));
                     attempts += 1;
                 }
@@ -384,7 +415,12 @@ impl IndexingService {
         while let Ok(command) = command_rx.recv() {
             match command {
                 IndexingCommand::Start { path, db_path, config } => {
-                    if matches!(*status.lock().unwrap(), IndexingStatus::RunningFileIndex { .. } | IndexingStatus::RunningTextIndex { .. }) {
+                    if matches!(
+                        *status.lock().unwrap(),
+                        IndexingStatus::CountingFiles { .. }
+                            | IndexingStatus::RunningFileIndex { .. }
+                            | IndexingStatus::RunningTextIndex { .. }
+                    ) {
                         continue; // Already running
                     }
 
@@ -394,11 +430,20 @@ impl IndexingService {
                     }
 
                     *stop_flag.lock().unwrap() = false;
-                    *status.lock().unwrap() = IndexingStatus::RunningFileIndex {
+                    *status.lock().unwrap() = if config.processing.precount_files_for_progress {
-                        files_processed: 0,
+                        IndexingStatus::CountingFiles {
-                        total_files: None,
+                            _entries_scanned: 0,
-                        current_file: None,
+                            _indexable_files_counted: 0,
-                        start_time: Instant::now(),
+                            current_file: Some("Preparing database...".to_string()),
+                            start_time: Instant::now(),
+                        }
+                    } else {
+                        IndexingStatus::RunningFileIndex {
+                            files_processed: 0,
+                            total_files: None,
+                            current_file: None,
+                            start_time: Instant::now(),
+                        }
                     };
 
                     // Run indexing in a separate thread
@@ -426,7 +471,12 @@ impl IndexingService {
                     }));
                 }
                 IndexingCommand::Stop => {
-                    if matches!(*status.lock().unwrap(), IndexingStatus::RunningFileIndex { .. } | IndexingStatus::RunningTextIndex { .. }) {
+                    if matches!(
+                        *status.lock().unwrap(),
+                        IndexingStatus::CountingFiles { .. }
+                            | IndexingStatus::RunningFileIndex { .. }
+                            | IndexingStatus::RunningTextIndex { .. }
+                    ) {
                         *status.lock().unwrap() = IndexingStatus::Stopping;
                         *stop_flag.lock().unwrap() = true;
                     }
@@ -440,6 +490,20 @@ impl IndexingService {
         }
     }
 
+    fn file_index_status_callback(
+        status: &Arc<Mutex<IndexingStatus>>,
+    ) -> Box<dyn Fn(&str) + Send + Sync> {
+        let st = status.clone();
+        Box::new(move |file_status: &str| {
+            if let Ok(mut status_guard) = st.lock() {
+                if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard
+                {
+                    *current_file = Some(file_status.to_string());
+                }
+            }
+        })
+    }
+
     fn run_indexing(
         status: &Arc<Mutex<IndexingStatus>>,
         path: &str,
@@ -452,7 +516,7 @@ impl IndexingService {
         let conn = Connection::open(db_path)
             .map_err(|e| format!("Failed to open database: {}", e))?;
 
-        conn.execute_batch(
+        conn.execute_batch( // Default SQLITE page size is 4kB, and our memory cache is in units of page count
             "PRAGMA journal_mode = OFF;
              PRAGMA synchronous = 0;
              PRAGMA cache_size = 10000;
@@ -471,13 +535,36 @@ impl IndexingService {
         )
         .map_err(|e| format!("Failed to create files table: {}", e))?;
 
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS documents (
+                id      INTEGER PRIMARY KEY,
+                name    TEXT,
+                path    TEXT NOT NULL UNIQUE,
+                text    TEXT NOT NULL)",
+            (),
+        )
+        .map_err(|e| format!("Failed to create documents table: {}", e))?;
+
+        let fts_external = Self::searchabletext_is_external_content(&conn)?;
+        if !fts_external {
+            conn.execute("DROP TABLE IF EXISTS searchabletext", ())
+                .map_err(|e| format!("Failed to drop legacy searchabletext: {}", e))?;
+            conn.execute("DROP TABLE IF EXISTS searchabletext_doc", ())
+                .map_err(|e| format!("Failed to drop legacy searchabletext_doc: {}", e))?;
+        }
+
         let create_fts_sql = format!(
-            "CREATE VIRTUAL TABLE IF NOT EXISTS searchabletext USING fts5 (name, path, text, tokenize = '{}');",
+            "CREATE VIRTUAL TABLE IF NOT EXISTS searchabletext USING fts5(name, text, content='documents', content_rowid='id', tokenize='{}');",
             config.processing.tokenize
         );
         conn.execute(&create_fts_sql, ())
             .map_err(|e| format!("Failed to create searchabletext table: {}", e))?;
 
+        if !fts_external {
+            conn.execute("INSERT INTO searchabletext(searchabletext) VALUES('rebuild')", ())
+                .map_err(|e| format!("Failed to rebuild searchabletext: {}", e))?;
+        }
+
         conn.execute(
             "CREATE TABLE IF NOT EXISTS config_validation (
                 key     TEXT PRIMARY KEY,
@@ -503,134 +590,166 @@ impl IndexingService {
             *db_opt = Some(conn_mutex.clone());
         }
 
-        // Collect all file entries
+        let progress_display_total: Option<usize> =
-        let walker = WalkDir::new(path).into_iter();
+            if config.processing.precount_files_for_progress {
-        let entries: Vec<_> = walker
+                if *stop_flag.lock().unwrap() {
-            .filter_map(|entry| entry.ok())
+                    if let Ok(mut status_guard) = status.lock() {
-            .filter(|entry| !entry.metadata().map(|m| m.is_dir()).unwrap_or(true))
+                        *status_guard = IndexingStatus::Idle;
+                    }
+                    return Ok(());
+                }
+                if let Ok(mut g) = status.lock() {
+                    if let IndexingStatus::CountingFiles { ref mut current_file, .. } = *g {
+                        *current_file = Some("Counting paths (shell)...".to_string());
+                    }
+                }
+                let n = count_tree_entries_fast(path).map_err(|e| format!("Precount: {}", e))?;
+                if *stop_flag.lock().unwrap() {
+                    if let Ok(mut status_guard) = status.lock() {
+                        *status_guard = IndexingStatus::Idle;
+                    }
+                    return Ok(());
+                }
+                if let Ok(mut status_guard) = status.lock() {
+                    *status_guard = IndexingStatus::RunningFileIndex {
+                        files_processed: 0,
+                        total_files: Some(n),
+                        current_file: None,
+                        start_time: Instant::now(),
+                    };
+                }
+                Some(n)
+            } else {
+                None
+            };
+
+        let batch_size = config.processing.batch_size;
+        let mut pending_updates: Vec<(DirEntry, usize)> = Vec::new();
+        let mut pending_inserts: Vec<(DirEntry, usize)> = Vec::new();
+        let mut seen_existing_paths: HashSet<String> = HashSet::new();
+        let mut visit: usize = 0;
+        let mut had_incremental_work = false;
+        let flush_updates = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> {
+            if buf.is_empty() {
+                return Ok(());
+            }
+            process_batch_updates_files_only(
+                &conn_mutex,
+                buf.as_slice(),
+                stop_flag,
+                Some(Self::file_index_status_callback(status)),
+                None,
+                config,
+                progress_display_total,
+            )?;
+            buf.clear();
+            Ok(())
+        };
+
+        let flush_inserts = |buf: &mut Vec<(DirEntry, usize)>| -> Result<(), String> {
+                if buf.is_empty() {
+                    return Ok(());
+                }
+                process_batch_inserts_files_only(
+                    &conn_mutex,
+                    buf.as_slice(),
+                    stop_flag,
+                    Some(Self::file_index_status_callback(status)),
+                    None,
+                    config,
+                    progress_display_total,
+                )?;
+                buf.clear();
+                Ok(())
+            };
+
+        for entry in indexed_walk_file_entries(path, config.processing.follow_symlinks) {
+            if *stop_flag.lock().unwrap() {
+                flush_updates(&mut pending_updates)?;
+                flush_inserts(&mut pending_inserts)?;
+                if let Ok(mut status_guard) = status.lock() {
+                    *status_guard = IndexingStatus::Idle;
+                }
+                return Ok(());
+            }
+
+            let action = classify_dir_entry_for_indexing(&entry, &existing_files);
+            let Some(action) = action else {
+                continue;
+            };
+            let current_path = entry
+                .path()
+                .canonicalize()
+                .ok()
+                .map(|fp| {
+                    let path_str = fp.to_string_lossy().to_string();
+                    if path_str.starts_with("\\\\?\\") {
+                        path_str[4..].to_string()
+                    } else {
+                        path_str
+                    }
+                });
+
+            visit += 1;
+            if let Ok(mut g) = status.lock() {
+                if let IndexingStatus::RunningFileIndex {
+                    ref mut files_processed,
+                    ..
+                } = *g
+                {
+                    *files_processed = visit;
+                }
+            }
+
+            match action {
+                FileIndexAction::Skip => {
+                    if let Some(path) = current_path {
+                        seen_existing_paths.insert(path);
+                    }
+                }
+                FileIndexAction::Update => {
+                    if let Some(path) = current_path {
+                        seen_existing_paths.insert(path);
+                    }
+                    had_incremental_work = true;
+                    pending_updates.push((entry, visit));
+                    if pending_updates.len() >= batch_size {
+                        flush_updates(&mut pending_updates)?;
+                    }
+                }
+                FileIndexAction::Insert => {
+                    had_incremental_work = true;
+                    pending_inserts.push((entry, visit));
+                    if pending_inserts.len() >= batch_size {
+                        flush_inserts(&mut pending_inserts)?;
+                    }
+                }
+            }
+        }
+
+        flush_updates(&mut pending_updates)?;
+        flush_inserts(&mut pending_inserts)?;
+
+        let stale_paths: Vec<String> = existing_files
+            .keys()
+            .filter(|p| !seen_existing_paths.contains(*p))
+            .cloned()
             .collect();
-
+        let stale_deleted = cleanup_stale_index_entries(
-        let total_file_count = entries.len();
+            &conn_mutex,
-
+            stale_paths.as_slice(),
-        // Update status with total file count
+            stop_flag,
-        if let Ok(mut status_guard) = status.lock() {
+            Some(Self::file_index_status_callback(status)),
-            if let IndexingStatus::RunningFileIndex { ref mut total_files, .. } = *status_guard {
+        )?;
-                *total_files = Some(total_file_count);
+        if stale_deleted > 0 {
-            }
+            had_incremental_work = true;
         }
 
-        // Analyze which files need updates vs inserts
+        if !had_incremental_work {
-        let batch_update = analyze_files_for_batch_update(&entries, &existing_files);
-        
-        let total_work = batch_update.files_to_update.len() + batch_update.files_to_insert.len();
-        
-        // Update status to show actual work needed
-        if let Ok(mut status_guard) = status.lock() {
-            if let IndexingStatus::RunningFileIndex { ref mut total_files, .. } = *status_guard {
-                *total_files = Some(total_work);
-            }
-        }
-
-        let mut work_completed = 0;
-
-        // Process updated files in batches
-        if !batch_update.files_to_update.is_empty() {
             if let Ok(mut status_guard) = status.lock() {
-                if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard {
+                if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard
-                    *current_file = Some(format!("Updating {} modified files...", batch_update.files_to_update.len()));
+                {
-                }
-            }
-
-            // Create status callback to update current file
-            let status_clone_1 = status.clone();
-            let status_callback = Box::new(move |file_status: &str| {
-                if let Ok(mut status_guard) = status_clone_1.lock() {
-                    if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard {
-                        *current_file = Some(file_status.to_string());
-                    }
-                }
-            });
-            
-            // Create progress callback to update files_processed
-            let status_clone_2 = status.clone();
-            let base_work_completed = work_completed;
-            let progress_callback = Box::new(move |current_index: usize| {
-                if let Ok(mut status_guard) = status_clone_2.lock() {
-                    if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard {
-                        *files_processed = base_work_completed + current_index;
-                    }
-                }
-            });
-
-            if let Err(e) = process_batch_updates_files_only(&conn_mutex, &batch_update.files_to_update, &stop_flag, Some(status_callback), Some(progress_callback), config) {
-                return Err(format!("Failed to process batch updates: {}", e));
-            }
-            
-            work_completed += batch_update.files_to_update.len();
-            
-            if let Ok(mut status_guard) = status.lock() {
-                if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard {
-                    *files_processed = work_completed;
-                }
-            }
-        }
-
-        // Check for stop signal
-        if *stop_flag.lock().unwrap() {
-            if let Ok(mut status_guard) = status.lock() {
-                *status_guard = IndexingStatus::Idle;
-            }
-            return Ok(());
-        }
-
-        // Process new files in batches
-        if !batch_update.files_to_insert.is_empty() {
-            if let Ok(mut status_guard) = status.lock() {
-                if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard {
-                    *current_file = Some(format!("Indexing {} new files...", batch_update.files_to_insert.len()));
-                }
-            }
-
-            // Create status callback for inserts
-            let status_clone_3 = status.clone();
-            let status_callback = Box::new(move |file_status: &str| {
-                if let Ok(mut status_guard) = status_clone_3.lock() {
-                    if let IndexingStatus::RunningFileIndex { ref mut current_file, .. } = *status_guard {
-                        *current_file = Some(file_status.to_string());
-                    }
-                }
-            });
-            
-            // Create progress callback for inserts
-            let status_clone_4 = status.clone();
-            let base_work_completed = work_completed;
-            let progress_callback = Box::new(move |current_index: usize| {
-                if let Ok(mut status_guard) = status_clone_4.lock() {
-                    if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard {
-                        *files_processed = base_work_completed + current_index;
-                    }
-                }
-            });
-
-            if let Err(e) = process_batch_inserts_files_only(&conn_mutex, &batch_update.files_to_insert, &stop_flag, Some(status_callback), Some(progress_callback), config) {
-                return Err(format!("Failed to process batch inserts: {}", e));
-            }
-            
-            work_completed += batch_update.files_to_insert.len();
-            
-            if let Ok(mut status_guard) = status.lock() {
-                if let IndexingStatus::RunningFileIndex { ref mut files_processed, .. } = *status_guard {
-                    *files_processed = work_completed;
-                }
-            }
-        }
-
-        // If no incremental work was needed, show completion status for file indexing phase
-        if total_work == 0 {
-            if let Ok(mut status_guard) = status.lock() {
-                if let IndexingStatus::RunningFileIndex { ref mut current_file, ref mut files_processed, .. } = *status_guard {
                     *current_file = Some("File index is up to date".to_string());
-                    *files_processed = total_file_count;
                 }
             }
         }
@@ -688,6 +807,25 @@ impl IndexingService {
         Ok(())
     }
 
+    fn searchabletext_is_external_content(conn: &Connection) -> Result<bool, String> {
+        let sql: Option<String> = conn
+            .query_row(
+                "SELECT sql FROM sqlite_master WHERE name='searchabletext'",
+                [],
+                |r| r.get(0),
+            )
+            .optional()
+            .map_err(|e| format!("sqlite_master searchabletext: {}", e))?;
+        Ok(sql
+            .as_deref()
+            .map(|s| {
+                s.contains("content='documents'")
+                    || s.contains("content=\"documents\"")
+                    || s.contains("content=documents")
+            })
+            .unwrap_or(false))
+    }
+
     /// Validates configuration against stored values and returns validation results.
     /// Critical configuration changes that require index recreation:
     /// - hash_length: affects file hash computation, invalidates existing file metadata

src/main.rs

@@ -64,7 +64,7 @@ Duplicate files:
 SELECT name, count(*) as cnt, path FROM files GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC;
 
 Full text search:
-SELECT name, path, text, snippet(searchabletext, 2 , "<b>", "</b>", "<b>...</b>", 64) as "snip" FROM searchabletext WHERE text MATCH 'searchstring'
+SELECT d.name, d.path, d.text, snippet(st, 1 , "<b>", "</b>", "<b>...</b>", 64) as "snip" FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE st.text MATCH 'searchstring'
 
 Filename search:
 SELECT name, path FROM files WHERE name LIKE '%searchstring%';

src/search.rs

@@ -20,6 +20,8 @@ impl PartialEq for SearchProps {
 pub fn Search(props: SearchProps) -> Element {
     let mut search_type = use_signal(|| "fulltext".to_string());
     let mut search_term = use_signal(|| String::new());
+    let mut fulltext_exact = use_signal(|| false);
+    let mut fulltext_case_sensitive = use_signal(|| false);
     let search_results = use_signal(|| Vec::<SearchResult>::new());
     let mut search_error = use_signal(|| None::<String>);
     let is_searching = use_signal(|| false);
@@ -35,6 +37,8 @@ pub fn Search(props: SearchProps) -> Element {
         let db_path = db_path.clone();
         let search_type = search_type.clone();
         let search_term = search_term.clone();
+        let fulltext_exact = fulltext_exact.clone();
+        let fulltext_case_sensitive = fulltext_case_sensitive.clone();
         let search_results = search_results.clone();
         let search_error = search_error.clone();
         let is_searching = is_searching.clone();
@@ -46,6 +50,8 @@ pub fn Search(props: SearchProps) -> Element {
             let db_clone = db_path.clone();
             let search_type_val = search_type().clone();
             let search_term_val = search_term().clone();
+            let fulltext_exact_val = fulltext_exact();
+            let fulltext_case_sensitive_val = fulltext_case_sensitive();
             
             let mut search_results_clone = search_results.clone();
             let mut search_error_clone = search_error.clone();
@@ -61,38 +67,81 @@ pub fn Search(props: SearchProps) -> Element {
                 
                 let query = match search_type_val.as_str() {
                     "fulltext" => {
-                        if search_term_val.trim().is_empty() {
+                        let trimmed = search_term_val.trim();
+                        if trimmed.is_empty() {
                             search_error_clone.set(Some("Please enter a search term".to_string()));
                             is_searching_clone.set(false);
                             return;
                         }
 
-                        // Sanitize search term for FTS5 by removing problematic characters
+                        let sanitized_term = trimmed
-                        let sanitized_term = search_term_val
+                            .replace(':', " ")
-                            .replace("'", "''")  // Escape single quotes for SQL
+                            .replace(';', " ")
-                            .replace(":", " ")   // Replace colons with spaces (common in file paths, times, etc.)
+                            .replace('(', " ")
-                            .replace(";", " ")   // Replace semicolons with spaces
+                            .replace(')', " ")
-                            .replace("(", " ")   // Replace parentheses with spaces
+                            .replace('[', " ")
-                            .replace(")", " ")
+                            .replace(']', " ")
-                            .replace("[", " ")   // Replace brackets with spaces
+                            .replace('{', " ")
-                            .replace("]", " ")
+                            .replace('}', " ")
-                            .replace("{", " ")   // Replace braces with spaces
+                            .replace('^', " ")
-                            .replace("}", " ")
+                            .replace('~', " ")
-                            .replace("^", " ")   // Replace carets with spaces
+                            .replace('"', " ");
-                            .replace("~", " ")   // Replace tildes with spaces
-                            .replace("\"", " ");  // Replace quotes with spaces to avoid nesting issues
 
-                        // Split into words and rejoin to handle multiple spaces and create a proper FTS5 query
+                        let tokens: Vec<&str> = sanitized_term.split_whitespace().collect();
-                        let words: Vec<&str> = sanitized_term.split_whitespace().collect();
+                        if tokens.is_empty() {
-                        if words.is_empty() {
                             search_error_clone.set(Some("Please enter a valid search term".to_string()));
                             is_searching_clone.set(false);
                             return;
                         }
 
-                        // Join words with AND for better matching
+                        let words: Vec<&str> = if fulltext_exact_val {
-                        let fts_query = words.join(" AND ");
+                            tokens
-                        format!("SELECT name, path, snippet(searchabletext, 2, '<b>', '</b>', '<b>...</b>', 64) as snippet FROM searchabletext WHERE text MATCH '{}'", fts_query)
+                        } else {
+                            let filtered: Vec<&str> = tokens
+                                .into_iter()
+                                .filter(|w| w.chars().count() >= 3)
+                                .collect();
+                            if filtered.is_empty() {
+                                search_error_clone.set(Some(
+                                    "Trigram index needs each word to be at least 3 characters unless you use exact phrase search.".to_string(),
+                                ));
+                                is_searching_clone.set(false);
+                                return;
+                            }
+                            filtered
+                        };
+
+                        let sql_quote = |s: &str| s.replace('\'', "''");
+
+                        let fts_match = if fulltext_exact_val {
+                            let phrase = words.join(" ");
+                            format!("\"{}\"", phrase.replace('"', "\"\""))
+                        } else {
+                            words.join(" AND ")
+                        };
+
+                        let mut where_clause = format!("st.text MATCH '{}'", sql_quote(&fts_match));
+                        if fulltext_case_sensitive_val {
+                            if fulltext_exact_val {
+                                let literal = words.join(" ");
+                                where_clause.push_str(&format!(
+                                    " AND instr(st.text, '{}') > 0",
+                                    sql_quote(&literal)
+                                ));
+                            } else {
+                                for w in &words {
+                                    where_clause.push_str(&format!(
+                                        " AND instr(st.text, '{}') > 0",
+                                        sql_quote(w)
+                                    ));
+                                }
+                            }
+                        }
+
+                        format!(
+                            "SELECT d.name, d.path, snippet(st, 1, '<b>', '</b>', '<b>...</b>', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank",
+                            where_clause
+                        )
                     },
                     "filename" => {
                         if search_term_val.trim().is_empty() {
@@ -161,6 +210,32 @@ pub fn Search(props: SearchProps) -> Element {
                 }
             }
 
+            if search_type() == "fulltext" {
+                div {
+                    class: "form-group",
+                    style: "display: flex; flex-direction: column; gap: 6px;",
+                    span { style: "font-weight: 600;", "Full text options" }
+                    label {
+                        style: "display: flex; align-items: center; gap: 8px; cursor: pointer;",
+                        input {
+                            r#type: "checkbox",
+                            checked: fulltext_exact(),
+                            onchange: move |evt| fulltext_exact.set(evt.checked()),
+                        }
+                        "Exact phrase match"
+                    }
+                    label {
+                        style: "display: flex; align-items: center; gap: 8px; cursor: pointer;",
+                        input {
+                            r#type: "checkbox",
+                            checked: fulltext_case_sensitive(),
+                            onchange: move |evt| fulltext_case_sensitive.set(evt.checked()),
+                        }
+                        "Case-sensitive match"
+                    }
+                }
+            }
+            
             if search_type() != "duplicates" {
                 div { 
                     class: "form-group",