jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

Added skipping of hidden files and fixed some bugs.

Browse source
This commit is contained in:
Jeremy Karst 2026-04-21 03:43:28 -04:00
parent 5c039206c6
commit f0edf3963b
6 changed files with 80 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config_example.toml
View file

@ -18,6 +18,8 @@ fts_update_batch_size = 1000
precount_files_for_progress = false
# If true, follow symbolic links during directory walks (indexing only; shell precount unchanged).
follow_symlinks = false
# If true, hidden files and directories will be indexed.
include_hidden = false
# FTS5 tokenization method (e.g., 'trigram', 'porter', 'unicode61')
# Look here for more information https://www.sqlite.org/fts5.html#tokenizers
tokenize = "trigram"

3
src/config.rs
View file

@ -31,6 +31,8 @@ pub struct ProcessingConfig {
pub precount_files_for_progress: bool,
#[serde(default)]
pub follow_symlinks: bool,
#[serde(default)]
pub include_hidden: bool,
}
impl Default for Config {
@ -49,6 +51,7 @@ impl Default for Config {
tokenize: "trigram".to_string(),
precount_files_for_progress: false,
follow_symlinks: false,
include_hidden: false,
},
}
}

19
src/file_handling.rs
View file

@ -2,6 +2,7 @@ use std::sync::{Mutex, Arc};
use std::ffi::OsString;
use std::fs::{File,read_to_string};
use std::io::{Read, Seek, SeekFrom};
use std::path::Component;
use std::process::{Command, Stdio};
use std::time::UNIX_EPOCH;
use std::collections::HashMap;
@ -78,7 +79,10 @@ pub fn load_existing_files(conn: &Connection) -> Result<HashMap<String, Existing
Ok(existing_files)
}
pub fn indexed_walk_file_entries(path: &str, follow_symlinks: bool) -> impl Iterator<Item = DirEntry> {
pub fn indexed_walk_file_entries(
path: &str,
follow_symlinks: bool,
) -> impl Iterator<Item = DirEntry> {
WalkDir::new(path)
.follow_links(follow_symlinks)
.into_iter()
@ -86,6 +90,15 @@ pub fn indexed_walk_file_entries(path: &str, follow_symlinks: bool) -> impl Iter
.filter(|entry| !entry.metadata().map(|m| m.is_dir()).unwrap_or(true))
}
pub fn path_has_hidden_component(path: &std::path::Path) -> bool {
path.components().any(|c| {
matches!(
c,
Component::Normal(name) if name.to_string_lossy().starts_with('.')
)
})
}
fn parse_wc_l_stdout(bytes: &[u8]) -> Result<usize, String> {
let s = String::from_utf8_lossy(bytes);
let token = s
@ -772,6 +785,10 @@ pub fn process_text_indexing(
}
}
if let Some(ref callback) = status_callback {
callback("Rebuilding FTS index after text addition...");
}
tx.commit()
.map_err(|e| format!("Failed to commit transaction: {}", e))?;
}

18
src/frontend.rs
View file

@ -164,7 +164,7 @@ pub fn App(props: AppProps) -> Element {
)
}
}
IndexingStatus::RunningTextIndex { files_processed, total_files, current_file, start_time } => {
IndexingStatus::RunningTextIndex { files_processed, current_file, start_time } => {
// Add data point to speed tracker
speed_tracker_clone.with_mut(|tracker| {
tracker.add_data_point(files_processed);
@ -182,21 +182,6 @@ pub fn App(props: AppProps) -> Element {
.map(|fps| format!(" - {:.1} files/sec", fps))
.unwrap_or_default()
});
if let Some(total) = total_files {
let percentage = if total > 0 {
(files_processed as f64 / total as f64 * 100.0) as u32
} else { 0 };
format!(
"Phase 2 - Text Index: {}/{} files ({}%) - {:.1}s elapsed{}\n{}",
files_processed,
total,
percentage,
elapsed.as_secs_f64(),
speed_display,
current_file_display
)
} else {
format!(
"Phase 2 - Text Index: {} files processed - {:.1}s elapsed{}\n{}",
files_processed,
@ -205,7 +190,6 @@ pub fn App(props: AppProps) -> Element {
current_file_display
)
}
}
IndexingStatus::Stopping => "Indexing Stopped".to_string(),
IndexingStatus::Error(ref e) => format!("Error: {}", e),
};

62
src/indexing.rs
View file

@ -12,6 +12,7 @@ use crate::file_handling::{
count_tree_entries_fast,
indexed_walk_file_entries,
load_existing_files,
path_has_hidden_component,
process_batch_inserts_files_only,
process_batch_updates_files_only,
process_text_indexing,
@ -47,7 +48,6 @@ pub enum IndexingStatus {
},
RunningTextIndex {
files_processed: usize,
total_files: Option<usize>,
current_file: Option<String>,
start_time: Instant,
},
@ -189,6 +189,12 @@ impl IndexingService {
}
})?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);",
(),
)
.map_err(|e| format!("Failed to prepare duplicate-search index: {}", e))?;
let mut stmt = conn.prepare(query)
.map_err(|e| {
let error_msg = e.to_string();
@ -535,6 +541,12 @@ impl IndexingService {
)
.map_err(|e| format!("Failed to create files table: {}", e))?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);",
(),
)
.map_err(|e| format!("Failed to create files hash index: {}", e))?;
conn.execute(
"CREATE TABLE IF NOT EXISTS documents (
id INTEGER PRIMARY KEY,
@ -673,6 +685,21 @@ impl IndexingService {
return Ok(());
}
visit += 1;
if let Ok(mut g) = status.lock() {
if let IndexingStatus::RunningFileIndex {
ref mut files_processed,
..
} = *g
{
*files_processed = visit;
}
}
if !config.processing.include_hidden && path_has_hidden_component(entry.path()) {
continue;
}
let action = classify_dir_entry_for_indexing(&entry, &existing_files);
let Some(action) = action else {
continue;
@ -690,17 +717,6 @@ impl IndexingService {
}
});
visit += 1;
if let Ok(mut g) = status.lock() {
if let IndexingStatus::RunningFileIndex {
ref mut files_processed,
..
} = *g
{
*files_processed = visit;
}
}
match action {
FileIndexAction::Skip => {
if let Some(path) = current_path {
@ -766,7 +782,6 @@ impl IndexingService {
if let Ok(mut status_guard) = status.lock() {
*status_guard = IndexingStatus::RunningTextIndex {
files_processed: 0,
total_files: None,
current_file: Some("Starting text indexing...".to_string()),
start_time: Instant::now(),
};
@ -835,6 +850,7 @@ impl IndexingService {
// Critical configuration values that require index recreation
let hash_length = config.processing.hash_length.to_string();
let tokenize = config.processing.tokenize.clone();
let include_hidden = config.processing.include_hidden.to_string();
let normalized_path = {
let path = std::path::Path::new(indexing_path)
.canonicalize()
@ -853,8 +869,9 @@ impl IndexingService {
let mut stored_hash_length: Option<String> = None;
let mut stored_indexing_path: Option<String> = None;
let mut stored_tokenize: Option<String> = None;
let mut stored_include_hidden: Option<String> = None;
if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path', 'tokenize')") {
if let Ok(mut stmt) = conn.prepare("SELECT key, value FROM config_validation WHERE key IN ('hash_length', 'indexing_path', 'tokenize', 'include_hidden')") {
if let Ok(rows) = stmt.query_map([], |row| {
let key: String = row.get(0)?;
let value: String = row.get(1)?;
@ -865,6 +882,7 @@ impl IndexingService {
"hash_length" => stored_hash_length = Some(row.1),
"indexing_path" => stored_indexing_path = Some(row.1),
"tokenize" => stored_tokenize = Some(row.1),
"include_hidden" => stored_include_hidden = Some(row.1),
_ => {}
}
}
@ -875,8 +893,9 @@ impl IndexingService {
let hash_length_changed = stored_hash_length.as_ref().map_or(false, |stored| stored != &hash_length);
let indexing_path_changed = stored_indexing_path.as_ref().map_or(false, |stored| stored != &normalized_path);
let tokenize_changed = stored_tokenize.as_ref().map_or(false, |stored| stored != &tokenize);
let include_hidden_changed = stored_include_hidden.as_ref().map_or(false, |stored| stored != &include_hidden);
if hash_length_changed || indexing_path_changed || tokenize_changed {
if hash_length_changed || indexing_path_changed || tokenize_changed || include_hidden_changed {
let mut changes = Vec::new();
if hash_length_changed {
changes.push(format!("hash_length: {} -> {}",
@ -890,6 +909,13 @@ impl IndexingService {
changes.push(format!("tokenize: {} -> {}",
stored_tokenize.unwrap_or_else(|| "unknown".to_string()), tokenize));
}
if include_hidden_changed {
changes.push(format!(
"include_hidden: {} -> {}",
stored_include_hidden.unwrap_or_else(|| "unknown".to_string()),
include_hidden
));
}
return Ok(Some(changes));
}
@ -903,6 +929,7 @@ impl IndexingService {
fn update_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result<(), String> {
let hash_length = config.processing.hash_length.to_string();
let tokenize = config.processing.tokenize.clone();
let include_hidden = config.processing.include_hidden.to_string();
let normalized_path = {
let path = std::path::Path::new(indexing_path)
.canonicalize()
@ -933,6 +960,11 @@ impl IndexingService {
params![tokenize],
).map_err(|e| format!("Failed to store tokenize config: {}", e))?;
conn.execute(
"INSERT OR REPLACE INTO config_validation (key, value) VALUES ('include_hidden', ?1)",
params![include_hidden],
).map_err(|e| format!("Failed to store include_hidden config: {}", e))?;
Ok(())
}
}

4
src/search.rs
View file

@ -139,7 +139,7 @@ pub fn Search(props: SearchProps) -> Element {
}
format!(
"SELECT d.name, d.path, snippet(st, 1, '<b>', '</b>', '<b>...</b>', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank",
"SELECT d.name, d.path, snippet(searchabletext, 1, '<b>', '</b>', '<b>...</b>', 64) as snippet FROM searchabletext AS st JOIN documents d ON d.id = st.rowid WHERE {} ORDER BY rank",
where_clause
)
},
@ -151,7 +151,7 @@ pub fn Search(props: SearchProps) -> Element {
}
format!("SELECT name, path FROM files WHERE name LIKE '%{}%'", search_term_val.replace("'", "''"))
},
"duplicates" => "SELECT name, count(*) as cnt, path FROM files GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC".to_string(),
"duplicates" => "SELECT name, count(*) as cnt, path FROM files WHERE hash IS NOT NULL GROUP BY hash HAVING cnt > 1 ORDER BY cnt DESC LIMIT 5000".to_string(),
_ => {
is_searching_clone.set(false);
return;