Backend seems to be working. Still need test cases and frontend.

Cargo.toml

@@ -4,8 +4,13 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+dioxus = { version = "0.5.1", features = ["desktop"] }
+dioxus-desktop = "0.5.1"
 dpc-pariter = "0.5.1"
 rusqlite = { version = "0.31.0", features = ["bundled"] }
 sha2 = "0.10.8"
 tqdm = "0.7.0"
 walkdir = "2.5.0"
+zip = "0.6"
+quick-xml = "0.31"
+tokio = { version = "1.0", features = ["full"] }

src/document_extraction.rs

@@ -0,0 +1,364 @@
+use std::ffi::OsString;
+use std::fs::File;
+use std::io::{Read, BufReader};
+
+use zip::ZipArchive;
+use quick_xml::Reader;
+use quick_xml::events::Event;
+
+/// Extract text from DOCX files by parsing the word/document.xml
+pub fn extract_text_from_docx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
+    let file = File::open(file_path)?;
+    let mut archive = ZipArchive::new(BufReader::new(file))?;
+    
+    let mut document_xml = archive.by_name("word/document.xml")?;
+    let mut content = String::new();
+    document_xml.read_to_string(&mut content)?;
+    
+    let mut reader = Reader::from_str(&content);
+    reader.trim_text(true);
+    
+    let mut text_content = String::new();
+    let mut buf = Vec::new();
+    let mut in_text = false;
+    
+    loop {
+        match reader.read_event_into(&mut buf) {
+            Ok(Event::Start(ref e)) => {
+                if e.name().as_ref() == b"w:t" {
+                    in_text = true;
+                }
+            }
+            Ok(Event::Text(e)) if in_text => {
+                text_content.push_str(&e.unescape()?.into_owned());
+            }
+            Ok(Event::End(ref e)) => {
+                if e.name().as_ref() == b"w:t" {
+                    in_text = false;
+                } else if e.name().as_ref() == b"w:p" {
+                    text_content.push('\n');
+                }
+            }
+            Ok(Event::Eof) => break,
+            Err(e) => return Err(format!("Error parsing XML: {}", e).into()),
+            _ => {}
+        }
+        buf.clear();
+    }
+    
+    Ok(text_content)
+}
+
+/// Extract text from XLSX files by parsing worksheet XML files
+pub fn extract_text_from_xlsx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
+    let file = File::open(file_path)?;
+    let mut archive = ZipArchive::new(BufReader::new(file))?;
+    
+    let mut text_content = String::new();
+    
+    // First, read shared strings if they exist
+    let mut shared_strings = Vec::new();
+    if let Ok(mut shared_strings_xml) = archive.by_name("xl/sharedStrings.xml") {
+        let mut content = String::new();
+        shared_strings_xml.read_to_string(&mut content)?;
+        
+        let mut reader = Reader::from_str(&content);
+        reader.trim_text(true);
+        let mut buf = Vec::new();
+        let mut in_text = false;
+        
+        loop {
+            match reader.read_event_into(&mut buf) {
+                Ok(Event::Start(ref e)) => {
+                    if e.name().as_ref() == b"t" {
+                        in_text = true;
+                    }
+                }
+                Ok(Event::Text(e)) if in_text => {
+                    shared_strings.push(e.unescape()?.into_owned());
+                }
+                Ok(Event::End(ref e)) => {
+                    if e.name().as_ref() == b"t" {
+                        in_text = false;
+                    }
+                }
+                Ok(Event::Eof) => break,
+                _ => {}
+            }
+            buf.clear();
+        }
+    }
+    
+    // Read worksheets
+    for i in 0..archive.len() {
+        let file_name = archive.by_index(i)?.name().to_string();
+        if file_name.starts_with("xl/worksheets/sheet") && file_name.ends_with(".xml") {
+            let mut sheet_xml = archive.by_index(i)?;
+            let mut content = String::new();
+            sheet_xml.read_to_string(&mut content)?;
+            
+            let mut reader = Reader::from_str(&content);
+            reader.trim_text(true);
+            let mut buf = Vec::new();
+            let mut in_cell = false;
+            let mut cell_type = String::new();
+            
+            loop {
+                match reader.read_event_into(&mut buf) {
+                    Ok(Event::Start(ref e)) => {
+                        if e.name().as_ref() == b"c" {
+                            in_cell = true;
+                            cell_type.clear();
+                            for attr in e.attributes() {
+                                let attr = attr?;
+                                if attr.key.as_ref() == b"t" {
+                                    cell_type = String::from_utf8_lossy(&attr.value).to_string();
+                                }
+                            }
+                        } else if e.name().as_ref() == b"v" && in_cell {
+                            // Value element
+                        }
+                    }
+                    Ok(Event::Text(e)) if in_cell => {
+                        let text = e.unescape()?.into_owned();
+                        if cell_type == "s" {
+                            // Shared string reference
+                            if let Ok(index) = text.parse::<usize>() {
+                                if index < shared_strings.len() {
+                                    text_content.push_str(&shared_strings[index]);
+                                    text_content.push(' ');
+                                }
+                            }
+                        } else {
+                            text_content.push_str(&text);
+                            text_content.push(' ');
+                        }
+                    }
+                    Ok(Event::End(ref e)) => {
+                        if e.name().as_ref() == b"c" {
+                            in_cell = false;
+                        } else if e.name().as_ref() == b"row" {
+                            text_content.push('\n');
+                        }
+                    }
+                    Ok(Event::Eof) => break,
+                    _ => {}
+                }
+                buf.clear();
+            }
+        }
+    }
+    
+    Ok(text_content)
+}
+
+/// Extract text from PPTX files by parsing slide XML files
+pub fn extract_text_from_pptx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
+    let file = File::open(file_path)?;
+    let mut archive = ZipArchive::new(BufReader::new(file))?;
+    
+    let mut text_content = String::new();
+    
+    // Read all slide files
+    for i in 0..archive.len() {
+        let file_name = archive.by_index(i)?.name().to_string();
+        if file_name.starts_with("ppt/slides/slide") && file_name.ends_with(".xml") {
+            let mut slide_xml = archive.by_index(i)?;
+            let mut content = String::new();
+            slide_xml.read_to_string(&mut content)?;
+            
+            let mut reader = Reader::from_str(&content);
+            reader.trim_text(true);
+            let mut buf = Vec::new();
+            let mut in_text = false;
+            
+            loop {
+                match reader.read_event_into(&mut buf) {
+                    Ok(Event::Start(ref e)) => {
+                        if e.name().as_ref() == b"a:t" {
+                            in_text = true;
+                        }
+                    }
+                    Ok(Event::Text(e)) if in_text => {
+                        text_content.push_str(&e.unescape()?.into_owned());
+                    }
+                    Ok(Event::End(ref e)) => {
+                        if e.name().as_ref() == b"a:t" {
+                            in_text = false;
+                        } else if e.name().as_ref() == b"a:p" {
+                            text_content.push('\n');
+                        }
+                    }
+                    Ok(Event::Eof) => break,
+                    _ => {}
+                }
+                buf.clear();
+            }
+            text_content.push_str("\n--- New Slide ---\n");
+        }
+    }
+    
+    Ok(text_content)
+}
+
+/// Extract text from ODT files (OpenDocument Text)
+pub fn extract_text_from_odt(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
+    let file = File::open(file_path)?;
+    let mut archive = ZipArchive::new(BufReader::new(file))?;
+    
+    let mut content_xml = archive.by_name("content.xml")?;
+    let mut content = String::new();
+    content_xml.read_to_string(&mut content)?;
+    
+    let mut reader = Reader::from_str(&content);
+    reader.trim_text(true);
+    
+    let mut text_content = String::new();
+    let mut buf = Vec::new();
+    let mut in_text = false;
+    
+    loop {
+        match reader.read_event_into(&mut buf) {
+            Ok(Event::Start(ref e)) => {
+                let name = e.name();
+                if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" || name.as_ref() == b"text:span" {
+                    in_text = true;
+                }
+            }
+            Ok(Event::Text(e)) if in_text => {
+                text_content.push_str(&e.unescape()?.into_owned());
+            }
+            Ok(Event::End(ref e)) => {
+                let name = e.name();
+                if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" {
+                    text_content.push('\n');
+                    in_text = false;
+                } else if name.as_ref() == b"text:span" {
+                    in_text = false;
+                }
+            }
+            Ok(Event::Eof) => break,
+            _ => {}
+        }
+        buf.clear();
+    }
+    
+    Ok(text_content)
+}
+
+/// Extract text from ODP files (OpenDocument Presentation)
+pub fn extract_text_from_odp(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
+    let file = File::open(file_path)?;
+    let mut archive = ZipArchive::new(BufReader::new(file))?;
+    
+    let mut content_xml = archive.by_name("content.xml")?;
+    let mut content = String::new();
+    content_xml.read_to_string(&mut content)?;
+    
+    let mut reader = Reader::from_str(&content);
+    reader.trim_text(true);
+    
+    let mut text_content = String::new();
+    let mut buf = Vec::new();
+    let mut in_text = false;
+    
+    loop {
+        match reader.read_event_into(&mut buf) {
+            Ok(Event::Start(ref e)) => {
+                let name = e.name();
+                if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" || name.as_ref() == b"text:span" {
+                    in_text = true;
+                }
+            }
+            Ok(Event::Text(e)) if in_text => {
+                text_content.push_str(&e.unescape()?.into_owned());
+            }
+            Ok(Event::End(ref e)) => {
+                let name = e.name();
+                if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" {
+                    text_content.push('\n');
+                    in_text = false;
+                } else if name.as_ref() == b"text:span" {
+                    in_text = false;
+                }
+            }
+            Ok(Event::Eof) => break,
+            _ => {}
+        }
+        buf.clear();
+    }
+    
+    Ok(text_content)
+}
+
+/// Extract text from ODS files (OpenDocument Spreadsheet)
+pub fn extract_text_from_ods(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
+    let file = File::open(file_path)?;
+    let mut archive = ZipArchive::new(BufReader::new(file))?;
+    
+    let mut content_xml = archive.by_name("content.xml")?;
+    let mut content = String::new();
+    content_xml.read_to_string(&mut content)?;
+    
+    let mut reader = Reader::from_str(&content);
+    reader.trim_text(true);
+    
+    let mut text_content = String::new();
+    let mut buf = Vec::new();
+    let mut in_text = false;
+    
+    loop {
+        match reader.read_event_into(&mut buf) {
+            Ok(Event::Start(ref e)) => {
+                let name = e.name();
+                if name.as_ref() == b"text:p" || name.as_ref() == b"text:span" {
+                    in_text = true;
+                }
+            }
+            Ok(Event::Text(e)) if in_text => {
+                text_content.push_str(&e.unescape()?.into_owned());
+                text_content.push(' ');
+            }
+            Ok(Event::End(ref e)) => {
+                let name = e.name();
+                if name.as_ref() == b"text:p" {
+                    text_content.push('\n');
+                    in_text = false;
+                } else if name.as_ref() == b"text:span" {
+                    in_text = false;
+                }
+            }
+            Ok(Event::Eof) => break,
+            _ => {}
+        }
+        buf.clear();
+    }
+    
+    Ok(text_content)
+}
+
+/// Extract text from various document formats
+pub fn extract_document_text(file_path: &OsString, extension: &str) -> Result<String, Box<dyn std::error::Error>> {
+    match extension {
+        "docx" => extract_text_from_docx(file_path),
+        "doc" => {
+            // DOC format is binary and complex to parse without external tools
+            // For now, return an empty result
+            Ok(String::new())
+        }
+        "xlsx" => extract_text_from_xlsx(file_path),
+        "xls" => {
+            // XLS format is binary and complex to parse without external tools
+            Ok(String::new())
+        }
+        "pptx" => extract_text_from_pptx(file_path),
+        "ppt" => {
+            // PPT format is binary and complex to parse without external tools
+            Ok(String::new())
+        }
+        "odt" => extract_text_from_odt(file_path),
+        "odp" => extract_text_from_odp(file_path),
+        "ods" => extract_text_from_ods(file_path),
+        _ => Ok(String::new())
+    }
+}

src/file_handling.rs

@@ -0,0 +1,181 @@
+use std::sync::{Mutex, Arc};
+use std::ffi::OsString;
+use std::fs::{File,read_to_string};
+use std::io::{Read, Seek, SeekFrom};
+use std::time::UNIX_EPOCH;
+
+use sha2::{Sha256, Digest};
+use walkdir::DirEntry;
+use rusqlite::{params, Connection};
+
+use crate::document_extraction::extract_document_text;
+
+
+const HASHLEN:usize = 1024 * 8;
+const MAXIMUM_TEXT_SIZE:usize = 1024 * 512;
+const MAXIMUM_FILE_SIZE:u64 = 1024 * 1024 * 50;
+const PLAINTEXT_EXTENSIONS_LIST: [&'static str; 86] = 
+    ["","txt","rtf","log", // Text Documents
+    "csv", // Spreadsheet
+    "sh","bat","cmd","bash","ps1","psm1","psd1","pssc","psrc", // Scripts
+    "c","cpp","i","cs","csx","caki", // C#
+    "cpp","cc","cxx","c++","hpp","hh","hxx","h","ii", // C++
+    "tex","bib","bbx","cbx", // LaTeX
+    "css","xml","md","json","yaml","yml", // Markup Languages and others
+    "html","htm","shtml","xhtml","xht","mdoc","jsp","asp","aspx","jshtm", // HTML
+    "js","cjs","mjs","es6","es","jsx","ts","tsx", // Javascript and TypeScript
+    "cfg","conf","ini","gitattributes","gitignore", // Config and related files
+    "java","jav", // Java
+    "pl","pm","pod","t","psgi", // Perl
+    "php","php4","php5","phtml","ctp", // PHP
+    "py","rpy","pyw","cpy","gyp","gypi","pyi","ipy","pyt","ipynb", // Python
+    "wasm","wat", // Web Assembly
+    ];
+
+const SUPPORTED_DOCUMENT_EXTENSIONS_LIST: [&'static str; 9] = 
+    ["odt", "docx", "doc", // Office Documents
+    "ppt", "pptx", "odp", // Presentation
+    "xls", "xlsx", "ods"]; // Spreadsheet
+
+/// Get a hash of a file by reading the first and last HASHLEN bytes of the file
+fn get_file_hash(size: u64, path: OsString) -> Result<Vec<u8>, std::io::Error> {
+    let mut hasher = Sha256::new();
+    let mut f: File = File::open(path)?;
+    hasher.update(&size.to_le_bytes());
+    if size > HASHLEN as u64 {
+        let mut file_start_block = [0u8; HASHLEN];
+        f.read_exact(&mut file_start_block)?;
+        hasher.update(file_start_block);
+        f.seek(SeekFrom::End(0 - HASHLEN as i64))?;
+        let mut file_end_block = [0u8; HASHLEN];
+        f.read_exact(&mut file_end_block)?;
+        hasher.update(file_end_block);
+    } else if size > 0 {
+        let mut file_block = Vec::new();
+        f.read_to_end(&mut file_block)?;
+        hasher.update(file_block);
+    }
+    drop(f);
+    Ok(hasher.finalize().to_vec()) 
+}
+
+
+
+pub fn process_entry(conn_mutex: &Arc<Mutex<Connection>>, entry: DirEntry) {
+    let meta = entry.metadata().unwrap();
+    if !meta.is_dir() {
+        // let fpath = entry.path().canonicalize()?.into_os_string();
+        let fpath_result = entry.path().canonicalize();
+        let fpath = match fpath_result {
+            Ok(fp) => fp.into_os_string(),
+            Err(error) => {
+                println!("Error converting fpath: {:?}", error);
+                return;
+            },
+        };
+        
+        // Get basic file properties
+        let fsize = meta.len();
+        let fmodified = meta.modified().unwrap().duration_since(UNIX_EPOCH).unwrap().as_secs();
+
+        // Get file hash
+        let fhash_result = get_file_hash(fsize, fpath.clone());
+        let fhash = match fhash_result {
+            Ok(fh) => fh,
+            Err(error) => {
+                println!("Error digesting hash: {:?}", error);
+                return;
+            },
+        };
+        // let fhash = b"";
+        
+        // Insert results into database
+        let query = "INSERT INTO files VALUES (?1,?2,?3,?4,?5)";
+        let conn = conn_mutex.lock().unwrap();
+        let mut stmt = conn.prepare_cached(query).unwrap();
+        let fname = entry.path().file_name().unwrap().to_os_string();
+
+        let stmt_result = stmt.execute(params![fname.to_str(), fpath.to_string_lossy(), fsize, fmodified, fhash]);
+        match stmt_result {
+            Ok(us) => us,
+            Err(error) => {
+                println!("Error with sqlite transaction: {:?}", error);
+                return;
+            },
+        };
+        std::mem::drop(stmt); // Free the mutex lock so that other threads can access the database
+        std::mem::drop(conn);
+
+        // Generate searchable plain text for file if applicable
+        if fsize <= MAXIMUM_FILE_SIZE {
+            let default_ext = OsString::new();
+            let file_extension = entry.path().extension().unwrap_or(&default_ext).to_ascii_lowercase().to_str().unwrap().to_string();
+            let ext_str = file_extension.as_str();
+            if PLAINTEXT_EXTENSIONS_LIST.contains(&ext_str) {
+                let file_contents_result = read_to_string(fpath.clone());
+                let file_string = match file_contents_result {
+                    Ok(fs) => fs,
+                    Err(_error) => {
+                        // println!("Error reading file to string: {:?}", error);
+                        return;
+                    },
+                };
+
+                let trimmed_file_string;
+                // Trim file contents if too large
+                if file_string.len() > MAXIMUM_TEXT_SIZE {
+                    trimmed_file_string = file_string[..MAXIMUM_TEXT_SIZE].to_string();
+                } else {
+                    trimmed_file_string = file_string;
+                }
+
+                // Insert file contents into database
+                let query2 = "INSERT INTO searchabletext VALUES (?1,?2,?3)";
+                let conn2 = conn_mutex.lock().unwrap();
+                let mut stmt2 = conn2.prepare_cached(query2).unwrap();
+
+                let stmt_result2 = stmt2.execute(params![fname.to_str(), fpath.to_string_lossy(), trimmed_file_string]);
+                match stmt_result2 {
+                    Ok(us) => us,
+                    Err(error) => {
+                        println!("Error with sqlite transaction 2: {:?}", error);
+                        return;
+                    },
+                };
+            }
+            else if SUPPORTED_DOCUMENT_EXTENSIONS_LIST.contains(&ext_str) {
+                // Extract text from office documents
+                match extract_document_text(&fpath, ext_str) {
+                    Ok(extracted_text) => {
+                        if !extracted_text.trim().is_empty() {
+                            let trimmed_file_string;
+                            // Trim file contents if too large
+                            if extracted_text.len() > MAXIMUM_TEXT_SIZE {
+                                trimmed_file_string = extracted_text[..MAXIMUM_TEXT_SIZE].to_string();
+                            } else {
+                                trimmed_file_string = extracted_text;
+                            }
+
+                            // Insert file contents into database
+                            let query2 = "INSERT INTO searchabletext VALUES (?1,?2,?3)";
+                            let conn2 = conn_mutex.lock().unwrap();
+                            let mut stmt2 = conn2.prepare_cached(query2).unwrap();
+
+                            let stmt_result2 = stmt2.execute(params![fname.to_str(), fpath.to_string_lossy(), trimmed_file_string]);
+                            match stmt_result2 {
+                                Ok(_) => {},
+                                Err(error) => {
+                                    println!("Error with sqlite transaction for document: {:?}", error);
+                                },
+                            };
+                        }
+                    }
+                    Err(error) => {
+                        println!("Error extracting text from document {}: {:?}", fpath.to_string_lossy(), error);
+                    }
+                }
+            }
+        }
+
+    }
+}

src/frontend.rs

@@ -0,0 +1,15 @@
+#![allow(non_snake_case)]
+
+use dioxus::prelude::*;
+
+
+// define a component that renders a div with the text "Hello, world!"
+pub fn App() -> Element {
+    let mut count = use_signal(|| 0);
+
+    rsx! {
+        h1 { "High-Five counter: {count}" }
+        button { onclick: move |_| count += 1, "Up high!" }
+        button { onclick: move |_| count -= 1, "Down low!" }
+    }
+}

src/main.rs

@@ -1,86 +1,19 @@
-use std::ffi::OsString;
-use std::fs::File;
-use std::io::{Read, Seek, SeekFrom};
-use std::time::UNIX_EPOCH;
 use std::sync::{Mutex, Arc};
 
-use sha2::{Sha256, Digest};
-use walkdir::{WalkDir, DirEntry};
+use walkdir::WalkDir;
 use tqdm;
-use rusqlite::{params, Connection};
+use rusqlite::Connection;
+use dioxus::prelude::*;
 use dpc_pariter::IteratorExt as _;
 
-const HASHLEN:usize = 1024*8;
-
-
-fn get_file_hash(size: u64, path: OsString) -> Result<Vec<u8>, std::io::Error> {
-    let mut hasher = Sha256::new();
-    let mut f: File = File::open(path)?;
-    hasher.update(&size.to_le_bytes());
-    if size > HASHLEN as u64 {
-        let mut file_start_block = [0u8; HASHLEN];
-        f.read_exact(&mut file_start_block)?;
-        hasher.update(file_start_block);
-        f.seek(SeekFrom::End(0 - HASHLEN as i64))?;
-        let mut file_end_block = [0u8; HASHLEN];
-        f.read_exact(&mut file_end_block)?;
-        hasher.update(file_end_block);
-    } else if size > 0 {
-        let mut file_block = Vec::new();
-        f.read_to_end(&mut file_block)?;
-        hasher.update(file_block);
-    }
-    drop(f);
-    Ok(hasher.finalize().to_vec()) 
-}
-
-fn process_entry(conn_mutex: &Arc<Mutex<Connection>>, entry: DirEntry) {
-    let meta = entry.metadata().unwrap();
-    if !meta.is_dir() {
-        // let fpath = entry.path().canonicalize()?.into_os_string();
-        let fpath_result = entry.path().canonicalize();
-        let fpath = match fpath_result {
-            Ok(fp) => fp.into_os_string(),
-            Err(error) => {
-                println!("Error converting fpath: {:?}", error);
-                return;
-            },
-        };
-        
-        let fsize = meta.len();
-        let fmodified = meta.modified().unwrap().duration_since(UNIX_EPOCH).unwrap().as_secs();
-
-        // let fhash = get_file_hash(fsize, fpath.clone())?;
-        let fhash_result = get_file_hash(fsize, fpath.clone());
-        let fhash = match fhash_result {
-            Ok(fh) => fh,
-            Err(error) => {
-                println!("Error digesting hash: {:?}", error);
-                return;
-            },
-        };
-        // let fhash = b"";
-        
-        let query = "INSERT INTO files VALUES (?1,?2,?3,?4,?5)";
-        let conn = conn_mutex.lock().unwrap();
-        let mut stmt = conn.prepare_cached(query).unwrap();
-        let fname = entry.path().file_name().unwrap().to_os_string();
-
-        // stmt.execute(params![fname.to_str(), fpath.to_str(), fsize, fmodified, fhash])?;
-        let stmt_result = stmt.execute(params![fname.to_str(), fpath.to_str(), fsize, fmodified, fhash]);
-        match stmt_result {
-            Ok(us) => us,
-            Err(error) => {
-                println!("Error with sqlite transaction: {:?}", error);
-                return;
-            },
-        };
-    }
-}
+mod frontend;
+mod file_handling;
+mod document_extraction;
 
 fn main() {
-    let path: &str = "Y:\\";
-    let db_path: &str = "YDrive.db";
+    // launch(frontend::App);
+    let path: &str = "G:\\datasets\\preprocessed_ch_100";
+    let db_path: &str = "GDrive.db";
 
     let conn = Connection::open(db_path).unwrap();
     // let conn = Connection::open_in_memory().unwrap();
@@ -89,22 +22,24 @@ fn main() {
         "PRAGMA journal_mode = OFF;
               PRAGMA synchronous = 0;
               PRAGMA cache_size = 10000;
-              PRAGMA locking_mode = EXCLUSIVE;
               PRAGMA temp_store = MEMORY;",
     )
+    //               PRAGMA locking_mode = EXCLUSIVE;
     .expect("PRAGMA");
     conn.execute("CREATE TABLE IF NOT EXISTS files (
                         name    TEXT,
                         path    TEXT,
                         size    INTEGER,
                         moddate INTEGER,
-                        hash    BLOB)", ()).unwrap();
+                        hash    BLOB);", ()).unwrap();
+    // https://sqlite.org/fts5.html
+    conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS searchabletext USING fts5 (name, path, text, tokenize = 'trigram');", ()).unwrap();
 
     let conn_mutex = Arc::new(Mutex::new(conn));
 
     tqdm::tqdm(WalkDir::new(path).into_iter()).parallel_map(move |entry| {
         match entry {
-            Ok(e) => process_entry(&conn_mutex, e),
+            Ok(e) => file_handling::process_entry(&conn_mutex, e),
             Err(error) => {
                 println!("Error with directory walk: {:?}", error);
                 return;
@@ -113,8 +48,9 @@ fn main() {
     }).for_each(drop);
 
 /*
-select name, hash, count(hash) as cnt from files group by hash
-ORDER BY cnt DESC;
+SELECT name, hash, count(*) as cnt FROM files GROUP BY hash ORDER BY cnt DESC;
+
+SELECT name, path, text, snippet(searchabletext, 2 , "<b>", "</b>", "...", 64) as "snip" FROM searchabletext WHERE text MATCH 'Terrasound' LIMIT 100;
 */
 
 }