jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

DB working

Browse source
This commit is contained in:
Jeremy Karst 2024-06-10 13:56:01 -04:00
parent dcfa0f8817
commit 464fc6e15a
1 changed files with 23 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
src/main.rs
View file

@ -12,7 +12,7 @@ use tqdm;
use rusqlite::{params, Connection}; use rusqlite::{params, Connection};
const HASHLEN:usize = 1024*8; const HASHLEN:usize = 1024*1;
// struct Finfo { // struct Finfo {
// name: String, // name: String,
@ -55,47 +55,49 @@ fn get_file_hash(size: u64, path: OsString) -> Result<Vec<u8>, std::io::Error> {
fn main() { fn main() {
let path: &str = "G:\\datasets\\preprocessed_ch_100"; let path: &str = "Y:\\";
let db_path: &str = "GDrive.db"; let db_path: &str = "YDrive.db";
let mut conn = Connection::open(db_path).unwrap(); let conn = Connection::open(db_path).unwrap();
// conn.execute_batch( // PRAGMA cache_size is in number of pages with 1024 byte page size by default
// "PRAGMA journal_mode = OFF; conn.execute_batch(
// PRAGMA synchronous = 0; "PRAGMA journal_mode = OFF;
// PRAGMA cache_size = 1000000; PRAGMA synchronous = 0;
// PRAGMA locking_mode = EXCLUSIVE; PRAGMA cache_size = 1000000;
// PRAGMA temp_store = MEMORY;", PRAGMA locking_mode = EXCLUSIVE;
// ) PRAGMA temp_store = MEMORY;",
// .expect("PRAGMA"); )
.expect("PRAGMA");
conn.execute("CREATE TABLE IF NOT EXISTS files ( conn.execute("CREATE TABLE IF NOT EXISTS files (
name TEXT, name TEXT,
path TEXT, path TEXT,
size INTEGER, size INTEGER,
moddate INTEGER, moddate INTEGER,
hash BLOB)", ()).unwrap(); hash BLOB)", ()).unwrap();
let tx = conn.transaction().unwrap();
let st1 = "INSERT INTO files VALUES (?1,?2,?3,?4,?5)";
let mut stmt = tx.prepare(st1).unwrap();
let mut filecount: i64 = 0; let mut filecount: i64 = 0;
let start_time = Instant::now(); let start_time = Instant::now();
for entry in tqdm::tqdm(WalkDir::new(path).into_iter().filter_map(|e| e.ok())) { for entry in tqdm::tqdm(WalkDir::new(path).into_iter().filter_map(|e| e.ok())) {
let meta = entry.metadata().unwrap(); let meta = entry.metadata().unwrap();
if !meta.is_dir() { if !meta.is_dir() {
let query = "INSERT INTO files VALUES (?1,?2,?3,?4,?5)";
let mut stmt = conn.prepare_cached(query).unwrap();
let fname = entry.path().file_name().unwrap().to_os_string(); let fname = entry.path().file_name().unwrap().to_os_string();
let fpath = entry.path().canonicalize().unwrap().into_os_string(); let fpath = entry.path().canonicalize().unwrap().into_os_string();
let fsize = meta.len(); let fsize = meta.len();
let fmodified = meta.modified().unwrap().duration_since(UNIX_EPOCH).unwrap().as_secs(); let fmodified = meta.modified().unwrap().duration_since(UNIX_EPOCH).unwrap().as_secs();
let fhash = get_file_hash(fsize, fpath.clone()).unwrap(); // let fhash = get_file_hash(fsize, fpath.clone()).unwrap();
let fhash = b"";
filecount += 1; filecount += 1;
stmt.execute(params![fname.to_str(), fpath.to_str(), fsize, fmodified, fhash]).unwrap(); stmt.execute(params![fname.to_str(), fpath.to_str(), fsize, fmodified, fhash]).unwrap();
if filecount >= 100 {
break;
}
} }
} }
let elapsed_time = start_time.elapsed(); let elapsed_time = start_time.elapsed();
println!("{} files enumerated in {} seconds", filecount, elapsed_time.as_secs()); println!("{} files enumerated in {} seconds", filecount, elapsed_time.as_secs());
println!("{} files per second", filecount as f32 / elapsed_time.as_secs() as f32); println!("{} files per second", filecount as f32 / elapsed_time.as_millis() as f32 * 1000.0);
// select name, hash, count(hash) as cnt from files group by hash
// ORDER BY cnt DESC;
} }