quick_search/crates/quicksearch-core/examples/walkprobe.rs

//! Times and syscall-counts the phase-1 walk, without touching a database.
//!
//! ```text
//! cargo build -p quicksearch-core --example walkprobe --release
//! ./target/release/examples/walkprobe <root> parallel   # the threaded walker
//! ./target/release/examples/walkprobe <root> serial     # one thread, for comparison
//! ```
//!
//! Indexing a network share is bound by round trips, not bandwidth: every
//! metadata operation that misses the client cache costs one, and throughput
//! is round-trips-in-flight divided by latency. So the number that matters is
//! syscalls per file, which this makes directly visible:
//!
//! ```text
//! strace -f -c -e trace=openat,statx,newfstatat,readlink,getdents64,read,lseek,close \
//!     ./target/release/examples/walkprobe <root> parallel
//! ```
//!
//! Expect roughly one `statx` per unchanged file, plus open/read/close for
//! files that are new or modified, and `readlink` only for resolving the roots
//! themselves — a per-file `readlink` count means a `canonicalize` has crept
//! back into the hot path.
//!
//! Those four syscalls also now cover the *whole* cost of a small text file:
//! the head read for the hash is the file's entire contents, so the walk
//! extracts its text there and the content pass never opens it again. That
//! work is CPU, not syscalls, so it shows up in files/sec here and not in the
//! trace. Use [`indexprobe`](indexprobe.rs) to see both phases together.
//!
//! Both modes report files/sec. Run each twice: the first pass warms the page
//! cache (or, on a share, the client's attribute cache), so the second is the
//! one to compare.
use std::collections::HashMap;
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Mutex};
use std::time::{Instant, UNIX_EPOCH};

use quicksearch_core::config::{Config, IgnoreSet};
use quicksearch_core::extract::Registry;
use quicksearch_core::file_handling::{
    classify_for_indexing, filtered_walk, prepare_file_record, ExistingFileEntry, FileIndexAction,
    UnreadableDirs,
};
use quicksearch_core::walk::walk_indexable_files;

fn main() {
    let root = std::env::args().nth(1).unwrap();
    let mode = std::env::args().nth(2).unwrap_or_else(|| "parallel".into());
    let config = Config::default();
    let existing: HashMap<String, ExistingFileEntry> = HashMap::new();

    let start = Instant::now();
    let (seen, prepared) = match mode.as_str() {
        "serial" => serial(&root, &config, &existing),
        _ => parallel(&root, &config, existing),
    };
    let elapsed = start.elapsed();

    eprintln!(
        "{mode}: {seen} files, {prepared} prepared in {:?} ({:.0} files/sec)",
        elapsed,
        seen as f64 / elapsed.as_secs_f64()
    );
}

fn serial(
    root: &str,
    config: &Config,
    existing: &HashMap<String, ExistingFileEntry>,
) -> (usize, usize) {
    let ignore = IgnoreSet::compile(&[]).unwrap();
    let registry = Registry::default_set();
    let (mut seen, mut prepared) = (0, 0);
    for entry in filtered_walk(root, false, false, &ignore, &UnreadableDirs::default()) {
        seen += 1;
        // Same rule as the real walk: a name that is not valid UTF-8 cannot be
        // stored in `files.path` and reopened by it, so it is skipped before
        // anything tries to hash it. Counted as seen, never prepared.
        let Some(path) = entry.path().to_str().map(str::to_owned) else {
            continue;
        };
        let Ok(meta) = std::fs::metadata(entry.path()) else {
            continue;
        };
        let Some(mtime) = meta
            .modified()
            .ok()
            .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
            .map(|d| d.as_secs())
        else {
            continue;
        };
        if classify_for_indexing(&path, mtime, existing) != FileIndexAction::Skip
            && prepare_file_record(&path, &meta, config, &registry).is_some()
        {
            prepared += 1;
        }
    }
    (seen, prepared)
}

fn parallel(
    root: &str,
    config: &Config,
    existing: HashMap<String, ExistingFileEntry>,
) -> (usize, usize) {
    let (mut seen, mut prepared) = (0, 0);
    for file in walk_indexable_files(
        &[root.to_string()],
        false,
        false,
        IgnoreSet::compile(&[]).unwrap(),
        Arc::new(existing),
        config.clone(),
        Arc::new(Registry::default_set()),
        Arc::new(Mutex::new(false)),
        Arc::new(AtomicBool::new(false)),
        4,
    ) {
        seen += 1;
        if file.record.is_some() {
            prepared += 1;
        }
    }
    (seen, prepared)
}