538 lines
18 KiB
Rust
538 lines
18 KiB
Rust
//! End-to-end timing and syscall accounting for a full indexing run.
|
|
//!
|
|
//! [`walkprobe`](walkprobe.rs) covers phase 1 alone; this covers the whole
|
|
//! pipeline — parallel walk, `files` writes, and content extraction.
|
|
//!
|
|
//! ```text
|
|
//! cargo build -p quicksearch-core --example indexprobe --release
|
|
//! ./target/release/examples/indexprobe gen /tmp/qs-bench
|
|
//! ./target/release/examples/indexprobe cold /tmp/qs-bench /tmp/qs-bench.db
|
|
//! ./target/release/examples/indexprobe warm /tmp/qs-bench /tmp/qs-bench.db
|
|
//! ```
|
|
//!
|
|
//! `cold` deletes the database first; `warm` re-runs untouched — the case
|
|
//! that must stay at one `stat` per file. The run modes inspect nothing
|
|
//! themselves, so every syscall a trace attributes to the tree is the indexer's:
|
|
//!
|
|
//! ```text
|
|
//! strace -f -y -o /tmp/t.log \
|
|
//! -e trace=openat,statx,newfstatat,fstat,read,pread64,readlink,close,getdents64,lseek \
|
|
//! ./target/release/examples/indexprobe cold /tmp/qs-bench /tmp/qs-bench.db
|
|
//! grep -oP '^\d+ \K[a-z0-9_]+' <(grep '/tmp/qs-bench/' /tmp/t.log) | sort | uniq -c
|
|
//! ```
|
|
//!
|
|
//! `QSB_HASH_LENGTH` overrides `[processing] hash_length` for a run — how
|
|
//! [`hashprobe`](hashprobe.rs) gets its end-to-end column.
|
|
//!
|
|
//! `QSB_KEY=<64 hex digits>` measures an encrypted index. It is not optional
|
|
//! dressing: without a key installed this probe polls the completion marker
|
|
//! through a plain open, which an encrypted index cannot answer, so the run
|
|
//! reports a three-hour hang instead of its actual time.
|
|
|
|
mod common;
|
|
|
|
use std::alloc::{GlobalAlloc, Layout};
|
|
|
|
// What `Counting` wraps: the allocator the shipped binaries install, or the
|
|
// throughput figures describe a build nobody runs.
|
|
use quicksearch_core::platform::Allocator as Inner;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::atomic::{AtomicU64, Ordering};
|
|
|
|
use common::{evict, mib, Io};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Allocation accounting
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// [`Inner`], counting — per binary, so the shipped `quicksearch` is
|
|
/// untouched. Global atomics, not `search_alloc`'s per-thread `Cell`s: the
|
|
/// work spreads over several pools and nothing else runs here, so a global
|
|
/// count is exactly the run. The contended RMW is fine when both sides of a
|
|
/// comparison carry it; never quote against an uninstrumented build.
|
|
struct Counting;
|
|
|
|
static ALLOCS: AtomicU64 = AtomicU64::new(0);
|
|
static ALLOC_BYTES: AtomicU64 = AtomicU64::new(0);
|
|
static LIVE: AtomicU64 = AtomicU64::new(0);
|
|
static PEAK_LIVE: AtomicU64 = AtomicU64::new(0);
|
|
|
|
#[inline]
|
|
fn note_alloc(size: usize) {
|
|
ALLOCS.fetch_add(1, Ordering::Relaxed);
|
|
ALLOC_BYTES.fetch_add(size as u64, Ordering::Relaxed);
|
|
let live = LIVE.fetch_add(size as u64, Ordering::Relaxed) + size as u64;
|
|
PEAK_LIVE.fetch_max(live, Ordering::Relaxed);
|
|
}
|
|
|
|
unsafe impl GlobalAlloc for Counting {
|
|
unsafe fn alloc(&self, l: Layout) -> *mut u8 {
|
|
let p = unsafe { Inner.alloc(l) };
|
|
if !p.is_null() {
|
|
note_alloc(l.size());
|
|
}
|
|
p
|
|
}
|
|
unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 {
|
|
let p = unsafe { Inner.alloc_zeroed(l) };
|
|
if !p.is_null() {
|
|
note_alloc(l.size());
|
|
}
|
|
p
|
|
}
|
|
unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
|
|
LIVE.fetch_sub(l.size() as u64, Ordering::Relaxed);
|
|
unsafe { Inner.dealloc(p, l) }
|
|
}
|
|
unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
|
|
let q = unsafe { Inner.realloc(p, l, new) };
|
|
if !q.is_null() {
|
|
let (old, new) = (l.size() as u64, new as u64);
|
|
ALLOC_BYTES.fetch_add(new.saturating_sub(old), Ordering::Relaxed);
|
|
let live = if new >= old {
|
|
LIVE.fetch_add(new - old, Ordering::Relaxed) + (new - old)
|
|
} else {
|
|
LIVE.fetch_sub(old - new, Ordering::Relaxed) - (old - new)
|
|
};
|
|
PEAK_LIVE.fetch_max(live, Ordering::Relaxed);
|
|
}
|
|
q
|
|
}
|
|
}
|
|
|
|
#[global_allocator]
|
|
static ALLOCATOR: Counting = Counting;
|
|
|
|
use std::time::{Duration, Instant};
|
|
|
|
use quicksearch_core::config::Config;
|
|
use quicksearch_core::indexing::{IndexingService, IndexingStatus};
|
|
|
|
/// Files whose head the walk reads in full — never reopened.
|
|
const SMALL_TEXT: usize = 800;
|
|
/// Text files past `hash_length`, which extraction must still read.
|
|
const LARGE_TEXT: usize = 100;
|
|
/// Unclaimed by any extractor — a control group whose cost must not move.
|
|
const BINARY: usize = 100;
|
|
|
|
/// Scale the generated tree (`QSB_SCALE`), keeping the mix fixed: the
|
|
/// default thousand files is dominated by fixed start-up, and the difference
|
|
/// between two scales is the only way to separate it from per-file cost.
|
|
fn scale() -> usize {
|
|
std::env::var("QSB_SCALE")
|
|
.ok()
|
|
.and_then(|v| v.parse().ok())
|
|
.filter(|n| *n >= 1)
|
|
.unwrap_or(1)
|
|
}
|
|
|
|
const WORDS: &[&str] = &[
|
|
"alpha",
|
|
"beta",
|
|
"gamma",
|
|
"delta",
|
|
"epsilon",
|
|
"zeta",
|
|
"eta",
|
|
"theta",
|
|
"quick",
|
|
"brown",
|
|
"fox",
|
|
"jumps",
|
|
"over",
|
|
"lazy",
|
|
"dog",
|
|
"indexer",
|
|
"rust",
|
|
"cargo",
|
|
"sqlite",
|
|
"baloo",
|
|
"tokenizer",
|
|
"trigram",
|
|
"snippet",
|
|
"ocean",
|
|
"forest",
|
|
"mountain",
|
|
"river",
|
|
"valley",
|
|
"bridge",
|
|
"tunnel",
|
|
"morning",
|
|
"afternoon",
|
|
"evening",
|
|
"midnight",
|
|
"yesterday",
|
|
"today",
|
|
];
|
|
|
|
/// Deterministic, so two runs index byte-identical trees.
|
|
struct Rng(u64);
|
|
|
|
impl Rng {
|
|
fn next(&mut self) -> u64 {
|
|
self.0 = self
|
|
.0
|
|
.wrapping_mul(6364136223846793005)
|
|
.wrapping_add(1442695040888963407);
|
|
self.0 >> 33
|
|
}
|
|
|
|
fn in_range(&mut self, lo: usize, hi: usize) -> usize {
|
|
lo + (self.next() as usize) % (hi - lo)
|
|
}
|
|
}
|
|
|
|
fn main() {
|
|
let mode = std::env::args().nth(1).unwrap_or_default();
|
|
let tree = PathBuf::from(
|
|
std::env::args()
|
|
.nth(2)
|
|
.expect("usage: indexprobe <gen|evict|cold|warm> <tree> [db]"),
|
|
);
|
|
|
|
match mode.as_str() {
|
|
"gen" => generate(&tree),
|
|
"evict" => {
|
|
let db = std::env::args().nth(3).map(PathBuf::from);
|
|
let (files, bytes) = evict(&tree, db.as_deref());
|
|
eprintln!(
|
|
"evicted {} files ({}) from the page cache",
|
|
files,
|
|
mib(bytes)
|
|
);
|
|
}
|
|
"cold" | "warm" => {
|
|
let db = PathBuf::from(
|
|
std::env::args()
|
|
.nth(3)
|
|
.expect("usage: indexprobe <cold|warm> <tree> <db>"),
|
|
);
|
|
if mode == "cold" {
|
|
for suffix in ["", "-wal", "-shm"] {
|
|
let _ = std::fs::remove_file(format!("{}{}", db.display(), suffix));
|
|
}
|
|
}
|
|
run(&mode, &tree, &db);
|
|
}
|
|
_ => {
|
|
eprintln!("usage: indexprobe <gen|evict|cold|warm> <tree> [db]");
|
|
std::process::exit(2);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn generate(tree: &Path) {
|
|
let _ = std::fs::remove_dir_all(tree);
|
|
std::fs::create_dir_all(tree).expect("create tree");
|
|
|
|
let mut rng = Rng(0x5eed);
|
|
let (mut small_bytes, mut large_bytes, mut bin_bytes) = (0usize, 0usize, 0usize);
|
|
let scale = scale();
|
|
let (small_text, large_text, binary) = (SMALL_TEXT * scale, LARGE_TEXT * scale, BINARY * scale);
|
|
|
|
for i in 0..small_text {
|
|
let dir = tree.join(format!("src/mod{}", i % (40 * scale)));
|
|
std::fs::create_dir_all(&dir).expect("mkdir");
|
|
let ext = ["txt", "md", "rs", "json"][i % 4];
|
|
let size = rng.in_range(200, 8 * 1024);
|
|
let body = prose(&mut rng, size);
|
|
small_bytes += body.len();
|
|
std::fs::write(dir.join(format!("f{}.{}", i, ext)), body).expect("write");
|
|
}
|
|
|
|
for i in 0..large_text {
|
|
let dir = tree.join(format!("docs/set{}", i % (10 * scale)));
|
|
std::fs::create_dir_all(&dir).expect("mkdir");
|
|
let size = rng.in_range(8 * 1024 + 1, 200 * 1024);
|
|
let body = prose(&mut rng, size);
|
|
large_bytes += body.len();
|
|
std::fs::write(dir.join(format!("doc{}.md", i)), body).expect("write");
|
|
}
|
|
|
|
for i in 0..binary {
|
|
let dir = tree.join(format!("assets/set{}", i % (10 * scale)));
|
|
std::fs::create_dir_all(&dir).expect("mkdir");
|
|
let n = rng.in_range(1024, 50 * 1024);
|
|
let blob: Vec<u8> = (0..n).map(|_| (rng.next() & 0xff) as u8).collect();
|
|
bin_bytes += blob.len();
|
|
std::fs::write(dir.join(format!("blob{}.bin", i)), blob).expect("write");
|
|
}
|
|
|
|
let total = small_text + large_text + binary;
|
|
eprintln!("generated {} files under {}", total, tree.display());
|
|
eprintln!(
|
|
" text <= 8 KiB : {:5} files, {:8.1} MiB (head covers the whole file)",
|
|
small_text,
|
|
small_bytes as f64 / (1024.0 * 1024.0)
|
|
);
|
|
eprintln!(
|
|
" text > 8 KiB : {:5} files, {:8.1} MiB (extraction must read it)",
|
|
large_text,
|
|
large_bytes as f64 / (1024.0 * 1024.0)
|
|
);
|
|
eprintln!(
|
|
" binary : {:5} files, {:8.1} MiB (no extractor; control group)",
|
|
binary,
|
|
bin_bytes as f64 / (1024.0 * 1024.0)
|
|
);
|
|
}
|
|
|
|
fn prose(rng: &mut Rng, target: usize) -> String {
|
|
let mut s = String::with_capacity(target + 16);
|
|
while s.len() < target {
|
|
s.push_str(WORDS[rng.next() as usize % WORDS.len()]);
|
|
s.push(if rng.next().is_multiple_of(12) {
|
|
'\n'
|
|
} else {
|
|
' '
|
|
});
|
|
}
|
|
s.truncate(target);
|
|
s
|
|
}
|
|
|
|
/// What the WAL did during a run, sampled from outside the process: growth
|
|
/// between samples is frames appended, a drop is a checkpoint, and the size
|
|
/// it dropped *from* bounds the copy-back. The split matters: more frames
|
|
/// means the load rewrites pages, more copy-back means checkpointing too
|
|
/// often — checkpointing less can be strictly cheaper.
|
|
#[derive(Default, Clone, Copy)]
|
|
struct WalStats {
|
|
peak: u64,
|
|
appended: u64,
|
|
/// Sum of sizes before each truncation — bounds checkpoint copy-back.
|
|
copied_back: u64,
|
|
checkpoints: u64,
|
|
}
|
|
|
|
/// Watch `path` until `stop`, at `SAMPLE` (1 ms): a sampler that misses a
|
|
/// small log's rise and fall reports neither.
|
|
fn sample_wal(
|
|
path: PathBuf,
|
|
stop: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
|
) -> std::thread::JoinHandle<WalStats> {
|
|
const SAMPLE: Duration = Duration::from_millis(1);
|
|
std::thread::spawn(move || {
|
|
let mut stats = WalStats::default();
|
|
let mut last = 0u64;
|
|
while !stop.load(Ordering::Relaxed) {
|
|
let now = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
|
|
if now > last {
|
|
stats.appended += now - last;
|
|
} else if now < last {
|
|
// A shrink is a checkpoint; `last` bounds its copy-back.
|
|
stats.checkpoints += 1;
|
|
stats.copied_back += last;
|
|
}
|
|
stats.peak = stats.peak.max(now);
|
|
last = now;
|
|
std::thread::sleep(SAMPLE);
|
|
}
|
|
stats
|
|
})
|
|
}
|
|
|
|
fn db_sizes(db: &Path) -> (u64, u64) {
|
|
let len = |p: PathBuf| std::fs::metadata(p).map(|m| m.len()).unwrap_or(0);
|
|
(
|
|
len(db.to_path_buf()),
|
|
len(PathBuf::from(format!("{}-wal", db.display()))),
|
|
)
|
|
}
|
|
|
|
/// `[processing] hash_length` for this run, from `QSB_HASH_LENGTH`: charged
|
|
/// to the walk, credited back by the content pass, so the knob has to be
|
|
/// swept end-to-end. Out-of-range values are clamped, with a warning.
|
|
fn hash_length_override() -> Option<usize> {
|
|
std::env::var("QSB_HASH_LENGTH")
|
|
.ok()
|
|
.and_then(|v| v.parse().ok())
|
|
}
|
|
|
|
/// `QSB_KEY=<64 hex digits>` measures an *encrypted* index: the key is
|
|
/// installed process-wide before any connection exists, exactly as the GUI
|
|
/// does after an unlock. Raw hex rather than a password, so no Argon2id
|
|
/// derivation lands inside a timed run.
|
|
fn install_key() -> bool {
|
|
match std::env::var("QSB_KEY") {
|
|
Ok(hex) => {
|
|
let key = quicksearch_core::security::IndexKey::from_hex(hex.trim())
|
|
.expect("QSB_KEY must be 64 hex digits");
|
|
quicksearch_core::db::set_process_key(Some(key));
|
|
true
|
|
}
|
|
Err(_) => false,
|
|
}
|
|
}
|
|
|
|
/// Open the index the way this run's key demands.
|
|
///
|
|
/// **A plain open cannot read an encrypted index**, and
|
|
/// [`get_last_full_index`](quicksearch_core::db::repo::get_last_full_index)
|
|
/// reports that failure as `None` — indistinguishable from "not finished
|
|
/// yet". Polling an encrypted run through a plain open therefore never
|
|
/// observes its own completion and sits here until the deadline, which reads
|
|
/// as an indexing slowdown of several orders of magnitude rather than as the
|
|
/// probe defect it is.
|
|
fn probe_open(db: &Path, keyed: bool) -> Option<rusqlite::Connection> {
|
|
if keyed {
|
|
quicksearch_core::db::open_existing(&db.to_string_lossy(), false).ok()
|
|
} else {
|
|
rusqlite::Connection::open(db).ok()
|
|
}
|
|
}
|
|
|
|
fn run(mode: &str, tree: &Path, db: &Path) {
|
|
let mut config = Config::default();
|
|
if let Some(n) = hash_length_override() {
|
|
config.processing.hash_length = n;
|
|
}
|
|
let hash_length = config.processing.hash_length;
|
|
let keyed = install_key();
|
|
|
|
// The marker is the one unambiguous completion signal; polling the
|
|
// status enum races on a small tree.
|
|
if db.exists() {
|
|
let conn = probe_open(db, keyed).expect("open db");
|
|
conn.execute("DELETE FROM schema_info WHERE key = 'last_full_index'", [])
|
|
.expect("clear marker");
|
|
}
|
|
|
|
// Cleared so the phase summaries below belong to this run alone.
|
|
quicksearch_core::log::clear();
|
|
let io_start = Io::read();
|
|
let (db_before, wal_before) = db_sizes(db);
|
|
|
|
let wal_path = PathBuf::from(format!("{}-wal", db.display()));
|
|
let wal_stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
|
let wal_sampler = sample_wal(wal_path, wal_stop.clone());
|
|
|
|
let service = IndexingService::new();
|
|
let start = Instant::now();
|
|
service
|
|
.start_indexing(
|
|
vec![tree.to_string_lossy().into_owned()],
|
|
db.to_string_lossy().into_owned(),
|
|
config,
|
|
)
|
|
.expect("start indexing");
|
|
|
|
// Generous, like `memprobe`'s: the scale sweep this probe exists for runs
|
|
// hundreds of thousands of files, and a run that times out reports
|
|
// nothing. Past this is a hang, not a slow disk.
|
|
let deadline = Instant::now() + Duration::from_secs(3 * 3600);
|
|
let mut done = false;
|
|
while Instant::now() < deadline {
|
|
if let IndexingStatus::Error(e) = service.get_status() {
|
|
panic!("indexing failed: {}", e);
|
|
}
|
|
if db.exists() {
|
|
if let Some(conn) = probe_open(db, keyed) {
|
|
if quicksearch_core::db::repo::get_last_full_index(&conn).is_some() {
|
|
done = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
std::thread::sleep(Duration::from_millis(5));
|
|
}
|
|
let elapsed = start.elapsed();
|
|
assert!(
|
|
done,
|
|
"indexing did not finish within the timeout (keyed = {})",
|
|
keyed
|
|
);
|
|
// The run's last checkpoint happens inside here, so the sampler outlives it.
|
|
service.stop_indexing().expect("stop");
|
|
wal_stop.store(true, Ordering::Relaxed);
|
|
let wal = wal_sampler.join().unwrap_or_default();
|
|
|
|
// Count what was actually indexed rather than assuming `gen`'s tree —
|
|
// pointing the probe elsewhere made the rate a fiction.
|
|
let total = probe_open(db, keyed)
|
|
.and_then(|c| quicksearch_core::db::repo::row_count(&c).ok())
|
|
.unwrap_or(0);
|
|
|
|
// Read after `stop_indexing`, so the optimize pass's checkpoint is inside the totals.
|
|
let io = Io::read().since(&io_start);
|
|
let (db_after, wal_after) = db_sizes(db);
|
|
let per_file = |n: u64| {
|
|
if total == 0 {
|
|
"-".to_string()
|
|
} else {
|
|
format!("{:.0} B/file", n as f64 / total as f64)
|
|
}
|
|
};
|
|
|
|
eprintln!(
|
|
"\n{}: {:?} ({:.0} files/sec over {} files, hash_length {})",
|
|
mode,
|
|
elapsed,
|
|
total as f64 / elapsed.as_secs_f64(),
|
|
total,
|
|
hash_length,
|
|
);
|
|
|
|
// One log line per root per phase: the walk/extract split without `perf`.
|
|
for line in quicksearch_core::log::snapshot() {
|
|
let m = &line.text;
|
|
if m.contains("walk done")
|
|
|| m.contains("walk ended early")
|
|
|| m.contains("content done")
|
|
|| m.contains("stale cleanup")
|
|
|| m.contains("indexing complete")
|
|
{
|
|
eprintln!(" phase {}", m);
|
|
}
|
|
}
|
|
|
|
let allocs = ALLOCS.load(Ordering::Relaxed);
|
|
eprintln!(
|
|
" wal peak {}, {} appended, {} copied back over {} checkpoint(s)",
|
|
mib(wal.peak),
|
|
mib(wal.appended),
|
|
mib(wal.copied_back),
|
|
wal.checkpoints,
|
|
);
|
|
eprintln!(
|
|
" memory {} allocations ({:.1} per file), {} churned, peak live {}, VmHWM {}",
|
|
allocs,
|
|
allocs as f64 / total.max(1) as f64,
|
|
mib(ALLOC_BYTES.load(Ordering::Relaxed)),
|
|
mib(PEAK_LIVE.load(Ordering::Relaxed)),
|
|
mib(common::vm_hwm().unwrap_or(0)),
|
|
);
|
|
eprintln!(
|
|
" index {} -> {} wal {} -> {}",
|
|
mib(db_before),
|
|
mib(db_after),
|
|
mib(wal_before),
|
|
mib(wal_after),
|
|
);
|
|
eprintln!(
|
|
" syscall {} reads, {} writes ({:.1} reads/file, {:.1} writes/file)",
|
|
io.syscr,
|
|
io.syscw,
|
|
io.syscr as f64 / total.max(1) as f64,
|
|
io.syscw as f64 / total.max(1) as f64,
|
|
);
|
|
eprintln!(
|
|
" bytes rchar {} / wchar {} (through the syscall layer, cache included)",
|
|
mib(io.rchar),
|
|
mib(io.wchar),
|
|
);
|
|
if io.read_bytes == 0 && io.write_bytes == 0 {
|
|
eprintln!(
|
|
" disk not reported for this filesystem (virtiofs/tmpfs); \
|
|
use rchar/wchar and the index sizes above"
|
|
);
|
|
} else {
|
|
eprintln!(
|
|
" disk read {} / written {} (cancelled {}) -> {} written",
|
|
mib(io.read_bytes),
|
|
mib(io.write_bytes),
|
|
mib(io.cancelled),
|
|
per_file(io.write_bytes.saturating_sub(io.cancelled)),
|
|
);
|
|
}
|
|
}
|