//! Shared scaffolding for the integration-test binaries. //! //! Cargo compiles this into each `tests/*.rs` that declares `mod common;`, so //! every binary gets its own copy and each one uses a different subset. #![allow(dead_code)] use std::path::Path; use std::time::{Duration, Instant}; use quicksearch_core::config::Config; use quicksearch_core::db; use quicksearch_core::indexing::{IndexingService, IndexingStatus}; #[allow(unused_imports)] pub use quicksearch_core::testutil::{scratch_dir, scratch_dir_canonical, touch}; /// A scratch database path under a fresh directory. The sidecars SQLite /// creates alongside it (`-wal`, `-shm`) land in the same directory. pub fn scratch_db(tag: &str) -> std::path::PathBuf { scratch_dir(tag).join("index.sqlite") } /// Deterministic pseudo-random word picker — the same LCG and constants /// `benches/corpus`, `examples/indexprobe` and `tests/search_perf` use, for the /// same reason: a fixed seed is what makes two runs comparable, so a number /// that moved is a real change rather than a different corpus. pub struct Lcg(pub u64); impl Lcg { pub fn new(seed: u64) -> Lcg { Lcg(seed) } pub fn next(&mut self) -> u64 { self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1); self.0 >> 33 } } /// The rare term a seeded index is searched for. /// /// Nine bytes — long enough to clear the trigram floor, and exactly /// `3 × (2 + 1)`, so it sits on the boundary where a `fuzzy_max_edits = 2` /// pigeonhole split into three-character chunks becomes legal. pub const NEEDLE: &str = "quartzite"; /// A term planted only in document *bodies*, never in a file name. /// /// The needle above reaches the index through both, so a query for it is /// answered mostly by the filename pass. This one forces the full-text pass to /// do real work: the filename `LIKE` finds nothing, and every candidate the /// trigram index returns has to be decompressed and verified. pub const BODY_TERM: &str = "chalcedony"; /// Filler vocabulary for seeded indexes. /// /// **Deliberately shares no trigram with [`NEEDLE`]** — no filler word contains /// so much as `qua`. That is what makes a needle query genuinely rare, and it /// matters more than it looks: with a vocabulary that merely *resembled* the /// needle, every query would fill the display limit within the first few /// hundred rows, the cascade would break out of pass A, and passes B, C and D /// would never run at all. A harness built that way reports the same figure for /// a literal and a fuzzy search and looks perfectly healthy doing it. pub const WORDS: &[&str] = &[ "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa", "lambda", "brown", "fox", "jumps", "lazy", "index", "search", "cascade", "snippet", "document", "content", "extract", "summary", "meeting", "invoice", "contract", "budget", "revenue", "planning", "review", "draft", "final", "notes", "appendix", "figure", ]; /// What [`seed_index`] should build. pub struct SeedSpec { pub files: usize, /// One file in every `content_every` gets extracted text. A tenth is the /// real shape — most files in a tree are not text — and it keeps the FTS /// index smaller than the table, as it is in practice. pub content_every: usize, /// Words in each stored document body. pub body_words: usize, /// Directories to spread the rows across, so `files.parent` has real /// variety and `idx_files_parent` has interior levels. pub dirs: usize, /// File names carrying [`NEEDLE`]. Kept far below any sane display limit so /// a needle query never fills it — an early-exiting query measures how fast /// the cascade gives up, not how fast it scans. pub needle_names: usize, /// Document bodies carrying [`NEEDLE`], on top of the names. pub needle_docs: usize, /// Document bodies carrying [`BODY_TERM`]. Sized by the caller to stay /// under the display limit, or the pass stops early and measures the /// give-up rather than the verification. pub body_term_docs: usize, } impl Default for SeedSpec { fn default() -> SeedSpec { SeedSpec { files: 50_000, content_every: 10, // ~2 KB of text per document. Not arbitrary: the fuzzy full-text // pass decompresses and scans every stored body, so a corpus of // 400-byte documents makes that pass look free when in production // it is the most expensive thing the cascade does. Still far under // `maximum_text_size` (256 KiB), which is the real worst case. body_words: 300, dirs: 500, needle_names: 50, needle_docs: 50, body_term_docs: 500, } } } /// Seed an index with synthetic rows, in one transaction. /// /// Shared by the measurement harnesses so they all describe the same corpus; /// what varies between them is the size, not the shape. pub fn seed_index(path: &Path, spec: &SeedSpec) { use quicksearch_core::db::repo::{insert_file, set_content_done, NewFile}; use quicksearch_core::mime::FileType; use quicksearch_core::testutil::zstd_of; let mut conn = db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap(); let mut rng = Lcg::new(0x5eed); // Spacing rather than a random draw, so the planted rows are spread across // the table instead of clustering in whatever prefix the scan reaches // first — a cluster at the front would let a pass stop early and report a // fraction of the work a real rare query costs. let name_stride = spec.files / spec.needle_names.max(1); let doc_stride = spec.files / spec.needle_docs.max(1); let body_stride = spec.files / spec.body_term_docs.max(1); let tx = conn.transaction().unwrap(); for i in 0..spec.files { let w1 = WORDS[(rng.next() as usize) % WORDS.len()]; let w2 = WORDS[(rng.next() as usize) % WORDS.len()]; let name = if spec.needle_names > 0 && i % name_stride.max(1) == 0 { format!("{}-{}-{:07}.txt", w1, NEEDLE, i) } else { format!("{}-{}-{:07}.txt", w1, w2, i) }; // Stored parents always end in a separator; see `dir_to_db_parent`. let dir = format!("/seed/{:03}/", i % spec.dirs.max(1)); let id = insert_file( &tx, &NewFile { name: &name, parent: &dir, size: 4096, mtime: 1_700_000_000 + i as u64, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, needs_content: i % spec.content_every.max(1) == 0, }, ) .unwrap() .expect("unique path"); if i % spec.content_every.max(1) == 0 { let mut body: Vec<&str> = (0..spec.body_words) .map(|_| WORDS[(rng.next() as usize) % WORDS.len()]) .collect(); if spec.needle_docs > 0 && i % doc_stride.max(1) == 0 { // Mid-body, so a snippet window has to be cut around it rather // than falling out of a head-of-file window for free. body[spec.body_words / 2] = NEEDLE; } if spec.body_term_docs > 0 && i % body_stride.max(1) == 0 { // Two thirds in, so verifying it means scanning most of the // document rather than stopping at the first few bytes. body[spec.body_words * 2 / 3] = BODY_TERM; } let body = body.join(" "); set_content_done(&tx, id, &body, zstd_of(&body).as_deref()).unwrap(); } } tx.commit().unwrap(); conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok(); } /// How long a single indexing run may take before the test gives up. Generous: /// CI runs these in a container against a cold page cache. const INDEX_TIMEOUT: Duration = Duration::from_secs(120); /// One full indexing run, awaited to completion. /// /// Completion is read from the `last_full_index` marker rather than the status /// enum, because `run_indexing` writes that marker only on a successful finish. /// Polling for `IndexingStatus::Idle` instead would race: a small tree finishes /// between two polls, leaving `Idle` ambiguous between "not started yet" and /// "already done". pub struct IndexOnce<'a> { pub db: &'a Path, pub roots: Vec, pub config: &'a Config, /// Delete any existing completion marker first, so a second run over the /// same index is distinguishable from the first. Off for suites that index /// into a database whose lifecycle they are themselves testing. pub fresh_marker: bool, /// Poll the marker through the keyed open. An encrypted index cannot be /// read by a plain `rusqlite::Connection::open`, so a run against one would /// otherwise never observe its own completion and time out. pub encrypted: bool, } impl IndexOnce<'_> { pub fn run(mut self) { if self.fresh_marker && self.db.exists() { let conn = rusqlite::Connection::open(self.db).unwrap(); conn.execute("DELETE FROM schema_info WHERE key = 'last_full_index'", []) .unwrap(); } let service = IndexingService::new(); service .start_indexing( std::mem::take(&mut self.roots), self.db.to_string_lossy().into_owned(), self.config.clone(), ) .unwrap(); let deadline = Instant::now() + INDEX_TIMEOUT; let mut done = false; while Instant::now() < deadline { if let IndexingStatus::Error(e) = service.get_status() { panic!("indexing failed: {}", e); } if self.db.exists() && self.completed() { done = true; break; } std::thread::sleep(Duration::from_millis(10)); } assert!(done, "indexing did not finish within {:?}", INDEX_TIMEOUT); service.stop_indexing().unwrap(); } /// Whether the completion marker is present. A database mid-creation is /// simply "not yet", not a failure — the poll comes round again. fn completed(&self) -> bool { let conn = if self.encrypted { db::open_existing(&self.db.to_string_lossy(), false).ok() } else { rusqlite::Connection::open(self.db).ok() }; conn.is_some_and(|c| db::repo::get_last_full_index(&c).is_some()) } }