//! Where the time goes when an added ignore pattern scrubs the index. //! //! Adding one pattern sets `IndexWork { prune_scope: true, reindex: false }` //! (`config::diff_actions`). `scope::advance` then reads every stored row //! under every root, decides each one, and deletes the losers by id. This //! probe attributes that time to a phase. //! //! ```text //! cargo build -p quicksearch-core --example pruneprobe --release //! ./target/release/examples/pruneprobe //! ``` //! //! `QSB_FILES`, `QSB_DIRS`, `QSB_DIR_DEPTH` and `QSB_CONTENT_EVERY` size the //! corpus; `QSB_ARMS=plain|keyed|both` picks the key states (see //! `common::arms`). //! //! The stages are cumulative, so consecutive rows subtract to a phase cost: //! //! ```text //! page read the rows, decide nothing //! +cover ...and run the scope test per row //! +files ...and delete the doomed `files` rows //! +fts(all) ...and tombstone every doomed id //! +fts(done) ...tombstoning only ids that can have an FTS row //! +subtree ...range-deleting a doomed directory instead of paging it //! ``` //! //! The stages decompose the cost; none is the shipped shape. `scope::advance` //! tombstones only the ids that can hold a posting (`+fts(done)`), commits per //! slice, and turns FTS5's delete-merging off for the pass //! (`file_handling::fts_begin_tombstone_burst`), so the `live` row lands under //! every stage. //! //! Measured and not adopted: `+subtree` bought nothing over `+fts(done)` //! (247 ms against 243 on the 40k corpus, with all 8,080 doomed rows skipping //! the page loop), and raising the page cache cut misses twelvefold while //! barely moving the clock. //! //! Read the `commit` column, not `fts`. FTS5 buffers a contentless delete in //! memory and writes the tombstone pages when the transaction flushes, so the //! `DELETE` times as nearly free and the cost lands in the commit. The //! transaction-size table sweeps pages per transaction for the same reason: //! if the flush costs per commit rather than per tombstone, transaction size //! matters and row count does not. //! //! The final table prices the other bulk withdrawal, a completed run's stale //! cleanup (`file_handling::cleanup_stale_index_entries`): the same doomed //! rows, deleted by path. Its stages are probe code, fixed like the stages //! above; its `live` row is the shipped function. mod common; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; use rusqlite::{Connection, OptionalExtension}; use quicksearch_core::config::Config; use quicksearch_core::db::{self, repo}; use quicksearch_core::extract::Registry; use quicksearch_core::file_handling::{ cleanup_stale_index_entries, fts_begin_tombstone_burst, fts_end_tombstone_burst, fts_finalize_after_text_indexing, split_db_path, RootRange, }; use quicksearch_core::scope::{self, Scope, WorkCursor}; use quicksearch_core::testutil::{self, Arm, SeedSpec}; use common::{ chunked, clone_arm, count, env_usize, merge_to_quiescence, open, print_merge, set_cache, Commit, Io, MERGE_PAGES, PAGE, SHIPPED_MIB, }; /// The root every `seed_index` row hangs under. It need not exist on disk: /// nothing in the prune path stats a stored path on Unix /// (`platform::entry_hidden_reason` short-circuits on the dot prefix). const ROOT: &str = "/seed"; /// The ignore patterns the probe can add, each excluding a fifth of the index. /// /// `seed_index` names its second path segment `WORDS[(d * 7 + 13) % 35]`, and /// because 7 and 35 share a factor only these five of the thirty-five words /// are reachable, one per residue of `d mod 5`. So each excludes a fifth of /// the directories, spread across the keyspace rather than gathered at one /// end, which is the shape a real `node_modules` or `.git` pattern has. The /// probe sweeps the excluded fraction by taking a prefix of this list. const PRUNE_PATTERNS: [&str; 5] = ["jumps", "content", "revenue", "figure", "eta"]; /// The single-pattern case, and the one the stage table runs. const PRUNE_PATTERN: &str = PRUNE_PATTERNS[0]; // --------------------------------------------------------------------------- // Stages // --------------------------------------------------------------------------- /// Cumulative slices of the prune. Each does everything the one before it does. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] enum Stage { Page, Cover, Files, FtsAll, FtsDone, Subtree, } impl Stage { const ALL: [Stage; 6] = [ Stage::Page, Stage::Cover, Stage::Files, Stage::FtsAll, Stage::FtsDone, Stage::Subtree, ]; fn label(self) -> &'static str { match self { Stage::Page => "page", Stage::Cover => "+cover", Stage::Files => "+files", Stage::FtsAll => "+fts(all)", Stage::FtsDone => "+fts(done)", Stage::Subtree => "+subtree", } } fn tag(self) -> &'static str { match self { Stage::Page => "page", Stage::Cover => "cover", Stage::Files => "files", Stage::FtsAll => "fts-all", Stage::FtsDone => "fts-done", Stage::Subtree => "subtree", } } /// Whether the doomed rows leave `files`. fn deletes_rows(self) -> bool { self >= Stage::Files } /// Whether the doomed ids are tombstoned in FTS. fn deletes_fts(self) -> bool { self >= Stage::FtsAll } /// Whether the FTS delete is narrowed to ids that can hold a posting. fn fts_filtered(self) -> bool { self >= Stage::FtsDone } } /// What one stage cost, split by phase. #[derive(Default)] struct Timing { page: Duration, cover: Duration, files: Duration, fts: Duration, commit: Duration, total: Duration, examined: usize, deleted: usize, /// Rows skipped by a range delete rather than paged through. skipped: usize, io: Io, misses: i64, } // --------------------------------------------------------------------------- // The measured loop // --------------------------------------------------------------------------- /// Whether the walker would still descend into the stored parent `parent`. /// /// `Scope::covers` judges a path component by component, which for a /// directory asks what the walker asks before recursing. The trailing /// separator every stored parent carries goes first, or the leaf component /// would be empty. fn dir_covered(scope: &Scope, root: &Path, parent: &str) -> bool { let trimmed = parent.trim_end_matches(['/', '\\']); scope.covers(root, Path::new(trimmed)) } /// Page the index as `scope::advance` does, doing `stage`'s share of the work /// and timing each phase apart. /// /// The deletes are spelled out here rather than routed through `repo`, so the /// filtered and unfiltered FTS variants compare in one run and the stages stay /// fixed when `repo` changes. fn run_stage(conn: &Connection, config: &Config, stage: Stage, commit: Commit) -> Timing { let scope = Scope::from_config(config).expect("compile the scope"); let root = PathBuf::from(ROOT); let range = RootRange::for_root(ROOT); let mut t = Timing::default(); let io_before = Io::read(); let (_, misses_before) = testutil::cache_stats(conn); let started = Instant::now(); let mut after = (range.lo.clone(), String::new()); let mut covered = scope::CoverCache::default(); // `unchecked_transaction` borrows shared, so one transaction can end and // the next begin inside the loop, as in `testutil::seed_index`. let mut tx = conn.unchecked_transaction().expect("begin"); let mut pages_open = 0usize; let mut tx_since = Instant::now(); loop { let at = Instant::now(); let rows = repo::rows_in_range_page(&tx, &after.0, &after.1, &range.hi, PAGE).expect("read a page"); t.page += at.elapsed(); let Some(last) = rows.last() else { break }; after = (last.parent.clone(), last.name.clone()); pages_open += 1; if stage == Stage::Page { t.examined += rows.len(); continue; } // The subtree short-circuit. A page spans several directories, so each // distinct parent is tested as it appears; the first doomed one is // range-deleted and the scan re-seeks past it. Testing only `rows[0]` // would fire only on pages that begin inside an excluded directory. let mut rows: &[repo::ScopeRow] = &rows; if stage == Stage::Subtree { let at = Instant::now(); let mut doomed_at = None; let mut last_dir: Option<(&str, bool)> = None; for (i, row) in rows.iter().enumerate() { let ok = match last_dir { Some((dir, ok)) if dir == row.parent => ok, _ => { let ok = dir_covered(&scope, &root, &row.parent); last_dir = Some((&row.parent, ok)); ok } }; if !ok { doomed_at = Some(i); break; } } t.cover += at.elapsed(); if let Some(i) = doomed_at { let dir = rows[i].parent.trim_end_matches(['/', '\\']).to_string(); let sub = RootRange::for_root(&dir); let (removed, fts, files) = delete_range(&tx, &sub.lo, &sub.hi); t.fts += fts; t.files += files; t.deleted += removed; t.skipped += removed; // The rows before the doomed directory still need deciding; // everything from it on is gone or re-read after the seek. // Both happen inside the open transaction, so the seek sees // the range delete without a commit. after = (sub.hi.clone(), String::new()); rows = &rows[..i]; } } let at = Instant::now(); let mut doomed: Vec = Vec::new(); let mut doomed_fts: Vec = Vec::new(); for row in rows { if scope.covers_cached(&root, Path::new(&row.path), &mut covered) { continue; } doomed.push(row.id); if row.content_state == repo::STATE_DONE { doomed_fts.push(row.id); } } t.cover += at.elapsed(); t.examined += rows.len(); if stage.deletes_rows() && !doomed.is_empty() { let fts_ids: &[i64] = if stage.fts_filtered() { &doomed_fts } else { &doomed }; if stage.deletes_fts() { let at = Instant::now(); chunked( &tx, "DELETE FROM searchabletext WHERE rowid IN", fts_ids, false, ); t.fts += at.elapsed(); } let at = Instant::now(); t.deleted += chunked(&tx, "DELETE FROM files WHERE id IN", &doomed, false); t.files += at.elapsed(); } if commit.due(pages_open, tx_since) { let at = Instant::now(); tx.commit().expect("commit"); t.commit += at.elapsed(); tx = conn.unchecked_transaction().expect("begin"); pages_open = 0; tx_since = Instant::now(); } } let at = Instant::now(); tx.commit().expect("commit"); t.commit += at.elapsed(); t.total = started.elapsed(); t.io = Io::read().since(&io_before); let (_, misses_after) = testutil::cache_stats(conn); t.misses = misses_after - misses_before; t } /// Delete a whole parent range, tombstoning only the ids that can hold a /// posting. Returns `(rows, fts time, files time)`. fn delete_range(tx: &Connection, lo: &str, hi: &str) -> (usize, Duration, Duration) { let at = Instant::now(); tx.execute( "DELETE FROM searchabletext WHERE rowid IN \ (SELECT id FROM files WHERE parent >= ?1 AND parent < ?2 AND content_state = ?3)", rusqlite::params![lo, hi, repo::STATE_DONE], ) .expect("tombstone range"); let fts = at.elapsed(); let at = Instant::now(); let removed = tx .execute( "DELETE FROM files WHERE parent >= ?1 AND parent < ?2", rusqlite::params![lo, hi], ) .expect("delete range"); (removed, fts, at.elapsed()) } // --------------------------------------------------------------------------- // Stale cleanup stages // --------------------------------------------------------------------------- /// Cumulative slices of a completed run's stale cleanup, which deletes by /// *path* rather than deciding rows from a page it already read. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] enum StaleShape { /// Per-path deletes, three statements each, with FTS handed every id. The /// control row: probe code, so it must not move when /// `cleanup_stale_index_entries` changes. PerRow, /// ...resolve `(id, content_state)` per path instead, then chunked /// `IN (...)` deletes with FTS narrowed to ids that can hold a posting. Ids, /// ...and hold FTS5's delete-merging off for the whole pass. Burst, } impl StaleShape { const ALL: [StaleShape; 3] = [StaleShape::PerRow, StaleShape::Ids, StaleShape::Burst]; fn label(self) -> &'static str { match self { StaleShape::PerRow => "per-row", StaleShape::Ids => "+ids", StaleShape::Burst => "+burst", } } fn tag(self) -> &'static str { match self { StaleShape::PerRow => "row", StaleShape::Ids => "ids", StaleShape::Burst => "burst", } } } /// Delete `paths` the way stale cleanup does, one transaction per `PAGE` /// chunk, timing each phase. Column mapping in the shared table: `cover` is /// the id resolution, `files` the `files` deletes, `fts` the tombstones plus /// the trailing consolidation. fn run_stale(conn: &Connection, paths: &[String], shape: StaleShape) -> Timing { let mut t = Timing::default(); let io_before = Io::read(); let (_, misses_before) = testutil::cache_stats(conn); let started = Instant::now(); if shape == StaleShape::Burst { fts_begin_tombstone_burst(conn); } for page in paths.chunks(PAGE as usize) { let tx = conn.unchecked_transaction().expect("begin"); if shape == StaleShape::PerRow { for path in page { let Some((parent, name)) = split_db_path(path) else { continue; }; let at = Instant::now(); let id: Option = tx .prepare_cached( "DELETE FROM files WHERE parent = ?1 AND name = ?2 RETURNING id", ) .expect("prepare") .query_row(rusqlite::params![parent, name], |r| r.get(0)) .optional() .expect("delete row"); t.files += at.elapsed(); let Some(id) = id else { continue }; t.deleted += 1; let at = Instant::now(); tx.prepare_cached("DELETE FROM searchabletext WHERE rowid = ?1") .expect("prepare") .execute([id]) .expect("tombstone"); t.fts += at.elapsed(); let at = Instant::now(); tx.prepare_cached("DELETE FROM documents_text WHERE file_id = ?1") .expect("prepare") .execute([id]) .expect("clear body"); t.files += at.elapsed(); } } else { let at = Instant::now(); let mut ids: Vec = Vec::new(); let mut with_postings: Vec = Vec::new(); { let mut sel = tx .prepare_cached( "SELECT id, content_state FROM files WHERE parent = ?1 AND name = ?2", ) .expect("prepare"); for path in page { let Some((parent, name)) = split_db_path(path) else { continue; }; let row: Option<(i64, i64)> = sel .query_row(rusqlite::params![parent, name], |r| { Ok((r.get(0)?, r.get(1)?)) }) .optional() .expect("resolve"); if let Some((id, state)) = row { ids.push(id); if state == repo::STATE_DONE { with_postings.push(id); } } } } t.cover += at.elapsed(); let at = Instant::now(); chunked( &tx, "DELETE FROM searchabletext WHERE rowid IN", &with_postings, false, ); t.fts += at.elapsed(); let at = Instant::now(); t.deleted += chunked(&tx, "DELETE FROM files WHERE id IN", &ids, false); t.files += at.elapsed(); } let at = Instant::now(); tx.commit().expect("commit"); t.commit += at.elapsed(); } let at = Instant::now(); if shape == StaleShape::Burst { fts_end_tombstone_burst(conn); } else { fts_finalize_after_text_indexing(conn); } t.fts += at.elapsed(); t.total = started.elapsed(); t.io = Io::read().since(&io_before); let (_, misses_after) = testutil::cache_stats(conn); t.misses = misses_after - misses_before; t } // --------------------------------------------------------------------------- // Corpus and arms // --------------------------------------------------------------------------- fn spec() -> SeedSpec { SeedSpec { files: env_usize("QSB_FILES", 200_000), dirs: env_usize("QSB_DIRS", 2_000), // Coprime with 5. `seed_index` places content every `content_every` // files and directories every `dirs`, and [`PRUNE_PATTERN`] excludes // the directories where `d mod 5 == 0`. The default of 10 shares the // factor 5, so every extracted document would land in an excluded // directory and the prune would delete all the postings. 7 puts a // fifth of the documents in the doomed set. `main` asserts the // fraction. content_every: env_usize("QSB_CONTENT_EVERY", 7), // At least two segments, so the second can carry the excluded name; // see `SeedSpec::dir_depth` for how depth sets row width. dir_depth: env_usize("QSB_DIR_DEPTH", 6).max(2), // A real run commits in slices, each flushing FTS5's hash to its own // segment; one transaction would leave a single segment and // understate every tombstone cost. commit_every: 500, ..SeedSpec::default() } } fn config_with(patterns: &[&str]) -> Config { let mut config = Config::default(); config.paths.indexing_paths = vec![ROOT.to_string()]; config.indexing.ignore_patterns = patterns.iter().map(|p| p.to_string()).collect(); config.processing.batch_size = PAGE as usize; config } /// Open `arm` on a run's own writer profile, `PRAGMAS_FAST`, which stale /// cleanup runs on. fn open_run_writer(arm: &Arm) -> Connection { arm.with_key(|| { db::open::open_existing(&arm.path.to_string_lossy(), true).expect("open the copy") }) } // --------------------------------------------------------------------------- fn main() { let spec = spec(); println!( "corpus: {} files across {} dirs, 1 in {} with content, pattern {:?}", spec.files, spec.dirs, spec.content_every, PRUNE_PATTERN ); for keyed in common::arms() { let label = if keyed { "keyed" } else { "plain" }; let master = Arm::seed(label, &format!("prune-{}", label), keyed, &spec); println!( "\n=== {} === seeded in {:.1}s, {}", label, master.seeded_in.as_secs_f64(), common::mib(master.size_bytes()) ); // Census: what the pattern takes, read off the index. { let conn = open(&master); let total = count(&conn, "SELECT COUNT(*) FROM files"); let doomed = count( &conn, &format!( "SELECT COUNT(*) FROM files WHERE parent LIKE '%/{}/%'", PRUNE_PATTERN ), ); let doomed_fts = count( &conn, &format!( "SELECT COUNT(*) FROM files WHERE parent LIKE '%/{}/%' AND content_state = 1", PRUNE_PATTERN ), ); let fts = count(&conn, "SELECT COUNT(*) FROM searchabletext"); let done = count(&conn, "SELECT COUNT(*) FROM files WHERE content_state = 1"); assert!( doomed > 0, "the pattern excludes nothing — seed_index's directory naming moved" ); assert_eq!(fts, done, "a posting exists exactly for content_state = 1"); assert!( doomed_fts * 2 < done, "the prune takes {} of {} postings — the content and directory \ strides have collided again; see `spec`'s content_every", doomed_fts, done ); println!( " {} rows, {} doomed ({:.0}%), of which {} hold a posting ({:.0}%) — \ the rest hold nothing to tombstone", total, doomed, 100.0 * doomed as f64 / total as f64, doomed_fts, 100.0 * doomed_fts as f64 / doomed as f64, ); println!( " plan: {}", conn.query_row( "EXPLAIN QUERY PLAN DELETE FROM searchabletext WHERE rowid IN (1,2,3)", [], |r| r.get::<_, String>(3), ) .unwrap_or_else(|e| format!("unavailable: {}", e)) ); common::print_fts_census(&conn); } let config = config_with(&[PRUNE_PATTERN]); let header = || { println!( "\n {:<11} {:>7} {:>7} {:>7} {:>7} {:>8} {:>8} {:>9} {:>8}", "stage", "page", "cover", "files", "fts", "commit", "total", "misses", "deleted" ); }; let row = |name: &str, t: &Timing| { let ms = |d: Duration| d.as_secs_f64() * 1000.0; println!( " {:<11} {:>7.0} {:>7.0} {:>7.0} {:>7.0} {:>8.0} {:>8.0} {:>9} {:>8}", name, ms(t.page), ms(t.cover), ms(t.files), ms(t.fts), ms(t.commit), ms(t.total), t.misses, t.deleted, ); }; header(); for stage in Stage::ALL { let arm = clone_arm(&master, &format!("prune-{}-{}", label, stage.tag())); let conn = open(&arm); let t = run_stage(&conn, &config, stage, Commit::PerPage); row(stage.label(), &t); if stage == Stage::Subtree { println!( " {:<11} {} of those rows never reached a page", "", t.skipped ); } // Consolidation, priced apart: merge until `%_data` stops changing, // as `fts_end_tombstone_burst` ends a burst. if stage.deletes_fts() { print_merge(&conn, 11, ""); } drop(conn); arm.discard(); } // Is the cost per commit or per row? The first line commits per page; // if the cost collapses as pages fold together, commit cadence is the // lever and none of the stages are. println!("\n transaction size, at +fts(all):"); header(); for pages in [1usize, 8, 64, 512] { let arm = clone_arm(&master, &format!("prune-{}-tx{}", label, pages)); let conn = open(&arm); let t = run_stage(&conn, &config, Stage::FtsAll, Commit::Pages(pages)); row(&format!("{} page/tx", pages), &t); drop(conn); arm.discard(); } // Every lever at once: commit per `advance` slice, skip doomed // directories wholesale, tombstone only what can hold a posting. The // slice is already `advance`'s cancellation point, so this needs no // longer uninterruptible window. println!("\n combined, committing once per {:?} slice:", scope::SLICE); header(); { let arm = clone_arm(&master, &format!("prune-{}-combined", label)); let conn = open(&arm); let t = run_stage(&conn, &config, Stage::Subtree, Commit::Slice(scope::SLICE)); row("combined", &t); print_merge(&conn, 11, ""); drop(conn); arm.discard(); } // The page cache. `PRAGMAS_INCREMENTAL` gives the reconcile a flat // 4 MiB for a pass across the whole index. A delete also maintains // `idx_files_mtime`, `idx_files_mime` and `idx_files_hash`, three // b-trees in orders uncorrelated with `(parent, name)`, so every // doomed row scatters ~3 page touches that a 512-page cache cannot // hold. Read the `misses` column. Both commit policies are swept, // because a cache that holds the working set may make commit cadence // stop mattering. println!("\n page cache, at +fts(all) [{} = shipped]:", SHIPPED_MIB); header(); for mib in [SHIPPED_MIB, 16, 32, 64, 128, 256] { for (name, commit) in [ ("per page", Commit::PerPage), ("per slice", Commit::Slice(scope::SLICE)), ] { let arm = clone_arm(&master, &format!("prune-{}-c{}-{}", label, mib, name.len())); let conn = open(&arm); set_cache(&conn, mib); let t = run_stage(&conn, &config, Stage::FtsAll, commit); row(&format!("{} MiB, {}", mib, name), &t); drop(conn); arm.discard(); } } // Measured and rejected: at its 1000-page default, autocheckpoint has // the committing thread copy the log back into the main file every // 8 MiB, re-encrypting every page on a keyed index. Deferring it to // one checkpoint at the end is a wash, because the pages move either // way. { let arm = clone_arm(&master, &format!("prune-{}-nockpt", label)); let conn = open(&arm); conn.execute_batch("PRAGMA wal_autocheckpoint = 0;") .expect("autocheckpoint"); let t = run_stage(&conn, &config, Stage::FtsAll, Commit::Slice(scope::SLICE)); let at = Instant::now(); conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok(); let ckpt = at.elapsed(); row( "no autockpt", &Timing { commit: t.commit + ckpt, total: t.total + ckpt, ..t }, ); drop(conn); arm.discard(); } // The excluded fraction, swept. Past some fraction FTS5 stops // tombstoning and starts rewriting segments, which costs what building // the index did. `baseline` is `+fts(all)` committing per page; // `fixed` is the combined row above. println!("\n excluded fraction (baseline -> fixed, both + merge to quiescence):"); println!( " {:<11} {:>8} {:>9} {:>9} {:>9} {:>11} {:>11}", "patterns", "deleted", "baseline", "fixed", "merge", "%_data", "postings" ); for n in 1..=PRUNE_PATTERNS.len() { let patterns = &PRUNE_PATTERNS[..n]; let config = config_with(patterns); let mut timings = Vec::new(); for (tag, stage, commit) in [ ("baseline", Stage::FtsAll, Commit::PerPage), ("fixed", Stage::Subtree, Commit::Slice(scope::SLICE)), ] { let arm = clone_arm(&master, &format!("prune-{}-f{}-{}", label, n, tag)); let conn = open(&arm); let before = common::fts_data_rows(&conn); let t = run_stage(&conn, &config, stage, commit); let (merge, _, _, after) = merge_to_quiescence(&conn, MERGE_PAGES); let left = count(&conn, "SELECT COUNT(*) FROM searchabletext"); timings.push((t, merge, before, after, left)); drop(conn); arm.discard(); } let ms = |d: Duration| d.as_secs_f64() * 1000.0; println!( " {:<11} {:>8} {:>8.0} {:>8.0} {:>8.0} {:>11} {:>11}", format!("{} of 5", n), timings[0].0.deleted, ms(timings[0].0.total), ms(timings[1].0.total), ms(timings[1].1), format!("{}->{}", timings[1].2, timings[1].3), timings[1].4, ); } // The real `scope::advance`, driven to completion as the coordinator // drives it. It lands under every stage; see the module doc. { let arm = clone_arm(&master, &format!("prune-{}-live", label)); let mut conn = open(&arm); let old = config_with(&[]); let actions = quicksearch_core::config::diff_actions(&old, &config); let mut cursor = WorkCursor::new(actions.work, &config).expect("plan"); let registry = Registry::default_set(); let run = std::sync::atomic::AtomicBool::new(false); let started = Instant::now(); while !cursor.done() { scope::advance( &mut conn, &config, ®istry, &mut cursor, Instant::now() + scope::SLICE, &run, ) .expect("advance"); } println!( " {:<11} {:>8} {:>8} {:>8} {:>8} {:>8} {:>8.0} {:>10} {:>9} <- scope::advance", "live", "", "", "", "", "", started.elapsed().as_secs_f64() * 1000.0, "", cursor.deleted, ); drop(conn); arm.discard(); } // Stale cleanup, the pass a completed run ends with: the same doomed // rows, deleted by path, because the walk hands // `cleanup_stale_index_entries` the paths it did not see, in directory // order. It runs on the run's own writer connection // (`open_run_writer`); `open` would measure the reconcile's 4 MiB // cache instead. { let stale: Vec = { let conn = open(&master); let mut stmt = conn .prepare( "SELECT parent || name FROM files WHERE parent LIKE ?1 \ ORDER BY parent, name", ) .expect("prepare"); let rows = stmt .query_map([format!("%/{}/%", PRUNE_PATTERN)], |r| r.get(0)) .expect("query stale paths"); rows.collect::, _>>().expect("read stale paths") }; println!("\n stale cleanup, {} doomed paths:", stale.len()); header(); for shape in StaleShape::ALL { let arm = clone_arm(&master, &format!("prune-{}-stale-{}", label, shape.tag())); let conn = open_run_writer(&arm); let t = run_stale(&conn, &stale, shape); row(shape.label(), &t); print_merge(&conn, 11, ""); drop(conn); arm.discard(); } // The shipped function, whole: the row that moves when // `file_handling::batch` changes. let arm = clone_arm(&master, &format!("prune-{}-stale-live", label)); let conn = open_run_writer(&arm); let (_, misses_before) = testutil::cache_stats(&conn); let conn_mutex = std::sync::Arc::new(std::sync::Mutex::new(conn)); let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); let started = Instant::now(); let deleted = cleanup_stale_index_entries(&conn_mutex, &stale, &stop, &config) .expect("cleanup stale"); let total = started.elapsed(); let conn = std::sync::Arc::try_unwrap(conn_mutex) .map_err(|_| ()) .expect("sole owner") .into_inner() .expect("unpoisoned"); let (_, misses_after) = testutil::cache_stats(&conn); row( "live", &Timing { total, deleted, misses: misses_after - misses_before, ..Timing::default() }, ); print_merge(&conn, 11, " <- cleanup_stale_index_entries"); drop(conn); arm.discard(); } master.discard(); } }