869 lines
34 KiB
Rust
869 lines
34 KiB
Rust
//! Where the time goes when an added ignore pattern scrubs the index.
|
|
//!
|
|
//! Adding one pattern sets `IndexWork { prune_scope: true, reindex: false }`
|
|
//! (`config::diff_actions`). `scope::advance` then reads every stored row
|
|
//! under every root, decides each one, and deletes the losers by id. This
|
|
//! probe attributes that time to a phase.
|
|
//!
|
|
//! ```text
|
|
//! cargo build -p quicksearch-core --example pruneprobe --release
|
|
//! ./target/release/examples/pruneprobe
|
|
//! ```
|
|
//!
|
|
//! `QSB_FILES`, `QSB_DIRS`, `QSB_DIR_DEPTH` and `QSB_CONTENT_EVERY` size the
|
|
//! corpus; `QSB_ARMS=plain|keyed|both` picks the key states (see
|
|
//! `common::arms`).
|
|
//!
|
|
//! The stages are cumulative, so consecutive rows subtract to a phase cost:
|
|
//!
|
|
//! ```text
|
|
//! page read the rows, decide nothing
|
|
//! +cover ...and run the scope test per row
|
|
//! +files ...and delete the doomed `files` rows
|
|
//! +fts(all) ...and tombstone every doomed id
|
|
//! +fts(done) ...tombstoning only ids that can have an FTS row
|
|
//! +subtree ...range-deleting a doomed directory instead of paging it
|
|
//! ```
|
|
//!
|
|
//! The stages decompose the cost; none is the shipped shape. `scope::advance`
|
|
//! tombstones only the ids that can hold a posting (`+fts(done)`), commits per
|
|
//! slice, and turns FTS5's delete-merging off for the pass
|
|
//! (`file_handling::fts_begin_tombstone_burst`), so the `live` row lands under
|
|
//! every stage.
|
|
//!
|
|
//! Measured and not adopted: `+subtree` bought nothing over `+fts(done)`
|
|
//! (247 ms against 243 on the 40k corpus, with all 8,080 doomed rows skipping
|
|
//! the page loop), and raising the page cache cut misses twelvefold while
|
|
//! barely moving the clock.
|
|
//!
|
|
//! Read the `commit` column, not `fts`. FTS5 buffers a contentless delete in
|
|
//! memory and writes the tombstone pages when the transaction flushes, so the
|
|
//! `DELETE` times as nearly free and the cost lands in the commit. The
|
|
//! transaction-size table sweeps pages per transaction for the same reason:
|
|
//! if the flush costs per commit rather than per tombstone, transaction size
|
|
//! matters and row count does not.
|
|
//!
|
|
//! The final table prices the other bulk withdrawal, a completed run's stale
|
|
//! cleanup (`file_handling::cleanup_stale_index_entries`): the same doomed
|
|
//! rows, deleted by path. Its stages are probe code, fixed like the stages
|
|
//! above; its `live` row is the shipped function.
|
|
|
|
mod common;
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use rusqlite::{Connection, OptionalExtension};
|
|
|
|
use quicksearch_core::config::Config;
|
|
use quicksearch_core::db::{self, repo};
|
|
use quicksearch_core::extract::Registry;
|
|
use quicksearch_core::file_handling::{
|
|
cleanup_stale_index_entries, fts_begin_tombstone_burst, fts_end_tombstone_burst,
|
|
fts_finalize_after_text_indexing, split_db_path, RootRange,
|
|
};
|
|
use quicksearch_core::scope::{self, Scope, WorkCursor};
|
|
use quicksearch_core::testutil::{self, Arm, SeedSpec};
|
|
|
|
use common::{
|
|
chunked, clone_arm, count, env_usize, merge_to_quiescence, open, print_merge, set_cache,
|
|
Commit, Io, MERGE_PAGES, PAGE, SHIPPED_MIB,
|
|
};
|
|
|
|
/// The root every `seed_index` row hangs under. It need not exist on disk:
|
|
/// nothing in the prune path stats a stored path on Unix
|
|
/// (`platform::entry_hidden_reason` short-circuits on the dot prefix).
|
|
const ROOT: &str = "/seed";
|
|
|
|
/// The ignore patterns the probe can add, each excluding a fifth of the index.
|
|
///
|
|
/// `seed_index` names its second path segment `WORDS[(d * 7 + 13) % 35]`, and
|
|
/// because 7 and 35 share a factor only these five of the thirty-five words
|
|
/// are reachable, one per residue of `d mod 5`. So each excludes a fifth of
|
|
/// the directories, spread across the keyspace rather than gathered at one
|
|
/// end, which is the shape a real `node_modules` or `.git` pattern has. The
|
|
/// probe sweeps the excluded fraction by taking a prefix of this list.
|
|
const PRUNE_PATTERNS: [&str; 5] = ["jumps", "content", "revenue", "figure", "eta"];
|
|
|
|
/// The single-pattern case, and the one the stage table runs.
|
|
const PRUNE_PATTERN: &str = PRUNE_PATTERNS[0];
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stages
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Cumulative slices of the prune. Each does everything the one before it does.
|
|
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
|
enum Stage {
|
|
Page,
|
|
Cover,
|
|
Files,
|
|
FtsAll,
|
|
FtsDone,
|
|
Subtree,
|
|
}
|
|
|
|
impl Stage {
|
|
const ALL: [Stage; 6] = [
|
|
Stage::Page,
|
|
Stage::Cover,
|
|
Stage::Files,
|
|
Stage::FtsAll,
|
|
Stage::FtsDone,
|
|
Stage::Subtree,
|
|
];
|
|
|
|
fn label(self) -> &'static str {
|
|
match self {
|
|
Stage::Page => "page",
|
|
Stage::Cover => "+cover",
|
|
Stage::Files => "+files",
|
|
Stage::FtsAll => "+fts(all)",
|
|
Stage::FtsDone => "+fts(done)",
|
|
Stage::Subtree => "+subtree",
|
|
}
|
|
}
|
|
|
|
fn tag(self) -> &'static str {
|
|
match self {
|
|
Stage::Page => "page",
|
|
Stage::Cover => "cover",
|
|
Stage::Files => "files",
|
|
Stage::FtsAll => "fts-all",
|
|
Stage::FtsDone => "fts-done",
|
|
Stage::Subtree => "subtree",
|
|
}
|
|
}
|
|
|
|
/// Whether the doomed rows leave `files`.
|
|
fn deletes_rows(self) -> bool {
|
|
self >= Stage::Files
|
|
}
|
|
|
|
/// Whether the doomed ids are tombstoned in FTS.
|
|
fn deletes_fts(self) -> bool {
|
|
self >= Stage::FtsAll
|
|
}
|
|
|
|
/// Whether the FTS delete is narrowed to ids that can hold a posting.
|
|
fn fts_filtered(self) -> bool {
|
|
self >= Stage::FtsDone
|
|
}
|
|
}
|
|
|
|
/// What one stage cost, split by phase.
|
|
#[derive(Default)]
|
|
struct Timing {
|
|
page: Duration,
|
|
cover: Duration,
|
|
files: Duration,
|
|
fts: Duration,
|
|
commit: Duration,
|
|
total: Duration,
|
|
examined: usize,
|
|
deleted: usize,
|
|
/// Rows skipped by a range delete rather than paged through.
|
|
skipped: usize,
|
|
io: Io,
|
|
misses: i64,
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The measured loop
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Whether the walker would still descend into the stored parent `parent`.
|
|
///
|
|
/// `Scope::covers` judges a path component by component, which for a
|
|
/// directory asks what the walker asks before recursing. The trailing
|
|
/// separator every stored parent carries goes first, or the leaf component
|
|
/// would be empty.
|
|
fn dir_covered(scope: &Scope, root: &Path, parent: &str) -> bool {
|
|
let trimmed = parent.trim_end_matches(['/', '\\']);
|
|
scope.covers(root, Path::new(trimmed))
|
|
}
|
|
|
|
/// Page the index as `scope::advance` does, doing `stage`'s share of the work
|
|
/// and timing each phase apart.
|
|
///
|
|
/// The deletes are spelled out here rather than routed through `repo`, so the
|
|
/// filtered and unfiltered FTS variants compare in one run and the stages stay
|
|
/// fixed when `repo` changes.
|
|
fn run_stage(conn: &Connection, config: &Config, stage: Stage, commit: Commit) -> Timing {
|
|
let scope = Scope::from_config(config).expect("compile the scope");
|
|
let root = PathBuf::from(ROOT);
|
|
let range = RootRange::for_root(ROOT);
|
|
|
|
let mut t = Timing::default();
|
|
let io_before = Io::read();
|
|
let (_, misses_before) = testutil::cache_stats(conn);
|
|
let started = Instant::now();
|
|
|
|
let mut after = (range.lo.clone(), String::new());
|
|
let mut covered = scope::CoverCache::default();
|
|
// `unchecked_transaction` borrows shared, so one transaction can end and
|
|
// the next begin inside the loop, as in `testutil::seed_index`.
|
|
let mut tx = conn.unchecked_transaction().expect("begin");
|
|
let mut pages_open = 0usize;
|
|
let mut tx_since = Instant::now();
|
|
loop {
|
|
let at = Instant::now();
|
|
let rows =
|
|
repo::rows_in_range_page(&tx, &after.0, &after.1, &range.hi, PAGE).expect("read a page");
|
|
t.page += at.elapsed();
|
|
let Some(last) = rows.last() else { break };
|
|
after = (last.parent.clone(), last.name.clone());
|
|
pages_open += 1;
|
|
|
|
if stage == Stage::Page {
|
|
t.examined += rows.len();
|
|
continue;
|
|
}
|
|
|
|
// The subtree short-circuit. A page spans several directories, so each
|
|
// distinct parent is tested as it appears; the first doomed one is
|
|
// range-deleted and the scan re-seeks past it. Testing only `rows[0]`
|
|
// would fire only on pages that begin inside an excluded directory.
|
|
let mut rows: &[repo::ScopeRow] = &rows;
|
|
if stage == Stage::Subtree {
|
|
let at = Instant::now();
|
|
let mut doomed_at = None;
|
|
let mut last_dir: Option<(&str, bool)> = None;
|
|
for (i, row) in rows.iter().enumerate() {
|
|
let ok = match last_dir {
|
|
Some((dir, ok)) if dir == row.parent => ok,
|
|
_ => {
|
|
let ok = dir_covered(&scope, &root, &row.parent);
|
|
last_dir = Some((&row.parent, ok));
|
|
ok
|
|
}
|
|
};
|
|
if !ok {
|
|
doomed_at = Some(i);
|
|
break;
|
|
}
|
|
}
|
|
t.cover += at.elapsed();
|
|
if let Some(i) = doomed_at {
|
|
let dir = rows[i].parent.trim_end_matches(['/', '\\']).to_string();
|
|
let sub = RootRange::for_root(&dir);
|
|
let (removed, fts, files) = delete_range(&tx, &sub.lo, &sub.hi);
|
|
t.fts += fts;
|
|
t.files += files;
|
|
t.deleted += removed;
|
|
t.skipped += removed;
|
|
// The rows before the doomed directory still need deciding;
|
|
// everything from it on is gone or re-read after the seek.
|
|
// Both happen inside the open transaction, so the seek sees
|
|
// the range delete without a commit.
|
|
after = (sub.hi.clone(), String::new());
|
|
rows = &rows[..i];
|
|
}
|
|
}
|
|
|
|
let at = Instant::now();
|
|
let mut doomed: Vec<i64> = Vec::new();
|
|
let mut doomed_fts: Vec<i64> = Vec::new();
|
|
for row in rows {
|
|
if scope.covers_cached(&root, Path::new(&row.path), &mut covered) {
|
|
continue;
|
|
}
|
|
doomed.push(row.id);
|
|
if row.content_state == repo::STATE_DONE {
|
|
doomed_fts.push(row.id);
|
|
}
|
|
}
|
|
t.cover += at.elapsed();
|
|
t.examined += rows.len();
|
|
|
|
if stage.deletes_rows() && !doomed.is_empty() {
|
|
let fts_ids: &[i64] = if stage.fts_filtered() {
|
|
&doomed_fts
|
|
} else {
|
|
&doomed
|
|
};
|
|
if stage.deletes_fts() {
|
|
let at = Instant::now();
|
|
chunked(
|
|
&tx,
|
|
"DELETE FROM searchabletext WHERE rowid IN",
|
|
fts_ids,
|
|
false,
|
|
);
|
|
t.fts += at.elapsed();
|
|
}
|
|
let at = Instant::now();
|
|
t.deleted += chunked(&tx, "DELETE FROM files WHERE id IN", &doomed, false);
|
|
t.files += at.elapsed();
|
|
}
|
|
|
|
if commit.due(pages_open, tx_since) {
|
|
let at = Instant::now();
|
|
tx.commit().expect("commit");
|
|
t.commit += at.elapsed();
|
|
tx = conn.unchecked_transaction().expect("begin");
|
|
pages_open = 0;
|
|
tx_since = Instant::now();
|
|
}
|
|
}
|
|
let at = Instant::now();
|
|
tx.commit().expect("commit");
|
|
t.commit += at.elapsed();
|
|
|
|
t.total = started.elapsed();
|
|
t.io = Io::read().since(&io_before);
|
|
let (_, misses_after) = testutil::cache_stats(conn);
|
|
t.misses = misses_after - misses_before;
|
|
t
|
|
}
|
|
|
|
/// Delete a whole parent range, tombstoning only the ids that can hold a
|
|
/// posting. Returns `(rows, fts time, files time)`.
|
|
fn delete_range(tx: &Connection, lo: &str, hi: &str) -> (usize, Duration, Duration) {
|
|
let at = Instant::now();
|
|
tx.execute(
|
|
"DELETE FROM searchabletext WHERE rowid IN \
|
|
(SELECT id FROM files WHERE parent >= ?1 AND parent < ?2 AND content_state = ?3)",
|
|
rusqlite::params![lo, hi, repo::STATE_DONE],
|
|
)
|
|
.expect("tombstone range");
|
|
let fts = at.elapsed();
|
|
let at = Instant::now();
|
|
let removed = tx
|
|
.execute(
|
|
"DELETE FROM files WHERE parent >= ?1 AND parent < ?2",
|
|
rusqlite::params![lo, hi],
|
|
)
|
|
.expect("delete range");
|
|
(removed, fts, at.elapsed())
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stale cleanup stages
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Cumulative slices of a completed run's stale cleanup, which deletes by
|
|
/// *path* rather than deciding rows from a page it already read.
|
|
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
|
enum StaleShape {
|
|
/// Per-path deletes, three statements each, with FTS handed every id. The
|
|
/// control row: probe code, so it must not move when
|
|
/// `cleanup_stale_index_entries` changes.
|
|
PerRow,
|
|
/// ...resolve `(id, content_state)` per path instead, then chunked
|
|
/// `IN (...)` deletes with FTS narrowed to ids that can hold a posting.
|
|
Ids,
|
|
/// ...and hold FTS5's delete-merging off for the whole pass.
|
|
Burst,
|
|
}
|
|
|
|
impl StaleShape {
|
|
const ALL: [StaleShape; 3] = [StaleShape::PerRow, StaleShape::Ids, StaleShape::Burst];
|
|
|
|
fn label(self) -> &'static str {
|
|
match self {
|
|
StaleShape::PerRow => "per-row",
|
|
StaleShape::Ids => "+ids",
|
|
StaleShape::Burst => "+burst",
|
|
}
|
|
}
|
|
|
|
fn tag(self) -> &'static str {
|
|
match self {
|
|
StaleShape::PerRow => "row",
|
|
StaleShape::Ids => "ids",
|
|
StaleShape::Burst => "burst",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Delete `paths` the way stale cleanup does, one transaction per `PAGE`
|
|
/// chunk, timing each phase. Column mapping in the shared table: `cover` is
|
|
/// the id resolution, `files` the `files` deletes, `fts` the tombstones plus
|
|
/// the trailing consolidation.
|
|
fn run_stale(conn: &Connection, paths: &[String], shape: StaleShape) -> Timing {
|
|
let mut t = Timing::default();
|
|
let io_before = Io::read();
|
|
let (_, misses_before) = testutil::cache_stats(conn);
|
|
let started = Instant::now();
|
|
|
|
if shape == StaleShape::Burst {
|
|
fts_begin_tombstone_burst(conn);
|
|
}
|
|
for page in paths.chunks(PAGE as usize) {
|
|
let tx = conn.unchecked_transaction().expect("begin");
|
|
if shape == StaleShape::PerRow {
|
|
for path in page {
|
|
let Some((parent, name)) = split_db_path(path) else {
|
|
continue;
|
|
};
|
|
let at = Instant::now();
|
|
let id: Option<i64> = tx
|
|
.prepare_cached(
|
|
"DELETE FROM files WHERE parent = ?1 AND name = ?2 RETURNING id",
|
|
)
|
|
.expect("prepare")
|
|
.query_row(rusqlite::params![parent, name], |r| r.get(0))
|
|
.optional()
|
|
.expect("delete row");
|
|
t.files += at.elapsed();
|
|
let Some(id) = id else { continue };
|
|
t.deleted += 1;
|
|
let at = Instant::now();
|
|
tx.prepare_cached("DELETE FROM searchabletext WHERE rowid = ?1")
|
|
.expect("prepare")
|
|
.execute([id])
|
|
.expect("tombstone");
|
|
t.fts += at.elapsed();
|
|
let at = Instant::now();
|
|
tx.prepare_cached("DELETE FROM documents_text WHERE file_id = ?1")
|
|
.expect("prepare")
|
|
.execute([id])
|
|
.expect("clear body");
|
|
t.files += at.elapsed();
|
|
}
|
|
} else {
|
|
let at = Instant::now();
|
|
let mut ids: Vec<i64> = Vec::new();
|
|
let mut with_postings: Vec<i64> = Vec::new();
|
|
{
|
|
let mut sel = tx
|
|
.prepare_cached(
|
|
"SELECT id, content_state FROM files WHERE parent = ?1 AND name = ?2",
|
|
)
|
|
.expect("prepare");
|
|
for path in page {
|
|
let Some((parent, name)) = split_db_path(path) else {
|
|
continue;
|
|
};
|
|
let row: Option<(i64, i64)> = sel
|
|
.query_row(rusqlite::params![parent, name], |r| {
|
|
Ok((r.get(0)?, r.get(1)?))
|
|
})
|
|
.optional()
|
|
.expect("resolve");
|
|
if let Some((id, state)) = row {
|
|
ids.push(id);
|
|
if state == repo::STATE_DONE {
|
|
with_postings.push(id);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
t.cover += at.elapsed();
|
|
let at = Instant::now();
|
|
chunked(
|
|
&tx,
|
|
"DELETE FROM searchabletext WHERE rowid IN",
|
|
&with_postings,
|
|
false,
|
|
);
|
|
t.fts += at.elapsed();
|
|
let at = Instant::now();
|
|
t.deleted += chunked(&tx, "DELETE FROM files WHERE id IN", &ids, false);
|
|
t.files += at.elapsed();
|
|
}
|
|
let at = Instant::now();
|
|
tx.commit().expect("commit");
|
|
t.commit += at.elapsed();
|
|
}
|
|
let at = Instant::now();
|
|
if shape == StaleShape::Burst {
|
|
fts_end_tombstone_burst(conn);
|
|
} else {
|
|
fts_finalize_after_text_indexing(conn);
|
|
}
|
|
t.fts += at.elapsed();
|
|
|
|
t.total = started.elapsed();
|
|
t.io = Io::read().since(&io_before);
|
|
let (_, misses_after) = testutil::cache_stats(conn);
|
|
t.misses = misses_after - misses_before;
|
|
t
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Corpus and arms
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn spec() -> SeedSpec {
|
|
SeedSpec {
|
|
files: env_usize("QSB_FILES", 200_000),
|
|
dirs: env_usize("QSB_DIRS", 2_000),
|
|
// Coprime with 5. `seed_index` places content every `content_every`
|
|
// files and directories every `dirs`, and [`PRUNE_PATTERN`] excludes
|
|
// the directories where `d mod 5 == 0`. The default of 10 shares the
|
|
// factor 5, so every extracted document would land in an excluded
|
|
// directory and the prune would delete all the postings. 7 puts a
|
|
// fifth of the documents in the doomed set. `main` asserts the
|
|
// fraction.
|
|
content_every: env_usize("QSB_CONTENT_EVERY", 7),
|
|
// At least two segments, so the second can carry the excluded name;
|
|
// see `SeedSpec::dir_depth` for how depth sets row width.
|
|
dir_depth: env_usize("QSB_DIR_DEPTH", 6).max(2),
|
|
// A real run commits in slices, each flushing FTS5's hash to its own
|
|
// segment; one transaction would leave a single segment and
|
|
// understate every tombstone cost.
|
|
commit_every: 500,
|
|
..SeedSpec::default()
|
|
}
|
|
}
|
|
|
|
fn config_with(patterns: &[&str]) -> Config {
|
|
let mut config = Config::default();
|
|
config.paths.indexing_paths = vec![ROOT.to_string()];
|
|
config.indexing.ignore_patterns = patterns.iter().map(|p| p.to_string()).collect();
|
|
config.processing.batch_size = PAGE as usize;
|
|
config
|
|
}
|
|
|
|
/// Open `arm` on a run's own writer profile, `PRAGMAS_FAST`, which stale
|
|
/// cleanup runs on.
|
|
fn open_run_writer(arm: &Arm) -> Connection {
|
|
arm.with_key(|| {
|
|
db::open::open_existing(&arm.path.to_string_lossy(), true).expect("open the copy")
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn main() {
|
|
let spec = spec();
|
|
|
|
println!(
|
|
"corpus: {} files across {} dirs, 1 in {} with content, pattern {:?}",
|
|
spec.files, spec.dirs, spec.content_every, PRUNE_PATTERN
|
|
);
|
|
|
|
for keyed in common::arms() {
|
|
let label = if keyed { "keyed" } else { "plain" };
|
|
let master = Arm::seed(label, &format!("prune-{}", label), keyed, &spec);
|
|
println!(
|
|
"\n=== {} === seeded in {:.1}s, {}",
|
|
label,
|
|
master.seeded_in.as_secs_f64(),
|
|
common::mib(master.size_bytes())
|
|
);
|
|
|
|
// Census: what the pattern takes, read off the index.
|
|
{
|
|
let conn = open(&master);
|
|
let total = count(&conn, "SELECT COUNT(*) FROM files");
|
|
let doomed = count(
|
|
&conn,
|
|
&format!(
|
|
"SELECT COUNT(*) FROM files WHERE parent LIKE '%/{}/%'",
|
|
PRUNE_PATTERN
|
|
),
|
|
);
|
|
let doomed_fts = count(
|
|
&conn,
|
|
&format!(
|
|
"SELECT COUNT(*) FROM files WHERE parent LIKE '%/{}/%' AND content_state = 1",
|
|
PRUNE_PATTERN
|
|
),
|
|
);
|
|
let fts = count(&conn, "SELECT COUNT(*) FROM searchabletext");
|
|
let done = count(&conn, "SELECT COUNT(*) FROM files WHERE content_state = 1");
|
|
assert!(
|
|
doomed > 0,
|
|
"the pattern excludes nothing — seed_index's directory naming moved"
|
|
);
|
|
assert_eq!(fts, done, "a posting exists exactly for content_state = 1");
|
|
assert!(
|
|
doomed_fts * 2 < done,
|
|
"the prune takes {} of {} postings — the content and directory \
|
|
strides have collided again; see `spec`'s content_every",
|
|
doomed_fts,
|
|
done
|
|
);
|
|
println!(
|
|
" {} rows, {} doomed ({:.0}%), of which {} hold a posting ({:.0}%) — \
|
|
the rest hold nothing to tombstone",
|
|
total,
|
|
doomed,
|
|
100.0 * doomed as f64 / total as f64,
|
|
doomed_fts,
|
|
100.0 * doomed_fts as f64 / doomed as f64,
|
|
);
|
|
println!(
|
|
" plan: {}",
|
|
conn.query_row(
|
|
"EXPLAIN QUERY PLAN DELETE FROM searchabletext WHERE rowid IN (1,2,3)",
|
|
[],
|
|
|r| r.get::<_, String>(3),
|
|
)
|
|
.unwrap_or_else(|e| format!("unavailable: {}", e))
|
|
);
|
|
common::print_fts_census(&conn);
|
|
}
|
|
|
|
let config = config_with(&[PRUNE_PATTERN]);
|
|
let header = || {
|
|
println!(
|
|
"\n {:<11} {:>7} {:>7} {:>7} {:>7} {:>8} {:>8} {:>9} {:>8}",
|
|
"stage", "page", "cover", "files", "fts", "commit", "total", "misses", "deleted"
|
|
);
|
|
};
|
|
let row = |name: &str, t: &Timing| {
|
|
let ms = |d: Duration| d.as_secs_f64() * 1000.0;
|
|
println!(
|
|
" {:<11} {:>7.0} {:>7.0} {:>7.0} {:>7.0} {:>8.0} {:>8.0} {:>9} {:>8}",
|
|
name,
|
|
ms(t.page),
|
|
ms(t.cover),
|
|
ms(t.files),
|
|
ms(t.fts),
|
|
ms(t.commit),
|
|
ms(t.total),
|
|
t.misses,
|
|
t.deleted,
|
|
);
|
|
};
|
|
|
|
header();
|
|
for stage in Stage::ALL {
|
|
let arm = clone_arm(&master, &format!("prune-{}-{}", label, stage.tag()));
|
|
let conn = open(&arm);
|
|
let t = run_stage(&conn, &config, stage, Commit::PerPage);
|
|
row(stage.label(), &t);
|
|
if stage == Stage::Subtree {
|
|
println!(
|
|
" {:<11} {} of those rows never reached a page",
|
|
"", t.skipped
|
|
);
|
|
}
|
|
// Consolidation, priced apart: merge until `%_data` stops changing,
|
|
// as `fts_end_tombstone_burst` ends a burst.
|
|
if stage.deletes_fts() {
|
|
print_merge(&conn, 11, "");
|
|
}
|
|
drop(conn);
|
|
arm.discard();
|
|
}
|
|
|
|
// Is the cost per commit or per row? The first line commits per page;
|
|
// if the cost collapses as pages fold together, commit cadence is the
|
|
// lever and none of the stages are.
|
|
println!("\n transaction size, at +fts(all):");
|
|
header();
|
|
for pages in [1usize, 8, 64, 512] {
|
|
let arm = clone_arm(&master, &format!("prune-{}-tx{}", label, pages));
|
|
let conn = open(&arm);
|
|
let t = run_stage(&conn, &config, Stage::FtsAll, Commit::Pages(pages));
|
|
row(&format!("{} page/tx", pages), &t);
|
|
drop(conn);
|
|
arm.discard();
|
|
}
|
|
|
|
// Every lever at once: commit per `advance` slice, skip doomed
|
|
// directories wholesale, tombstone only what can hold a posting. The
|
|
// slice is already `advance`'s cancellation point, so this needs no
|
|
// longer uninterruptible window.
|
|
println!("\n combined, committing once per {:?} slice:", scope::SLICE);
|
|
header();
|
|
{
|
|
let arm = clone_arm(&master, &format!("prune-{}-combined", label));
|
|
let conn = open(&arm);
|
|
let t = run_stage(&conn, &config, Stage::Subtree, Commit::Slice(scope::SLICE));
|
|
row("combined", &t);
|
|
print_merge(&conn, 11, "");
|
|
drop(conn);
|
|
arm.discard();
|
|
}
|
|
|
|
// The page cache. `PRAGMAS_INCREMENTAL` gives the reconcile a flat
|
|
// 4 MiB for a pass across the whole index. A delete also maintains
|
|
// `idx_files_hash`, a b-tree in an order uncorrelated with
|
|
// `(parent, name)`, so every doomed row scatters a page touch that a
|
|
// 512-page cache cannot hold (three, before `idx_files_mtime` and
|
|
// `idx_files_mime` were dropped). Read the `misses` column. Both commit policies are swept,
|
|
// because a cache that holds the working set may make commit cadence
|
|
// stop mattering.
|
|
println!("\n page cache, at +fts(all) [{} = shipped]:", SHIPPED_MIB);
|
|
header();
|
|
for mib in [SHIPPED_MIB, 16, 32, 64, 128, 256] {
|
|
for (name, commit) in [
|
|
("per page", Commit::PerPage),
|
|
("per slice", Commit::Slice(scope::SLICE)),
|
|
] {
|
|
let arm = clone_arm(&master, &format!("prune-{}-c{}-{}", label, mib, name.len()));
|
|
let conn = open(&arm);
|
|
set_cache(&conn, mib);
|
|
let t = run_stage(&conn, &config, Stage::FtsAll, commit);
|
|
row(&format!("{} MiB, {}", mib, name), &t);
|
|
drop(conn);
|
|
arm.discard();
|
|
}
|
|
}
|
|
|
|
// Measured and rejected: at its 1000-page default, autocheckpoint has
|
|
// the committing thread copy the log back into the main file every
|
|
// 8 MiB, re-encrypting every page on a keyed index. Deferring it to
|
|
// one checkpoint at the end is a wash, because the pages move either
|
|
// way.
|
|
{
|
|
let arm = clone_arm(&master, &format!("prune-{}-nockpt", label));
|
|
let conn = open(&arm);
|
|
conn.execute_batch("PRAGMA wal_autocheckpoint = 0;")
|
|
.expect("autocheckpoint");
|
|
let t = run_stage(&conn, &config, Stage::FtsAll, Commit::Slice(scope::SLICE));
|
|
let at = Instant::now();
|
|
conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok();
|
|
let ckpt = at.elapsed();
|
|
row(
|
|
"no autockpt",
|
|
&Timing {
|
|
commit: t.commit + ckpt,
|
|
total: t.total + ckpt,
|
|
..t
|
|
},
|
|
);
|
|
drop(conn);
|
|
arm.discard();
|
|
}
|
|
|
|
// The excluded fraction, swept. Past some fraction FTS5 stops
|
|
// tombstoning and starts rewriting segments, which costs what building
|
|
// the index did. `baseline` is `+fts(all)` committing per page;
|
|
// `fixed` is the combined row above.
|
|
println!("\n excluded fraction (baseline -> fixed, both + merge to quiescence):");
|
|
println!(
|
|
" {:<11} {:>8} {:>9} {:>9} {:>9} {:>11} {:>11}",
|
|
"patterns", "deleted", "baseline", "fixed", "merge", "%_data", "postings"
|
|
);
|
|
for n in 1..=PRUNE_PATTERNS.len() {
|
|
let patterns = &PRUNE_PATTERNS[..n];
|
|
let config = config_with(patterns);
|
|
let mut timings = Vec::new();
|
|
for (tag, stage, commit) in [
|
|
("baseline", Stage::FtsAll, Commit::PerPage),
|
|
("fixed", Stage::Subtree, Commit::Slice(scope::SLICE)),
|
|
] {
|
|
let arm = clone_arm(&master, &format!("prune-{}-f{}-{}", label, n, tag));
|
|
let conn = open(&arm);
|
|
let before = common::fts_data_rows(&conn);
|
|
let t = run_stage(&conn, &config, stage, commit);
|
|
let (merge, _, _, after) = merge_to_quiescence(&conn, MERGE_PAGES);
|
|
let left = count(&conn, "SELECT COUNT(*) FROM searchabletext");
|
|
timings.push((t, merge, before, after, left));
|
|
drop(conn);
|
|
arm.discard();
|
|
}
|
|
let ms = |d: Duration| d.as_secs_f64() * 1000.0;
|
|
println!(
|
|
" {:<11} {:>8} {:>8.0} {:>8.0} {:>8.0} {:>11} {:>11}",
|
|
format!("{} of 5", n),
|
|
timings[0].0.deleted,
|
|
ms(timings[0].0.total),
|
|
ms(timings[1].0.total),
|
|
ms(timings[1].1),
|
|
format!("{}->{}", timings[1].2, timings[1].3),
|
|
timings[1].4,
|
|
);
|
|
}
|
|
|
|
// The real `scope::advance`, driven to completion as the coordinator
|
|
// drives it. It lands under every stage; see the module doc.
|
|
{
|
|
let arm = clone_arm(&master, &format!("prune-{}-live", label));
|
|
let mut conn = open(&arm);
|
|
let old = config_with(&[]);
|
|
let actions = quicksearch_core::config::diff_actions(&old, &config);
|
|
let mut cursor = WorkCursor::new(actions.work, &config).expect("plan");
|
|
let registry = Registry::default_set();
|
|
let run = std::sync::atomic::AtomicBool::new(false);
|
|
let started = Instant::now();
|
|
while !cursor.done() {
|
|
scope::advance(
|
|
&mut conn,
|
|
&config,
|
|
®istry,
|
|
&mut cursor,
|
|
Instant::now() + scope::SLICE,
|
|
&run,
|
|
)
|
|
.expect("advance");
|
|
}
|
|
println!(
|
|
" {:<11} {:>8} {:>8} {:>8} {:>8} {:>8} {:>8.0} {:>10} {:>9} <- scope::advance",
|
|
"live",
|
|
"",
|
|
"",
|
|
"",
|
|
"",
|
|
"",
|
|
started.elapsed().as_secs_f64() * 1000.0,
|
|
"",
|
|
cursor.deleted,
|
|
);
|
|
drop(conn);
|
|
arm.discard();
|
|
}
|
|
|
|
// Stale cleanup, the pass a completed run ends with: the same doomed
|
|
// rows, deleted by path, because the walk hands
|
|
// `cleanup_stale_index_entries` the paths it did not see, in directory
|
|
// order. It runs on the run's own writer connection
|
|
// (`open_run_writer`); `open` would measure the reconcile's 4 MiB
|
|
// cache instead.
|
|
{
|
|
let stale: Vec<String> = {
|
|
let conn = open(&master);
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"SELECT parent || name FROM files WHERE parent LIKE ?1 \
|
|
ORDER BY parent, name",
|
|
)
|
|
.expect("prepare");
|
|
let rows = stmt
|
|
.query_map([format!("%/{}/%", PRUNE_PATTERN)], |r| r.get(0))
|
|
.expect("query stale paths");
|
|
rows.collect::<Result<Vec<_>, _>>().expect("read stale paths")
|
|
};
|
|
println!("\n stale cleanup, {} doomed paths:", stale.len());
|
|
header();
|
|
for shape in StaleShape::ALL {
|
|
let arm = clone_arm(&master, &format!("prune-{}-stale-{}", label, shape.tag()));
|
|
let conn = open_run_writer(&arm);
|
|
let t = run_stale(&conn, &stale, shape);
|
|
row(shape.label(), &t);
|
|
print_merge(&conn, 11, "");
|
|
drop(conn);
|
|
arm.discard();
|
|
}
|
|
|
|
// The shipped function, whole: the row that moves when
|
|
// `file_handling::batch` changes.
|
|
let arm = clone_arm(&master, &format!("prune-{}-stale-live", label));
|
|
let conn = open_run_writer(&arm);
|
|
let (_, misses_before) = testutil::cache_stats(&conn);
|
|
let conn_mutex = std::sync::Arc::new(std::sync::Mutex::new(conn));
|
|
let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
|
let started = Instant::now();
|
|
let deleted = cleanup_stale_index_entries(&conn_mutex, &stale, &stop, &config)
|
|
.expect("cleanup stale");
|
|
let total = started.elapsed();
|
|
let conn = std::sync::Arc::try_unwrap(conn_mutex)
|
|
.map_err(|_| ())
|
|
.expect("sole owner")
|
|
.into_inner()
|
|
.expect("unpoisoned");
|
|
let (_, misses_after) = testutil::cache_stats(&conn);
|
|
row(
|
|
"live",
|
|
&Timing {
|
|
total,
|
|
deleted,
|
|
misses: misses_after - misses_before,
|
|
..Timing::default()
|
|
},
|
|
);
|
|
print_merge(&conn, 11, " <- cleanup_stale_index_entries");
|
|
drop(conn);
|
|
arm.discard();
|
|
}
|
|
|
|
master.discard();
|
|
}
|
|
}
|