quick_search/crates/quicksearch-core/tests/full_index.rs

//! End-to-end phase-1 tests over a real tree and a real database.

use std::path::Path;
use std::time::{Duration, Instant, SystemTime};

use quicksearch_core::config::Config;
use quicksearch_core::file_handling::{count_extract_scope, mark_oversize_pending_na, RootRange};
use quicksearch_core::indexing::{IndexingService, IndexingStatus, RootPhase};
use quicksearch_core::testutil::Scratch;

mod common;
use common::touch;

fn index_roots_once(roots: &[&Path], db: &Path, config: &Config) {
    common::IndexOnce {
        db,
        roots: roots
            .iter()
            .map(|r| r.to_string_lossy().into_owned())
            .collect(),
        config,
        fresh_marker: true,
        encrypted: false,
    }
    .run()
}

fn index_once(root: &Path, db: &Path, config: &Config) {
    index_roots_once(&[root], db, config)
}

/// Start a run indexing `root` into `db`.
fn start(root: &Path, db: &Path, config: &Config) -> IndexingService {
    let service = IndexingService::new();
    service
        .start_indexing(
            vec![root.to_string_lossy().into_owned()],
            db.to_string_lossy().into_owned(),
            config.clone(),
        )
        .unwrap();
    service
}

/// Poll `service` until `Idle`, calling `each_poll` first every time. `Idle`
/// is the end of the post-run maintenance pass and a resting state, so waiting
/// for it cannot miss it however fast the pass was.
fn run_to_idle(service: &IndexingService, mut each_poll: impl FnMut()) {
    let deadline = Instant::now() + Duration::from_secs(180);
    loop {
        each_poll();
        match service.get_status() {
            IndexingStatus::Idle => return,
            IndexingStatus::Error(e) => panic!("indexing failed: {}", e),
            _ => {}
        }
        assert!(Instant::now() < deadline, "the run never reached Idle");
        std::thread::sleep(Duration::from_millis(1));
    }
}

/// The single integer `sql` returns.
fn count(db: &Path, sql: &str) -> i64 {
    rusqlite::Connection::open(db)
        .unwrap()
        .query_row(sql, [], |r| r.get(0))
        .unwrap()
}

/// FTS rows matching `word`.
fn hits(db: &Path, word: &str) -> i64 {
    quicksearch_core::testutil::fts_count(&rusqlite::Connection::open(db).unwrap(), word)
}

/// Bytes in `db`'s write-ahead log; 0 when there is none.
fn wal_bytes(db: &Path) -> u64 {
    let mut wal = db.as_os_str().to_owned();
    wal.push("-wal");
    std::fs::metadata(wal).map(|m| m.len()).unwrap_or(0)
}

fn rows(db: &Path) -> Vec<(String, i64, i64)> {
    let conn = rusqlite::Connection::open(db).unwrap();
    let mut stmt = conn
        .prepare("SELECT parent || name, mtime, content_state FROM files ORDER BY parent, name")
        .unwrap();
    let out = stmt
        .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
        .unwrap()
        .map(|r| r.unwrap())
        .collect();
    out
}

/// The file names of [`rows`], in the same order.
fn names(db: &Path) -> Vec<String> {
    rows(db)
        .into_iter()
        .map(|(p, _, _)| {
            Path::new(&p)
                .file_name()
                .unwrap()
                .to_string_lossy()
                .into_owned()
        })
        .collect()
}

#[test]
fn reindexing_an_unchanged_tree_changes_nothing() {
    let root = Scratch::dir("stable");
    let (_db_dir, db) = Scratch::db("stable-db");
    let config = Config::default();

    touch(&root.join("a.txt"), b"alpha");
    touch(&root.join("sub/b.txt"), b"bravo");
    touch(&root.join("sub/deep/c.txt"), b"charlie");
    touch(&root.join("other/d.md"), b"delta");

    index_once(&root, &db, &config);
    let first = rows(&db);
    assert_eq!(first.len(), 4, "all four files indexed");

    index_once(&root, &db, &config);
    let second = rows(&db);

    assert_eq!(
        first, second,
        "an unchanged tree must re-index to an identical set"
    );
}

#[test]
fn deleted_files_are_removed_and_new_ones_added() {
    let root = Scratch::dir("churn");
    let (_db_dir, db) = Scratch::db("churn-db");
    let config = Config::default();

    touch(&root.join("keep.txt"), b"keep");
    touch(&root.join("remove.txt"), b"remove");
    index_once(&root, &db, &config);
    assert_eq!(rows(&db).len(), 2);

    std::fs::remove_file(root.join("remove.txt")).unwrap();
    touch(&root.join("added.txt"), b"added");
    index_once(&root, &db, &config);

    assert_eq!(
        names(&db),
        vec!["added.txt", "keep.txt"],
        "stale cleanup still works"
    );
}

#[test]
fn a_modified_file_is_updated_in_place() {
    let root = Scratch::dir("modify");
    let (_db_dir, db) = Scratch::db("modify-db");
    let config = Config::default();

    let target = root.join("doc.txt");
    touch(&target, b"first");
    index_once(&root, &db, &config);
    let before = rows(&db);
    assert_eq!(before.len(), 1);

    touch(&target, b"second body, clearly different");
    let later = SystemTime::now() + Duration::from_secs(5);
    filetime_set(&target, later);

    index_once(&root, &db, &config);
    let after = rows(&db);
    assert_eq!(after.len(), 1, "still exactly one row");
    assert_ne!(before[0].1, after[0].1, "mtime was refreshed");
    assert_eq!(before[0].0, after[0].0, "same path");
}

fn filetime_set(path: &Path, when: SystemTime) {
    let f = std::fs::OpenOptions::new().write(true).open(path).unwrap();
    f.set_modified(when).unwrap();
    f.sync_all().unwrap();
}

#[test]
#[cfg(unix)]
fn an_unreadable_directory_does_not_delete_its_rows() {
    use std::os::unix::fs::PermissionsExt;

    let root = Scratch::dir("blip");
    let (_db_dir, db) = Scratch::db("blip-db");
    let config = Config::default();

    touch(&root.join("visible.txt"), b"visible");
    let vault = root.join("vault");
    touch(&vault.join("secret.txt"), b"secret");
    touch(&vault.join("nested/deeper.txt"), b"deeper");

    index_once(&root, &db, &config);
    assert_eq!(rows(&db).len(), 3, "all three indexed while readable");

    std::fs::set_permissions(&vault, std::fs::Permissions::from_mode(0o000)).unwrap();
    index_once(&root, &db, &config);
    let during = rows(&db);
    std::fs::set_permissions(&vault, std::fs::Permissions::from_mode(0o755)).unwrap();

    assert_eq!(
        during.len(),
        3,
        "rows under an unreadable directory must survive, not be deleted"
    );

    index_once(&root, &db, &config);
    assert_eq!(rows(&db).len(), 3);
}

#[test]
fn stopping_mid_run_deletes_nothing() {
    let root = Scratch::dir("stop");
    let (_db_dir, db) = Scratch::db("stop-db");
    let config = Config::default();

    for i in 0..1500 {
        touch(&root.join(format!("d{}/f{:04}.txt", i % 25, i)), b"body");
    }

    index_once(&root, &db, &config);
    let full = rows(&db);
    assert_eq!(full.len(), 1500);

    // Stop only once a snapshot shows a root still in flight, which proves the
    // stop landed mid-run.
    let service = start(&root, &db, &config);
    let deadline = Instant::now() + Duration::from_secs(120);
    let stopped_in_flight = loop {
        assert!(Instant::now() < deadline, "the run never became observable");
        match service.get_status() {
            IndexingStatus::Error(e) => panic!("indexing failed: {}", e),
            IndexingStatus::Preparing { .. } => {}
            IndexingStatus::Running { roots, .. }
                if roots.iter().any(|r| r.phase != RootPhase::Done) =>
            {
                break true;
            }
            _ => break false,
        }
        std::thread::sleep(Duration::from_micros(100));
    };
    assert!(
        stopped_in_flight,
        "the run finished before a stop could land mid-run; grow the fixture"
    );
    service.stop_indexing().unwrap();
    drop(service);
    std::thread::sleep(Duration::from_millis(250));

    let after = rows(&db);
    assert_eq!(
        after.len(),
        1500,
        "an interrupted run must not delete the rows it never got to"
    );
}

/// Let a started run go for `delay`, stop it, and give its tail time to land:
/// the stamp lands past the point a stop can still cut the run short.
fn run_then_stop(service: IndexingService, delay: Duration) {
    std::thread::sleep(delay);
    service.stop_indexing().unwrap();
    drop(service);
    std::thread::sleep(Duration::from_millis(300));
}

#[test]
fn a_stamped_run_has_finished_its_stale_cleanup() {
    // A stamp always means stale cleanup finished. The assertion is one-sided,
    // so timing cannot fail it spuriously, whether the stop landed inside the
    // window or not at all.
    let root = Scratch::dir("stop-stamp");
    let (_db_dir, db) = Scratch::db("stop-stamp-db");
    let mut config = Config::default();
    // No extraction, so the run's whole tail is the stale cleanup to interrupt.
    config.processing.maximum_text_file_size = 0;

    const FILES: usize = 8000;
    let file = |i: usize| root.join(format!("d{}/f{:05}.txt", i % 25, i));
    for i in 0..FILES {
        touch(&file(i), b"body");
    }
    index_once(&root, &db, &config);
    assert_eq!(rows(&db).len(), FILES);
    for i in 0..FILES {
        std::fs::remove_file(file(i)).unwrap();
    }

    let stamped = |db: &Path| -> bool {
        let conn = rusqlite::Connection::open(db).unwrap();
        quicksearch_core::db::repo::get_last_full_index(&conn).is_some()
    };

    // The ladder samples the race. The trailing `None` lets the run finish, so
    // the loop ends stamped however loaded the host: the suite runs in
    // parallel, and under load no stopped run may stamp within 200 ms.
    for delay_ms in [2, 5, 10, 20, 35, 60, 100, 200]
        .map(Some)
        .into_iter()
        .chain([None])
    {
        rusqlite::Connection::open(&db)
            .unwrap()
            .execute("DELETE FROM schema_info WHERE key = 'last_full_index'", [])
            .unwrap();
        assert!(!stamped(&db), "stamp cleared before the run");

        let service = start(&root, &db, &config);
        match delay_ms {
            Some(ms) => run_then_stop(service, Duration::from_millis(ms)),
            None => {
                run_to_idle(&service, || {});
                drop(service);
            }
        }

        if stamped(&db) {
            assert_eq!(
                rows(&db).len(),
                0,
                "delay {:?}: the run stamped itself complete but left stale rows behind",
                delay_ms
            );
            return;
        }
    }
    // The `None` pass is never stopped, so arriving here unstamped is a defect
    // in the stamp itself.
    panic!(
        "a run allowed to finish did not stamp itself; the fixture never \
         exercised the stamp-means-clean invariant"
    );
}

#[test]
fn starting_a_run_claims_the_status_before_it_returns() {
    let root = Scratch::dir("start-claims");
    let (_db_dir, db) = Scratch::db("start-claims-db");
    let config = Config::default();
    touch(&root.join("a.txt"), b"body");

    let service = start(&root, &db, &config);

    // No sleep, no poll. `Preparing` is a claim: it holds the index exactly
    // as `Running` does.
    assert!(
        matches!(service.get_status(), IndexingStatus::Preparing { .. }),
        "status must be claimed synchronously, got {:?}",
        service.get_status()
    );

    let err = service
        .start_indexing(
            vec![root.to_string_lossy().into_owned()],
            db.to_string_lossy().into_owned(),
            config.clone(),
        )
        .unwrap_err();
    assert!(err.contains("already running"), "got: {}", err);

    service.stop_indexing().unwrap();
    drop(service);
    std::thread::sleep(Duration::from_millis(250));
}

#[test]
fn a_wide_tree_indexes_every_file_exactly_once() {
    let root = Scratch::dir("wide");
    let (_db_dir, db) = Scratch::db("wide-db");
    let config = Config::default();

    let count = 900;
    for i in 0..count {
        touch(&root.join(format!("d{}/f{:04}.txt", i % 13, i)), b"body");
    }

    index_once(&root, &db, &config);
    assert_eq!(rows(&db).len(), count, "every file indexed exactly once");

    index_once(&root, &db, &config);
    assert_eq!(rows(&db).len(), count, "and the second run is stable");
}

#[test]
fn two_roots_walk_extract_and_clean_independently() {
    let root_a = Scratch::dir("multi-a");
    let root_b = Scratch::dir("multi-b");
    let (_db_dir, db) = Scratch::db("multi-db");
    let config = Config::default();

    for i in 0..60 {
        touch(
            &root_a.join(format!("a{:03}.txt", i)),
            b"alpha corpus xylophone",
        );
    }
    for i in 0..5 {
        touch(
            &root_b.join(format!("b{:03}.txt", i)),
            b"bravo corpus quagmire",
        );
    }

    index_roots_once(&[&root_a, &root_b], &db, &config);

    assert_eq!(
        count(&db, "SELECT COUNT(*) FROM files"),
        65,
        "both roots fully walked"
    );
    assert_eq!(
        count(&db, "SELECT COUNT(*) FROM files WHERE content_state = 0"),
        0,
        "per-root extraction drained both roots"
    );
    for word in ["xylophone", "quagmire"] {
        assert!(
            hits(&db, word) > 0,
            "content from both roots must be indexed ({})",
            word
        );
    }

    std::fs::remove_file(root_b.join("b000.txt")).unwrap();
    index_roots_once(&[&root_a, &root_b], &db, &config);
    assert_eq!(
        count(&db, "SELECT COUNT(*) FROM files"),
        64,
        "stale row swept across roots"
    );
}

// ---------------------------------------------------------------------------
// Reconciliation without a global path set: classification and stale
// detection are per-directory, so these cover the cases a single directory
// read cannot see.
// ---------------------------------------------------------------------------

#[test]
fn a_deleted_directory_takes_its_whole_subtree_out_of_the_index() {
    let root = Scratch::dir("gone-dir");
    let (_db_dir, db) = Scratch::db("gone-dir-db");
    let config = Config::default();

    touch(&root.join("keep.txt"), b"stays");
    touch(&root.join("doomed/a.txt"), b"goes");
    touch(&root.join("doomed/b.txt"), b"goes");
    touch(&root.join("doomed/deeper/c.txt"), b"goes too");

    index_once(&root, &db, &config);
    assert_eq!(rows(&db).len(), 4, "all four indexed");

    std::fs::remove_dir_all(root.join("doomed")).unwrap();
    index_once(&root, &db, &config);

    assert_eq!(names(&db), vec!["keep.txt"], "the whole subtree is swept");
}

/// Only the target under the pruned directory exercises the alias exemption:
/// its parent is in the sweep's range yet legitimately absent from
/// `seen_dirs`. An out-of-root target is already outside the sweep's range.
#[test]
#[cfg(unix)]
fn a_symlink_target_in_an_unwalked_directory_survives_reindexing() {
    let root = Scratch::dir("alias-root");
    let outside = Scratch::dir("alias-outside");
    let (db_dir, db) = Scratch::db("alias-db");
    let mut config = Config::default();
    config.indexing.follow_symlinks = true;

    touch(&root.join("normal.txt"), b"inside the root");

    // In range, but under a hidden directory the walk prunes.
    let hidden_target = root.join(".pruned/inner.txt");
    touch(&hidden_target, b"only reachable through the link");
    std::os::unix::fs::symlink(&hidden_target, root.join("hidden_link.txt")).unwrap();

    // Out of range entirely.
    let outer_target = outside.join("target.txt");
    touch(&outer_target, b"outside the root entirely");
    std::os::unix::fs::symlink(&outer_target, root.join("outside_link.txt")).unwrap();

    index_once(&root, &db, &config);
    let first = rows(&db);
    assert_eq!(first.len(), 3, "both targets indexed under their own paths");
    assert!(
        first
            .iter()
            .any(|(p, _, _)| p.ends_with(".pruned/inner.txt")),
        "the pruned-directory target is stored under its canonical path"
    );

    index_once(&root, &db, &config);
    assert_eq!(rows(&db), first, "an aliased row must survive a re-index");

    // The other half of the setting: with links off, neither target is indexed.
    let db2 = db_dir.join("links-off.sqlite");
    index_once(&root, &db2, &Config::default());
    let off: Vec<String> = rows(&db2).into_iter().map(|(p, _, _)| p).collect();
    assert_eq!(off.len(), 1, "only the ordinary file: {:?}", off);
    assert!(off[0].ends_with("normal.txt"));
}

/// A target inside the root, so the walk reaches it twice: once by reading its
/// own directory, once by resolving the link. The other symlink tests use
/// targets the walk never reaches directly, so only this one makes a duplicate
/// visit.
///
/// The writer keeps no duplicate-visit set, which would hold a digest per
/// walked file and be the largest thing a run keeps: `insert_file` is
/// `INSERT OR IGNORE` and returns `None` for a row that exists, so a repeat
/// visit produces neither a second row nor a second FTS entry. This test fails
/// if `insert_file` ever stops ignoring conflicts.
#[test]
#[cfg(unix)]
fn a_file_reachable_both_directly_and_through_a_link_gets_one_row() {
    let root = Scratch::dir("alias-dup-root");
    let db_dir = Scratch::dir("alias-dup-db");
    let mut config = Config::default();
    config.indexing.follow_symlinks = true;

    let target = root.join("real/file.txt");
    touch(&target, b"reachable two ways");
    std::os::unix::fs::symlink(&target, root.join("link.txt")).unwrap();

    let db = db_dir.join("links-on.sqlite");
    index_once(&root, &db, &config);
    let on = rows(&db);
    assert_eq!(
        on.len(),
        1,
        "one canonical path, one row, however many ways it was reached: {:?}",
        on
    );
    assert!(
        on[0].0.ends_with("real/file.txt"),
        "stored under the target"
    );

    // The link is not a second document: a duplicated visit that reached the
    // writer twice would tokenize the body twice into a contentless FTS
    // table, where nothing would later collapse the two.
    assert_eq!(hits(&db, "reachable"), 1, "the body is indexed once");

    // Links off: the link is never followed, so the same tree yields the one
    // row.
    let db2 = db_dir.join("links-off.sqlite");
    index_once(&root, &db2, &Config::default());
    let off = rows(&db2);
    assert_eq!(off.len(), 1, "only the real file: {:?}", off);
    assert_eq!(off[0].0, on[0].0, "and the same path as with links on");
}

#[test]
#[cfg(unix)]
fn a_modified_symlink_target_is_updated_not_silently_ignored() {
    let root = Scratch::dir("alias-mod-root");
    let outside = Scratch::dir("alias-mod-outside");
    let (_db_dir, db) = Scratch::db("alias-mod-db");
    let mut config = Config::default();
    config.indexing.follow_symlinks = true;

    let target = outside.join("target.txt");
    touch(&target, b"first body");
    std::os::unix::fs::symlink(&target, root.join("link.txt")).unwrap();

    index_once(&root, &db, &config);
    let before = rows(&db);
    assert_eq!(before.len(), 1);

    std::fs::write(&target, b"second body, quite different").unwrap();
    filetime_set(&target, SystemTime::now() + Duration::from_secs(120));

    index_once(&root, &db, &config);
    let after = rows(&db);
    assert_eq!(after.len(), 1, "still exactly one row");
    assert_eq!(after[0].0, before[0].0, "same path");
    assert_ne!(
        after[0].1, before[0].1,
        "mtime was refreshed, so it was re-read"
    );
}

#[test]
fn overlapping_roots_index_each_file_exactly_once() {
    let outer = Scratch::dir("overlap-outer");
    let (_db_dir, db) = Scratch::db("overlap-db");
    let config = Config::default();

    let inner = outer.join("inner");
    touch(&outer.join("top.txt"), b"in the outer root only");
    touch(&inner.join("shared.txt"), b"reachable from both roots");
    touch(&inner.join("also.txt"), b"likewise");

    index_roots_once(&[&outer, &inner], &db, &config);

    let all = rows(&db);
    assert_eq!(all.len(), 3, "three files, however many roots reach them");
    let shared: Vec<&(String, i64, i64)> = all
        .iter()
        .filter(|(p, _, _)| p.ends_with("shared.txt"))
        .collect();
    assert_eq!(
        shared.len(),
        1,
        "the doubly-reachable file has exactly one row"
    );

    index_roots_once(&[&outer, &inner], &db, &config);
    assert_eq!(rows(&db), all, "a second overlapping run changes nothing");
}

#[test]
#[cfg(unix)]
fn a_directory_that_becomes_unreadable_deletes_nothing() {
    use std::os::unix::fs::PermissionsExt;

    let root = Scratch::dir("locked-later");
    let (_db_dir, db) = Scratch::db("locked-later-db");
    let config = Config::default();

    touch(&root.join("open.txt"), b"always readable");
    let vault = root.join("vault");
    touch(&vault.join("secret.txt"), b"readable for now");
    touch(&vault.join("deeper/also.txt"), b"and this one");

    index_once(&root, &db, &config);
    let before = rows(&db);
    assert_eq!(before.len(), 3, "all three indexed while readable");

    std::fs::set_permissions(&vault, std::fs::Permissions::from_mode(0o000)).unwrap();
    index_once(&root, &db, &config);
    let after = rows(&db);
    std::fs::set_permissions(&vault, std::fs::Permissions::from_mode(0o755)).ok();

    assert_eq!(after, before, "an unreadable directory is not an empty one");
}

// ---------------------------------------------------------------------------
// Inline extraction: the walk finishes files whose head is the whole file.
// `hash_length` decides how much the walk reads, so 0 leaves nothing to
// inline and serves as the control the optimised path must match.
// ---------------------------------------------------------------------------

type ContentRow = (String, i64, Option<String>, Option<i64>);

fn content_rows(db: &Path) -> Vec<ContentRow> {
    let conn = rusqlite::Connection::open(db).unwrap();
    let mut stmt = conn
        .prepare(
            "SELECT f.parent || f.name, f.content_state, ff.reason, LENGTH(d.text_zstd)
               FROM files f
               LEFT JOIN documents_text d ON d.file_id = f.id
               LEFT JOIN failed_files ff ON ff.file_id = f.id
              ORDER BY f.parent, f.name",
        )
        .unwrap();
    let out = stmt
        .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))
        .unwrap()
        .map(|r| r.unwrap())
        .collect();
    out
}

fn stored_text(db: &Path, suffix: &str) -> Option<String> {
    let conn = rusqlite::Connection::open(db).unwrap();
    let blob: Option<Vec<u8>> = conn
        .query_row(
            "SELECT d.text_zstd FROM documents_text d
               JOIN files f ON f.id = d.file_id
              WHERE f.parent || f.name LIKE '%' || ?1",
            [suffix],
            |r| r.get(0),
        )
        .ok();
    blob.map(|b| String::from_utf8(zstd::decode_all(&b[..]).unwrap()).unwrap())
}

fn seed_mixed_tree(root: &Path) {
    let big = "lorem ipsum dolor sit amet ".repeat(600); // ~16 KiB, past any head
    touch(
        &root.join("small.txt"),
        b"a small plaintext body with xylophone in it",
    );
    touch(&root.join("large.txt"), big.as_bytes());
    touch(&root.join("empty.txt"), b"");
    // The NUL fails the binary guard, and the FF FE pair is not at offset 0,
    // so it is no BOM.
    touch(&root.join("bad.txt"), &[0x68, 0x69, 0xff, 0xfe, 0x00, 0x41]);
    // NUL soup: no extension table, magic, or text sniff claims it.
    touch(
        &root.join("blob.bin"),
        &[0x00, 0x01, 0x02, 0xfd, 0xfe, 0xff],
    );
    touch(
        &root.join("nested/deep/note.md"),
        b"# heading\n\nquagmire body text\n",
    );
}

#[test]
fn inline_extraction_produces_an_identical_index_to_the_two_pass_path() {
    let root = Scratch::dir("inline-equiv");
    let db_dir = Scratch::dir("inline-equiv-db");
    seed_mixed_tree(&root);

    // Control: hash_length 0 => empty head => nothing can be inlined.
    let mut control = Config::default();
    control.processing.hash_length = 0;
    let db_control = db_dir.join("control.sqlite");
    index_once(&root, &db_control, &control);

    // Optimised: the default head covers every small file in the tree.
    let optimised = Config::default();
    let db_opt = db_dir.join("optimised.sqlite");
    index_once(&root, &db_opt, &optimised);

    assert_eq!(
        content_rows(&db_control),
        content_rows(&db_opt),
        "inlining during the walk must not change a single indexed byte"
    );

    // And the bodies themselves round-trip identically, not just their lengths.
    for f in ["small.txt", "large.txt", "note.md"] {
        assert_eq!(
            stored_text(&db_control, f),
            stored_text(&db_opt, f),
            "stored body differs for {}",
            f
        );
    }
}

#[test]
fn the_head_boundary_decides_inlining_without_changing_the_result() {
    let root = Scratch::dir("inline-boundary");
    let (_db_dir, db) = Scratch::db("inline-boundary-db");

    let mut config = Config::default();
    config.processing.hash_length = 64;
    let at = "x".repeat(64);
    let past = "y".repeat(65);
    touch(&root.join("at.txt"), at.as_bytes());
    touch(&root.join("past.txt"), past.as_bytes());
    index_once(&root, &db, &config);

    assert_eq!(
        count(&db, "SELECT COUNT(*) FROM files WHERE content_state != 1"),
        0,
        "both sides of the boundary end up extracted"
    );

    assert_eq!(stored_text(&db, "at.txt").as_deref(), Some(at.as_str()));
    assert_eq!(stored_text(&db, "past.txt").as_deref(), Some(past.as_str()));
}

/// The content state and failure reason of the one file ending in `suffix`.
fn failure(db: &Path, suffix: &str) -> (i64, Option<String>) {
    rusqlite::Connection::open(db)
        .unwrap()
        .query_row(
            "SELECT f.content_state, ff.reason FROM files f \
               LEFT JOIN failed_files ff ON ff.file_id = f.id \
              WHERE f.parent || f.name LIKE '%' || ?1",
            [suffix],
            |r| Ok((r.get(0)?, r.get(1)?)),
        )
        .unwrap()
}

#[test]
fn undecodable_small_files_are_reported_as_failures_not_silently_skipped() {
    let root = Scratch::dir("inline-badutf8");
    let (_db_dir, db) = Scratch::db("inline-badutf8-db");

    // The NUL keeps this undecodable: without it these bytes decode as
    // windows-1252 and the test asserts nothing.
    touch(&root.join("bad.txt"), &[0x68, 0x00, 0x69, 0xff]);
    index_once(&root, &db, &Config::default());

    let (state, msg) = failure(&db, "bad.txt");
    assert_eq!(state, 2, "undecodable content is FAILED, not DONE or NA");
    assert!(
        msg.unwrap_or_default().contains("bad.txt"),
        "the failure names the file"
    );
}

#[test]
fn an_unreadable_legacy_office_file_fails_with_a_reason() {
    let root = Scratch::dir("legacy-doc");
    let (_db_dir, db) = Scratch::db("legacy-doc-db");

    touch(
        &root.join("broken.doc"),
        b"D0CF11E0 this is not really a compound file",
    );
    index_once(&root, &db, &Config::default());

    let (state, msg) = failure(&db, "broken.doc");
    assert_eq!(
        state, 2,
        "an unreadable .doc is FAILED, not DONE-with-no-text"
    );
    let msg = msg.unwrap_or_default();
    assert!(msg.contains("broken.doc"), "names the file: {msg}");
    assert!(msg.contains("compound file"), "says what went wrong: {msg}");
}

/// The fix keeps size and mtime identical, so the walk sees an unchanged file
/// and only the run-start retry of failed files can rescue it.
#[test]
fn a_failed_file_is_retried_on_the_next_run() {
    let root = Scratch::dir("retry-failed");
    let (_db_dir, db) = Scratch::db("retry-failed-db");

    let flaky = root.join("flaky.txt");
    // The NUL keeps this undecodable (see the inline-badutf8 test).
    touch(&flaky, &[0x68, 0x00, 0x69, 0xff]);
    index_once(&root, &db, &Config::default());

    let probe = |db: &Path| -> (i64, i64) {
        let conn = rusqlite::Connection::open(db).unwrap();
        conn.query_row(
            "SELECT f.content_state,
                    (SELECT COUNT(*) FROM failed_files) FROM files f \
              WHERE f.name = 'flaky.txt'",
            [],
            |r| Ok((r.get(0).unwrap(), r.get(1).unwrap())),
        )
        .unwrap()
    };
    assert_eq!(probe(&db), (2, 1), "the first run records the failure");

    let mtime = std::fs::metadata(&flaky).unwrap().modified().unwrap();
    touch(&flaky, b"hiya");
    std::fs::File::options()
        .write(true)
        .open(&flaky)
        .unwrap()
        .set_times(std::fs::FileTimes::new().set_modified(mtime))
        .unwrap();

    index_once(&root, &db, &Config::default());
    assert_eq!(
        probe(&db),
        (1, 0),
        "the retry re-extracted the fixed file and cleared the record"
    );
    assert_eq!(stored_text(&db, "flaky.txt").as_deref(), Some("hiya"));
}

#[test]
fn extensionless_text_files_are_indexed() {
    let root = Scratch::dir("extless");
    let (_db_dir, db) = Scratch::db("extless-db");

    touch(
        &root.join("README"),
        b"QuickSearch indexes zanzibar contents.\n",
    );
    touch(&root.join("Makefile"), b"all:\n\tcargo build --release\n");
    touch(&root.join("go.sum"), b"example.com/x v1.0.0 h1:abcdef=\n");
    touch(&root.join("blob"), &[0x00, 0x01, 0xfe, 0xff]);
    index_once(&root, &db, &Config::default());

    let conn = rusqlite::Connection::open(&db).unwrap();
    let state_of = |name: &str| -> i64 {
        conn.query_row(
            "SELECT content_state FROM files WHERE parent || name LIKE '%' || ?1",
            [name],
            |r| r.get(0),
        )
        .unwrap()
    };
    for name in ["README", "Makefile", "go.sum"] {
        assert_eq!(state_of(name), 1, "{} should be content-indexed", name);
    }
    assert_eq!(state_of("blob"), 3, "binary blob stays not-applicable");
    drop(conn);

    assert_eq!(
        stored_text(&db, "README").as_deref(),
        Some("QuickSearch indexes zanzibar contents.\n"),
        "the stored body round-trips"
    );
}

/// `stored_text` decodes the zstd sidecar with `String::from_utf8`, so a
/// `Some` result *is* the storage-is-UTF-8 assertion.
#[test]
fn utf16_files_are_stored_as_utf8() {
    let root = Scratch::dir("charset");
    let (_db_dir, db) = Scratch::db("charset-db");

    let reg_src =
        "Windows Registry Editor Version 5.00\r\n\r\n[HKEY_CURRENT_USER\\Software\\Xylograph]\r\n";
    let mut reg_body = vec![0xFF, 0xFE];
    reg_body.extend(reg_src.encode_utf16().flat_map(|u| u.to_le_bytes()));
    touch(&root.join("export.reg"), &reg_body);

    let mut extless = vec![0xFF, 0xFE];
    extless.extend(
        "utf16 notes about quokkas"
            .encode_utf16()
            .flat_map(|u| u.to_le_bytes()),
    );
    touch(&root.join("NOTES16"), &extless);

    touch(
        &root.join("legacy.txt"),
        b"un caf\xe9 tr\xe8s agr\xe9able pr\xe8s du mus\xe9e",
    );
    index_once(&root, &db, &Config::default());

    assert_eq!(stored_text(&db, "export.reg").as_deref(), Some(reg_src));
    assert_eq!(
        stored_text(&db, "NOTES16").as_deref(),
        Some("utf16 notes about quokkas")
    );
    assert_eq!(
        stored_text(&db, "legacy.txt").as_deref(),
        Some("un café très agréable près du musée")
    );
}

/// Covers both extraction paths: a small file the walk inlines, and one past
/// `hash_length` that the content pass opens.
#[test]
fn rtf_files_are_extracted() {
    let root = Scratch::dir("rtf");
    let (_db_dir, db) = Scratch::db("rtf-db");

    touch(
        &root.join("small.rtf"),
        br"{\rtf1\ansi Meeting notes about the pangolin budget.}",
    );
    let big_body = format!(
        r"{{\rtf1\ansi {}}}",
        r"paragraphs about the pangolin budget \par ".repeat(400)
    );
    assert!(big_body.len() > 8192, "must exceed the default head");
    touch(&root.join("big.rtf"), big_body.as_bytes());
    index_once(&root, &db, &Config::default());

    for name in ["small.rtf", "big.rtf"] {
        let text = stored_text(&db, name).unwrap_or_else(|| panic!("{} has no stored text", name));
        assert!(
            text.contains("pangolin budget"),
            "{}: {:?}",
            name,
            &text[..text.len().min(80)]
        );
        assert!(!text.contains(r"\rtf"), "{} stored control words", name);
    }
}

#[test]
fn the_extraction_denominator_counts_only_files_that_need_text() {
    let root = Scratch::dir("denominator");
    let (_db_dir, db) = Scratch::db("denominator-db");

    // `big.txt` exceeds `hash_length`, so it is the only row the content pass
    // opens; the NUL-bearing seven are claimed by nothing.
    for name in ["a.txt", "b.json"] {
        touch(&root.join(name), b"body bytes with no magic");
    }
    touch(&root.join("big.txt"), &vec![b'z'; 32 * 1024]);
    for name in ["d.mp4", "e.zip", "f.bin", "g.exe", "h.iso", "i.so", "j"] {
        touch(&root.join(name), b"\x00\x01body bytes\x00");
    }

    let config = Config::default();
    index_once(&root, &db, &config);

    let in_state = |state: i64| {
        count(
            &db,
            &format!("SELECT COUNT(*) FROM files WHERE content_state = {}", state),
        )
    };
    assert_eq!(in_state(0), 0, "a finished run leaves nothing pending");
    assert_eq!(in_state(1), 3, "the claimed files have text");
    assert_eq!(
        in_state(3),
        7,
        "the rest are NA, and were NA from the walk on"
    );

    // The calls the production path makes to fill
    // `RootProgress::extract_total`, in production order, against the finished
    // index: a ten-file tree finishes between two polls, so the live status
    // cannot be sampled.
    let conn = quicksearch_core::db::open_existing(db.to_str().unwrap(), true).unwrap();
    let range = RootRange::for_root(root.to_str().unwrap());
    mark_oversize_pending_na(&conn, &range, &config).unwrap();
    let scope = count_extract_scope(&conn, &range, &config).unwrap();
    assert_eq!(
        (scope.pending, scope.already_done),
        (0, 3),
        "extract_total is the searchable set, not the file count"
    );
    assert_eq!(scope.pending + scope.already_done, 3);
    drop(conn);
}

#[test]
fn an_empty_file_is_done_with_no_snippet_sidecar() {
    let root = Scratch::dir("inline-empty");
    let (_db_dir, db) = Scratch::db("inline-empty-db");

    touch(&root.join("empty.txt"), b"");
    index_once(&root, &db, &Config::default());

    let conn = rusqlite::Connection::open(&db).unwrap();
    let (state, sidecars): (i64, i64) = conn
        .query_row(
            "SELECT f.content_state, (SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id)
               FROM files f WHERE f.parent || f.name LIKE '%empty.txt'",
            [],
            |r| Ok((r.get(0)?, r.get(1)?)),
        )
        .unwrap();
    assert_eq!(state, 1, "an empty file is extracted, not failed");
    assert_eq!(sidecars, 0, "no zstd frame for an empty body");
}

#[test]
fn the_content_extension_filter_still_excludes_small_text_files() {
    let root = Scratch::dir("inline-filter");
    let (_db_dir, db) = Scratch::db("inline-filter-db");

    let mut config = Config::default();
    config.indexing.content_extensions = vec!["md".into()];
    touch(&root.join("kept.md"), b"kept quagmire body");
    touch(&root.join("skipped.txt"), b"skipped xylophone body");
    index_once(&root, &db, &config);

    for (path, _, state) in rows(&db) {
        if path.ends_with("kept.md") {
            assert_eq!(state, 1, "an allowed extension is extracted");
        } else {
            assert_eq!(state, 3, "a filtered extension is NA, never inlined");
        }
    }
    assert_eq!(
        stored_text(&db, "skipped.txt"),
        None,
        "no body stored for a filtered file"
    );
}

/// Both layouts index the same files the same way; only the FTS table
/// differs, and every file with text keeps its body for verification.
#[test]
fn either_index_layout_stores_and_finds_the_same_files() {
    for compact in [true, false] {
        let root = Scratch::dir("layout");
        let (_db_dir, db) = Scratch::db("layout-db");
        let mut config = Config::default();
        config.processing.compact_text_index = compact;
        touch(&root.join("small.txt"), b"searchable xylophone body");
        touch(&root.join("other.txt"), b"nothing to see");
        index_once(&root, &db, &config);

        assert_eq!(
            count(&db, "SELECT COUNT(*) FROM documents_text"),
            2,
            "every body is stored (compact = {compact})"
        );
        assert_eq!(hits(&db, "xylophone"), 1, "compact = {compact}");
        let detail_none = count(
            &db,
            "SELECT COUNT(*) FROM sqlite_master \
             WHERE name = 'searchabletext' AND sql LIKE '%detail=none%'",
        );
        assert_eq!(detail_none == 1, compact);
    }
}

// ---------------------------------------------------------------------------
// How the stall tests measure interleaving; `observe_overlap` collects the
// counters.
//
// A stall is counted in work, not milliseconds: while the heavy root's
// content pass runs, how many files the light root's walk was drained of,
// against how many heavy rows landed. The same writer loop advances both
// counters (`service_walking` / `service_extracting` in
// `indexing/pipeline.rs`), each root's turn bounded by one slice, so their
// ratio is the interleaving, and a slow host freezes both together.
//
// `writer_turn_slice_ms = 0` makes the verdict arithmetic rather than a
// measurement of the host: with no time in a turn, a writer round is one
// bounded piece of work per root (`service_walking` runs one `batch_size`
// batch, `store_extracted` consumes one row). The round is the unit, the
// interleave floor is `BATCH : 1` on any host, and load can only raise it.
// The ratio is also uniform across the pass, so the observed window may be a
// partial sample of it.
//
// The bound: a serialised writer, which reads the heavy batch itself and
// drains nobody meanwhile, manages one batch of each per round, so it obeys
// `light < heavy + BATCH`. Each test demands
// `light_drained ≥ MIN_INTERLEAVE × (heavy_stored + BATCH)`: three times a
// bound the serialised design cannot reach, against a floor of 16:1.
//
// `MIN_ROWS_SAMPLED`: a window of `n` rows drains `BATCH × n` light files at
// the floor and must beat `MIN_INTERLEAVE × (n + BATCH)`, so under four rows
// the constant `+ BATCH` term decides the comparison instead of the design.
// ---------------------------------------------------------------------------

/// What one watch of a heavy/light overlap saw; see [`observe_overlap`].
struct Overlap {
    /// Light files drained and heavy rows stored, across the window in which
    /// the heavy root extracted while the light root walked.
    light_drained: usize,
    heavy_stored: usize,
    /// The heavy root's `extract_total` and pool size, for the fixture guards.
    heavy_pending: usize,
    heavy_pool: usize,
    /// Publications in which the heavy root's row count moved. Reported, never
    /// asserted: it is `min(writer rounds, watcher polls)`, there to make a
    /// surprising run legible.
    heavy_steps: usize,
    /// Longest this watcher went between polls: distinguishes a writer that
    /// gulped the pass in one turn from a watcher descheduled past it.
    worst_gap: Duration,
}

/// Watch a two-root run until the heavy root has finished extracting, report
/// how the two counters moved while both were in flight, then stop the run.
/// Removing the fixture is the caller's job.
///
/// The window opens on the first snapshot holding both roots in flight and
/// closes when the heavy root leaves `Extracting`, not when the light root
/// finishes walking, which is a race between two rates with no fixed ratio
/// across hosts. A light walk that ends early just stops contributing,
/// understating the interleaving, never overstating it.
/// Panics if the window never opened.
fn observe_overlap(service: &IndexingService, heavy_tag: &str, light_tag: &str) -> Overlap {
    let mut opened: Option<(usize, usize)> = None; // (light.walked, heavy.extracted)
    let mut last = (0usize, 0usize);
    let mut heavy_pending = 0usize;
    let mut heavy_pool = 0usize;
    let mut heavy_steps = 0usize;
    let mut stepped_at = 0usize;
    let mut worst_gap = Duration::ZERO;
    let mut polled_at = Instant::now();
    // Published (heavy, light) phase pairs, kept only for the failure diagnosis.
    let mut phases: Vec<(RootPhase, RootPhase)> = Vec::new();
    let deadline = Instant::now() + Duration::from_secs(120);
    while Instant::now() < deadline {
        let mut in_window = false;
        worst_gap = worst_gap.max(polled_at.elapsed());
        polled_at = Instant::now();
        match service.get_status() {
            IndexingStatus::Running { roots, .. } => {
                let heavy_p = roots.iter().find(|r| r.root.contains(heavy_tag));
                let light_p = roots.iter().find(|r| r.root.contains(light_tag));
                if let (Some(h), Some(l)) = (heavy_p, light_p) {
                    if phases.last() != Some(&(h.phase, l.phase)) {
                        phases.push((h.phase, l.phase));
                    }
                    // Opening takes both roots in flight; staying open takes
                    // only the heavy root's pass.
                    in_window = if opened.is_none() {
                        h.phase == RootPhase::Extracting && l.phase == RootPhase::Walking
                    } else {
                        h.phase == RootPhase::Extracting
                    };
                    if in_window {
                        last = (l.walked, h.extracted);
                        if opened.is_none() {
                            opened = Some(last);
                            stepped_at = h.extracted;
                        }
                        // A fresh publication, not a fresh poll.
                        if h.extracted > stepped_at {
                            stepped_at = h.extracted;
                            heavy_steps += 1;
                        }
                        if let Some(total) = h.extract_total {
                            heavy_pending = total;
                        }
                        heavy_pool = h.total_workers;
                    }
                }
            }
            // Claimed but not yet walking; nothing to sample yet.
            IndexingStatus::Preparing { .. } => {}
            IndexingStatus::Error(e) => panic!("indexing failed: {}", e),
            _ => break,
        }
        // Both phases are monotone, so a closed window will not reopen.
        if opened.is_some() && !in_window {
            break;
        }
        // Poll finer than a writer round, or this loop sets the window's edges.
        std::thread::sleep(Duration::from_micros(500));
    }
    service.stop_indexing().unwrap();

    let Some((light_open, heavy_open)) = opened else {
        panic!(
            "never observed the heavy root extracting while the light root walked. \
             Published (heavy, light) phases: {:?}. An empty list means neither \
             root matched the tags {:?}/{:?}; a list with no heavy Extracting in \
             it means the heavy root's content pass began and ended between two \
             status publications, so lower `writer_turn_slice_ms` until a writer \
             round is shorter than that pass.",
            phases, heavy_tag, light_tag
        );
    };
    Overlap {
        light_drained: last.0 - light_open,
        heavy_stored: last.1 - heavy_open,
        heavy_pending,
        heavy_pool,
        heavy_steps,
        worst_gap,
    }
}

/// The writer's batch in the stall tests: the bound is arithmetic in it, and a
/// default 500-file round is a coarse enough publish interval to look like a
/// stall on its own.
const BATCH: usize = 16;
/// A wide tree of tiny files, inlined by their walk workers, so the light root
/// has no extraction phase of its own to confuse the window with, and wide
/// enough that it is still walking when the heavy pass starts.
const LIGHT_FILES: usize = 16_000;
/// Light files drained per (heavy row + batch); derived above `Overlap`.
const MIN_INTERLEAVE: usize = 3;
/// Fewest heavy rows a window needs; derived above `Overlap`.
const MIN_ROWS_SAMPLED: usize = 4;

/// Index `heavy_files` copies of `body`, spread over `heavy_dirs` directories
/// and extracted on `workers` threads, beside [`LIGHT_FILES`] one-byte files,
/// and watch the overlap under `config` plus the writer settings derived above
/// `Overlap`. Asserts the fixture guards, each of which silently costs a
/// factor of the margin if it stops holding.
///
/// Removes the fixture before asserting, unlike the rest of this file: it is
/// generated and identical every run, and leaving tens of MB in a RAM-backed
/// /tmp behind a failure is itself a reason to fail.
#[allow(clippy::too_many_arguments)]
fn watch_heavy_beside_light(
    tag: &str,
    heavy_files: usize,
    heavy_dirs: usize,
    body: &[u8],
    ext: &str,
    workers: usize,
    mut config: Config,
) -> Overlap {
    let (heavy_tag, light_tag) = (format!("{}-heavy", tag), format!("{}-light", tag));
    let heavy = Scratch::dir(&heavy_tag);
    for i in 0..heavy_files {
        touch(
            &heavy.join(format!("d{}/big{:04}.{}", i % heavy_dirs, i, ext)),
            body,
        );
    }
    let light = Scratch::dir(&light_tag);
    for i in 0..LIGHT_FILES {
        touch(&light.join(format!("d{}/f{:05}.txt", i % 60, i)), b"x");
    }
    let (db_dir, db) = Scratch::db(&format!("{}-db", tag));
    let roots = vec![
        heavy.to_string_lossy().into_owned(),
        light.to_string_lossy().into_owned(),
    ];

    config.processing.batch_size = BATCH;
    config.processing.writer_turn_slice_ms = 0;
    // `root_workers` is keyed by the `indexing_paths` spelling; both sides
    // canonicalize before matching.
    config.paths.indexing_paths = roots.clone();
    config
        .indexing
        .root_workers
        .insert(roots[0].clone(), workers);
    // The default WAL cap is far above anything these runs write; that stops
    // being true if a fixture grows by an order of magnitude.

    let service = IndexingService::new();
    service
        .start_indexing(roots, db.to_string_lossy().into_owned(), config)
        .unwrap();
    let seen = observe_overlap(&service, &heavy_tag, &light_tag);
    drop(service);
    drop((heavy, light, db_dir));

    assert_eq!(
        seen.heavy_pool, workers,
        "the heavy root must extract on the {} workers root_workers asked for",
        workers
    );
    assert_eq!(
        seen.heavy_pending, heavy_files,
        "every heavy file must reach the content pass; one inlined by its walk \
         worker never produces an extraction phase to overlap with"
    );
    assert!(
        seen.heavy_stored >= MIN_ROWS_SAMPLED,
        "only {} of {} heavy rows landed inside the observed window, fewer than \
         the {} a verdict needs (worst watcher gap {:?}, rows seen over {} \
         rounds) — a gap near the pass's own length means this watcher was \
         descheduled past it, not that the writer gulped it",
        seen.heavy_stored,
        heavy_files,
        MIN_ROWS_SAMPLED,
        seen.worst_gap,
        seen.heavy_steps
    );
    seen
}

/// A slow root does not stall the others: the single writer thread keeps
/// draining the light root's walk while the heavy root extracts.
///
/// The assertion is about stalls, not throughput: writing is serial by
/// construction (one SQLite connection), so a wall-clock comparison would
/// mostly measure the machine. The measurement and bound are derived above
/// [`Overlap`].
#[test]
fn a_heavy_root_does_not_stall_a_light_one() {
    // Few files, each big enough that reading one is real work, with a small
    // `maximum_text_size` so the cost lands in extraction: the bound grows
    // with the row count, while what the light root drains does not. Sized
    // for the guards: enough that the rows arrive over separate writer
    // rounds, cheap enough that a loaded runner still finishes in seconds.
    // RTF, because its extractor reads and parses the whole file, where
    // plaintext stops at twice `maximum_text_size`.
    const HEAVY_FILES: usize = 12;
    // ~2 MiB each, 24 MB total.
    let body = format!(
        "{{\\rtf1\\ansi {}}}",
        "sphinx of black quartz judge my vow ".repeat(58_000)
    );

    let mut config = Config::default();
    config.processing.maximum_text_size = 1024;
    // Above the heavy files, or `mark_oversize_pending_na` writes them off as
    // N/A before the pass starts and there is no extraction phase at all.
    config.processing.maximum_text_file_size = 4 * 1024 * 1024;
    // One extraction thread: with the default four the pass is four times
    // shorter, and so is the margin.
    let seen = watch_heavy_beside_light("stall", HEAVY_FILES, 4, body.as_bytes(), "rtf", 1, config);

    eprintln!(
        "light files drained while the heavy root extracted: {} against {} heavy \
         rows (batch {}) landing over {} rounds, worst watcher gap {:?} — \
         {}x the {}x required; the \
         serialised design cannot exceed 1x",
        seen.light_drained,
        seen.heavy_stored,
        BATCH,
        seen.heavy_steps,
        seen.worst_gap,
        seen.light_drained / (seen.heavy_stored + BATCH),
        MIN_INTERLEAVE
    );
    assert!(
        seen.light_drained >= MIN_INTERLEAVE * (seen.heavy_stored + BATCH),
        "the light root was drained of only {} files while the heavy root landed \
         {} rows; one batch of each per round is all a writer that extracts \
         inline can manage, so anything near {} means the extraction is back on \
         the writer thread",
        seen.light_drained,
        seen.heavy_stored,
        seen.heavy_stored + BATCH
    );
}

/// [`a_heavy_root_does_not_stall_a_light_one`] for the cost that test keeps
/// small: the writer's own tokenising. Each heavy row carries the default
/// 256 KiB of text, and its FTS5 trigram insert runs on the writer thread
/// inside the transaction, so only the turn's slice bounds it. An extraction
/// turn that writes everything it finds brings the ratio below one.
#[test]
fn a_heavy_root_does_not_stall_a_light_one_at_the_writer() {
    // Over the walk's inline threshold, with the full 256 KiB of stored text,
    // so the tokenising is what is measured.
    const HEAVY_FILES: usize = 32;
    let body = "sphinx of black quartz judge my vow ".repeat(9_000);

    // Four readers, so the ready channel is full when the writer's turn comes.
    let seen = watch_heavy_beside_light(
        "stall-writer",
        HEAVY_FILES,
        8,
        body.as_bytes(),
        "txt",
        4,
        Config::default(),
    );

    eprintln!(
        "light files drained while the heavy root tokenised: {} against {} heavy \
         rows (batch {}) landing over {} rounds, worst watcher gap {:?} — \
         {}x the {}x required",
        seen.light_drained,
        seen.heavy_stored,
        BATCH,
        seen.heavy_steps,
        seen.worst_gap,
        seen.light_drained / (seen.heavy_stored + BATCH),
        MIN_INTERLEAVE
    );
    assert!(
        seen.light_drained >= MIN_INTERLEAVE * (seen.heavy_stored + BATCH),
        "the light root was drained of only {} files while the heavy root landed \
         {} rows; an extraction turn is writing to the end of its batch again \
         instead of yielding at its slice",
        seen.light_drained,
        seen.heavy_stored
    );
}

/// A text-heavy tree: the FTS index fills the log and dominates the database,
/// so a log measured against the database size means something.
fn seed_text_tree(tag: &str) -> Scratch {
    let root = Scratch::dir(tag);
    let body: Vec<u8> = "sphinx of black quartz judge my vow "
        .repeat(200)
        .into_bytes();
    for i in 0..4000 {
        touch(&root.join(format!("d{}/f{:05}.txt", i % 40, i)), &body);
    }
    root
}

/// The write-ahead log does not grow for the length of a run. Autocheckpoint
/// can reset the log only at an instant no reader holds a read mark, and a run
/// keeps a reader per root querying continuously (the prompting case: a
/// 12.5 GiB index carrying a 21.6 GiB log).
///
/// The assertion is about the peak while running: `stop_indexing` and the
/// post-run maintenance both truncate the log on the way out, so a reading
/// taken afterwards proves nothing.
#[test]
fn the_wal_stays_bounded_during_a_run() {
    let root = seed_text_tree("wal-bound");
    let (_db_dir, db) = Scratch::db("wal-bound-db");
    let mut config = Config::default();
    // The floor `MINIMUM_WAL_SIZE` clamps to, so the cap is exercised many
    // times rather than once at the end.
    config.processing.maximum_wal_size = 16 * 1024 * 1024;

    let service = start(&root, &db, &config);

    let mut peak = 0u64;
    let mut checkpointed = false;
    let mut last = 0u64;
    let deadline = Instant::now() + Duration::from_secs(120);
    while Instant::now() < deadline {
        let len = wal_bytes(&db);
        peak = peak.max(len);
        // Vacuity guard: without a mid-run checkpoint the bound could be met
        // by the fixture being too small.
        if len + 1024 * 1024 < last {
            checkpointed = true;
        }
        last = len;
        match service.get_status() {
            IndexingStatus::Running { .. } | IndexingStatus::Preparing { .. } => {}
            IndexingStatus::Error(e) => panic!("indexing failed: {}", e),
            _ => break,
        }
        std::thread::sleep(Duration::from_millis(2));
    }
    let idle_by = Instant::now() + Duration::from_secs(120);
    while Instant::now() < idle_by && !matches!(service.get_status(), IndexingStatus::Idle) {
        std::thread::sleep(Duration::from_millis(10));
    }
    let after = wal_bytes(&db);
    drop(service);

    eprintln!("peak WAL during the run: {} bytes", peak);
    assert!(
        checkpointed,
        "the log never shrank mid-run; the fixture is not exercising the cap"
    );
    // Generously above the cap: the check runs between rounds, so a round's
    // commits can land on top, and a checkpoint that loses its lock race
    // defers to the next cap of growth.
    assert!(
        peak < 96 * 1024 * 1024,
        "the log peaked at {} bytes against a 16 MiB cap",
        peak
    );
    assert_eq!(after, 0, "the optimize pass leaves an empty log behind");
}

/// A stopped run still runs the optimize pass: a run cut short is when the log
/// is largest, and nothing else lands it.
///
/// Asserted through `repo::optimize_count`, a latch, because the transient
/// `Optimizing` status is gone too fast to poll for.
#[test]
fn a_stopped_run_is_still_optimized() {
    let root = seed_text_tree("stop-optimize");
    let (db_dir, db) = Scratch::db("stop-optimize-db");
    let dir_key = db_dir.to_string_lossy().into_owned();

    // Per-directory, so a sibling test optimizing its own scratch index on
    // another thread cannot satisfy this.
    let before = quicksearch_core::db::repo::optimize_count(&dir_key);

    let service = start(&root, &db, &Config::default());
    let deadline = Instant::now() + Duration::from_secs(120);
    while Instant::now() < deadline {
        if wal_bytes(&db) > 512 * 1024 {
            break;
        }
        if let IndexingStatus::Error(e) = service.get_status() {
            panic!("indexing failed: {}", e);
        }
        std::thread::sleep(Duration::from_millis(2));
    }
    service.request_stop();
    run_to_idle(&service, || {});

    assert_eq!(
        quicksearch_core::db::repo::optimize_count(&dir_key),
        before + 1,
        "a stopped run must still run PRAGMA optimize against its index"
    );
    assert_eq!(
        wal_bytes(&db),
        0,
        "the optimize pass must land the stopped run's log"
    );
    drop(service);
}

/// High-byte binaries (protobuf and friends: no NUL, no control bytes) clear
/// the binary guard; the tightened sniff must still refuse them. The row must
/// survive in `files` while acquiring no `documents_text` sidecar and no
/// `failed_files` entry — no text is not a failure — and the `.txt` twin with
/// the same bytes must still extract.
#[test]
fn high_byte_binaries_are_listed_but_not_text_extracted() {
    let root = Scratch::dir("sniff-binary");
    let (_db_dir, db) = Scratch::db("sniff-binary-db");

    // Head of a real protobuf-framed GPS log: varint framing around ASCII
    // NMEA sentences. No NUL, no control-byte density — it clears the binary
    // guard on its own.
    let mut pb = b"\x10\n\x02v1\x10\x01\x18\xe2\xe3\xfc\xd3\x9d\xca\x97\xe4\x189\x08".to_vec();
    pb.extend_from_slice(b"\x12*$GNGGA,181558.00,,,,,0,00,99.99,,,,,,*78\r\n");
    assert!(!pb.contains(&0u8), "fixture must not trip the NUL guard");

    let legacy = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.";

    touch(&root.join("rtk.pb"), &pb);
    touch(&root.join("legacy.txt"), legacy);
    touch(&root.join("notes.md"), b"ordinary utf-8 prose");
    index_once(&root, &db, &Config::default());

    let conn = rusqlite::Connection::open(&db).unwrap();
    let probe = |suffix: &str| -> (i64, i64, i64) {
        conn.query_row(
            "SELECT f.content_state,
                    (SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id),
                    (SELECT COUNT(*) FROM failed_files x WHERE x.file_id = f.id)
               FROM files f WHERE f.parent || f.name LIKE '%' || ?1",
            [suffix],
            |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
        )
        .unwrap_or_else(|e| panic!("{suffix} must be indexed: {e}"))
    };

    // 3 = not applicable.
    assert_eq!(
        probe("rtk.pb"),
        (3, 0, 0),
        "a high-byte binary must be listed, not extracted, and not a failure"
    );

    let (state, sidecars, failures) = probe("legacy.txt");
    assert_eq!(
        (state, failures),
        (1, 0),
        "a legacy-charset .txt must still extract"
    );
    assert_eq!(sidecars, 1, "and must still store its text");

    assert_eq!(probe("notes.md"), (1, 1, 0), "ordinary UTF-8 is unaffected");
}

/// Index `root` into `db` and run to `Idle`, sampling the log throughout;
/// returns its peak. Waits for `Idle` rather than the completion marker, which
/// lands inside `run_indexing` before the FTS merge, the tail checkpoints and
/// `repo::maintain`: the window the tail tests measure.
fn reindex_sampling_the_log(root: &Path, db: &Path, config: &Config) -> u64 {
    let service = start(root, db, config);
    let mut peak = 0u64;
    run_to_idle(&service, || peak = peak.max(wal_bytes(db)));
    drop(service);
    peak
}

/// The log stays bounded after the writer loop, the window
/// [`the_wal_stays_bounded_during_a_run`] does not cover.
///
/// Everything after the writer loop is database work with no checkpoint of its
/// own: the FTS merge, the completion stamp, the per-root counts, then
/// `repo::maintain`'s VACUUM, whose copy-back pushes the whole database
/// through the log. With autocheckpoint off for the run and every per-root
/// reader holding a read mark, all of it lands on one log unless the tail
/// checkpoints. A warm reindex writes almost nothing during the loop, so the
/// in-loop checkpoint never fires; without the tail's, one left a `-wal`
/// several times the size of the index.
///
/// A reader is held across both runs, and the test is vacuous without it. The
/// application always has one: the search worker keeps its connection for
/// `IDLE_RELEASE`. Without it the indexer's connection is the last handle on
/// the file, and SQLite checkpoints and deletes the log when the last one
/// closes, hiding anything the run failed to land.
///
/// Two assertions. The mechanism: the tail's checkpoints hand `repo::maintain`
/// an empty log, read back through `repo::log_on_entry_to_maintain`, a latch,
/// because the value is gone before a test could sample it; this fails without
/// the tail checkpoints. The peak guards the reported symptom, but a fixture
/// this size cannot build a tail large enough to breach the ceiling on its own,
/// so it pins the current behaviour rather than reproducing the bug.
#[test]
fn the_wal_stays_bounded_through_the_tail_of_a_warm_reindex() {
    let root = seed_text_tree("wal-tail");
    let (db_dir, db) = Scratch::db("wal-tail-db");
    let dir_key = db_dir.to_string_lossy().into_owned();
    // With positions: the log is measured against the index's size, and
    // position lists make this fixture's index several times larger.
    let mut config = Config::default();
    config.processing.compact_text_index = false;

    // An empty root first, purely to bring the index into existence so the
    // reader below can be opened before the run that matters.
    let empty = Scratch::dir("wal-tail-empty");
    reindex_sampling_the_log(&empty, &db, &config);

    let reader = rusqlite::Connection::open(&db).unwrap();
    // Lazily attached: without a statement there is no handle on the file yet,
    // and the point of this connection is to be one.
    reader
        .query_row("SELECT COUNT(*) FROM files", [], |r| r.get::<_, i64>(0))
        .unwrap();

    // The cold run is where the mechanism is visible: it fills the log, and
    // 30-odd MiB against a 512 MiB default cap means the in-loop checkpoint
    // never fires, so the tail's is the only one there is.
    reindex_sampling_the_log(&root, &db, &config);
    let indexed = std::fs::metadata(&db).unwrap().len();
    assert!(
        indexed > 8 * 1024 * 1024,
        "the fixture built a {} byte index; too small to measure a log against",
        indexed
    );
    assert_eq!(
        quicksearch_core::db::repo::log_on_entry_to_maintain(&dir_key),
        Some(0),
        "the tail must land its log before the pass that VACUUMs through it"
    );

    // Nothing on disk has changed, so every byte of log below is the tail's.
    let peak = reindex_sampling_the_log(&root, &db, &config);

    assert_eq!(
        quicksearch_core::db::repo::log_on_entry_to_maintain(&dir_key),
        Some(0),
        "and a warm reindex's tail must land its own"
    );
    drop(reader);

    // One VACUUM's copy-back is the largest thing the tail may legitimately
    // write, plus the log's own 16 MiB floor. The bug cleared twice the index.
    let ceiling = indexed + 16 * 1024 * 1024;
    assert!(
        peak < ceiling,
        "the tail peaked at {} bytes of log against a {} byte index",
        peak,
        indexed
    );
    assert_eq!(wal_bytes(&db), 0, "and the tail leaves no log behind");
}

/// A reindex that finds nothing to do does not grow the FTS index.
///
/// The end-of-run merge takes a positive page budget. A negative one routes
/// through `fts5IndexOptimizeStruct`, a rate-limited `optimize` that hoists
/// every segment into one level and leaves the structure mid-merge in
/// `%_data`; called once per run, each run would restart it and churn pages
/// for an unchanged corpus.
#[test]
fn repeated_warm_reindexes_do_not_grow_the_fts_index() {
    let root = seed_text_tree("fts-churn");
    let (_db_dir, db) = Scratch::db("fts-churn-db");
    let config = Config::default();

    let fts_pages = |db: &Path| {
        count(
            db,
            "SELECT COUNT(*) FROM dbstat WHERE name = 'searchabletext_data'",
        )
    };

    reindex_sampling_the_log(&root, &db, &config);
    let first = fts_pages(&db);
    assert!(first > 0, "the fixture built no FTS index");

    let mut sizes = vec![first];
    for _ in 0..3 {
        reindex_sampling_the_log(&root, &db, &config);
        sizes.push(fts_pages(&db));
    }

    // Not equality: `PRAGMA optimize` and a merge that consolidates real
    // segments may move the figure either way once. Monotonic growth over
    // three no-op runs is the signature of a structure that never settles.
    let last = *sizes.last().unwrap();
    assert!(
        last <= first,
        "searchabletext_data grew across no-op reindexes: {:?} pages",
        sizes
    );
}

// ---------------------------------------------------------------------------
// Container size against extraction cost
// ---------------------------------------------------------------------------

/// A presentation carrying media: one slide of text inside megabytes of
/// picture, which is the shape every real deck has.
fn media_heavy_deck(path: &Path, media_bytes: usize) {
    let file = std::fs::File::create(path).unwrap();
    let mut zip = zip::ZipWriter::new(file);
    zip.start_file("ppt/slides/slide1.xml", zip::write::FileOptions::default())
        .unwrap();
    std::io::Write::write_all(
        &mut zip,
        b"<p:sld><p:cSld><p:spTree><p:sp><p:txBody>\
          <a:p><a:r><a:t>Quarterly chalcedony forecast</a:t></a:r></a:p>\
          </p:txBody></p:sp></p:spTree></p:cSld></p:sld>",
    )
    .unwrap();
    // Stored, not deflated: the file on disk has to really be this large, or
    // the size gate under test never sees it.
    zip.start_file(
        "ppt/media/image1.png",
        zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored),
    )
    .unwrap();
    std::io::Write::write_all(&mut zip, &vec![0u8; media_bytes]).unwrap();
    zip.finish().unwrap();
}

/// The content pass gates on a file's size, which for a container is its media,
/// not its text: a 5 MiB deck holds about 10 KiB of words, and reading them
/// costs a seek through the central directory plus the one member named. A cap
/// tuned to prose indexes every deck by name only, with nothing inside
/// findable and no failure to show for it.
#[test]
fn a_media_heavy_presentation_is_indexed_under_the_default_cap() {
    let root = Scratch::dir("deck-media");
    let (_db_dir, db) = Scratch::db("deck-media-db");
    let deck = root.join("quarterly.pptx");
    media_heavy_deck(&deck, 5 * 1024 * 1024);

    let config = Config::default();
    let size = std::fs::metadata(&deck).unwrap().len();
    assert!(
        size > 2 * 1024 * 1024,
        "the fixture is only a test of the cap if it is large: {} bytes",
        size
    );
    assert!(
        size <= config.processing.maximum_text_file_size,
        "a {} byte deck is over the {} byte default cap, so its text is skipped",
        size,
        config.processing.maximum_text_file_size
    );

    index_once(&root, &db, &config);

    let text = stored_text(&db, "quarterly.pptx").expect("the deck stored no text at all");
    assert!(
        text.contains("Quarterly chalcedony forecast"),
        "the slide's text is not what was stored: {:?}",
        text
    );

    assert_eq!(
        hits(&db, "chalcedony"),
        1,
        "a word inside the deck is not searchable"
    );
}

/// Raising the cap has to reach an index that already wrote a file off. The
/// row is `NA` and nothing re-asks about a file until the file itself
/// changes, so without a reconcile the setting looks like it does nothing:
/// the deck stays findable by name only, with no failure to explain it.
#[test]
fn raising_the_size_cap_reaches_files_already_written_off() {
    use std::sync::atomic::AtomicBool;

    let root = Scratch::dir("cap-raise");
    let (_db_dir, db) = Scratch::db("cap-raise-db");
    media_heavy_deck(&root.join("quarterly.pptx"), 5 * 1024 * 1024);

    // A cap under the deck's size: its text is never read. The root goes in
    // the config too — `WorkCursor` scans the roots the config names, not
    // the ones an indexing run was handed.
    let mut tight = Config::default();
    tight.paths.indexing_paths = vec![root.to_string_lossy().into_owned()];
    tight.processing.maximum_text_file_size = 1024 * 1024;
    index_once(&root, &db, &tight);
    assert!(
        stored_text(&db, "quarterly.pptx").is_none(),
        "the fixture has to start out skipped, or this proves nothing"
    );

    // Raised, the way Settings raises it: one field, nothing else.
    let mut raised = tight.clone();
    raised.processing.maximum_text_file_size = Config::default().processing.maximum_text_file_size;
    let work = quicksearch_core::config::diff_actions(&tight, &raised).work;
    assert!(
        work.reconcile_content && work.reindex,
        "a raised cap must ask for both halves: {:?}",
        work
    );

    let mut conn = rusqlite::Connection::open(&db).unwrap();
    let registry = quicksearch_core::extract::Registry::default_set();
    let mut cursor = quicksearch_core::scope::WorkCursor::new(work, &raised).unwrap();
    let cancel = AtomicBool::new(false);
    while !cursor.done() {
        quicksearch_core::scope::advance(
            &mut conn,
            &raised,
            &registry,
            &mut cursor,
            Instant::now(),
            &cancel,
        )
        .unwrap();
    }
    drop(conn);

    index_once(&root, &db, &raised);

    let text = stored_text(&db, "quarterly.pptx")
        .expect("the deck is still skipped after the cap was raised");
    assert!(
        text.contains("Quarterly chalcedony forecast"),
        "the deck's text is not what was stored: {:?}",
        text
    );
}