quick_search/crates/quicksearch-core/src/db/repo.rs

//! Row-level write helpers that keep the FTS5 contentless table in sync with
//! `files`/`documents`/`properties`.
//!
//! FTS5 contentless tables store postings only. Updating them requires the
//! old row values to compute which terms to remove. These helpers centralize
//! that bookkeeping so callers never have to remember the order of operations.
//!
//! States (mirrors `basic_state` / `content_state` columns):
//!
//! | value | meaning |
//! |------:|---------|
//! |     0 | pending |
//! |     1 | done    |
//! |     2 | failed  |
//! |     3 | not applicable (content only) |

use rusqlite::{params, params_from_iter, Connection, OptionalExtension, Transaction};

use crate::mime::FileType;

pub const STATE_PENDING: i64 = 0;
pub const STATE_DONE: i64 = 1;
pub const STATE_FAILED: i64 = 2;
pub const STATE_NA: i64 = 3;

/// `prepare_cached` + `execute`, returning the affected row count.
///
/// `what` is a closure rather than a string so the message is built only on
/// the error path — several callers run this per property or per file, where
/// formatting a description that is then discarded is the dominant cost.
///
/// Takes `&Connection`; a `Transaction` derefs to one, so the same call works
/// inside and outside a transaction.
fn exec(
    conn: &Connection,
    sql: &str,
    params: impl rusqlite::Params,
    what: impl FnOnce() -> String,
) -> Result<usize, String> {
    conn.prepare_cached(sql)
        .and_then(|mut stmt| stmt.execute(params))
        .map_err(|e| format!("{}: {}", what(), e))
}

/// Move a file to a content state that means "nothing is currently wrong with
/// this row", clearing any failure record along with it.
///
/// The pair is one operation, not two that happen to be adjacent: a row that
/// succeeded, that nothing can extract, or that has been queued for another
/// attempt must not keep a `failed_files` entry explaining a failure that no
/// longer applies. `list-failed` reads that table directly, so a stale entry
/// is a file the user is told is broken after it stopped being broken.
///
/// [`set_content_failed`] is deliberately not routed through here — it is the
/// one transition that *writes* a failure record.
fn set_state_clearing_failure(
    tx: &Transaction<'_>,
    file_id: i64,
    state: i64,
    transition: &'static str,
) -> Result<(), String> {
    exec(
        tx,
        "UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2",
        params![state, file_id],
        || format!("{} content_state {}", transition, file_id),
    )?;
    exec(
        tx,
        "DELETE FROM failed_files WHERE file_id = ?1",
        params![file_id],
        || format!("clear failed_files {}", file_id),
    )?;
    Ok(())
}

/// Everything needed to insert a fresh file row.
#[derive(Debug, Clone)]
pub struct NewFile<'a> {
    pub name: &'a str,
    pub path: &'a str,
    pub parent: &'a str,
    pub size: u64,
    pub mtime: u64,
    pub inode: Option<u64>,
    pub device_id: Option<u64>,
    pub mime: Option<&'a str>,
    pub ftype: FileType,
    pub hash: Option<&'a [u8]>,
    /// Whether the content pass has work to do for this file. `false` means
    /// the row is born `STATE_NA` — nothing claims its MIME, the
    /// `content_extensions` filter excludes it, or it is over
    /// `maximum_text_file_size`. Decided at walk time by
    /// [`crate::file_handling::content_extractable`].
    pub needs_content: bool,
}

/// Insert a new file row, returning its id. `basic_state` is set to DONE
/// (row existing *is* the basic-index state); `content_state` comes from
/// `needs_content` — PENDING for a file an extractor will claim, NA for one it
/// won't. Deciding here rather than on a content worker is what keeps PENDING
/// meaning "real work outstanding", which the extraction progress denominator
/// and [`pending_content_page`] both depend on.
///
/// Uses `INSERT OR IGNORE` so a UNIQUE(path) collision (which indicates the
/// caller fed the same path twice in one run) becomes a silent no-op
/// returning `None` rather than aborting the whole batch. The walker is
/// expected to dedupe visits upstream; this is a defense-in-depth backstop.
pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> {
    let rows = tx
        .prepare_cached(
            "INSERT OR IGNORE INTO files (
                name, path, parent, size, mtime, inode, device_id,
                mime, type, basic_state, content_state, hash
             ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
        )
        .and_then(|mut stmt| {
            stmt.execute(params![
                f.name,
                f.path,
                f.parent,
                f.size as i64,
                f.mtime as i64,
                f.inode.map(|x| x as i64),
                f.device_id.map(|x| x as i64),
                f.mime,
                f.ftype.bits() as i64,
                STATE_DONE,
                initial_content_state(f),
                f.hash,
            ])
        })
        .map_err(|e| format!("insert file {}: {}", f.path, e))?;
    if rows == 0 {
        // Existing row with the same path (e.g. duplicate visit within the run).
        return Ok(None);
    }
    Ok(Some(tx.last_insert_rowid()))
}

/// The `content_state` a freshly written row starts in. Shared by
/// [`insert_file`] and [`update_file_basic`] so a file that reappears as an
/// update lands in the same state it would have as an insert.
fn initial_content_state(f: &NewFile<'_>) -> i64 {
    if f.needs_content {
        STATE_PENDING
    } else {
        STATE_NA
    }
}

/// Update a file's metadata in place (same path, changed size/mtime/hash) and
/// reset its content state from `f.needs_content`, clearing any extracted
/// content so the text-indexing pass re-processes it.
///
/// Writes `size`, `mtime`, `hash`, `mime`, `type`, `content_state` and
/// `failure_msg` — and only those. `name`, `parent`, `inode` and `device_id`
/// are not refreshed here despite being present on the `NewFile`: the row is
/// found by path, and the first two are functions of it.
pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> {
    // One statement, not a lookup then an update: `RETURNING` hands back the
    // id of the row it just wrote, and a miss is simply no row returned.
    let id: Option<i64> = tx
        .prepare_cached(
            "UPDATE files
                SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5,
                    content_state = ?6, failure_msg = NULL
              WHERE path = ?7
          RETURNING id",
        )
        .and_then(|mut stmt| {
            stmt.query_row(
                params![
                    f.size as i64,
                    f.mtime as i64,
                    f.hash,
                    f.mime,
                    f.ftype.bits() as i64,
                    initial_content_state(f),
                    f.path,
                ],
                |r| r.get(0),
            )
            .optional()
        })
        .map_err(|e| format!("update file {}: {}", f.path, e))?;
    let Some(id) = id else {
        return Ok(None);
    };
    // Any prior extracted content is stale — remove it along with its FTS row.
    remove_content_for_id(tx, id)?;
    Ok(Some(id))
}

/// Mark a file's content indexing as complete and write the extracted text +
/// properties atomically. The plaintext is fed to the contentless FTS5
/// tokenizer (which keeps only the inverted index) and — when `store_text`
/// is `true` — separately stored zstd-compressed in `documents_text` for
/// on-demand snippet rendering. When `store_text=false` the sidecar INSERT
/// is skipped: queries still match the right files but result rows can't
/// render snippets. `properties` are stored both as a structured side-
/// table (for exact retrieval) and concatenated into the FTS `properties`
/// column (for MATCH).
pub fn set_content_done(
    tx: &Transaction<'_>,
    file_id: i64,
    name: &str,
    text: &str,
    properties: &[(String, String)],
    store_text: bool,
) -> Result<(), String> {
    // Clear any previous extraction (in case of re-run).
    remove_content_for_id(tx, file_id)?;

    for (k, v) in properties {
        exec(
            tx,
            "INSERT INTO properties(file_id, key, value) VALUES (?1, ?2, ?3)",
            params![file_id, k, v],
            || format!("insert property {}={}", k, v),
        )?;
    }
    let props_blob = encode_properties_for_fts(properties);
    // Contentless FTS5 still accepts values on INSERT — the tokenizer needs
    // them — it simply doesn't persist the raw column values.
    exec(
        tx,
        "INSERT INTO searchabletext(rowid, name, text, properties) VALUES (?1, ?2, ?3, ?4)",
        params![file_id, name, text, props_blob],
        || format!("insert FTS row {}", file_id),
    )?;

    // Skip the compressed sidecar when: the config disables snippet storage
    // outright, or there's no body text (e.g. an image whose extractor
    // returned only EXIF properties). The second case saves a zstd frame
    // on what would otherwise be an empty blob.
    if store_text && !text.is_empty() {
        let compressed = zstd::encode_all(text.as_bytes(), ZSTD_LEVEL)
            .map_err(|e| format!("zstd encode for file {}: {}", file_id, e))?;
        exec(
            tx,
            "INSERT INTO documents_text(file_id, text_zstd, text_len) VALUES (?1, ?2, ?3)",
            params![file_id, compressed, text.len() as i64],
            || format!("insert documents_text {}", file_id),
        )?;
    }

    set_state_clearing_failure(tx, file_id, STATE_DONE, "update DONE")
}

/// zstd level tuned for extracted-text prose. Level 3 hits ~3-5× on English
/// prose at high throughput (hundreds of MB/s) — level 9+ would shave a few
/// percent more but at 10× the CPU cost during indexing. Wrong knob to
/// tune: readers decompress far faster than writers compress, so keep
/// write-side cost low.
const ZSTD_LEVEL: i32 = 3;

/// Mark a file's content extraction as failed. Keeps the basic row in place.
pub fn set_content_failed(tx: &Transaction<'_>, file_id: i64, reason: &str) -> Result<(), String> {
    let now = crate::log::now_unix() as i64;
    exec(
        tx,
        "UPDATE files SET content_state = ?1, failure_msg = ?2 WHERE id = ?3",
        params![STATE_FAILED, reason, file_id],
        || format!("update content_state FAILED {}", file_id),
    )?;
    exec(
        tx,
        "INSERT OR REPLACE INTO failed_files(file_id, reason, ts) VALUES (?1, ?2, ?3)",
        params![file_id, reason, now],
        || format!("insert failed_files {}", file_id),
    )?;
    Ok(())
}

/// Mark content extraction as not applicable (e.g. binary format we don't
/// support). The file row still contributes to filename search.
pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
    set_state_clearing_failure(tx, file_id, STATE_NA, "update NA")
}

/// Delete a file row by path, keeping FTS in sync. Returns whether a row was
/// removed.
pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result<bool, String> {
    // `RETURNING` gives the id of the row that went, so the dependent tables
    // can be cleared without a separate lookup first.
    let id: Option<i64> = tx
        .prepare_cached("DELETE FROM files WHERE path = ?1 RETURNING id")
        .and_then(|mut stmt| stmt.query_row(params![path], |r| r.get(0)).optional())
        .map_err(|e| format!("delete file {}: {}", path, e))?;
    let Some(id) = id else { return Ok(false) };
    remove_content_for_id(tx, id)?;
    Ok(true)
}

/// Delete every row whose path falls in the half-open range `[lo, hi)`,
/// keeping FTS, `documents_text`, `properties` and `failed_files` in step.
/// Returns how many `files` rows went.
///
/// The bulk counterpart to [`delete_file_by_path`], for removing a whole
/// directory at once. Five statements regardless of how many files the range
/// holds — where deleting them one at a time costs ~5 *per file* — and the
/// range is a plain index seek on `UNIQUE(files.path)`, so this is
/// `SEARCH … (path>? AND path<?)` rather than a scan. Build the bounds with
/// [`crate::file_handling::ExtractCursor::for_root`], which is what makes them
/// separator-correct.
///
/// All four dependent tables are deleted explicitly rather than left to their
/// `ON DELETE CASCADE` foreign keys. `searchabletext` is an FTS5 virtual table
/// and has no foreign key at all, so relying on cascade would leave the rule
/// half-declarative and half-manual, free to drift the moment someone opens a
/// connection without `PRAGMA foreign_keys`. (`contentless_delete=1` is what
/// lets the FTS rows go by rowid alone.)
pub fn delete_subtree(tx: &Transaction<'_>, lo: &str, hi: &str) -> Result<usize, String> {
    // Every dependent table is keyed by the file id, so they share one
    // sub-select; `files` itself goes last, once nothing references it.
    for (table, key) in DEPENDENT_TABLES {
        let sql = format!(
            "DELETE FROM {} WHERE {} IN \
             (SELECT id FROM files WHERE path >= ?1 AND path < ?2)",
            table, key
        );
        exec(tx, &sql, params![lo, hi], || {
            format!("delete {} under {}", table, lo)
        })?;
    }
    exec(
        tx,
        "DELETE FROM files WHERE path >= ?1 AND path < ?2",
        params![lo, hi],
        || format!("delete files under {}", lo),
    )
}

/// Delete every row whose path falls in *none* of `ranges`, keeping the four
/// dependent tables in step. Returns how many `files` rows went. An empty
/// `ranges` deletes nothing.
///
/// The complement of [`delete_subtree`], for the one case that needs it:
/// with `follow_symlinks` off, a file outside every configured root cannot be
/// produced by any walk, so rows left there by a followed symlink are
/// unreachable — no root's range covers them, so no scan will ever visit
/// them again either. This is a scan of `files` rather than a seek, which is
/// why it is reserved for that one transition instead of being the general
/// prune.
///
/// An empty `ranges` means no roots are configured. Deleting everything would
/// be the literal reading and is certainly not what a half-written config
/// means, so it is a no-op.
pub fn delete_outside_ranges(
    tx: &Transaction<'_>,
    ranges: &[(String, String)],
) -> Result<usize, String> {
    if ranges.is_empty() {
        return Ok(0);
    }
    let mut predicate = String::new();
    for i in 0..ranges.len() {
        if i > 0 {
            predicate.push_str(" AND ");
        }
        predicate.push_str(&format!(
            "NOT (path >= ?{} AND path < ?{})",
            i * 2 + 1,
            i * 2 + 2
        ));
    }
    let bounds: Vec<&String> = ranges.iter().flat_map(|(lo, hi)| [lo, hi]).collect();
    for (table, key) in DEPENDENT_TABLES {
        let sql = format!(
            "DELETE FROM {} WHERE {} IN (SELECT id FROM files WHERE {})",
            table, key, predicate
        );
        exec(tx, &sql, params_from_iter(bounds.iter()), || {
            format!("delete {} outside the roots", table)
        })?;
    }
    let sql = format!("DELETE FROM files WHERE {}", predicate);
    exec(tx, &sql, params_from_iter(bounds.iter()), || {
        "delete files outside the roots".to_string()
    })
}

/// The tables a file id owns, in the order they must be cleared: everything
/// keyed to `files.id` first, then `files` itself once nothing references it.
///
/// `searchabletext` is an FTS5 virtual table with no foreign key at all, so
/// none of this can be left to `ON DELETE CASCADE` without splitting one rule
/// across two mechanisms — half declarative, half manual, free to drift the
/// moment someone opens a connection without `PRAGMA foreign_keys`. Shared by
/// [`delete_subtree`] and [`delete_ids`] so the two ways of removing a file
/// cannot disagree about what a file owns.
const DEPENDENT_TABLES: [(&str, &str); 4] = [
    ("searchabletext", "rowid"),
    ("documents_text", "file_id"),
    ("properties", "file_id"),
    ("failed_files", "file_id"),
];

/// How many ids [`delete_ids`] binds into one statement.
///
/// Fixed so `prepare_cached` sees a bounded set of distinct SQL texts: every
/// full chunk of a batch shares one statement and only the short final chunk
/// varies, where a per-call length would mint a new prepared statement each
/// time.
const DELETE_IDS_CHUNK: usize = 512;

/// Delete the given file ids and everything keyed to them, keeping FTS,
/// `documents_text`, `properties` and `failed_files` in step. Returns how many
/// `files` rows went.
///
/// The id counterpart to [`delete_subtree`], for rows chosen by a predicate no
/// SQL range can express — a glob ignore pattern, say. Five statements per
/// [`DELETE_IDS_CHUNK`] ids rather than five per file, which is the difference
/// that lets a newly-added ignore pattern prune a large index instead of
/// forcing a rebuild.
pub fn delete_ids(tx: &Transaction<'_>, ids: &[i64]) -> Result<usize, String> {
    let mut removed = 0;
    for chunk in ids.chunks(DELETE_IDS_CHUNK) {
        let mut placeholders = String::with_capacity(chunk.len() * 2);
        for i in 0..chunk.len() {
            if i > 0 {
                placeholders.push(',');
            }
            placeholders.push('?');
        }
        for (table, key) in DEPENDENT_TABLES {
            let sql = format!("DELETE FROM {} WHERE {} IN ({})", table, key, placeholders);
            exec(tx, &sql, params_from_iter(chunk.iter()), || {
                format!("delete {} for {} ids", table, chunk.len())
            })?;
        }
        let sql = format!("DELETE FROM files WHERE id IN ({})", placeholders);
        removed += exec(tx, &sql, params_from_iter(chunk.iter()), || {
            format!("delete {} file rows", chunk.len())
        })?;
    }
    Ok(removed)
}

/// Every indexed file directly inside `parent`, as `name -> mtime`.
///
/// The walk's unit of classification. Keyed by name rather than full path
/// because the parent is implied — at millions of files, not storing the
/// directory prefix once per entry is the difference this whole path exists
/// to make. Served by `idx_files_parent`, so this is one index range lookup.
pub fn dir_rows(
    conn: &Connection,
    parent: &str,
) -> Result<std::collections::HashMap<String, u64>, String> {
    let mut stmt = conn
        .prepare_cached("SELECT name, mtime FROM files WHERE parent = ?1")
        .map_err(|e| format!("prepare dir rows for {}: {}", parent, e))?;
    let rows = stmt
        .query_map(params![parent], |r| {
            Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?.max(0) as u64))
        })
        .map_err(|e| format!("query dir rows for {}: {}", parent, e))?;
    let mut out = std::collections::HashMap::new();
    for row in rows {
        let (name, mtime) = row.map_err(|e| format!("read dir row under {}: {}", parent, e))?;
        out.insert(name, mtime);
    }
    Ok(out)
}

/// A row the content pass has yet to extract: `(id, name, path, mime)`.
pub type PendingContentRow = (i64, String, String, Option<String>);

/// One page of rows still awaiting content extraction under `cursor`'s range,
/// ordered by id.
///
/// Keyset, not `OFFSET`: `id > cursor.last_id` means each page is an index
/// seek rather than a re-scan of everything already handed out, and — because
/// the cursor only moves forward — a row is served exactly once even though
/// the writer is concurrently flipping `content_state` behind the reader. The
/// `content_state = 0` predicate is the belt to that braces, not the
/// mechanism.
pub fn pending_content_page(
    conn: &Connection,
    cursor: &crate::file_handling::ExtractCursor,
    max_size: i64,
    limit: i64,
) -> Result<Vec<PendingContentRow>, String> {
    let mut stmt = conn
        .prepare_cached(
            "SELECT id, name, path, mime FROM files
              WHERE content_state = 0 AND size <= ?1 AND id > ?2
                AND path >= ?3 AND path < ?4
              ORDER BY id
              LIMIT ?5",
        )
        .map_err(|e| format!("prepare pending content query: {}", e))?;
    let rows = stmt
        .query_map(
            params![max_size, cursor.last_id, cursor.lo, cursor.hi, limit],
            |row| {
                Ok((
                    row.get::<_, i64>(0)?,
                    row.get::<_, String>(1)?,
                    row.get::<_, String>(2)?,
                    row.get::<_, Option<String>>(3)?,
                ))
            },
        )
        .map_err(|e| format!("query pending content: {}", e))?;
    rows.collect::<Result<Vec<_>, _>>()
        .map_err(|e| format!("read pending content row: {}", e))
}

/// A stored row as the scope reconciler sees it: enough to decide both
/// whether the path is still in scope and whether its content still is.
#[derive(Debug, Clone)]
pub struct ScopeRow {
    pub id: i64,
    pub path: String,
    pub size: u64,
    pub mime: Option<String>,
    pub content_state: i64,
}

/// How many files the index holds.
///
/// A denominator for a scan that pages over all of them, so the cost is paid
/// once against work measured in pages. SQLite answers it from the smallest
/// index rather than the table, so it is a key scan and not a row fetch.
pub fn row_count(conn: &Connection) -> Result<usize, String> {
    conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get::<_, i64>(0))
        .map(|n| n.max(0) as usize)
        .map_err(|e| format!("count indexed files: {}", e))
}

/// One page of rows whose path is `> after` and `< hi`, in path order.
///
/// Keyset on `path` rather than on `id`: the range is already a seek on
/// `UNIQUE(files.path)`, so paging by the same column keeps every page an
/// index walk with no sort step, and — because the cursor only moves forward
/// — a row is served at most once even though the caller is deleting behind
/// the reader. Seed `after` with the range's `lo` bound, which is
/// `root + separator` and so can never equal a stored path.
pub fn rows_in_range_page(
    conn: &Connection,
    after: &str,
    hi: &str,
    limit: i64,
) -> Result<Vec<ScopeRow>, String> {
    let mut stmt = conn
        .prepare_cached(
            "SELECT id, path, size, mime, content_state FROM files
              WHERE path > ?1 AND path < ?2
              ORDER BY path
              LIMIT ?3",
        )
        .map_err(|e| format!("prepare range page: {}", e))?;
    let rows = stmt
        .query_map(params![after, hi, limit], |row| {
            Ok(ScopeRow {
                id: row.get(0)?,
                path: row.get(1)?,
                size: row.get::<_, i64>(2)?.max(0) as u64,
                mime: row.get(3)?,
                content_state: row.get(4)?,
            })
        })
        .map_err(|e| format!("query range page after {}: {}", after, e))?;
    rows.collect::<Result<Vec<_>, _>>()
        .map_err(|e| format!("read range page row: {}", e))
}

/// Drop the stored text of the given file ids, leaving their FTS row and
/// `files` row intact.
///
/// Turning `store_text_for_snippets` off means exactly this: full-text search
/// keeps working from the FTS index, and only the snippet/occurrence source
/// goes away. Re-extracting to achieve it would re-read every file for nothing.
pub fn drop_stored_text(tx: &Transaction<'_>, ids: &[i64]) -> Result<usize, String> {
    let mut removed = 0;
    for chunk in ids.chunks(DELETE_IDS_CHUNK) {
        let mut placeholders = String::with_capacity(chunk.len() * 2);
        for i in 0..chunk.len() {
            if i > 0 {
                placeholders.push(',');
            }
            placeholders.push('?');
        }
        let sql = format!(
            "DELETE FROM documents_text WHERE file_id IN ({})",
            placeholders
        );
        removed += exec(tx, &sql, params_from_iter(chunk.iter()), || {
            format!("drop stored text for {} ids", chunk.len())
        })?;
    }
    Ok(removed)
}

/// Put a file's content back in the pending queue without touching its row's
/// metadata, clearing whatever the last extraction left behind.
///
/// For a config change that widens what gets extracted: the file itself has
/// not changed, so `update_file_basic` would be wrong (it rewrites size, mtime
/// and hash from a fresh stat), but its content must be produced again.
pub fn reset_content_pending(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
    remove_content_for_id(tx, file_id)?;
    set_state_clearing_failure(tx, file_id, STATE_PENDING, "reset pending")
}

/// The stored mtime for one exact path, or `None` if it isn't indexed.
///
/// For files the walk reaches by a spelling whose parent isn't the directory
/// being read — a resolved symlink target — where [`dir_rows`] would not
/// have them.
pub fn mtime_for_path(conn: &Connection, path: &str) -> Result<Option<u64>, String> {
    let mut stmt = conn
        .prepare_cached("SELECT mtime FROM files WHERE path = ?1")
        .map_err(|e| format!("prepare mtime lookup for {}: {}", path, e))?;
    stmt.query_row(params![path], |r| r.get::<_, i64>(0))
        .optional()
        .map(|o| o.map(|m| m.max(0) as u64))
        .map_err(|e| format!("mtime lookup for {}: {}", path, e))
}

/// Distinct `parent` values within the half-open path range `[lo, hi)`,
/// streamed to `f` rather than collected.
///
/// Callers use this to find directories the walk never visited, so it must
/// not itself materialize a list proportional to the tree — the whole point
/// of the change that introduced it. `idx_files_parent` makes this an
/// index-only scan.
pub fn for_each_parent_in_range<F: FnMut(String)>(
    conn: &Connection,
    lo: &str,
    hi: &str,
    mut f: F,
) -> Result<(), String> {
    let mut stmt = conn
        .prepare("SELECT DISTINCT parent FROM files WHERE parent >= ?1 AND parent < ?2")
        .map_err(|e| format!("prepare parent scan: {}", e))?;
    let rows = stmt
        .query_map(params![lo, hi], |r| r.get::<_, String>(0))
        .map_err(|e| format!("parent scan: {}", e))?;
    for row in rows {
        f(row.map_err(|e| format!("read parent row: {}", e))?);
    }
    Ok(())
}

/// Paths of every file directly inside `parent`.
///
/// The companion to [`for_each_parent_in_range`]: once a parent is known to
/// be unvisited, this is what its rows are.
pub fn paths_in_dir(conn: &Connection, parent: &str) -> Result<Vec<String>, String> {
    let mut stmt = conn
        .prepare_cached("SELECT path FROM files WHERE parent = ?1")
        .map_err(|e| format!("prepare paths in {}: {}", parent, e))?;
    let rows = stmt
        .query_map(params![parent], |r| r.get::<_, String>(0))
        .map_err(|e| format!("query paths in {}: {}", parent, e))?;
    rows.collect::<Result<Vec<_>, _>>()
        .map_err(|e| format!("read path under {}: {}", parent, e))
}

/// Remove the FTS row, compressed text blob, and any `properties` rows for
/// a given file id. Does not touch the `files` row itself. Idempotent — a
/// missing row is fine.
pub fn remove_content_for_id(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
    // `contentless_delete=1` on the FTS5 table makes this work without
    // re-supplying the old column values (it tombstones the rowid).
    for (table, key) in [
        ("searchabletext", "rowid"),
        ("documents_text", "file_id"),
        ("properties", "file_id"),
    ] {
        let sql = format!("DELETE FROM {} WHERE {} = ?1", table, key);
        exec(tx, &sql, params![file_id], || {
            format!("delete {} for {}", table, file_id)
        })?;
    }
    Ok(())
}

/// Serialize properties for the FTS `properties` column. `key:value` pairs
/// separated by spaces so `MATCH 'properties:artist:beatles'` works.
fn encode_properties_for_fts(props: &[(String, String)]) -> String {
    let mut buf = String::new();
    for (i, (k, v)) in props.iter().enumerate() {
        if i > 0 {
            buf.push(' ');
        }
        buf.push_str(k);
        buf.push(':');
        buf.push_str(v);
    }
    buf
}

/// Free pages, as a percentage of the file, that make a [`maintain`] VACUUM
/// worth its cost. Below this the file is rewritten for nothing: a run that
/// changed little leaves little slack, and rewriting a multi-gigabyte index
/// to reclaim a few megabytes is minutes of I/O for no gain.
const VACUUM_MIN_SLACK_PERCENT: i64 = 10;

/// Flush the whole WAL into the main database and truncate the log to zero
/// bytes. `Err` means the log was *not* emptied.
///
/// `execute`/`execute_batch` discard the pragma's result row, and that row is
/// the only place SQLite reports that a checkpoint gave up — an incomplete
/// checkpoint is not an error, it is a number in a row nobody read. That is
/// how a log can grow past the index it journals without a word in the logs.
///
/// The signal is the *log* column (frames left in the WAL), not `busy`: a
/// TRUNCATE that cannot take the writer lock silently downgrades itself to
/// PASSIVE and still reports `busy = 0` with the log untouched. A real restart
/// sets `mxFrame` to 0. A database not in WAL mode reports -1, hence `<= 0`.
pub fn checkpoint_truncate(conn: &Connection) -> Result<(), String> {
    let (busy, log): (i64, i64) = conn
        .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| {
            Ok((r.get(0)?, r.get(1)?))
        })
        .map_err(|e| format!("wal checkpoint: {}", e))?;
    if log <= 0 {
        Ok(())
    } else {
        Err(format!(
            "wal checkpoint incomplete: busy={}, {} frames left in the log",
            busy, log
        ))
    }
}

/// Flush the WAL into the main DB file and close. Call on clean shutdown so
/// the next open starts with an empty log. WAL mode itself is persistent in
/// the file — deliberately left on.
pub fn checkpoint_and_close(conn: Connection) {
    if let Err(e) = checkpoint_truncate(&conn) {
        crate::log_warn!("{}", e);
    }
    drop(conn);
}

/// Land the log, reclaim the file's slack, and refresh the query planner's
/// statistics. Returns whether it vacuumed.
///
/// The sequence is checkpoint → VACUUM → `PRAGMA optimize` → checkpoint. The
/// trailing checkpoint is not a repeat of the first: VACUUM's copy-back pushes
/// every page of the rebuilt file through the log, and `optimize` writes
/// `sqlite_stat1`, so leaving without one would trade the slack just reclaimed
/// for a log the size of the index.
///
/// Run on a connection from [`crate::db::open::open_maintenance`], never on
/// the indexer's: VACUUM builds the replacement database in the temp store,
/// and under the indexer's `temp_store = MEMORY` that means building the whole
/// index in RAM.
///
/// `db_dir` is where the temporary database goes, and it must be the index's
/// own directory. `temp_store = FILE` alone resolves through `SQLITE_TMPDIR`,
/// `TMPDIR`, `/var/tmp`, then `/tmp` — and `/tmp` is a RAM-backed tmpfs on many
/// Linux systems, which would put us straight back where we started. Keeping
/// the temporary database beside the index also puts it on a volume already
/// known to hold one.
///
/// Peak transient space on that volume is roughly three times the index: the
/// original, the replacement being built beside it, and the log that VACUUM's
/// copy-back runs through. Running out is a failed VACUUM, not a damaged
/// index — the transaction rolls back.
pub fn maintain(conn: &Connection, db_dir: &str) -> Result<bool, String> {
    // Best-effort: a reader that briefly pins the log is no reason to skip
    // the compaction below, which does not need the log empty to start.
    if let Err(e) = checkpoint_truncate(conn) {
        crate::log_warn!("{}", e);
    }

    let page_count: i64 = conn
        .query_row("PRAGMA page_count", [], |r| r.get(0))
        .map_err(|e| format!("read page_count: {}", e))?;
    let freelist: i64 = conn
        .query_row("PRAGMA freelist_count", [], |r| r.get(0))
        .map_err(|e| format!("read freelist_count: {}", e))?;

    let vacuumed = freelist * 100 >= page_count * VACUUM_MIN_SLACK_PERCENT;
    if vacuumed {
        // `temp_store_directory` is a deprecated pragma that writes a global,
        // so it is set for the VACUUM and cleared straight after rather than
        // left standing for every other connection in the process.
        let escaped = db_dir.replace('\'', "''");
        conn.execute_batch(&format!("PRAGMA temp_store_directory = '{}';", escaped))
            .map_err(|e| format!("set temp dir for vacuum: {}", e))?;
        let outcome = conn
            .execute_batch("VACUUM;")
            .map_err(|e| format!("vacuum: {}", e));
        let _ = conn.execute_batch("PRAGMA temp_store_directory = '';");
        outcome?;
    }

    // Re-analyses only the tables whose shape has drifted far enough to matter,
    // so it is close to free on a run that changed little. A run that added
    // millions of rows is exactly when the planner's old statistics start
    // choosing the wrong index for a search.
    conn.execute_batch("PRAGMA optimize;")
        .map_err(|e| format!("optimize: {}", e))?;

    checkpoint_truncate(conn)?;
    Ok(vacuumed)
}

/// Read the `last_full_index` marker (unix seconds of the last *successful*
/// full indexing run) from `schema_info`. Absent key — fresh DB, or a DB
/// from before this marker existed — means "never".
pub fn get_last_full_index(conn: &Connection) -> Option<u64> {
    conn.query_row(
        "SELECT value FROM schema_info WHERE key = 'last_full_index'",
        [],
        |r| r.get::<_, String>(0),
    )
    .optional()
    .ok()
    .flatten()
    .and_then(|v| v.parse().ok())
}

/// Stamp `last_full_index` with `ts` (unix seconds). Called at the end of
/// every successful full indexing run; the coordinator reads it to schedule
/// periodic reindexing.
pub fn set_last_full_index(conn: &Connection, ts: u64) -> Result<(), String> {
    conn.execute(
        "INSERT OR REPLACE INTO schema_info(key, value) VALUES ('last_full_index', ?1)",
        params![ts.to_string()],
    )
    .map_err(|e| format!("write last_full_index: {}", e))?;
    Ok(())
}

/// `schema_info` key holding one root's last known file count.
fn walk_count_key(root: &str) -> String {
    format!("walk_count:{}", root)
}

/// How many files the last clean walk of `root` reported.
///
/// The progress bar's denominator. Absent means this root has never been walked
/// to completion — a new root, a fresh index, or a run that was stopped — and
/// the caller falls back to counting the tree as it goes.
///
/// Deliberately last run's *file* count rather than a tree-entry count: it is
/// the same quantity the numerator counts, where an entry count includes
/// directories and ignore-pruned subtrees and so reads high (over 1.6x on a home
/// directory). See [`crate::indexing::RootProgress::walk_denominator`].
pub fn get_root_walk_count(conn: &Connection, root: &str) -> Option<usize> {
    conn.query_row(
        "SELECT value FROM schema_info WHERE key = ?1",
        params![walk_count_key(root)],
        |r| r.get::<_, String>(0),
    )
    .optional()
    .ok()
    .flatten()
    .and_then(|v| v.parse().ok())
}

/// Record `n` as `root`'s file count, for the next run's progress bar.
///
/// Written only after a walk that finished cleanly: a stopped or partially
/// unreadable walk saw only part of the tree, and storing its count would leave
/// every later run dividing by a number that is too small.
pub fn set_root_walk_count(conn: &Connection, root: &str, n: usize) -> Result<(), String> {
    conn.execute(
        "INSERT OR REPLACE INTO schema_info(key, value) VALUES (?1, ?2)",
        params![walk_count_key(root), n.to_string()],
    )
    .map_err(|e| format!("write walk count for {}: {}", root, e))?;
    Ok(())
}

/// Forget the stored counts of roots that are no longer configured.
///
/// Without this, `schema_info` accumulates a row per root the user ever had —
/// harmless in size, but it also means a root removed and later re-added would
/// start from a count that predates everything that happened in between.
pub fn prune_root_walk_counts(conn: &Connection, keep: &[String]) -> Result<(), String> {
    let keep: std::collections::HashSet<String> = keep.iter().map(|r| walk_count_key(r)).collect();
    let mut stmt = conn
        .prepare("SELECT key FROM schema_info WHERE key LIKE 'walk_count:%'")
        .map_err(|e| format!("read walk counts: {}", e))?;
    let stored: Vec<String> = stmt
        .query_map([], |r| r.get::<_, String>(0))
        .map_err(|e| format!("read walk counts: {}", e))?
        .filter_map(|r| r.ok())
        .collect();
    drop(stmt);
    for key in stored.iter().filter(|k| !keep.contains(*k)) {
        conn.execute("DELETE FROM schema_info WHERE key = ?1", params![key])
            .map_err(|e| format!("drop walk count {}: {}", key, e))?;
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use std::sync::atomic::{AtomicBool, Ordering};
    use std::sync::Arc;

    use super::*;
    use crate::db::open_or_recreate;

    fn tmp_path() -> std::path::PathBuf {
        crate::testutil::scratch_dir("repo").join("index.sqlite")
    }

    #[test]
    fn insert_update_delete_round_trip() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        {
            let tx = conn.transaction().unwrap();
            let id = insert_file(
                &tx,
                &NewFile {
                    name: "a.txt",
                    path: "/tmp/a.txt",
                    parent: "/tmp",
                    size: 42,
                    mtime: 1_700_000_000,
                    inode: Some(7),
                    device_id: Some(64768),
                    mime: Some("text/plain"),
                    ftype: FileType::TEXT,
                    hash: Some(&[1, 2, 3]),
                    needs_content: true,
                },
            )
            .unwrap()
            .expect("unique path");
            set_content_done(
                &tx,
                id,
                "a.txt",
                "hello world",
                &[("title".to_string(), "hi".to_string())],
                true,
            )
            .unwrap();
            tx.commit().unwrap();
        }

        // Text is findable via FTS.
        let hit: i64 = conn
            .query_row(
                "SELECT rowid FROM searchabletext WHERE searchabletext MATCH 'hello'",
                [],
                |r| r.get(0),
            )
            .unwrap();
        assert!(hit > 0);

        // Delete cleans up.
        {
            let tx = conn.transaction().unwrap();
            assert!(delete_file_by_path(&tx, "/tmp/a.txt").unwrap());
            tx.commit().unwrap();
        }
        let count: i64 = conn
            .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
            .unwrap();
        assert_eq!(count, 0);
        let fts_count: i64 = conn
            .query_row("SELECT COUNT(*) FROM searchabletext", [], |r| r.get(0))
            .unwrap();
        assert_eq!(fts_count, 0);

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn insert_writes_content_state_from_needs_content() {
        // The whole point of the field: a row nothing will extract is born NA,
        // so "pending" downstream — the extraction denominator, the feeder's
        // page query, the Baloo pending count — means real outstanding work.
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let tx = conn.transaction().unwrap();
        let mut row = NewFile {
            name: "claimed.txt",
            path: "/tmp/claimed.txt",
            parent: "/tmp",
            size: 1,
            mtime: 1,
            inode: None,
            device_id: None,
            mime: Some("text/plain"),
            ftype: FileType::TEXT,
            hash: None,
            needs_content: true,
        };
        let claimed = insert_file(&tx, &row).unwrap().expect("unique path");
        row.name = "unclaimed.mp4";
        row.path = "/tmp/unclaimed.mp4";
        row.mime = Some("video/mp4");
        row.needs_content = false;
        let unclaimed = insert_file(&tx, &row).unwrap().expect("unique path");

        let state = |id: i64| -> i64 {
            tx.query_row(
                "SELECT content_state FROM files WHERE id = ?1",
                params![id],
                |r| r.get(0),
            )
            .unwrap()
        };
        assert_eq!(state(claimed), STATE_PENDING);
        assert_eq!(state(unclaimed), STATE_NA);

        drop(tx);
        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn update_writes_content_state_from_needs_content() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let mut row = NewFile {
            name: "a.txt",
            path: "/tmp/a.txt",
            parent: "/tmp",
            size: 10,
            mtime: 1,
            inode: None,
            device_id: None,
            mime: None,
            ftype: FileType::EMPTY,
            hash: None,
            needs_content: false,
        };
        let id = {
            let tx = conn.transaction().unwrap();
            let id = insert_file(&tx, &row).unwrap().expect("unique path");
            set_content_done(&tx, id, "a.txt", "old text", &[], true).unwrap();
            tx.commit().unwrap();
            id
        };

        let content_state = |conn: &Connection| -> i64 {
            conn.query_row(
                "SELECT content_state FROM files WHERE id = ?1",
                params![id],
                |r| r.get(0),
            )
            .unwrap()
        };

        // Rewritten as something an extractor claims: back to pending, and the
        // stale FTS row goes with it.
        {
            let tx = conn.transaction().unwrap();
            row.size = 20;
            row.mtime = 2;
            row.mime = Some("text/plain");
            row.ftype = FileType::TEXT;
            row.needs_content = true;
            let got = update_file_basic(&tx, &row).unwrap();
            assert_eq!(got, Some(id));
            tx.commit().unwrap();
        }
        let basic: i64 = conn
            .query_row(
                "SELECT basic_state FROM files WHERE id = ?1",
                params![id],
                |r| r.get(0),
            )
            .unwrap();
        assert_eq!(basic, STATE_DONE);
        assert_eq!(content_state(&conn), STATE_PENDING);

        let fts_hits: i64 = conn
            .query_row(
                "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'old'",
                [],
                |r| r.get(0),
            )
            .unwrap();
        assert_eq!(fts_hits, 0);

        // And rewritten as something nothing claims: NA, not pending. Without
        // this the row would re-enter the content pass on every run.
        {
            let tx = conn.transaction().unwrap();
            row.mtime = 3;
            row.mime = Some("video/mp4");
            row.needs_content = false;
            update_file_basic(&tx, &row)
                .unwrap()
                .expect("row still there");
            tx.commit().unwrap();
        }
        assert_eq!(content_state(&conn), STATE_NA);

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn insert_file_twice_on_same_path_is_idempotent() {
        // Defense-in-depth: when the walker visits a canonical path twice
        // (overlapping roots, symlink resolution quirks), the second INSERT
        // must be a silent no-op, not a run-ending error.
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let tx = conn.transaction().unwrap();
        let row = NewFile {
            name: "dup.txt",
            path: "/tmp/dup.txt",
            parent: "/tmp",
            size: 1,
            mtime: 1,
            inode: None,
            device_id: None,
            mime: Some("text/plain"),
            ftype: FileType::TEXT,
            hash: None,
            needs_content: true,
        };
        let id1 = insert_file(&tx, &row).unwrap().expect("first insert");
        let id2 = insert_file(&tx, &row).unwrap();
        assert!(id2.is_none(), "second insert of same path must return None");
        // Only one row exists.
        let count: i64 = tx
            .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
            .unwrap();
        assert_eq!(count, 1);
        let (id_read,): (i64,) = tx
            .query_row(
                "SELECT id FROM files WHERE path = ?1",
                params!["/tmp/dup.txt"],
                |r| Ok((r.get(0)?,)),
            )
            .unwrap();
        assert_eq!(id_read, id1);
        tx.commit().unwrap();
        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn delete_subtree_clears_every_dependent_table() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let add = |tx: &Transaction<'_>, path: &str| -> i64 {
            let name = path.rsplit('/').next().unwrap();
            let parent = &path[..path.rfind('/').unwrap()];
            let id = insert_file(
                tx,
                &NewFile {
                    name,
                    path,
                    parent,
                    size: 1,
                    mtime: 1,
                    inode: None,
                    device_id: None,
                    mime: Some("text/plain"),
                    ftype: FileType::TEXT,
                    hash: None,
                    needs_content: true,
                },
            )
            .unwrap()
            .expect("unique path");
            set_content_done(tx, id, name, "body text", &[("k".into(), "v".into())], true).unwrap();
            id
        };

        {
            let tx = conn.transaction().unwrap();
            add(&tx, "/tree/a.txt");
            add(&tx, "/tree/deep/b.txt");
            let failed = add(&tx, "/tree/deep/c.txt");
            set_content_failed(&tx, failed, "bad parse").unwrap();
            // Outside the range: a prefix sibling, and a LIKE-metacharacter
            // neighbour that a `LIKE 'tree_%'` sweep would have swallowed.
            add(&tx, "/tree2/keep.txt");
            add(&tx, "/treeX/keep.txt");
            tx.commit().unwrap();
        }

        let range = crate::file_handling::ExtractCursor::for_root("/tree");
        let removed = {
            let tx = conn.transaction().unwrap();
            // The directory row itself does not exist (only files are indexed),
            // so everything here comes from the range.
            let n = delete_subtree(&tx, &range.lo, &range.hi).unwrap();
            tx.commit().unwrap();
            n
        };
        assert_eq!(removed, 3, "three files under /tree");

        let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() };
        assert_eq!(count("SELECT COUNT(*) FROM files"), 2, "siblings survive");
        assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 2);
        assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 2);
        assert_eq!(count("SELECT COUNT(*) FROM properties"), 2);
        assert_eq!(
            count("SELECT COUNT(*) FROM failed_files"),
            0,
            "the failed row went with its file"
        );

        let survivors: Vec<String> = {
            let mut stmt = conn
                .prepare("SELECT path FROM files ORDER BY path")
                .unwrap();
            let v = stmt
                .query_map([], |r| r.get::<_, String>(0))
                .unwrap()
                .map(|r| r.unwrap())
                .collect();
            v
        };
        assert_eq!(survivors, vec!["/tree2/keep.txt", "/treeX/keep.txt"]);

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// Seed a database with `paths` as fully-indexed rows, each carrying an
    /// FTS entry, stored text and a property. Returns `path -> id`.
    fn seeded(conn: &mut Connection, paths: &[&str]) -> std::collections::HashMap<String, i64> {
        let tx = conn.transaction().unwrap();
        let mut ids = std::collections::HashMap::new();
        for path in paths {
            let name = path.rsplit('/').next().unwrap();
            let parent = &path[..path.rfind('/').unwrap()];
            let id = insert_file(
                &tx,
                &NewFile {
                    name,
                    path,
                    parent,
                    size: 1,
                    mtime: 1,
                    inode: None,
                    device_id: None,
                    mime: Some("text/plain"),
                    ftype: FileType::TEXT,
                    hash: None,
                    needs_content: true,
                },
            )
            .unwrap()
            .expect("unique path");
            set_content_done(
                &tx,
                id,
                name,
                "body text",
                &[("k".into(), "v".into())],
                true,
            )
            .unwrap();
            ids.insert((*path).to_string(), id);
        }
        tx.commit().unwrap();
        ids
    }

    /// The out-of-root sweep must keep every configured root's rows and take
    /// everything else — including a path that merely *starts* with a root's
    /// name, which is a different folder.
    #[test]
    fn delete_outside_ranges_keeps_exactly_the_configured_roots() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        seeded(
            &mut conn,
            &[
                "/roots/one/a.txt",
                "/roots/one/deep/b.txt",
                "/roots/two/c.txt",
                "/roots/onefold/d.txt",
                "/elsewhere/target.txt",
            ],
        );

        let ranges: Vec<(String, String)> = ["/roots/one", "/roots/two"]
            .iter()
            .map(|r| {
                let range = crate::file_handling::ExtractCursor::for_root(r);
                (range.lo, range.hi)
            })
            .collect();
        let removed = {
            let tx = conn.transaction().unwrap();
            let n = delete_outside_ranges(&tx, &ranges).unwrap();
            tx.commit().unwrap();
            n
        };
        assert_eq!(removed, 2, "the name-prefix sibling and the outsider");

        let survivors: Vec<String> = {
            let mut stmt = conn
                .prepare("SELECT path FROM files ORDER BY path")
                .unwrap();
            let v = stmt
                .query_map([], |r| r.get::<_, String>(0))
                .unwrap()
                .map(|r| r.unwrap())
                .collect();
            v
        };
        assert_eq!(
            survivors,
            vec![
                "/roots/one/a.txt",
                "/roots/one/deep/b.txt",
                "/roots/two/c.txt"
            ]
        );
        assert_eq!(
            conn.query_row("SELECT COUNT(*) FROM searchabletext", [], |r| r
                .get::<_, i64>(0))
                .unwrap(),
            3
        );
        assert_eq!(
            conn.query_row("SELECT COUNT(*) FROM documents_text", [], |r| r
                .get::<_, i64>(0))
                .unwrap(),
            3
        );

        // No roots configured is a half-written config, not an instruction to
        // delete the entire index.
        let tx = conn.transaction().unwrap();
        assert_eq!(delete_outside_ranges(&tx, &[]).unwrap(), 0);
        tx.commit().unwrap();
        assert_eq!(
            conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get::<_, i64>(0))
                .unwrap(),
            3
        );

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// `delete_ids` is what a newly-added ignore pattern prunes with, and a
    /// file is only really gone when its name row, its FTS postings, its
    /// stored text, its properties and any failure record all go. A survivor
    /// in any one of them keeps the file findable, which is the whole thing
    /// the user asked to stop.
    #[test]
    fn delete_ids_clears_every_dependent_table() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let ids = seeded(
            &mut conn,
            &["/t/a.txt", "/t/b.log", "/t/deep/c.log", "/t/keep.txt"],
        );
        {
            let tx = conn.transaction().unwrap();
            set_content_failed(&tx, ids["/t/deep/c.log"], "bad parse").unwrap();
            tx.commit().unwrap();
        }

        let doomed = vec![ids["/t/b.log"], ids["/t/deep/c.log"]];
        let removed = {
            let tx = conn.transaction().unwrap();
            let n = delete_ids(&tx, &doomed).unwrap();
            tx.commit().unwrap();
            n
        };
        assert_eq!(removed, 2);

        let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() };
        assert_eq!(count("SELECT COUNT(*) FROM files"), 2);
        assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 2);
        assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 2);
        assert_eq!(count("SELECT COUNT(*) FROM properties"), 2);
        assert_eq!(count("SELECT COUNT(*) FROM failed_files"), 0);

        // The FTS index really lost them, not just the `files` row: a
        // contentless table keeps serving deleted rowids without the tombstone.
        let hits: i64 = conn
            .query_row(
                "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'body'",
                [],
                |r| r.get(0),
            )
            .unwrap();
        assert_eq!(hits, 2);

        // Empty input is a no-op, not a statement with an empty `IN ()`.
        let tx = conn.transaction().unwrap();
        assert_eq!(delete_ids(&tx, &[]).unwrap(), 0);
        tx.commit().unwrap();

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// More ids than `DELETE_IDS_CHUNK`, so the short final chunk and the
    /// full ones both run — the boundary a fixed-size placeholder list makes
    /// easy to get wrong.
    #[test]
    fn delete_ids_spans_chunk_boundaries() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let paths: Vec<String> = (0..DELETE_IDS_CHUNK + 7)
            .map(|i| format!("/t/f{:05}.txt", i))
            .collect();
        let refs: Vec<&str> = paths.iter().map(String::as_str).collect();
        let ids = seeded(&mut conn, &refs);

        let mut all: Vec<i64> = ids.values().copied().collect();
        all.sort_unstable();
        let keep = all.pop().unwrap();
        let removed = {
            let tx = conn.transaction().unwrap();
            let n = delete_ids(&tx, &all).unwrap();
            tx.commit().unwrap();
            n
        };
        assert_eq!(removed, DELETE_IDS_CHUNK + 6);
        let left: i64 = conn
            .query_row("SELECT id FROM files", [], |r| r.get(0))
            .unwrap();
        assert_eq!(left, keep);

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// Dropping stored text must cost the file its snippets and nothing else:
    /// the FTS postings are what full-text search runs on, and re-extracting
    /// every file to turn a storage setting off would be absurd.
    #[test]
    fn drop_stored_text_keeps_the_file_searchable() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let ids = seeded(&mut conn, &["/t/a.txt", "/t/b.txt"]);

        {
            let tx = conn.transaction().unwrap();
            drop_stored_text(&tx, &[ids["/t/a.txt"]]).unwrap();
            tx.commit().unwrap();
        }

        let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() };
        assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 1);
        assert_eq!(count("SELECT COUNT(*) FROM files"), 2);
        assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 2);
        assert_eq!(
            count("SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'body'"),
            2,
            "both files still match on content"
        );
        assert_eq!(count("SELECT COUNT(*) FROM properties"), 2);

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// Re-queuing content leaves the row's metadata alone — the file has not
    /// changed, the configuration has — but clears what the last extraction
    /// produced, so a second pass cannot double-insert into the FTS table.
    #[test]
    fn reset_content_pending_clears_the_last_extraction() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let ids = seeded(&mut conn, &["/t/a.txt", "/t/b.txt"]);
        let id = ids["/t/a.txt"];
        {
            let tx = conn.transaction().unwrap();
            set_content_failed(&tx, ids["/t/b.txt"], "bad parse").unwrap();
            tx.commit().unwrap();
        }

        {
            let tx = conn.transaction().unwrap();
            reset_content_pending(&tx, id).unwrap();
            reset_content_pending(&tx, ids["/t/b.txt"]).unwrap();
            tx.commit().unwrap();
        }

        let row: (i64, i64, Option<String>) = conn
            .query_row(
                "SELECT content_state, mtime, failure_msg FROM files WHERE id = ?1",
                params![id],
                |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
            )
            .unwrap();
        assert_eq!(row.0, STATE_PENDING);
        assert_eq!(row.1, 1, "metadata untouched — the file did not change");
        assert_eq!(row.2, None);

        let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() };
        assert_eq!(count("SELECT COUNT(*) FROM files"), 2, "rows stay");
        assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 0);
        assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 0);
        assert_eq!(count("SELECT COUNT(*) FROM properties"), 0);
        assert_eq!(
            count("SELECT COUNT(*) FROM failed_files"),
            0,
            "a stale failure must not outlive the retry it was queued for"
        );

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// The reconciliation scan pages by path, so it must serve every row in
    /// the range exactly once, in order, and stop at the range bound rather
    /// than at a name prefix.
    #[test]
    fn rows_in_range_page_walks_the_range_once() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        seeded(
            &mut conn,
            &[
                "/t/a.txt",
                "/t/deep/b.txt",
                "/t/deep/deeper/c.txt",
                "/t2/outside.txt",
                "/tX/outside.txt",
            ],
        );

        let range = crate::file_handling::ExtractCursor::for_root("/t");
        let mut seen = Vec::new();
        let mut after = range.lo.clone();
        loop {
            let page = rows_in_range_page(&conn, &after, &range.hi, 2).unwrap();
            let Some(last) = page.last() else { break };
            after = last.path.clone();
            seen.extend(page.into_iter().map(|r| r.path));
        }
        assert_eq!(
            seen,
            vec!["/t/a.txt", "/t/deep/b.txt", "/t/deep/deeper/c.txt"],
            "in path order, once each, and the prefix siblings are outside"
        );

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// The walk's row prefetcher runs `dir_rows` once per directory, against a
    /// deliberately tiny page cache, so it must not have to touch the table
    /// heap at all. `idx_files_parent` carries `name` and `mtime` for exactly
    /// this; trimming it back to `(parent)` would silently reintroduce a row
    /// fetch per entry.
    #[test]
    fn dir_rows_is_served_entirely_from_the_index() {
        let p = tmp_path();
        let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let plan: String = conn
            .query_row(
                "EXPLAIN QUERY PLAN SELECT name, mtime FROM files WHERE parent = ?1",
                params!["/some/dir"],
                |r| r.get(3),
            )
            .unwrap();
        assert!(
            plan.contains("COVERING INDEX idx_files_parent"),
            "dir_rows must be index-only, got: {}",
            plan
        );
        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// The whole point of the range form: it is an index seek. `LIKE … ESCAPE`
    /// cannot use the index (SQLite disables the LIKE optimisation whenever an
    /// ESCAPE clause is present), so the old sweep read every path in the table
    /// on every deletion event.
    #[test]
    fn the_subtree_range_is_an_index_seek_not_a_scan() {
        let p = tmp_path();
        let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let plan: String = conn
            .query_row(
                "EXPLAIN QUERY PLAN DELETE FROM files WHERE path >= ?1 AND path < ?2",
                params!["/tree/", "/tree0"],
                |r| r.get(3),
            )
            .unwrap();
        assert!(
            plan.contains("SEARCH") && !plan.contains("SCAN"),
            "range delete must seek, got: {}",
            plan
        );
        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn last_full_index_round_trip() {
        let p = tmp_path();
        let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        assert_eq!(get_last_full_index(&conn), None, "fresh DB has no marker");
        set_last_full_index(&conn, 1_700_000_123).unwrap();
        assert_eq!(get_last_full_index(&conn), Some(1_700_000_123));
        // Overwrite, not accumulate.
        set_last_full_index(&conn, 1_700_000_999).unwrap();
        assert_eq!(get_last_full_index(&conn), Some(1_700_000_999));
        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn checkpoint_and_close_truncates_wal() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        {
            let tx = conn.transaction().unwrap();
            insert_file(
                &tx,
                &NewFile {
                    name: "w.txt",
                    path: "/tmp/w.txt",
                    parent: "/tmp",
                    size: 1,
                    mtime: 1,
                    inode: None,
                    device_id: None,
                    mime: None,
                    ftype: FileType::EMPTY,
                    hash: None,
                    needs_content: false,
                },
            )
            .unwrap();
            tx.commit().unwrap();
        }
        checkpoint_and_close(conn);
        // After a TRUNCATE checkpoint + close of the last connection the WAL
        // sidecar is gone or empty; the row lives in the main file.
        let wal = std::path::PathBuf::from(format!("{}-wal", p.display()));
        let wal_len = std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0);
        assert_eq!(wal_len, 0, "WAL should be truncated on clean close");
        let conn = crate::db::open_existing(p.to_str().unwrap(), false).unwrap();
        let n: i64 = conn
            .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
            .unwrap();
        assert_eq!(n, 1);
        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// Bulk rows, cheap to write, enough of them to make the file grow.
    fn seed_rows(conn: &mut Connection, range: std::ops::Range<usize>) {
        let tx = conn.transaction().unwrap();
        for i in range {
            let path = format!("/tmp/bulk/{}.txt", i);
            let name = format!("{}.txt", i);
            let id = insert_file(
                &tx,
                &NewFile {
                    name: &name,
                    path: &path,
                    parent: "/tmp/bulk",
                    size: 1,
                    mtime: 1,
                    inode: None,
                    device_id: None,
                    mime: Some("text/plain"),
                    ftype: FileType::TEXT,
                    hash: None,
                    needs_content: true,
                },
            )
            .unwrap()
            .unwrap();
            set_content_done(
                &tx,
                id,
                &name,
                &"lorem ipsum dolor sit amet ".repeat(64),
                &[],
                true,
            )
            .unwrap();
        }
        tx.commit().unwrap();
    }

    fn wal_bytes(p: &std::path::Path) -> u64 {
        std::fs::metadata(format!("{}-wal", p.display()))
            .map(|m| m.len())
            .unwrap_or(0)
    }

    #[test]
    fn checkpoint_truncate_empties_the_log() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        seed_rows(&mut conn, 0..200);
        assert!(wal_bytes(&p) > 0, "the writes should be sitting in the log");

        checkpoint_truncate(&conn).expect("nothing is holding the log back");
        assert_eq!(wal_bytes(&p), 0, "a completed TRUNCATE leaves no log");

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    /// The case the old `execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")`
    /// reported as success: a reader pins the log, SQLite declines to reset
    /// it, and the only word of it is in the result row.
    #[test]
    fn checkpoint_truncate_reports_an_incomplete_checkpoint() {
        let p = tmp_path();
        let mut writer = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        seed_rows(&mut writer, 0..200);
        // Long enough to prove the point, short enough not to stall the suite.
        writer
            .busy_timeout(std::time::Duration::from_millis(100))
            .unwrap();

        let reader = crate::db::open_existing(p.to_str().unwrap(), false).unwrap();
        let mut stmt = reader.prepare("SELECT id FROM files").unwrap();
        let mut rows = stmt.query([]).unwrap();
        rows.next().unwrap().expect("a row to hold the snapshot on");

        let err = checkpoint_truncate(&writer).expect_err("a reader holds the log open");
        assert!(err.contains("incomplete"), "unexpected message: {}", err);
        assert!(wal_bytes(&p) > 0, "and the log is still there");

        drop(rows);
        drop(stmt);
        drop(reader);
        drop(writer);
        std::fs::remove_file(&p).ok();
    }

    /// The mechanism the whole in-run checkpoint exists for.
    ///
    /// SQLite's autocheckpoint copies committed frames into the database
    /// continuously, but the log is only *reset* when the writer opens a
    /// transaction at an instant no reader holds a read mark — and it tries
    /// that lock exactly once, with no retry. A reader querying back to back,
    /// which is what an indexing run's per-root prefetcher is, keeps that
    /// instant from arriving, so the log appends for as long as the run lasts.
    /// An explicit checkpoint retries the same lock under `busy_timeout` and
    /// gets it.
    #[test]
    fn a_busy_reader_defeats_the_autocheckpoint_but_not_a_forced_one() {
        // Bare `files` rows: no text, so no zstd and no tokenising. The log
        // grows from committed frames, and frames are what starves the reset —
        // paying for extraction here would only make the test slow.
        fn seed_bare(conn: &mut Connection, range: std::ops::Range<usize>) {
            let tx = conn.transaction().unwrap();
            for i in range {
                let path = format!("/tmp/bare/{}.txt", i);
                let name = format!("{}.txt", i);
                insert_file(
                    &tx,
                    &NewFile {
                        name: &name,
                        path: &path,
                        parent: "/tmp/bare",
                        size: i as u64,
                        mtime: 1,
                        inode: None,
                        device_id: None,
                        mime: None,
                        ftype: FileType::TEXT,
                        hash: Some(&[0u8; 32]),
                        needs_content: false,
                    },
                )
                .unwrap();
            }
            tx.commit().unwrap();
        }

        fn run(p: &std::path::Path, force_every: usize) -> u64 {
            let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
            seed_bare(&mut conn, 0..500);

            // Stands in for a walk prefetcher: short reads, no gaps.
            let stop: Arc<AtomicBool> = Arc::new(AtomicBool::new(false));
            let reader = {
                let (path, stop) = (p.to_path_buf(), stop.clone());
                std::thread::spawn(move || {
                    let conn = crate::db::open_existing(path.to_str().unwrap(), false).unwrap();
                    while !stop.load(Ordering::Relaxed) {
                        let _: i64 = conn
                            .query_row(
                                "SELECT COUNT(*) FROM files WHERE parent = '/tmp/bare'",
                                [],
                                |r| r.get(0),
                            )
                            .unwrap();
                    }
                })
            };

            let mut peak = 0u64;
            for batch in 0..60 {
                let lo = 1000 + batch * 300;
                seed_bare(&mut conn, lo..lo + 300);
                if force_every > 0 && batch % force_every == force_every - 1 {
                    let _ = checkpoint_truncate(&conn);
                }
                peak = peak.max(wal_bytes(p));
            }

            stop.store(true, Ordering::Relaxed);
            reader.join().unwrap();
            drop(conn);
            peak
        }

        let unbounded = tmp_path();
        let left_alone = run(&unbounded, 0);
        std::fs::remove_file(&unbounded).ok();

        let bounded = tmp_path();
        let forced = run(&bounded, 4);
        std::fs::remove_file(&bounded).ok();

        eprintln!(
            "peak WAL: autocheckpoint only {}, forced {}",
            left_alone, forced
        );
        assert!(
            forced * 2 < left_alone,
            "forcing checkpoints did not bound the log: {} vs {}",
            forced,
            left_alone
        );
    }

    #[test]
    fn maintain_vacuums_when_slack_is_significant() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        seed_rows(&mut conn, 0..2000);
        checkpoint_truncate(&conn).unwrap();
        let before = std::fs::metadata(&p).unwrap().len();

        {
            let tx = conn.transaction().unwrap();
            for i in 0..1900 {
                delete_file_by_path(&tx, &format!("/tmp/bulk/{}.txt", i)).unwrap();
            }
            tx.commit().unwrap();
        }
        drop(conn);

        let conn = crate::db::open::open_maintenance(p.to_str().unwrap()).unwrap();
        let freelist: i64 = conn
            .query_row("PRAGMA freelist_count", [], |r| r.get(0))
            .unwrap();
        assert!(freelist > 0, "the deletions should have freed pages");

        let dir = p.parent().unwrap().to_string_lossy().into_owned();
        assert!(
            maintain(&conn, &dir).unwrap(),
            "that much slack is worth a vacuum"
        );
        assert_eq!(
            wal_bytes(&p),
            0,
            "the vacuum's own writes are checkpointed too"
        );
        assert!(
            std::fs::metadata(&p).unwrap().len() < before,
            "the file should have shrunk"
        );
        // The surviving rows are still there and still searchable.
        let n: i64 = conn
            .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
            .unwrap();
        assert_eq!(n, 100);
        let hits: i64 = conn
            .query_row(
                "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'lorem'",
                [],
                |r| r.get(0),
            )
            .unwrap();
        assert_eq!(hits, 100, "the FTS index survived the rewrite");

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn maintain_skips_vacuum_on_a_tight_file() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        seed_rows(&mut conn, 0..200);
        drop(conn);

        let conn = crate::db::open::open_maintenance(p.to_str().unwrap()).unwrap();
        let dir = p.parent().unwrap().to_string_lossy().into_owned();
        assert!(
            !maintain(&conn, &dir).unwrap(),
            "a file with no slack is not worth rewriting"
        );
        // The checkpoint is not conditional on the vacuum, though.
        assert_eq!(wal_bytes(&p), 0);

        drop(conn);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn set_content_failed_writes_failed_table() {
        let p = tmp_path();
        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
        let id = {
            let tx = conn.transaction().unwrap();
            let id = insert_file(
                &tx,
                &NewFile {
                    name: "oops.bin",
                    path: "/tmp/oops.bin",
                    parent: "/tmp",
                    size: 0,
                    mtime: 1,
                    inode: None,
                    device_id: None,
                    mime: None,
                    ftype: FileType::EMPTY,
                    hash: None,
                    needs_content: false,
                },
            )
            .unwrap()
            .expect("unique path");
            set_content_failed(&tx, id, "bad parse").unwrap();
            tx.commit().unwrap();
            id
        };

        let reason: String = conn
            .query_row(
                "SELECT reason FROM failed_files WHERE file_id = ?1",
                params![id],
                |r| r.get(0),
            )
            .unwrap();
        assert_eq!(reason, "bad parse");
        let content_state: i64 = conn
            .query_row(
                "SELECT content_state FROM files WHERE id = ?1",
                params![id],
                |r| r.get(0),
            )
            .unwrap();
        assert_eq!(content_state, STATE_FAILED);

        drop(conn);
        std::fs::remove_file(&p).ok();
    }
}