quick_search/crates/quicksearch-core/src/db/schema.rs

//! SQL strings for the current schema. Versioned; [`migrate`](super::migrate)
//! drives the upgrade path.

/// Pragmas applied on every writable connection open.
///
/// WAL, not journal-off: auto-indexing writes continuously while searches
/// stream from their own read-only connections, and WAL is what lets those
/// readers proceed without ever blocking the writer (or vice versa).
/// `synchronous = NORMAL` under WAL risks only the last commit on power
/// loss — acceptable for an index that is re-derivable from disk. Only two
/// writers exist (full index runs and the coordinator's incremental
/// updates) and they're serialized by design; `busy_timeout` is a backstop,
/// not a coordination mechanism. A clean shutdown truncates the log via
/// [`super::repo::checkpoint_and_close`], and a long run truncates it
/// periodically as it goes (see `maximum_wal_size`) — SQLite's own
/// autocheckpoint backfills but cannot reset a log that readers are touching.
pub const PRAGMAS_FAST: &str = "
    PRAGMA journal_mode = WAL;
    PRAGMA synchronous = NORMAL;
    PRAGMA busy_timeout = 5000;
    PRAGMA cache_size = 10000;
    PRAGMA temp_store = MEMORY;
    PRAGMA foreign_keys = ON;
";

/// Pragmas for the connection that compacts the index after a run.
///
/// [`PRAGMAS_FAST`] but for `temp_store`, and that one difference is the whole
/// reason this profile exists. VACUUM builds the replacement database in the
/// temp store; SQLCipher is compiled `-DSQLITE_TEMP_STORE=2`, under which
/// anything but an explicit `FILE` puts that database in memory. Vacuuming a
/// multi-gigabyte index on the indexer's connection would try to hold the
/// entire rebuilt index in RAM. See [`super::repo::maintain`], which also
/// points the temp directory at the index's own volume.
///
/// The smaller page cache is because this connection does one bulk copy and
/// then closes; the 40 MiB the indexer keeps hot buys it nothing.
pub const PRAGMAS_MAINTENANCE: &str = "
    PRAGMA journal_mode = WAL;
    PRAGMA synchronous = NORMAL;
    PRAGMA busy_timeout = 5000;
    PRAGMA cache_size = 2000;
    PRAGMA temp_store = FILE;
    PRAGMA foreign_keys = ON;
";

/// Pragmas safe to apply on a read-only connection, where `journal_mode`
/// and `synchronous` can't be changed on the file. Used by
/// [`super::open::open_existing`] for read-only opens; write paths get the
/// full [`PRAGMAS_FAST`] set.
pub const PRAGMAS_READONLY: &str = "
    PRAGMA busy_timeout = 5000;
    PRAGMA cache_size = 10000;
    PRAGMA temp_store = MEMORY;
    PRAGMA foreign_keys = ON;
";

/// Pragmas for a walk's row-prefetch connection.
///
/// Identical to [`PRAGMAS_READONLY`] but for `cache_size`, and that one
/// difference is the point. One of these connections exists per indexing
/// root, so the 10000-page (~40 MiB) cache the other profiles take would
/// cost ~200 MiB across five roots — more than the per-directory
/// classification this connection exists to serve was meant to save.
///
/// 256 pages (~1 MiB) is enough to hold the upper levels of
/// `idx_files_parent` hot, which is all these queries touch: each one is a
/// single index range lookup, and the pages under it are read once and not
/// revisited.
pub const PRAGMAS_WALK_READER: &str = "
    PRAGMA busy_timeout = 5000;
    PRAGMA cache_size = 256;
    PRAGMA temp_store = MEMORY;
    PRAGMA foreign_keys = ON;
";

/// The full current schema. Applied by [`super::open::open_or_recreate`]
/// when the DB is fresh or has just been wiped because it drifted from
/// [`super::open::CURRENT_SCHEMA_VERSION`].
///
/// FTS5 is *contentless* (see [`fts_create_sql`]): the inverted index is kept
/// but the column values aren't stored. The canonical extracted text lives
/// in a separate `documents_text` table, zstd-compressed. Snippet rendering
/// for search results decompresses on demand and highlights matches in Rust
/// (see `crate::snippet`). This keeps the on-disk footprint close to Baloo's
/// LMDB-only size while still supporting snippet/highlight features.
pub const SCHEMA_CURRENT: &str = r#"
CREATE TABLE schema_info (
    key   TEXT PRIMARY KEY,
    value TEXT NOT NULL
);

CREATE TABLE files (
    id            INTEGER PRIMARY KEY,
    name          TEXT    NOT NULL,
    path          TEXT    NOT NULL UNIQUE,
    parent        TEXT    NOT NULL,
    size          INTEGER NOT NULL,
    mtime         INTEGER NOT NULL,
    inode         INTEGER,
    device_id     INTEGER,
    mime          TEXT,
    type          INTEGER NOT NULL DEFAULT 0,
    basic_state   INTEGER NOT NULL DEFAULT 0,   -- 0=pending 1=done 2=failed
    content_state INTEGER NOT NULL DEFAULT 0,   -- 0=pending 1=done 2=failed 3=n/a
    failure_msg   TEXT,
    hash          BLOB
);

-- Covering, not just `(parent)`. The walk's row prefetcher issues
-- `SELECT name, mtime FROM files WHERE parent = ?` once per directory — the
-- hottest read in a full run — and with the bare index that is an index probe
-- plus a table-row fetch per entry. Those fetches are cold by design: the walk
-- reader deliberately runs on a 1 MiB page cache (see `PRAGMAS_WALK_READER`).
-- Carrying `name` and `mtime` in the index makes it an index-only scan.
-- `parent` stays leading, so `SELECT DISTINCT parent` range scans and
-- `paths_in_dir` are unaffected.
CREATE INDEX idx_files_parent ON files(parent, name, mtime);
CREATE INDEX idx_files_mtime  ON files(mtime);
CREATE INDEX idx_files_type   ON files(type);
CREATE INDEX idx_files_mime   ON files(mime);
CREATE INDEX idx_files_hash   ON files(hash);
CREATE INDEX idx_files_content_pending ON files(id) WHERE content_state = 0;

CREATE TABLE properties (
    file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
    key     TEXT    NOT NULL,
    value   TEXT    NOT NULL,
    PRIMARY KEY (file_id, key)
);

CREATE TABLE failed_files (
    file_id INTEGER PRIMARY KEY REFERENCES files(id) ON DELETE CASCADE,
    reason  TEXT,
    ts      INTEGER NOT NULL
);

-- Canonical extracted text for every successfully content-indexed file.
-- Compressed with zstd (see `crate::db::repo::set_content_done`). Only
-- written when the extractor produced text; absent rows mean "no body
-- text" (e.g. an image with only EXIF properties).
CREATE TABLE documents_text (
    file_id    INTEGER PRIMARY KEY REFERENCES files(id) ON DELETE CASCADE,
    text_zstd  BLOB    NOT NULL,
    text_len   INTEGER NOT NULL   -- original byte length pre-compression
);

CREATE TABLE config_validation (
    key   TEXT PRIMARY KEY,
    value TEXT NOT NULL
);
"#;

/// FTS5 virtual table DDL. Separate because the tokenizer is config-driven.
///
/// *Contentless* FTS5 (`content=''`): the inverted index is built from the
/// column values supplied on INSERT, but those values are not stored. This
/// is the main lever that pulls our on-disk footprint down toward Baloo's.
/// `contentless_delete=1` (SQLite 3.43+) lets us `DELETE FROM … WHERE
/// rowid=?` without replaying the original row text, at the cost of a
/// modest tombstone bitmap. The tokenizer is config-driven; by default
/// (`trigram`) we append `remove_diacritics 1` so queries and stored text
/// fold the same way.
///
/// Snippet/highlight aren't available through SQLite's built-in `snippet()`
/// in contentless mode — we render them in Rust from the zstd-compressed
/// `documents_text` sidecar instead.
pub fn fts_create_sql(tokenizer: &str) -> String {
    let effective = effective_tokenizer(tokenizer);
    format!(
        "CREATE VIRTUAL TABLE searchabletext USING fts5(\
            name, text, properties, \
            tokenize='{}', \
            content='', \
            contentless_delete=1\
        );",
        effective.replace('\'', "''")
    )
}

/// Map a user-facing tokenizer name to the actual FTS5 option string we
/// apply. The default `trigram` gets `remove_diacritics 1` appended so an
/// ASCII query like `cafe` matches indexed `café`, and vice versa.
/// Without this, the default trigram tokenizer would emit disjoint
/// trigram sets for the two spellings and `MATCH` would miss one of them.
/// Users who want precise match semantics can pass the full option string
/// (e.g. `"trigram case_sensitive 1 remove_diacritics 0"`) and we'll use
/// it verbatim.
pub fn effective_tokenizer(tokenizer: &str) -> String {
    let trimmed = tokenizer.trim();
    if trimmed.eq_ignore_ascii_case("trigram") {
        // Explicit default includes accent stripping. `case_sensitive 0`
        // is FTS5's default too; we repeat it here for clarity.
        "trigram remove_diacritics 1".to_string()
    } else {
        trimmed.to_string()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn plain_trigram_gets_accent_stripping() {
        assert_eq!(effective_tokenizer("trigram"), "trigram remove_diacritics 1");
        assert_eq!(effective_tokenizer("  trigram  "), "trigram remove_diacritics 1");
    }

    #[test]
    fn explicit_tokenizers_pass_through() {
        assert_eq!(
            effective_tokenizer("trigram remove_diacritics 0"),
            "trigram remove_diacritics 0"
        );
        assert_eq!(effective_tokenizer("porter"), "porter");
        assert_eq!(effective_tokenizer("unicode61"), "unicode61");
    }
}