453 lines
16 KiB
Rust
453 lines
16 KiB
Rust
//! Programmatic read-only query helpers.
|
||
//!
|
||
//! Pure functions that open a DB, run a query, and return structured data.
|
||
//! No stdout, no CLI framing — callers (GUI, future CLI binaries, Set B
|
||
//! `balooctl`) format the result as they see fit. Mutating operations live
|
||
//! on [`crate::indexing::IndexingService`] since they require a running
|
||
//! worker thread.
|
||
|
||
use rusqlite::{params, OptionalExtension};
|
||
|
||
use crate::db::open_existing;
|
||
use crate::db::repo::{STATE_DONE, STATE_FAILED, STATE_NA, STATE_PENDING};
|
||
|
||
/// Per-file indexing status, mirroring Baloo's multi-state reporting.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub enum IndexState {
|
||
NotIndexed,
|
||
Pending,
|
||
Done,
|
||
Failed,
|
||
NotApplicable,
|
||
}
|
||
|
||
impl From<i64> for IndexState {
|
||
fn from(v: i64) -> Self {
|
||
match v {
|
||
x if x == STATE_PENDING => IndexState::Pending,
|
||
x if x == STATE_DONE => IndexState::Done,
|
||
x if x == STATE_FAILED => IndexState::Failed,
|
||
x if x == STATE_NA => IndexState::NotApplicable,
|
||
_ => IndexState::Pending,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Indexing status for a single file. `basic` is the metadata row state
|
||
/// (indexed or not); `content` is the extractor state.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub struct FileStatus {
|
||
pub path: String,
|
||
pub basic: IndexState,
|
||
pub content: IndexState,
|
||
pub failure_reason: Option<String>,
|
||
}
|
||
|
||
/// Per-file entry returned by [`list_failed`].
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub struct FailedEntry {
|
||
pub file_id: i64,
|
||
pub path: String,
|
||
pub reason: Option<String>,
|
||
pub ts: i64,
|
||
}
|
||
|
||
/// Storage footprint report. "Partitions" correspond to SQL tables for our
|
||
/// SQLite layout (Baloo's LMDB has named sub-DBs; our equivalent is per-table
|
||
/// row/size counts).
|
||
///
|
||
/// `documents_text_*` fields cover the zstd-compressed extracted-text
|
||
/// sidecar that replaced the regular FTS5 stored text in schema v3. Ratio
|
||
/// is `compressed / raw` so a value of ~0.3 means we saved ~70% vs storing
|
||
/// the plaintext verbatim.
|
||
#[derive(Debug, Clone, PartialEq)]
|
||
pub struct SizeReport {
|
||
pub file_size_bytes: u64,
|
||
pub files_row_count: i64,
|
||
pub properties_row_count: i64,
|
||
pub failed_files_row_count: i64,
|
||
pub searchabletext_row_count: i64,
|
||
pub documents_text_row_count: i64,
|
||
pub documents_text_raw_bytes: i64,
|
||
pub documents_text_compressed_bytes: i64,
|
||
}
|
||
|
||
impl SizeReport {
|
||
/// Compressed:raw ratio for the stored extracted text. `None` when no
|
||
/// rows have been written yet (avoids divide-by-zero).
|
||
pub fn documents_text_ratio(&self) -> Option<f64> {
|
||
if self.documents_text_raw_bytes <= 0 {
|
||
return None;
|
||
}
|
||
Some(self.documents_text_compressed_bytes as f64 / self.documents_text_raw_bytes as f64)
|
||
}
|
||
}
|
||
|
||
/// Query the per-file indexing status. Returns `FileStatus` with
|
||
/// `basic == NotIndexed` if the path isn't in the database.
|
||
pub fn status_for_path(db_path: &str, path: &str) -> Result<FileStatus, String> {
|
||
let conn = open_existing(db_path, false)?;
|
||
let row: Option<(i64, i64, Option<String>)> = conn
|
||
.query_row(
|
||
"SELECT basic_state, content_state, failure_msg FROM files WHERE path = ?1",
|
||
params![path],
|
||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||
)
|
||
.optional()
|
||
.map_err(|e| format!("status_for_path({}): {}", path, e))?;
|
||
Ok(match row {
|
||
None => FileStatus {
|
||
path: path.to_string(),
|
||
basic: IndexState::NotIndexed,
|
||
content: IndexState::NotIndexed,
|
||
failure_reason: None,
|
||
},
|
||
Some((basic, content, reason)) => FileStatus {
|
||
path: path.to_string(),
|
||
basic: IndexState::from(basic),
|
||
content: IndexState::from(content),
|
||
failure_reason: reason,
|
||
},
|
||
})
|
||
}
|
||
|
||
/// Return every file that failed content extraction, newest first.
|
||
pub fn list_failed(db_path: &str, limit: Option<u32>) -> Result<Vec<FailedEntry>, String> {
|
||
let conn = open_existing(db_path, false)?;
|
||
let limit_sql = match limit {
|
||
Some(n) => format!(" LIMIT {}", n),
|
||
None => String::new(),
|
||
};
|
||
let sql = format!(
|
||
"SELECT ff.file_id, f.path, ff.reason, ff.ts \
|
||
FROM failed_files ff \
|
||
JOIN files f ON f.id = ff.file_id \
|
||
ORDER BY ff.ts DESC{}",
|
||
limit_sql
|
||
);
|
||
let mut stmt = conn
|
||
.prepare(&sql)
|
||
.map_err(|e| format!("list_failed prepare: {}", e))?;
|
||
let rows = stmt
|
||
.query_map([], |r| {
|
||
Ok(FailedEntry {
|
||
file_id: r.get(0)?,
|
||
path: r.get(1)?,
|
||
reason: r.get(2)?,
|
||
ts: r.get(3)?,
|
||
})
|
||
})
|
||
.map_err(|e| format!("list_failed query: {}", e))?;
|
||
rows.collect::<Result<Vec<_>, _>>()
|
||
.map_err(|e| format!("list_failed row: {}", e))
|
||
}
|
||
|
||
/// Return a rough size breakdown of the database on disk and by table.
|
||
pub fn index_size_breakdown(db_path: &str) -> Result<SizeReport, String> {
|
||
let file_size_bytes = std::fs::metadata(db_path)
|
||
.map(|m| m.len())
|
||
.unwrap_or(0);
|
||
let conn = open_existing(db_path, false)?;
|
||
let count = |table: &str| -> Result<i64, String> {
|
||
conn.query_row(&format!("SELECT COUNT(*) FROM {}", table), [], |r| r.get(0))
|
||
.map_err(|e| format!("count {}: {}", table, e))
|
||
};
|
||
let dt_row_count: i64 = count("documents_text")?;
|
||
let (dt_raw, dt_compressed): (i64, i64) = conn
|
||
.query_row(
|
||
"SELECT COALESCE(SUM(text_len), 0), COALESCE(SUM(LENGTH(text_zstd)), 0) FROM documents_text",
|
||
[],
|
||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||
)
|
||
.map_err(|e| format!("documents_text size sum: {}", e))?;
|
||
|
||
Ok(SizeReport {
|
||
file_size_bytes,
|
||
files_row_count: count("files")?,
|
||
properties_row_count: count("properties")?,
|
||
failed_files_row_count: count("failed_files")?,
|
||
searchabletext_row_count: count("searchabletext")?,
|
||
documents_text_row_count: dt_row_count,
|
||
documents_text_raw_bytes: dt_raw,
|
||
documents_text_compressed_bytes: dt_compressed,
|
||
})
|
||
}
|
||
|
||
/// Count files with `content_state = 0` (pending) — files an extractor claims
|
||
/// whose text has not been read yet. Files nothing extracts (binary formats,
|
||
/// too-large files) are written `content_state = 3` (NA) when the walk records
|
||
/// them and are never counted here, so this is outstanding work rather than
|
||
/// `files_row_count − searchabletext_row_count`, which counts those forever.
|
||
///
|
||
/// Used by the Baloo compat daemon to report the "Files waiting for content
|
||
/// indexing" figure both to balooctl and to the LMDB mirror.
|
||
pub fn pending_content_count(db_path: &str) -> Result<i64, String> {
|
||
let conn = open_existing(db_path, false)?;
|
||
conn.query_row(
|
||
"SELECT COUNT(*) FROM files WHERE content_state = ?1",
|
||
rusqlite::params![crate::db::repo::STATE_PENDING],
|
||
|r| r.get(0),
|
||
)
|
||
.map_err(|e| format!("pending_content_count: {}", e))
|
||
}
|
||
|
||
/// Cheap aggregate counts for the GUI's idle status bar ("N files
|
||
/// indexed"). Callers cache the result; it's three COUNT scans, not
|
||
/// something to run per frame.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||
pub struct IndexCounts {
|
||
pub files: i64,
|
||
pub content_done: i64,
|
||
pub content_pending: i64,
|
||
}
|
||
|
||
pub fn index_counts(db_path: &str) -> Result<IndexCounts, String> {
|
||
let conn = open_existing(db_path, false)?;
|
||
let count = |sql: &str| -> Result<i64, String> {
|
||
conn.query_row(sql, [], |r| r.get(0))
|
||
.map_err(|e| format!("index_counts: {}", e))
|
||
};
|
||
Ok(IndexCounts {
|
||
files: count("SELECT COUNT(*) FROM files")?,
|
||
content_done: count("SELECT COUNT(*) FROM files WHERE content_state = 1")?,
|
||
content_pending: count("SELECT COUNT(*) FROM files WHERE content_state = 0")?,
|
||
})
|
||
}
|
||
|
||
/// Remove a single file from the index. Returns whether a row was deleted.
|
||
/// Keeps FTS/documents/properties in sync via the repo helpers.
|
||
pub fn clear_path(db_path: &str, path: &str) -> Result<bool, String> {
|
||
let mut conn = open_existing(db_path, true)?;
|
||
let tx = conn
|
||
.transaction()
|
||
.map_err(|e| format!("clear_path begin tx: {}", e))?;
|
||
let removed = crate::db::repo::delete_file_by_path(&tx, path)?;
|
||
tx.commit()
|
||
.map_err(|e| format!("clear_path commit: {}", e))?;
|
||
Ok(removed)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use crate::db::open_or_recreate;
|
||
use crate::db::repo::{insert_file, set_content_done, set_content_failed, NewFile};
|
||
use crate::mime::FileType;
|
||
|
||
fn tmp_path() -> std::path::PathBuf {
|
||
let mut p = std::env::temp_dir();
|
||
p.push(format!(
|
||
"qs-cli-test-{}-{}.sqlite",
|
||
std::process::id(),
|
||
std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap()
|
||
.as_nanos()
|
||
));
|
||
p
|
||
}
|
||
|
||
fn seed_fixture(db_path: &str) -> (i64, i64) {
|
||
let mut conn = open_or_recreate(db_path, "trigram").unwrap();
|
||
let (a, b) = {
|
||
let tx = conn.transaction().unwrap();
|
||
let a = insert_file(
|
||
&tx,
|
||
&NewFile {
|
||
name: "a.txt",
|
||
path: "/tmp/a.txt",
|
||
parent: "/tmp",
|
||
size: 1,
|
||
mtime: 1,
|
||
inode: None,
|
||
device_id: None,
|
||
mime: Some("text/plain"),
|
||
ftype: FileType::TEXT,
|
||
hash: None,
|
||
needs_content: true,
|
||
},
|
||
)
|
||
.unwrap()
|
||
.expect("unique path");
|
||
set_content_done(&tx, a, "a.txt", "hello", &[], true).unwrap();
|
||
let b = insert_file(
|
||
&tx,
|
||
&NewFile {
|
||
name: "b.bin",
|
||
path: "/tmp/b.bin",
|
||
parent: "/tmp",
|
||
size: 1,
|
||
mtime: 1,
|
||
inode: None,
|
||
device_id: None,
|
||
mime: None,
|
||
ftype: FileType::EMPTY,
|
||
hash: None,
|
||
needs_content: false,
|
||
},
|
||
)
|
||
.unwrap()
|
||
.expect("unique path");
|
||
set_content_failed(&tx, b, "bad extract").unwrap();
|
||
tx.commit().unwrap();
|
||
(a, b)
|
||
};
|
||
drop(conn);
|
||
(a, b)
|
||
}
|
||
|
||
#[test]
|
||
fn status_for_path_returns_states() {
|
||
let p = tmp_path();
|
||
let (_a, _b) = seed_fixture(p.to_str().unwrap());
|
||
|
||
let st_a = status_for_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap();
|
||
assert_eq!(st_a.basic, IndexState::Done);
|
||
assert_eq!(st_a.content, IndexState::Done);
|
||
|
||
let st_b = status_for_path(p.to_str().unwrap(), "/tmp/b.bin").unwrap();
|
||
assert_eq!(st_b.basic, IndexState::Done);
|
||
assert_eq!(st_b.content, IndexState::Failed);
|
||
assert_eq!(st_b.failure_reason.as_deref(), Some("bad extract"));
|
||
|
||
let st_missing = status_for_path(p.to_str().unwrap(), "/tmp/never.txt").unwrap();
|
||
assert_eq!(st_missing.basic, IndexState::NotIndexed);
|
||
|
||
std::fs::remove_file(&p).ok();
|
||
}
|
||
|
||
#[test]
|
||
fn list_failed_returns_failed_rows() {
|
||
let p = tmp_path();
|
||
let (_a, b) = seed_fixture(p.to_str().unwrap());
|
||
|
||
let failed = list_failed(p.to_str().unwrap(), None).unwrap();
|
||
assert_eq!(failed.len(), 1);
|
||
assert_eq!(failed[0].file_id, b);
|
||
assert_eq!(failed[0].path, "/tmp/b.bin");
|
||
assert_eq!(failed[0].reason.as_deref(), Some("bad extract"));
|
||
|
||
std::fs::remove_file(&p).ok();
|
||
}
|
||
|
||
#[test]
|
||
fn index_size_breakdown_counts_rows() {
|
||
let p = tmp_path();
|
||
let _ = seed_fixture(p.to_str().unwrap());
|
||
|
||
let r = index_size_breakdown(p.to_str().unwrap()).unwrap();
|
||
assert!(r.file_size_bytes > 0);
|
||
assert_eq!(r.files_row_count, 2);
|
||
assert_eq!(r.failed_files_row_count, 1);
|
||
// File a got content ("hello"); file b failed. Only one documents_text row.
|
||
assert_eq!(r.documents_text_row_count, 1);
|
||
assert_eq!(r.documents_text_raw_bytes, "hello".len() as i64);
|
||
assert!(r.documents_text_compressed_bytes > 0);
|
||
|
||
std::fs::remove_file(&p).ok();
|
||
}
|
||
|
||
#[test]
|
||
fn documents_text_ratio_reports_savings_on_compressible_prose() {
|
||
// Feed highly-compressible prose (lots of repeated words) and verify
|
||
// the reported ratio reflects real savings. Guards against anyone
|
||
// silently swapping the compression step for a pass-through.
|
||
let p = tmp_path();
|
||
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
|
||
{
|
||
let tx = conn.transaction().unwrap();
|
||
let id = insert_file(
|
||
&tx,
|
||
&NewFile {
|
||
name: "big.txt",
|
||
path: "/tmp/big.txt",
|
||
parent: "/tmp",
|
||
size: 1,
|
||
mtime: 1,
|
||
inode: None,
|
||
device_id: None,
|
||
mime: Some("text/plain"),
|
||
ftype: FileType::TEXT,
|
||
hash: None,
|
||
needs_content: true,
|
||
},
|
||
)
|
||
.unwrap()
|
||
.expect("unique path");
|
||
let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500);
|
||
set_content_done(&tx, id, "big.txt", &prose, &[], true).unwrap();
|
||
tx.commit().unwrap();
|
||
}
|
||
drop(conn);
|
||
|
||
let r = index_size_breakdown(p.to_str().unwrap()).unwrap();
|
||
let ratio = r.documents_text_ratio().expect("has rows");
|
||
// Repeating a 44-byte sentence 500x → zstd should hit <20% ratio
|
||
// trivially. Loose bound protects the test from zstd version churn.
|
||
assert!(ratio < 0.3, "ratio too high: {ratio} raw={} comp={}",
|
||
r.documents_text_raw_bytes, r.documents_text_compressed_bytes);
|
||
|
||
std::fs::remove_file(&p).ok();
|
||
}
|
||
|
||
#[test]
|
||
fn clear_path_removes_file_and_returns_true() {
|
||
let p = tmp_path();
|
||
let _ = seed_fixture(p.to_str().unwrap());
|
||
|
||
assert!(clear_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap());
|
||
let st = status_for_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap();
|
||
assert_eq!(st.basic, IndexState::NotIndexed);
|
||
assert!(!clear_path(p.to_str().unwrap(), "/tmp/a.txt").unwrap());
|
||
|
||
std::fs::remove_file(&p).ok();
|
||
}
|
||
|
||
#[test]
|
||
fn clear_path_on_nondefault_tokenizer_db_removes_only_target() {
|
||
// Regression: clear_path used to open with a hardcoded "trigram", so on
|
||
// an index built with a non-default tokenizer the schema-mismatch wipe
|
||
// destroyed the WHOLE index instead of deleting one row. With
|
||
// open_existing it must delete only the target and leave the rest.
|
||
let p = tmp_path();
|
||
let dbp = p.to_str().unwrap();
|
||
{
|
||
let mut conn = open_or_recreate(dbp, "unicode61").unwrap();
|
||
let tx = conn.transaction().unwrap();
|
||
for (name, path) in [("a.txt", "/tmp/a.txt"), ("b.txt", "/tmp/b.txt")] {
|
||
insert_file(
|
||
&tx,
|
||
&NewFile {
|
||
name,
|
||
path,
|
||
parent: "/tmp",
|
||
size: 1,
|
||
mtime: 1,
|
||
inode: None,
|
||
device_id: None,
|
||
mime: Some("text/plain"),
|
||
ftype: FileType::TEXT,
|
||
hash: None,
|
||
needs_content: true,
|
||
},
|
||
)
|
||
.unwrap()
|
||
.expect("unique path");
|
||
}
|
||
tx.commit().unwrap();
|
||
}
|
||
|
||
assert!(clear_path(dbp, "/tmp/a.txt").unwrap());
|
||
// The other row must survive — proof we deleted one row, not wiped.
|
||
assert_eq!(
|
||
status_for_path(dbp, "/tmp/b.txt").unwrap().basic,
|
||
IndexState::Done
|
||
);
|
||
assert_eq!(
|
||
status_for_path(dbp, "/tmp/a.txt").unwrap().basic,
|
||
IndexState::NotIndexed
|
||
);
|
||
|
||
std::fs::remove_file(&p).ok();
|
||
}
|
||
}
|