jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

Improved text extraction for more file types

Browse source
This commit is contained in:
= 2026-08-03 12:33:47 -04:00
parent ccbe4f1c8c
commit c7e11c57d5
16 changed files with 1130 additions and 130 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
Cargo.lock generated
View file

@ -127,7 +127,7 @@ dependencies = [
"objc2-foundation 0.3.2",
"parking_lot",
"percent-encoding",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
"x11rb",
]
@ -626,6 +626,17 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "chardetng"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53"
dependencies = [
"cfg-if",
"encoding_rs",
"memchr",
]
[[package]]
name = "chrono"
version = "0.4.44"
@ -3182,7 +3193,9 @@ name = "quicksearch-core"
version = "0.8.8"
dependencies = [
"argon2",
"chardetng",
"ctrlc",
"encoding_rs",
"getrandom 0.2.15",
"globset",
"infer",
@ -3195,6 +3208,7 @@ dependencies = [
"pdf-extract",
"quick-xml 0.31.0",
"regex",
"rtf-parser",
"rusqlite",
"serde",
"sha2",
@ -3457,6 +3471,15 @@ dependencies = [
"thiserror 2.0.19",
]
[[package]]
name = "rtf-parser"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c3147b4eb521eae5e29b781bdc30ab98bbaed784bfcb8376cda60ba4b2e0e3d"
dependencies = [
"serde",
]
[[package]]
name = "rtoolbox"
version = "0.0.5"

19
README.md
View file

@ -287,11 +287,17 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
- **Indexing** (`indexing.rs`, `file_handling.rs`): full runs walk each
root (`filtered_walk` prunes hidden/ignored subtrees before descending),
classify files by mtime into insert/update/skip, batch-write metadata,
sweep stale rows, then extract content (plaintext, Office, PDF, audio
tags, EXIF; see `extract/`) for FTS. Files no larger than
`processing.hash_length` skip that second pass entirely: the head the walk
reads to hash them is already their whole content, so a plaintext body is
extracted in the same `read` and stored complete. Every run ends — whether
sweep stale rows, then extract content (plaintext, RTF, Office, PDF,
audio tags, EXIF; see `extract/`) for FTS. Files whose extension no MIME
table knows — including extensionless ones like `README` or `Makefile` —
are sniffed from their head bytes and indexed as text when they read as
text (`mime.rs`, `textenc.rs`); non-UTF-8 text (UTF-16 with BOM, legacy
charsets via chardetng) is decoded and stored as UTF-8. More claimed
files means a bigger index — `indexing.content_extensions` remains the
throttle. Files no larger than `processing.hash_length` skip that second
pass entirely: the head the walk reads to hash them is already their
whole content, so a plaintext body is extracted in the same `read` and
stored complete. Every run ends — whether
it completed or was stopped — with an optimize pass on its own connection:
checkpoint, VACUUM if the file has at least 10% slack to reclaim, `PRAGMA
optimize`, checkpoint again. Progress streams through a polled
@ -377,5 +383,6 @@ pagination: the table is virtualized, so a single scroll list capped at
- `QSB_SNIPPET_PERF=1 cargo test --release -p quicksearch-core --test
snippet_perf -- --nocapture`: snippet pipeline benchmark.
- New extractors: implement `extract::Extractor` and register it in
`Registry::default_set()`. New cascade behavior: `search/cascade.rs`
`Registry::default_set()` — order matters, the first extractor whose
`supports` accepts a MIME wins. New cascade behavior: `search/cascade.rs`
documents the rank invariants that keep streamed results append-only.

10
config_example.toml
View file

@ -46,6 +46,11 @@ include_hidden = false
# Empty = extract text from every supported format. Non-empty = content
# indexing only for these extensions; other files are still listed for
# filename search. Entries are case-insensitive, leading dot optional.
# The reserved entry "(none)" whitelists files that have no extension at
# all (Makefile, README, .bashrc); a non-empty list without it skips them.
# Inside an entry, "#" starts a comment that runs to its end, so entries may
# be annotated or commented out:
# content_extensions = ["txt", "md # docs", "# pdf — too slow", "(none)"]
content_extensions = []
# Excluded from the index entirely. A pattern without a separator matches
# any single path component (so ".git" prunes whole subtrees); patterns
@ -74,7 +79,10 @@ ignore_patterns = [".git", "node_modules", "*.tmp", ".venv", "venv"]
# Bytes read from the start of each file for its content hash, which is
# `sha256(size || first hash_length bytes)` and backs duplicate detection.
# Only the head is read: seeking to the end for a second block costs an
# extra round trip per file on network shares.
# extra round trip per file on network shares. The same bytes are also the
# detection window that decides what a file is: magic-byte matching, and
# the text sniff that lets extensionless or unknown-extension files be
# indexed as text — so shrinking this judges files on less evidence.
#
# Known limitation: files of identical size whose heads match will be
# reported as duplicates. In practice that means pre-allocated VM disk

11
crates/quicksearch-core/Cargo.toml
View file

@ -33,6 +33,17 @@ serde = { version = "1.0", features = ["derive"] }
toml = "0.8"
mime_guess = "2.0"
infer = "0.15"
# Charset decoding for non-UTF-8 text (UTF-16 .reg exports, legacy
# single-byte and CJK encodings). Already in the lockfile transitively via
# pdf-extract, so naming it directly compiles nothing new.
encoding_rs = "0.8"
# Statistical charset detection (Firefox's detector) for text that is neither
# UTF-8 nor BOM-marked. Its mandatory deps beyond encoding_rs are tiny
# (cfg-if, memchr, detone).
chardetng = "1.0"
# RTF text extraction. Pure Rust; with the default `jsbindings` feature off
# (it exists for the crate's WASM build) it depends only on serde.
rtf-parser = { version = "0.4", default-features = false }
pdf-extract = "0.12"
lopdf = "0.32"
lofty = "0.19"

137
crates/quicksearch-core/src/config.rs
View file

@ -61,8 +61,12 @@ pub struct IndexingConfig {
/// supports. Non-empty = only files with these extensions get content
/// extraction/FTS; everything else is still listed for filename search
/// (`content_state = NA`). Entries are case-insensitive, with or
/// without a leading dot. Applied at walk time, when the row is written —
/// which is why changing this forces a rebuild (see [`diff_actions`]).
/// without a leading dot. The reserved entry [`EXTENSIONLESS`] whitelists
/// files that have no extension at all (`Makefile`, `README`, `.bashrc`),
/// which are otherwise excluded by any non-empty filter. `#` starts a
/// comment — whole-entry or trailing — see [`content_filter_entries`].
/// Applied at walk time, when the row is written — which is why changing
/// what it matches forces a rebuild (see [`diff_actions`]).
pub content_extensions: Vec<String>,
/// Excluded from the index entirely — never even listed. A pattern
/// without `/` matches any single path component (so `.git` prunes
@ -82,14 +86,17 @@ pub struct IndexingConfig {
#[serde(default)]
pub struct ProcessingConfig {
/// Bytes read from the head of each new or changed file. Those bytes do
/// three jobs, so this one number sets more than the hash:
/// four jobs, so this one number sets more than the hash:
///
/// 1. with the size, they identify the file (see `get_file_hash`);
/// 2. they are the magic-byte window for MIME detection — `infer` reads
/// 8 KiB from a path and its longest matcher needs 262 bytes, so the
/// default is exactly as good as opening the file, and a value under
/// 262 makes some formats undetectable except by extension;
/// 3. any plaintext file no larger than this is extracted during the
/// 3. they are the text-sniff window (`textenc::looks_like_text`) that
/// decides whether an unknown-extension or extensionless file is
/// text — small values judge files on less evidence;
/// 4. any plaintext file no larger than this is extracted during the
/// walk, sparing the content pass an open/read/close.
///
/// Changing it invalidates stored hashes and forces a rebuild.
@ -504,21 +511,43 @@ impl Config {
}
}
/// Reserved `content_extensions` entry standing for "files with no
/// extension". Matched case-insensitively, and it cannot collide with a real
/// extension because the parentheses are not part of one.
pub const EXTENSIONLESS: &str = "(none)";
/// The `content_extensions` entries that actually filter: `#` starts a
/// comment and runs to the end of the entry, so a whole-line comment drops
/// out entirely and `md # notes` filters on `md`. Surrounding space is
/// trimmed; what is left of a `#` never is.
pub fn content_filter_entries(list: &[String]) -> impl Iterator<Item = &str> {
list.iter().filter_map(|raw| {
let entry = raw.split('#').next().unwrap_or_default().trim();
(!entry.is_empty()).then_some(entry)
})
}
/// Whether a file's content (text extraction + FTS) should be indexed under
/// the `content_extensions` filter. Files that fail this are still listed
/// for filename search. Empty filter = everything allowed.
/// for filename search. No entries (empty, or nothing but comments) =
/// everything allowed.
pub fn content_allowed(path: &Path, cfg: &Config) -> bool {
if cfg.indexing.content_extensions.is_empty() {
let list = &cfg.indexing.content_extensions;
if content_filter_entries(list).next().is_none() {
return true;
}
let ext = match path.extension().and_then(|e| e.to_str()) {
Some(e) => e.to_ascii_lowercase(),
None => return false,
};
cfg.indexing
.content_extensions
.iter()
.any(|allowed| allowed.trim_start_matches('.').eq_ignore_ascii_case(&ext))
// `Path::extension` is None for `Makefile` and for dot-only names like
// `.bashrc`, so without the sentinel a non-empty filter always skips them.
match path.extension().and_then(|e| e.to_str()) {
// The sentinel is reserved: it never doubles as an extension, so a
// file named `x.(none)` is not whitelisted by it.
Some(ext) => content_filter_entries(list)
.filter(|allowed| !allowed.eq_ignore_ascii_case(EXTENSIONLESS))
.any(|allowed| allowed.trim_start_matches('.').eq_ignore_ascii_case(ext)),
None => {
content_filter_entries(list).any(|allowed| allowed.eq_ignore_ascii_case(EXTENSIONLESS))
}
}
}
/// Compiled ignore patterns, split by matching scope: patterns without a
@ -678,7 +707,10 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
// outside every root, which no sweep will ever reach.
|| old.indexing.follow_symlinks != new.indexing.follow_symlinks
|| old.indexing.ignore_patterns != new.indexing.ignore_patterns
|| old.indexing.content_extensions != new.indexing.content_extensions
// Comments are not part of the filter, so annotating the list is not
// a reason to rebuild — only a change to what it actually matches is.
|| !content_filter_entries(&old.indexing.content_extensions)
.eq(content_filter_entries(&new.indexing.content_extensions))
|| old.security.password_protected != new.security.password_protected
|| old.security.salt != new.security.salt
|| roots_changed;
@ -815,6 +847,81 @@ mod tests {
assert!(content_allowed(Path::new("/a/readme.md"), &cfg), "leading dot + case in filter");
assert!(!content_allowed(Path::new("/a/b.pdf"), &cfg));
assert!(!content_allowed(Path::new("/a/noext"), &cfg));
assert!(!content_allowed(Path::new("/a/.bashrc"), &cfg), "dot-only name has no ext");
}
#[test]
fn content_allowed_extensionless_sentinel() {
let mut cfg = Config::default();
cfg.indexing.content_extensions = vec!["txt".into(), " (NonE) ".into()];
assert!(content_allowed(Path::new("/a/Makefile"), &cfg));
assert!(content_allowed(Path::new("/a/.bashrc"), &cfg), "dot-only name");
assert!(content_allowed(Path::new("/a/b.txt"), &cfg), "real extensions still work");
assert!(!content_allowed(Path::new("/a/b.pdf"), &cfg), "sentinel is not a wildcard");
// The sentinel is not itself an extension: a file literally named
// `x.none` is not whitelisted by it.
assert!(!content_allowed(Path::new("/a/x.none"), &cfg));
assert!(!content_allowed(Path::new("/a/x.(none)"), &cfg));
// Every capitalisation of the word means the same thing.
for spelling in ["(none)", "(NONE)", "(NonE)", "(nOnE)"] {
let mut c = Config::default();
c.indexing.content_extensions = vec![spelling.to_string()];
assert!(content_allowed(Path::new("/a/README"), &c), "{spelling}");
}
// A leading dot is stripped for extensions but must not turn some
// other entry into the sentinel.
let mut only_txt = Config::default();
only_txt.indexing.content_extensions = vec!["txt".into()];
assert!(!content_allowed(Path::new("/a/Makefile"), &only_txt));
}
#[test]
fn content_allowed_comments() {
let mut cfg = Config::default();
cfg.indexing.content_extensions = vec![
"# source files only".into(),
"rs # rust".into(),
" .MD\t# docs ".into(),
" # indented whole-line comment".into(),
"(none) # Makefile, LICENSE, ...".into(),
];
assert!(content_allowed(Path::new("/a/b.rs"), &cfg));
assert!(content_allowed(Path::new("/a/b.md"), &cfg), "dot + trailing comment");
assert!(content_allowed(Path::new("/a/Makefile"), &cfg), "sentinel + comment");
assert!(!content_allowed(Path::new("/a/b.pdf"), &cfg));
// Comment text is not itself a filter entry.
assert!(!content_allowed(Path::new("/a/b.rust"), &cfg));
assert!(!content_allowed(Path::new("/a/b.only"), &cfg));
assert!(!content_allowed(Path::new("/a/b.docs"), &cfg));
// Nothing but comments filters nothing — same as an empty list.
let mut all_comments = Config::default();
all_comments.indexing.content_extensions =
vec!["# nothing enabled yet".into(), " ".into(), "#".into()];
assert!(content_allowed(Path::new("/a/b.pdf"), &all_comments));
assert!(content_allowed(Path::new("/a/Makefile"), &all_comments));
}
#[test]
fn comment_only_edit_does_not_force_rebuild() {
let mut old = Config::default();
old.indexing.content_extensions = vec!["txt".into(), "md".into()];
let mut new = old.clone();
new.indexing.content_extensions =
vec!["# my notes".into(), "txt".into(), "md # markdown".into()];
assert!(!diff_actions(&old, &new).requires_rebuild);
// Changing what the list matches still does.
let mut changed = old.clone();
changed.indexing.content_extensions = vec!["txt".into(), "md".into(), "(none)".into()];
assert!(diff_actions(&old, &changed).requires_rebuild);
// ... including commenting an entry out.
let mut disabled = old.clone();
disabled.indexing.content_extensions = vec!["txt".into(), "# md".into()];
assert!(diff_actions(&old, &disabled).requires_rebuild);
}
#[test]

6
crates/quicksearch-core/src/coordinator.rs
View file

@ -821,6 +821,12 @@ impl Inner {
let idle = matches!(status, IndexingStatus::Idle | IndexingStatus::Error(_));
if !idle {
self.indexing.request_stop();
// Dropping the service joins its worker, and a VACUUM answers to
// nothing but `sqlite3_interrupt` — without this, closing the
// window during an optimize pass would wait out a rewrite of the
// whole index. The interrupted VACUUM rolls back, and the next
// run's checkpoints land the log.
self.indexing.cancel_optimizing();
}
if let Some(conn) = self.write_conn.take() {
if idle {

12
crates/quicksearch-core/src/db/open.rs
View file

@ -30,10 +30,14 @@ use crate::security::IndexKey;
pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: ";
/// Bump this whenever [`SCHEMA_CURRENT`] or [`fts_create_sql`] changes in
/// a way that makes an old DB unreadable by new code. Any such bump
/// causes existing indexes to be wiped on next open — there's no
/// migration path by design.
pub const CURRENT_SCHEMA_VERSION: u32 = 4;
/// a way that makes an old DB unreadable by new code — or when stored,
/// classifier-derived values go stale: `files.mime`, `files.type` and
/// `content_state` are computed at walk time and never re-derived for
/// unchanged files, so a classification change (v5: text sniffing, charset
/// decoding, RTF) needs the wipe to apply everywhere. Any such bump causes
/// existing indexes to be wiped on next open — there's no migration path
/// by design.
pub const CURRENT_SCHEMA_VERSION: u32 = 5;
/// Open `db_path`, applying fast-path pragmas, and ensure the on-disk
/// schema matches what this build expects. If it doesn't, delete the

33
crates/quicksearch-core/src/extract/mod.rs
View file

@ -20,6 +20,7 @@ pub mod image;
pub mod office;
pub mod pdf;
pub mod plaintext;
pub mod rtf;
/// Result of a successful extraction. `text` feeds the FTS5 `text` column;
/// `properties` feeds both the `properties` FTS5 column (as `key:value`
@ -169,10 +170,18 @@ impl Registry {
self.find(mime).and_then(|e| e.extract_from_head(path, head))
}
/// The default set wired up for Set A: plaintext, office docs, PDF,
/// audio tags, image EXIF.
/// The default set: RTF, plaintext, office docs, PDF, audio tags,
/// image EXIF.
///
/// Order matters — the first extractor whose `supports` accepts a MIME
/// wins. RTF precedes plaintext because plaintext claims every `text/*`
/// and would swallow `text/rtf` as raw control words. Plaintext
/// precedes audio and image because it deliberately claims playlist
/// (`audio/x-mpegurl`, `audio/scpls`) and SVG MIMEs whose text is worth
/// more than their tags.
pub fn default_set() -> Self {
Self::new()
.with(rtf::RtfExtractor)
.with(plaintext::PlaintextExtractor)
.with(office::OfficeExtractor)
.with(pdf::PdfExtractor)
@ -230,6 +239,26 @@ mod tests {
"{} should extract from a head", mime
);
}
for mime in ["application/rtf", "text/rtf"] {
assert!(
r.extract_complete_head(p, mime, br"{\rtf1 x}").is_some(),
"{} should extract from a head", mime
);
}
}
/// `text/rtf` must dispatch to the RTF extractor, not to plaintext's
/// `text/*` claim — i.e. the registration order does its job. The RTF
/// parser strips control words; plaintext would keep them.
#[test]
fn text_rtf_reaches_the_rtf_extractor_not_plaintext() {
let r = Registry::default_set();
let p = Path::new("/tmp/whatever.rtf");
let out = r
.extract_complete_head(p, "text/rtf", br"{\rtf1\ansi Hello {\b World}}")
.expect("claimed")
.expect("parsed");
assert_eq!(out.text, "Hello World");
}
#[test]

111
crates/quicksearch-core/src/extract/plaintext.rs
View file

@ -1,5 +1,7 @@
//! Read the file as UTF-8 text. Handles text/plain, text/x-*, application/json
//! and most source-code MIMEs.
//! Read the file as text, decoding UTF-8, BOM-marked UTF-16, and detected
//! legacy charsets to UTF-8 for storage (see [`crate::textenc`]). Handles
//! text/plain, text/x-*, and the non-`text/*` formats in
//! [`EXTRA_TEXT_MIMES`].
use std::fs::File;
use std::io::Read;
@ -7,36 +9,57 @@ use std::path::Path;
use super::{ExtractError, ExtractedContent, Extractor};
/// Non-`text/*` MIMEs the plaintext extractor claims. Every entry must be
/// reachable — emitted by [`crate::mime::guess_mime_from_head`] via the
/// override table, `mime_guess`, `infer`, or the text sniff — and must map
/// to a [`crate::mime::FileType`] containing TEXT; the cross-check tests in
/// `mime.rs` enforce both.
///
/// The `audio/*` and `image/*` entries (playlists, SVG) rely on this
/// extractor being registered before the audio and image extractors in
/// [`super::Registry::default_set`] — first match wins, and their text
/// content is worth more than their tags. `.svgz` also resolves to
/// `image/svg+xml`; its gzip body fails the binary guard and is recorded as
/// a failure rather than silently skipped.
pub(crate) const EXTRA_TEXT_MIMES: &[&str] = &[
"application/geo+json",
"application/javascript",
"application/json",
"application/json5",
"application/mbox",
"application/vnd.dart",
"application/x-csh",
"application/x-httpd-php",
"application/x-perl",
"application/x-sh",
// `.sql` resolves here rather than to `text/*`, so without it schema
// dumps are listed by name but never full-text indexed.
"application/x-sql",
"application/x-subrip",
"application/x-tcl",
"application/x-tex",
"application/x-texinfo",
"application/x-troff",
"application/x-troff-man",
"application/xhtml+xml",
"application/xml",
"audio/scpls",
"audio/x-mpegurl",
"image/svg+xml",
"message/rfc822",
];
/// Decode bytes that are known to be a complete file. Shared by both entry
/// points so on-disk and already-in-memory extraction cannot drift apart.
fn decode(bytes: Vec<u8>, path: &Path) -> Result<ExtractedContent, ExtractError> {
match String::from_utf8(bytes) {
Ok(text) => Ok(ExtractedContent::with_text(text)),
Err(e) => Err(format!("plaintext read {}: {}", path.display(), e.utf8_error())),
}
crate::textenc::decode_text(bytes, path).map(ExtractedContent::with_text)
}
pub struct PlaintextExtractor;
impl Extractor for PlaintextExtractor {
fn supports(&self, mime: &str) -> bool {
if mime.starts_with("text/") {
return true;
}
matches!(
mime,
"application/json"
| "application/xml"
| "application/javascript"
| "application/x-shellscript"
| "application/x-python"
| "application/toml"
| "application/yaml"
| "application/x-yaml"
// `.sql` resolves here rather than to `text/*`, so without it
// schema dumps are listed by name but never full-text indexed.
| "application/x-sql"
)
mime.starts_with("text/") || EXTRA_TEXT_MIMES.contains(&mime)
}
/// Read the whole file, sized from the handle we just opened.
@ -135,15 +158,41 @@ mod tests {
}
#[test]
fn both_paths_reject_invalid_utf8_and_name_the_file() {
let p = tmp("badutf8", &[0x68, 0x69, 0xff, 0xfe]);
fn both_paths_reject_binary_and_name_the_file() {
// A NUL keeps this undecodable now that legacy charsets decode.
let body = [0x68, 0x69, 0x00, 0xff];
let p = tmp("binary", &body);
let disk_err = PlaintextExtractor.extract(&p).unwrap_err();
let head_err = PlaintextExtractor
.extract_from_head(&p, &[0x68, 0x69, 0xff, 0xfe])
.extract_from_head(&p, &body)
.unwrap()
.unwrap_err();
assert_eq!(disk_err, head_err, "one decode path, one message");
assert!(disk_err.contains("badutf8"), "the failure names the file: {}", disk_err);
assert!(disk_err.contains("binary"), "the failure names the file: {}", disk_err);
std::fs::remove_file(&p).ok();
}
#[test]
fn latin1_decodes_via_both_paths() {
let body = b"une journ\xe9e agr\xe9able pr\xe8s de la rivi\xe8re";
let p = tmp("latin1", body);
let from_disk = PlaintextExtractor.extract(&p).unwrap();
let from_head = PlaintextExtractor.extract_from_head(&p, body).unwrap().unwrap();
assert_eq!(from_disk.text, from_head.text);
assert_eq!(from_disk.text, "une journée agréable près de la rivière");
std::fs::remove_file(&p).ok();
}
#[test]
fn utf16le_bom_decodes_via_both_paths() {
let src = "Windows Registry Editor Version 5.00\r\n[HKEY_CURRENT_USER\\Software]\r\n";
let mut body = vec![0xFF, 0xFE];
body.extend(src.encode_utf16().flat_map(|u| u.to_le_bytes()));
let p = tmp("utf16", &body);
let from_disk = PlaintextExtractor.extract(&p).unwrap();
let from_head = PlaintextExtractor.extract_from_head(&p, &body).unwrap().unwrap();
assert_eq!(from_disk.text, from_head.text);
assert_eq!(from_disk.text, src, "stored text is the UTF-8 decode, BOM stripped");
std::fs::remove_file(&p).ok();
}
@ -215,6 +264,14 @@ mod tests {
assert!(e.supports("text/plain"));
assert!(e.supports("text/x-rust"));
assert!(e.supports("application/json"));
assert!(e.supports("application/x-sh"));
assert!(e.supports("image/svg+xml"));
assert!(e.supports("audio/x-mpegurl"));
assert!(e.supports("message/rfc822"));
// Never emitted by any MIME source; removed as dead.
assert!(!e.supports("application/x-shellscript"));
// RTF belongs to the RTF extractor, which registers first.
assert!(!e.supports("application/rtf"));
assert!(!e.supports("application/pdf"));
assert!(!e.supports("image/png"));
}

113
crates/quicksearch-core/src/extract/rtf.rs Normal file
View file

@ -0,0 +1,113 @@
//! RTF text extraction via the `rtf-parser` crate.
//!
//! Claims `application/rtf` (what both `mime_guess` and `infer`'s magic
//! matcher emit) and `text/rtf` (a common alias). Registered *before* the
//! plaintext extractor in [`super::Registry::default_set`], because
//! plaintext claims every `text/*` and would otherwise swallow `text/rtf`
//! and index the control-word noise raw.
use std::path::Path;
use rtf_parser::document::RtfDocument;
use super::{ExtractError, ExtractedContent, Extractor};
/// Parse a complete RTF file's bytes. Shared by both entry points so
/// on-disk and already-in-memory extraction cannot drift apart.
///
/// RTF is 7-bit ASCII by design — non-ASCII characters travel as `\'hh` and
/// `\uN` escapes — so a lossy UTF-8 view loses nothing from a well-formed
/// document, and a malformed one fails in the parser with a real reason
/// rather than in the decode.
fn parse(bytes: Vec<u8>, path: &Path) -> Result<ExtractedContent, ExtractError> {
let source = String::from_utf8_lossy(&bytes);
match RtfDocument::try_from(source.as_ref()) {
Ok(doc) => Ok(ExtractedContent::with_text(doc.get_text())),
Err(e) => Err(format!("rtf parse {}: {}", path.display(), e)),
}
}
pub struct RtfExtractor;
impl Extractor for RtfExtractor {
fn supports(&self, mime: &str) -> bool {
mime == "application/rtf" || mime == "text/rtf"
}
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
// Plain read: RTF files are rare and small enough that plaintext's
// sized-read syscall trimming would be tuning without a workload.
let bytes = std::fs::read(path)
.map_err(|e| format!("rtf read {}: {}", path.display(), e))?;
parse(bytes, path)
}
/// RTF has no trailer and needs no seeking, so a head that is the whole
/// file parses exactly like the on-disk path.
fn extract_from_head(
&self,
path: &Path,
head: &[u8],
) -> Option<Result<ExtractedContent, ExtractError>> {
Some(parse(head.to_vec(), path))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn tmp(tag: &str, body: &[u8]) -> std::path::PathBuf {
let mut p = std::env::temp_dir();
p.push(format!(
"qs-rtf-{}-{}-{}.rtf",
tag,
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::write(&p, body).unwrap();
p
}
#[test]
fn extracts_text_without_control_words() {
let body = br"{\rtf1\ansi Hello {\b World}!}";
let p = tmp("basic", body);
let c = RtfExtractor.extract(&p).unwrap();
assert_eq!(c.text, "Hello World!");
std::fs::remove_file(&p).ok();
}
#[test]
fn head_extraction_matches_reading_the_file() {
// `\'e9` is the RTF hex escape for an e-acute: the literal itself
// stays 7-bit ASCII while the extracted text does not.
let body = br"{\rtf1\ansi caf\'e9 at noon}";
let p = tmp("agree", body);
let from_disk = RtfExtractor.extract(&p).unwrap();
let from_head = RtfExtractor.extract_from_head(&p, body).unwrap().unwrap();
assert_eq!(from_disk.text, from_head.text);
assert!(from_disk.text.contains("café"), "{:?}", from_disk.text);
std::fs::remove_file(&p).ok();
}
#[test]
fn malformed_input_errors_and_names_the_file() {
let p = tmp("broken", br"{\rtf1 truncated");
let err = RtfExtractor.extract(&p).unwrap_err();
assert!(err.contains("qs-rtf-broken"), "must name the file: {}", err);
std::fs::remove_file(&p).ok();
}
#[test]
fn supports_rtf_mimes_only() {
let e = RtfExtractor;
assert!(e.supports("application/rtf"));
assert!(e.supports("text/rtf"));
assert!(!e.supports("text/plain"));
assert!(!e.supports("application/pdf"));
}
}

60
crates/quicksearch-core/src/file_handling.rs
View file

@ -1629,31 +1629,34 @@ mod count_and_extract_tests {
let registry = Registry::default_set();
// Real files, because `decide_content` runs the extractor for anything
// it claims and its answer must be a genuine one.
let cases = [
"notes.txt",
"data.json",
"schema.sql",
"song.mp3",
"photo.jpg",
"movie.mp4",
"archive.zip",
"blob.bin",
"noextension",
// it claims and its answer must be a genuine one. A text body sniffs
// as text for the extension-less names; the NUL body keeps the
// unclaimed side of the invariant exercised too.
let cases: [(&str, &[u8]); 10] = [
("notes.txt", b"plain bytes with no magic"),
("data.json", b"plain bytes with no magic"),
("schema.sql", b"plain bytes with no magic"),
("song.mp3", b"plain bytes with no magic"),
("photo.jpg", b"plain bytes with no magic"),
("movie.mp4", b"plain bytes with no magic"),
("archive.zip", b"plain bytes with no magic"),
("blob.bin", b"plain bytes with no magic"),
("noextension", b"plain bytes with no magic"),
("real.bin", b"\x00\x01\x02\x03"),
];
for name in cases {
for (name, body) in cases {
let p = root.join(name);
std::fs::write(&p, b"plain bytes with no magic").unwrap();
std::fs::write(&p, body).unwrap();
}
// Once with the filter off (the default: everything the registry
// claims), once with it narrowed to `.txt`.
for filter in [Vec::new(), vec!["txt".to_string()]] {
cfg.indexing.content_extensions = filter.clone();
for name in cases {
for (name, body) in cases {
let p = root.join(name);
let path = p.to_str().unwrap();
let mime = guess_mime_from_head(&p, b"plain bytes with no magic");
let mime = guess_mime_from_head(&p, body);
let claimed = content_extractable(&p, mime.as_deref(), &cfg, &registry);
let outcome = decide_content(path, mime.as_deref(), &registry, &cfg);
assert_eq!(
@ -1685,17 +1688,20 @@ mod count_and_extract_tests {
.needs_content
};
for name in ["notes.txt", "song.mp3", "movie.mp4", "blob.bin"] {
for name in ["notes.txt", "song.mp3", "movie.mp4", "README"] {
std::fs::write(root.join(name), b"body").unwrap();
}
// NUL bytes: the text sniff must not rescue a genuinely binary blob.
std::fs::write(root.join("blob.bin"), b"\x00\x01\x02\x03").unwrap();
let big = root.join("huge.txt");
std::fs::write(&big, vec![b'x'; 4096]).unwrap();
let cfg = Config::default();
assert!(needs(&cfg, "notes.txt"), "plaintext is claimed");
assert!(needs(&cfg, "song.mp3"), "audio tags are content too");
assert!(needs(&cfg, "README"), "an extensionless text head sniffs as text/plain");
assert!(!needs(&cfg, "movie.mp4"), "no extractor claims video");
assert!(!needs(&cfg, "blob.bin"), "unsniffable: no MIME, no extractor");
assert!(!needs(&cfg, "blob.bin"), "binary content: no MIME, no extractor");
// Over `maximum_text_file_size`, so the content pass would never read
// it even though plaintext claims the MIME.
@ -1722,13 +1728,19 @@ mod count_and_extract_tests {
let mut db = root.clone();
db.set_extension("sqlite");
// 3 files an extractor claims, 5 it never will. Big enough that the
// old behaviour (every row pending) can't coincide with the new one.
let claimed = ["a.txt", "b.json", "c.mp3"];
// 4 files an extractor claims (README via the extensionless text
// sniff), 5 it never will — the unclaimed set gets NUL bodies so
// neither the extension tables nor the sniff have anything to say.
// Big enough that the old behaviour (every row pending) can't
// coincide with the new one.
let claimed = ["a.txt", "b.json", "c.mp3", "README"];
let unclaimed = ["d.mp4", "e.zip", "f.bin", "g.exe", "h"];
for name in claimed.iter().chain(unclaimed.iter()) {
for name in claimed.iter() {
std::fs::write(root.join(name), b"body bytes, no magic").unwrap();
}
for name in unclaimed.iter() {
std::fs::write(root.join(name), b"\x00\x01body\x00").unwrap();
}
let config = Config::default();
let registry = Registry::default_set();
@ -1752,13 +1764,13 @@ mod count_and_extract_tests {
let cursor = ExtractCursor::for_root(root.to_str().unwrap());
let scope = extract_scope_prepare(&conn_mutex, &cursor, &config).unwrap();
// `extract_total` in the GUI. The two small text-ish files were
// `extract_total` in the GUI. The three small text-ish files were
// finished inline by the walk so they land in `already_done`; the mp3
// needs the disk pass. Either way the denominator is the claimed set —
// before this was decided at walk time it read 8, every indexed file.
// before this was decided at walk time it read 9, every indexed file.
assert_eq!(
(scope.pending, scope.already_done),
(1, 2),
(1, 3),
"denominator must be the files needing text, not every indexed file"
);
assert_eq!(scope.pending + scope.already_done, claimed.len());

1
crates/quicksearch-core/src/lib.rs
View file

@ -16,5 +16,6 @@ pub mod search;
pub mod security;
pub mod shutdown;
pub mod snippet;
pub mod textenc;
pub mod walk;
pub mod watcher;

290
crates/quicksearch-core/src/mime.rs
View file

@ -1,16 +1,23 @@
//! MIME type guessing and `FileType` bitmask classification.
//!
//! Two stages:
//! 1. [`guess_mime_from_head`] infers a MIME type — extension first via
//! `mime_guess`, falling back to magic-byte sniffing via `infer` for files
//! whose extension is missing or ambiguous.
//! 2. [`mime_to_type`] maps a MIME string to a [`FileType`] bitmask so a single
//! file can belong to multiple categories (e.g. a `.docx` is Document|Text).
//! [`guess_mime_from_head`] infers a MIME type in three stages: extension
//! first (an override table, then `mime_guess`), magic-byte sniffing via
//! `infer` next, and finally a text sniff ([`crate::textenc`]) that answers
//! `text/plain` for anything whose head reads as text — which is how
//! extensionless files (README, Makefile) and source extensions no MIME
//! table knows (`.go`, `.zig`) get their contents indexed. Extensions in
//! [`AMBIGUOUS_EXTENSIONS`] invert the order: content decides, and the
//! extension's MIME is only a fallback.
//!
//! The magic bytes are always ones the caller already holds. Indexing reads
//! [`mime_to_type`] then maps a MIME string to a [`FileType`] bitmask so a
//! single file can belong to multiple categories (e.g. a `.docx` is
//! Document|Text).
//!
//! The head bytes are always ones the caller already holds. Indexing reads
//! the head of every new or changed file to hash it, and those are the same
//! bytes `infer` wants, so there is no path-based variant that goes back to
//! disk for them — that was a second open/read/close per undetectable file.
//! bytes `infer` and the text sniff want, so there is no path-based variant
//! that goes back to disk for them — that was a second open/read/close per
//! undetectable file.
use std::path::Path;
@ -70,22 +77,21 @@ impl std::ops::BitOrAssign for FileType {
}
}
/// Extensions `mime_guess` gets wrong or does not know, and what they really
/// are.
/// Extensions whose MIME is pinned regardless of what `mime_guess` or the
/// file's bytes say.
///
/// Consulted *before* `mime_guess`, because for these the table is not a
/// fallback but a correction. Everything here is plain text that would
/// otherwise get no content indexing at all:
/// Consulted *before* everything else, because for these the table is not a
/// fallback but a correction or a guarantee:
///
/// - `.ps1`/`.psm1`/`.psd1` and `.url` are simply absent from `mime_guess`,
/// and `infer` only knows binary magic, so they end up with no MIME — and
/// [`crate::extract::Registry`] has no extractor to offer, so the file is
/// marked "not applicable". PowerShell is the most common script type on a
/// Windows machine.
/// - `.bat` maps to `application/x-msdownload`, i.e. an executable. It is a
/// text file, and the plaintext extractor rightly refuses the executable
/// type. (`.cmd` already resolves to `text/plain`; it is listed so the pair
/// - `.bat` maps in `mime_guess` to `application/x-msdownload`, i.e. an
/// executable, and a non-empty `mime_guess` answer would preempt the text
/// sniff — so without this entry batch files are never content-indexed.
/// (`.cmd` already resolves to `text/plain`; it is listed so the pair
/// cannot drift.)
/// - `.ps1`/`.psm1`/`.psd1`, `.inf` and `.url` are absent from `mime_guess`.
/// The text sniff would usually catch them, but pinning them costs
/// nothing and classifies them deterministically, whatever their head
/// bytes happen to look like.
///
/// Platform-neutral on purpose: a `.ps1` copied to a Linux box should classify
/// the same way.
@ -99,6 +105,27 @@ const EXTENSION_OVERRIDES: &[(&str, &str)] = &[
("url", "text/plain"),
];
/// Extensions `mime_guess` maps to a binary format that is, on a modern
/// disk, at least as often a text file: `.ts`/`.mts` TypeScript vs MPEG
/// transport stream, `.mod` go.mod vs `video/mpeg`, `.org` Org-mode vs
/// Lotus Organizer, `.scm` Scheme vs Lotus ScreenCam, `.pot` gettext
/// template vs PowerPoint template, `.vhd` VHDL source vs VirtualBox disk
/// image.
///
/// For these the content decides: magic bytes first, then the text sniff,
/// and only if both decline does `mime_guess`'s extension answer stand — so
/// a real MPEG-TS recording still classifies as video.
const AMBIGUOUS_EXTENSIONS: &[&str] = &["mod", "mts", "org", "pot", "scm", "ts", "vhd"];
/// Whether `path`'s extension is in [`AMBIGUOUS_EXTENSIONS`].
/// ASCII-case-insensitive, like [`extension_override`].
fn extension_is_ambiguous(path: &Path) -> bool {
path.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase())
.is_some_and(|e| AMBIGUOUS_EXTENSIONS.contains(&e.as_str()))
}
/// Look up [`EXTENSION_OVERRIDES`] for `path`. Extension comparison is
/// ASCII-case-insensitive, which matters more on Windows where `REPORT.BAT` is
/// as common as the lowercase spelling.
@ -112,15 +139,21 @@ fn extension_override(path: &Path) -> Option<&'static str> {
/// Infer a MIME type from a path plus the file's leading bytes.
///
/// Extension first — an override table, then `mime_guess` — and magic bytes
/// only when those come up empty or say `application/octet-stream`.
/// Extension first — an override table, then `mime_guess` — then magic
/// bytes when those come up empty or say `application/octet-stream`, and
/// finally a text sniff that answers `text/plain` for any head that reads
/// as text ([`crate::textenc::looks_like_text`]). For
/// [`AMBIGUOUS_EXTENSIONS`] the `mime_guess` answer is demoted to a last
/// resort behind both content checks.
///
/// `head` is whatever the caller already read; indexing passes the same buffer
/// it hashes. It bounds magic-byte detection, so a caller that supplies fewer
/// than 262 bytes (`infer`'s longest signature) can get `None` where a longer
/// head would have matched. The indexer's `hash_length` defaults to 8 KiB —
/// exactly what `infer` itself reads from a path — so at default config this
/// is as good as opening the file, and strictly cheaper.
/// is as good as opening the file, and strictly cheaper. The same buffer
/// bounds the text sniff, which tolerates a multibyte character cut off at
/// the buffer's end.
///
/// A `None` result is a real answer, not a "don't know": the content pass
/// stores it and does not re-derive it (see
@ -129,13 +162,29 @@ pub fn guess_mime_from_head(path: &Path, head: &[u8]) -> Option<String> {
if let Some(m) = extension_override(path) {
return Some(m.to_string());
}
if let Some(g) = mime_guess::from_path(path).first() {
let by_extension = mime_guess::from_path(path).first().and_then(|g| {
let s = g.essence_str();
if !s.is_empty() && s != "application/octet-stream" {
return Some(s.to_string());
(!s.is_empty() && s != "application/octet-stream").then(|| s.to_string())
});
if !extension_is_ambiguous(path) && by_extension.is_some() {
return by_extension;
}
if let Some(t) = infer::get(head) {
let magic = t.mime_type();
// For an ambiguous extension, `infer`'s generic OLE-container answer
// is less specific than the extension's: a real PowerPoint `.pot`
// template must resolve to vnd.ms-powerpoint (which the office
// extractor claims), not to a container MIME nothing claims.
if magic == "application/x-ole-storage" && by_extension.is_some() {
return by_extension;
}
infer::get(head).map(|t| t.mime_type().to_string())
return Some(magic.to_string());
}
if crate::textenc::looks_like_text(head) {
return Some("text/plain".to_string());
}
// Only an ambiguous extension still has an answer left to fall back on.
by_extension
}
/// Map a MIME string to a [`FileType`] bitmask. Ported from Baloo's
@ -153,8 +202,9 @@ pub fn mime_to_type(mime: &str) -> FileType {
"video" => t |= FileType::VIDEO,
"text" => {
t |= FileType::TEXT;
// HTML counts as a document too in Baloo.
if sub == "html" || sub == "xhtml+xml" {
// HTML counts as a document too in Baloo. (xhtml+xml is handled
// in the subtype match below, whatever its top level.)
if sub == "html" {
t |= FileType::DOCUMENT;
}
}
@ -204,8 +254,19 @@ pub fn mime_to_type(mime: &str) -> FileType {
| "x-msi" => {
t |= FileType::ARCHIVE;
}
// application/xml is structured text
"xml" | "json" | "javascript" | "x-shellscript" | "x-python" => {
// XHTML is text and, like HTML above, a document in Baloo's model —
// whichever top level it arrives under.
"xhtml+xml" => {
t |= FileType::TEXT | FileType::DOCUMENT;
}
// Structured text: everything the plaintext extractor claims beyond
// `text/*` (see `extract::plaintext::EXTRA_TEXT_MIMES` and the
// cross-check test below). Keyed on the subtype alone, so playlists
// stay AUDIO|TEXT and SVG stays IMAGE|TEXT.
"xml" | "json" | "json5" | "geo+json" | "javascript" | "mbox" | "rfc822"
| "vnd.dart" | "x-csh" | "x-httpd-php" | "x-perl" | "x-sh" | "x-sql"
| "x-subrip" | "x-tcl" | "x-tex" | "x-texinfo" | "x-troff" | "x-troff-man"
| "x-mpegurl" | "scpls" | "svg+xml" => {
t |= FileType::TEXT;
}
_ => {}
@ -396,15 +457,19 @@ mod tests {
}
/// The other side of that bound: starve the head below `infer`'s longest
/// signature and detection legitimately degrades. Documented behaviour of
/// a non-default `hash_length`, not a bug — but it must stay a `None`
/// rather than a wrong guess.
/// signature and magic detection legitimately degrades. Documented
/// behaviour of a non-default `hash_length`, not a bug — but a binary
/// head must stay a `None` rather than become a wrong guess. (A head
/// that *reads as text* is a different case: the text sniff answers for
/// it, however short.)
#[test]
fn a_head_shorter_than_the_signature_declines_rather_than_guessing() {
use std::path::PathBuf;
let path = PathBuf::from("/tmp/qs-sniff-truncated");
assert_eq!(guess_mime_from_head(&path, b"").as_deref(), None);
assert_eq!(guess_mime_from_head(&path, &[0x89]).as_deref(), None);
// A PNG magic truncated to two bytes: not a magic match, and the
// NUL fails the binary guard, so no text guess either.
assert_eq!(guess_mime_from_head(&path, &[0x89, 0x00]).as_deref(), None);
// Enough bytes, and it resolves.
assert_eq!(
guess_mime_from_head(&path, &[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a])
@ -420,4 +485,155 @@ mod tests {
assert!(mime_to_type("application/vnd.ms-outlook").contains(FileType::DOCUMENT));
assert!(mime_to_type("application/vnd.ms-htmlhelp").contains(FileType::DOCUMENT));
}
/// Every extension fixed by this round of coverage work must reach the
/// plaintext extractor through real dispatch — `extract_complete_head`
/// rather than `supports` — so the svg/m3u/pls cases prove the
/// plaintext-first registration *order*, not just the MIME claim.
#[test]
fn newly_claimed_extensions_reach_the_plaintext_extractor() {
use crate::extract::Registry;
use std::path::PathBuf;
let registry = Registry::default_set();
let samples: &[(&str, &[u8])] = &[
("deploy.sh", b"echo hi"),
("env.csh", b"setenv X 1"),
("script.pl", b"print 1;"),
("Module.pm", b"package M;"),
("index.php", b"<?php echo 1;"),
("paper.tex", b"\\documentclass{article}"),
("page.xhtml", b"<html/>"),
("notes.json5", b"{a: 1}"),
("map.geojson", b"{}"),
("subs.srt", b"1\n00:00:01 --> 00:00:02\nhi\n"),
("run.tcl", b"puts hi"),
("main.dart", b"void main() {}"),
("page.man", b".TH TEST 1"),
("test.t", b"use Test::More;"),
("doc.texi", b"@node Top"),
("mail.eml", b"Subject: hi\n\nbody"),
("inbox.mbox", b"From a@b\n\nbody"),
("icon.svg", b"<svg xmlns='x'/>"),
("list.m3u", b"#EXTM3U\ntrack.mp3"),
("radio.pls", b"[playlist]"),
];
for (name, head) in samples {
let path = PathBuf::from(name);
let mime = guess_mime_from_head(&path, head)
.unwrap_or_else(|| panic!("{} has no MIME", name));
let extracted = registry
.extract_complete_head(&path, &mime, head)
.unwrap_or_else(|| panic!("{} -> {} not claimed by a head-capable extractor", name, mime))
.unwrap_or_else(|e| panic!("{} -> {} failed to extract: {}", name, mime, e));
assert!(
!extracted.text.is_empty(),
"{} -> {} extracted no text",
name,
mime
);
}
}
/// Extensionless files are decided by their bytes: text heads index,
/// binary heads stay unclassified.
#[test]
fn extensionless_files_sniff_by_content() {
use std::path::PathBuf;
let readme = PathBuf::from("README");
assert_eq!(
guess_mime_from_head(&readme, b"QuickSearch indexes your files.\n").as_deref(),
Some("text/plain")
);
let makefile = PathBuf::from("Makefile");
assert_eq!(
guess_mime_from_head(&makefile, b"all:\n\tcargo build\n").as_deref(),
Some("text/plain")
);
let blob = PathBuf::from("blob");
assert_eq!(guess_mime_from_head(&blob, &[0x00, 0x01, 0x02, 0xFF]).as_deref(), None);
}
/// Ambiguous extensions resolve by content in both directions: source
/// code beats the extension table, real binary keeps the extension's
/// MIME as the fallback.
#[test]
fn ambiguous_extensions_resolve_by_content_both_ways() {
use std::path::PathBuf;
let ts_source = b"export function hi(): string { return 'hi'; }\n";
assert_eq!(
guess_mime_from_head(&PathBuf::from("app.ts"), ts_source).as_deref(),
Some("text/plain")
);
// Uppercase, as Windows likes it.
assert_eq!(
guess_mime_from_head(&PathBuf::from("APP.TS"), ts_source).as_deref(),
Some("text/plain")
);
// An MPEG transport stream: 0x47 sync bytes with NUL-heavy payloads.
// No magic matcher, fails the text sniff, so the extension answers.
let mut ts_video = vec![0u8; 376];
ts_video[0] = 0x47;
ts_video[188] = 0x47;
assert_eq!(
guess_mime_from_head(&PathBuf::from("clip.ts"), &ts_video).as_deref(),
Some("video/vnd.dlna.mpeg-tts")
);
assert_eq!(
guess_mime_from_head(&PathBuf::from("go.mod"), b"module example.com/x\n\ngo 1.22\n")
.as_deref(),
Some("text/plain")
);
// gettext template vs PowerPoint template: text decides one way,
// binary bytes fall back to the extension's office MIME (whether
// infer's OLE matcher fires or the guard rejects, the answer agrees).
assert_eq!(
guess_mime_from_head(&PathBuf::from("app.pot"), b"msgid \"hello\"\nmsgstr \"\"\n")
.as_deref(),
Some("text/plain")
);
let ole = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1, 0x00, 0x00];
assert_eq!(
guess_mime_from_head(&PathBuf::from("slides.pot"), &ole).as_deref(),
Some("application/vnd.ms-powerpoint")
);
assert_eq!(
guess_mime_from_head(&PathBuf::from("cpu.vhd"), b"entity cpu is\nend cpu;\n")
.as_deref(),
Some("text/plain")
);
assert_eq!(
guess_mime_from_head(&PathBuf::from("disk.vhd"), &[0x00, 0x01, 0x02, 0x03])
.as_deref(),
Some("application/x-virtualbox-vhd")
);
}
/// Everything the plaintext extractor claims must carry the TEXT bit,
/// or `type:Text` silently misses content-indexed files (the pre-fix
/// state of `.sql`). Iterates the actual claim list so the two can
/// never drift apart.
#[test]
fn every_plaintext_claim_carries_the_text_bit() {
for mime in crate::extract::plaintext::EXTRA_TEXT_MIMES {
assert!(
mime_to_type(mime).contains(FileType::TEXT),
"{} is extractable as text but lacks FileType::TEXT",
mime
);
}
// The multi-category cases keep their native category too.
let svg = mime_to_type("image/svg+xml");
assert!(svg.contains(FileType::IMAGE) && svg.contains(FileType::TEXT));
let m3u = mime_to_type("audio/x-mpegurl");
assert!(m3u.contains(FileType::AUDIO) && m3u.contains(FileType::TEXT));
let xhtml = mime_to_type("application/xhtml+xml");
assert!(xhtml.contains(FileType::TEXT) && xhtml.contains(FileType::DOCUMENT));
// And the `text/` prefix arm still covers the rest.
assert!(mime_to_type("text/x-toml").contains(FileType::TEXT));
}
}

282
crates/quicksearch-core/src/textenc.rs Normal file
View file

@ -0,0 +1,282 @@
//! Text detection and charset decoding, shared by the MIME sniff and the
//! plaintext extractor.
//!
//! Both callers route through one classifier so they cannot drift: a head
//! that [`looks_like_text`] accepts is guaranteed to decode via
//! [`decode_text`] — every class except `Binary` decodes unconditionally
//! (strict UTF-8 after validation, BOM'd and legacy decodes are
//! lossy-with-replacement and never fail).
//!
//! Classification order is load-bearing:
//!
//! 1. **BOM** ([`encoding_rs::Encoding::for_bom`]) — before the binary
//! guard, because UTF-16 text is full of NUL bytes the guard would
//! reject.
//! 2. **Binary guard** — any NUL byte, or control bytes (outside
//! `\t \n \r`, with ESC tolerated for ANSI-colored logs) above 10% of
//! the buffer. UTF-16 without a BOM fails here by design.
//! 3. **Strict UTF-8** — with one tolerance for the sniff: a head is a
//! prefix of the file (indexing hashes the first `hash_length` bytes and
//! sniffs the same buffer), so a multibyte sequence cut off by the end
//! of the buffer does not disqualify it.
//! 4. **Legacy** — everything else. [`decode_text`] runs charset detection
//! (chardetng, windows-1252 floor) and decodes with replacement.
//!
//! The sniff sees only the head, so a file with a text head and a binary
//! tail classifies as text and then fails the whole-file decode; that lands
//! as a FAILED row with a reason, the normal shape of head-based
//! classification.
use std::path::Path;
enum TextClass {
/// Strict UTF-8 (modulo the truncated-tail tolerance).
Utf8,
/// Starts with a BOM; decode with this encoding.
Bom(&'static encoding_rs::Encoding),
/// Not UTF-8 but passes the binary guard: charset detection will decode.
Legacy,
/// Fails the binary guard.
Binary,
}
/// Control bytes tolerated in text: ordinary whitespace, plus ESC because
/// ANSI-colored logs are text worth indexing.
fn is_benign_control(b: u8) -> bool {
matches!(b, b'\t' | b'\n' | b'\r' | 0x1B)
}
/// Classify `bytes` as text or binary. `truncated` marks a buffer that may
/// be a prefix of the file rather than its entirety.
fn classify(bytes: &[u8], truncated: bool) -> TextClass {
if let Some((enc, _bom_len)) = encoding_rs::Encoding::for_bom(bytes) {
return TextClass::Bom(enc);
}
// Binary guard. NUL never appears in text of any supported encoding
// (UTF-16 was handled above, by BOM or not at all); a run of other
// control bytes marks compressed or machine data that merely lacks NULs.
let mut suspect = 0usize;
for &b in bytes {
if b == 0 {
return TextClass::Binary;
}
if (b < 0x20 && !is_benign_control(b)) || b == 0x7F {
suspect += 1;
}
}
if suspect * 10 > bytes.len() {
return TextClass::Binary;
}
match std::str::from_utf8(bytes) {
Ok(_) => TextClass::Utf8,
// `error_len() == None` means the only defect is a multibyte
// sequence running off the end of the buffer — for a truncated head
// that is the file boundary's fault, not the file's.
Err(e) if truncated && e.error_len().is_none() => TextClass::Utf8,
Err(_) => TextClass::Legacy,
}
}
/// Whether `head` — a possibly-truncated prefix of a file — reads as text.
///
/// Cheap: a byte scan plus UTF-8 validation, no charset detection. An empty
/// head proves nothing and answers `false`; that keeps zero-size
/// extensionless files (procfs included) unclassified rather than blanket
/// `text/plain`.
pub fn looks_like_text(head: &[u8]) -> bool {
!head.is_empty() && !matches!(classify(head, true), TextClass::Binary)
}
/// Decode a complete file's bytes to UTF-8 for storage.
///
/// Takes ownership so the dominant valid-UTF-8 case is a zero-copy move.
/// `path` is used only to name the file in the error, matching the
/// extractor error convention.
pub fn decode_text(bytes: Vec<u8>, path: &Path) -> Result<String, String> {
if bytes.is_empty() {
return Ok(String::new());
}
match classify(&bytes, false) {
// Cannot fail: classify ran strict validation with truncated=false.
TextClass::Utf8 => Ok(String::from_utf8(bytes).expect("classified as UTF-8")),
TextClass::Bom(enc) => {
// BOM-aware decode: strips the BOM, replaces malformed
// sequences (e.g. a truncated trailing code unit) with U+FFFD.
let (text, _, _) = enc.decode(&bytes);
Ok(text.into_owned())
}
TextClass::Legacy => {
// ISO-2022-JP detection is safe here: the browser caveat about
// it concerns script-running web content, not indexed files.
let mut det =
chardetng::EncodingDetector::new(chardetng::Iso2022JpDetection::Allow);
det.feed(&bytes, true);
// Deny UTF-8: strict UTF-8 was already ruled out, so a UTF-8
// guess could only mean malformed UTF-8.
let enc = det.guess(None, chardetng::Utf8Detection::Deny);
let (text, _, _) = enc.decode(&bytes);
Ok(text.into_owned())
}
TextClass::Binary => Err(format!(
"plaintext read {}: binary content",
path.display()
)),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
fn p() -> PathBuf {
PathBuf::from("/tmp/textenc-test-file")
}
#[test]
fn utf8_decodes_unchanged() {
let body = "plain ascii and café über 日本語".as_bytes().to_vec();
assert!(looks_like_text(&body));
assert_eq!(decode_text(body, &p()).unwrap(), "plain ascii and café über 日本語");
}
#[test]
fn utf8_bom_is_stripped() {
let mut body = vec![0xEF, 0xBB, 0xBF];
body.extend_from_slice("hello".as_bytes());
assert!(looks_like_text(&body));
let text = decode_text(body, &p()).unwrap();
assert_eq!(text, "hello", "BOM must not survive into stored text");
}
/// The shape of a Windows registry export: UTF-16LE with BOM.
#[test]
fn utf16le_bom_decodes() {
let src = "Windows Registry Editor Version 5.00\r\n";
let mut body = vec![0xFF, 0xFE];
for unit in src.encode_utf16() {
body.extend_from_slice(&unit.to_le_bytes());
}
assert!(looks_like_text(&body));
assert_eq!(decode_text(body, &p()).unwrap(), src);
}
#[test]
fn utf16be_bom_decodes() {
let src = "big endian text";
let mut body = vec![0xFE, 0xFF];
for unit in src.encode_utf16() {
body.extend_from_slice(&unit.to_be_bytes());
}
assert!(looks_like_text(&body));
assert_eq!(decode_text(body, &p()).unwrap(), src);
}
#[test]
fn windows_1252_decodes() {
// A sentence long enough for chardetng to settle on a Western
// single-byte encoding.
let body = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.".to_vec();
assert!(looks_like_text(&body));
assert_eq!(
decode_text(body, &p()).unwrap(),
"Le café près de la fenêtre est agréable en été."
);
}
#[test]
fn shift_jis_decodes() {
// "日本語のテキストです。これはシフトJISでエンコードされています。"
let src = "日本語のテキストです。これはシフトJISでエンコードされています。";
let (encoded, _, had_errors) = encoding_rs::SHIFT_JIS.encode(src);
assert!(!had_errors);
let body = encoded.into_owned();
assert!(looks_like_text(&body));
assert_eq!(decode_text(body, &p()).unwrap(), src);
}
#[test]
fn nul_bytes_are_binary() {
let body = b"looks like text until\x00it does not".to_vec();
assert!(!looks_like_text(&body));
let err = decode_text(body, &p()).unwrap_err();
assert!(err.contains("binary content"), "{err}");
assert!(err.contains("textenc-test-file"), "error must name the file: {err}");
}
#[test]
fn control_density_is_binary() {
// 4 control bytes in 24 total = 16% > 10%.
let body = b"abcdefghijklmnopqrst\x01\x02\x03\x04".to_vec();
assert!(!looks_like_text(&body));
assert!(decode_text(body, &p()).is_err());
}
#[test]
fn ansi_log_is_text() {
// ESC-heavy colored log output stays text.
let body = b"\x1b[31mERROR\x1b[0m something failed\n\x1b[33mWARN\x1b[0m retrying\n".to_vec();
assert!(looks_like_text(&body));
assert!(decode_text(body, &p()).is_ok());
}
#[test]
fn truncated_utf8_tail_still_sniffs_as_text() {
let mut head = "ends mid-char: caf".as_bytes().to_vec();
head.push(0xC3); // first byte of a two-byte sequence, cut off
assert!(looks_like_text(&head));
// The same bytes as a *complete* file are not valid UTF-8, so the
// decoder treats them as legacy-encoded — still Ok, never a panic.
assert!(decode_text(head, &p()).is_ok());
}
#[test]
fn empty_head_is_not_text_but_empty_file_decodes() {
assert!(!looks_like_text(b""));
assert_eq!(decode_text(Vec::new(), &p()).unwrap(), "");
}
/// UTF-16 without a BOM is out of scope: its NULs trip the binary
/// guard. This test documents the decision rather than a limitation we
/// intend to lift.
#[test]
fn utf16_without_bom_is_rejected() {
let mut body = Vec::new();
for unit in "no bom here".encode_utf16() {
body.extend_from_slice(&unit.to_le_bytes());
}
assert!(!looks_like_text(&body));
assert!(decode_text(body, &p()).is_err());
}
/// Every head the sniff accepts must decode — the invariant that makes
/// "sniffed as text/plain" safe to act on.
#[test]
fn sniffed_text_is_guaranteed_decodable() {
let heads: Vec<Vec<u8>> = vec![
b"ordinary ascii".to_vec(),
"utf-8 caf\u{e9}".as_bytes().to_vec(),
b"latin-1 caf\xe9 body".to_vec(),
{
let mut v = vec![0xFF, 0xFE];
v.extend("utf16".encode_utf16().flat_map(|u| u.to_le_bytes()));
v
},
{
let mut v = "truncated tail caf".as_bytes().to_vec();
v.push(0xC3);
v
},
];
for head in heads {
if looks_like_text(&head) {
assert!(
decode_text(head.clone(), &p()).is_ok(),
"sniffed-as-text head failed to decode: {head:?}"
);
}
}
}
}

127
crates/quicksearch-core/tests/full_index.rs
View file

@ -777,10 +777,13 @@ fn seed_mixed_tree(root: &Path) {
touch(&root.join("small.txt"), b"a small plaintext body with xylophone in it");
touch(&root.join("large.txt"), big.as_bytes());
touch(&root.join("empty.txt"), b"");
// Invalid UTF-8 with a .txt extension: claimed by the plaintext extractor,
// but not decodable, so it must be reported as a failure either way.
// Binary bytes with a .txt extension: claimed by the plaintext
// extractor, but the NUL fails the binary guard (and the FF FE pair is
// not at offset 0, so it is no BOM), so it must be reported as a
// failure either way.
touch(&root.join("bad.txt"), &[0x68, 0x69, 0xff, 0xfe, 0x00, 0x41]);
// No extension `infer` or `mime_guess` recognises: no extractor claims it.
// No extension table, magic, or text sniff has an answer for NUL soup:
// no MIME, no extractor.
touch(&root.join("blob.bin"), &[0x00, 0x01, 0x02, 0xfd, 0xfe, 0xff]);
touch(&root.join("nested/deep/note.md"), b"# heading\n\nquagmire body text\n");
}
@ -859,7 +862,9 @@ fn undecodable_small_files_are_reported_as_failures_not_silently_skipped() {
let db_dir = tmp_dir("inline-badutf8-db");
let db = db_dir.join("index.sqlite");
touch(&root.join("bad.txt"), &[0x68, 0x69, 0xff, 0xfe]);
// The NUL keeps this undecodable: without it these bytes would now
// decode as windows-1252 and the test would assert nothing.
touch(&root.join("bad.txt"), &[0x68, 0x00, 0x69, 0xff]);
index_once(&root, &db, &Config::default());
let conn = rusqlite::Connection::open(&db).unwrap();
@ -882,6 +887,114 @@ fn undecodable_small_files_are_reported_as_failures_not_silently_skipped() {
std::fs::remove_dir_all(&db_dir).ok();
}
/// The text sniff end-to-end: extensionless text files (README, Makefile,
/// go.sum) are content-indexed off their head bytes, while an extensionless
/// binary blob stays NA.
#[test]
fn extensionless_text_files_are_indexed() {
let root = tmp_dir("extless");
let db_dir = tmp_dir("extless-db");
let db = db_dir.join("index.sqlite");
touch(&root.join("README"), b"QuickSearch indexes zanzibar contents.\n");
touch(&root.join("Makefile"), b"all:\n\tcargo build --release\n");
touch(&root.join("go.sum"), b"example.com/x v1.0.0 h1:abcdef=\n");
touch(&root.join("blob"), &[0x00, 0x01, 0xfe, 0xff]);
index_once(&root, &db, &Config::default());
let conn = rusqlite::Connection::open(&db).unwrap();
let state_of = |name: &str| -> i64 {
conn.query_row(
"SELECT content_state FROM files WHERE path LIKE '%' || ?1",
[name],
|r| r.get(0),
)
.unwrap()
};
for name in ["README", "Makefile", "go.sum"] {
assert_eq!(state_of(name), 1, "{} should be content-indexed", name);
}
assert_eq!(state_of("blob"), 3, "binary blob stays not-applicable");
drop(conn);
assert_eq!(
stored_text(&db, "README").as_deref(),
Some("QuickSearch indexes zanzibar contents.\n"),
"the stored body round-trips"
);
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&db_dir).ok();
}
/// Charset decoding end-to-end: UTF-16LE files (the shape of a Windows
/// registry export) and legacy single-byte text are stored as UTF-8 —
/// `stored_text` decodes the zstd sidecar with `String::from_utf8`, so a
/// `Some` result *is* the storage-is-UTF-8 assertion.
#[test]
fn utf16_files_are_stored_as_utf8() {
let root = tmp_dir("charset");
let db_dir = tmp_dir("charset-db");
let db = db_dir.join("index.sqlite");
let reg_src = "Windows Registry Editor Version 5.00\r\n\r\n[HKEY_CURRENT_USER\\Software\\Xylograph]\r\n";
let mut reg_body = vec![0xFF, 0xFE];
reg_body.extend(reg_src.encode_utf16().flat_map(|u| u.to_le_bytes()));
touch(&root.join("export.reg"), &reg_body);
// The same encoding behind no extension at all: BOM first, sniff after.
let mut extless = vec![0xFF, 0xFE];
extless.extend("utf16 notes about quokkas".encode_utf16().flat_map(|u| u.to_le_bytes()));
touch(&root.join("NOTES16"), &extless);
touch(&root.join("legacy.txt"), b"un caf\xe9 tr\xe8s agr\xe9able pr\xe8s du mus\xe9e");
index_once(&root, &db, &Config::default());
assert_eq!(stored_text(&db, "export.reg").as_deref(), Some(reg_src));
assert_eq!(
stored_text(&db, "NOTES16").as_deref(),
Some("utf16 notes about quokkas")
);
assert_eq!(
stored_text(&db, "legacy.txt").as_deref(),
Some("un café très agréable près du musée")
);
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&db_dir).ok();
}
/// RTF end-to-end through both extraction paths: a small file the walk
/// finishes inline, and one past `hash_length` that the content pass opens.
/// Stored text is the parsed prose, not RTF control words.
#[test]
fn rtf_files_are_extracted() {
let root = tmp_dir("rtf");
let db_dir = tmp_dir("rtf-db");
let db = db_dir.join("index.sqlite");
touch(
&root.join("small.rtf"),
br"{\rtf1\ansi Meeting notes about the pangolin budget.}",
);
let big_body = format!(
r"{{\rtf1\ansi {}}}",
r"paragraphs about the pangolin budget \par ".repeat(400)
);
assert!(big_body.len() > 8192, "must exceed the default head");
touch(&root.join("big.rtf"), big_body.as_bytes());
index_once(&root, &db, &Config::default());
for name in ["small.rtf", "big.rtf"] {
let text = stored_text(&db, name).unwrap_or_else(|| panic!("{} has no stored text", name));
assert!(text.contains("pangolin budget"), "{}: {:?}", name, &text[..text.len().min(80)]);
assert!(!text.contains(r"\rtf"), "{} stored control words", name);
}
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&db_dir).ok();
}
/// End-to-end version of the fix: the extraction denominator the manage-index
/// tab renders is `extract_total`, and it must count files that need text —
/// not every indexed file. Asserted through a real `IndexingService` run so it
@ -894,13 +1007,15 @@ fn the_extraction_denominator_counts_only_files_that_need_text() {
// Three files an extractor claims, seven it never will. `big.txt` is the
// interesting one: larger than `hash_length`, so the walk cannot finish it
// inline and it is the only row the content pass actually opens.
// inline and it is the only row the content pass actually opens. The
// unclaimed seven get NUL-bearing bodies so neither the extension tables
// nor the text sniff have anything to say about them.
for name in ["a.txt", "b.json"] {
touch(&root.join(name), b"body bytes with no magic");
}
touch(&root.join("big.txt"), &vec![b'z'; 32 * 1024]);
for name in ["d.mp4", "e.zip", "f.bin", "g.exe", "h.iso", "i.so", "j"] {
touch(&root.join(name), b"body bytes with no magic");
touch(&root.join(name), b"\x00\x01body bytes\x00");
}
let config = Config::default();

13
crates/quicksearch-gui/src/manage_tab.rs
View file

@ -308,11 +308,20 @@ impl ManageTab {
ui.heading(egui::RichText::new("Content filters").strong());
ui.columns(2, |cols| {
cols[0].label("Full-text extensions whitelist (empty = all supported):");
cols[0].add(
cols[0]
.add(
egui::TextEdit::multiline(&mut self.ext_filter_text)
.desired_rows(4)
.desired_width(f32::INFINITY)
.hint_text("txt\nmd\npdf"),
.hint_text("txt\nmd\npdf # comments allowed\n(none)"),
)
.on_hover_text(
"One extension per line, leading dot optional. A non-empty \
list also excludes files that have no extension at all \
(Makefile, README, .bashrc) — add the line \"(none)\" to \
keep extracting text from those.\n\n\
\"#\" starts a comment, either on its own line or after an \
entry, so a type can be commented out without losing it.",
);
cols[1].label("Ignore patterns (excluded entirely):");
let mut remove_pat: Option<usize> = None;