From c7e11c57d5940d24162b07e058ae56c1b0e542f2 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 3 Aug 2026 12:33:47 -0400 Subject: [PATCH] Improved text extraction for more file types --- Cargo.lock | 25 +- README.md | 19 +- config_example.toml | 10 +- crates/quicksearch-core/Cargo.toml | 11 + crates/quicksearch-core/src/config.rs | 137 +++++++- crates/quicksearch-core/src/coordinator.rs | 6 + crates/quicksearch-core/src/db/open.rs | 12 +- crates/quicksearch-core/src/extract/mod.rs | 33 +- .../quicksearch-core/src/extract/plaintext.rs | 111 +++++-- crates/quicksearch-core/src/extract/rtf.rs | 113 +++++++ crates/quicksearch-core/src/file_handling.rs | 60 ++-- crates/quicksearch-core/src/lib.rs | 1 + crates/quicksearch-core/src/mime.rs | 292 +++++++++++++++--- crates/quicksearch-core/src/textenc.rs | 282 +++++++++++++++++ crates/quicksearch-core/tests/full_index.rs | 127 +++++++- crates/quicksearch-gui/src/manage_tab.rs | 21 +- 16 files changed, 1130 insertions(+), 130 deletions(-) create mode 100644 crates/quicksearch-core/src/extract/rtf.rs create mode 100644 crates/quicksearch-core/src/textenc.rs diff --git a/Cargo.lock b/Cargo.lock index f9e00ea..4f90cfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,7 +127,7 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "windows-sys 0.59.0", + "windows-sys 0.60.2", "x11rb", ] @@ -626,6 +626,17 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chardetng" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + [[package]] name = "chrono" version = "0.4.44" @@ -3182,7 +3193,9 @@ name = "quicksearch-core" version = "0.8.8" dependencies = [ "argon2", + "chardetng", "ctrlc", + "encoding_rs", "getrandom 0.2.15", "globset", "infer", @@ -3195,6 +3208,7 @@ dependencies = [ "pdf-extract", "quick-xml 0.31.0", "regex", + "rtf-parser", "rusqlite", "serde", "sha2", @@ -3457,6 +3471,15 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "rtf-parser" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3147b4eb521eae5e29b781bdc30ab98bbaed784bfcb8376cda60ba4b2e0e3d" +dependencies = [ + "serde", +] + [[package]] name = "rtoolbox" version = "0.0.5" diff --git a/README.md b/README.md index a514bf6..b8c871f 100644 --- a/README.md +++ b/README.md @@ -287,11 +287,17 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. - **Indexing** (`indexing.rs`, `file_handling.rs`): full runs walk each root (`filtered_walk` prunes hidden/ignored subtrees before descending), classify files by mtime into insert/update/skip, batch-write metadata, - sweep stale rows, then extract content (plaintext, Office, PDF, audio - tags, EXIF; see `extract/`) for FTS. Files no larger than - `processing.hash_length` skip that second pass entirely: the head the walk - reads to hash them is already their whole content, so a plaintext body is - extracted in the same `read` and stored complete. Every run ends — whether + sweep stale rows, then extract content (plaintext, RTF, Office, PDF, + audio tags, EXIF; see `extract/`) for FTS. Files whose extension no MIME + table knows — including extensionless ones like `README` or `Makefile` — + are sniffed from their head bytes and indexed as text when they read as + text (`mime.rs`, `textenc.rs`); non-UTF-8 text (UTF-16 with BOM, legacy + charsets via chardetng) is decoded and stored as UTF-8. More claimed + files means a bigger index — `indexing.content_extensions` remains the + throttle. Files no larger than `processing.hash_length` skip that second + pass entirely: the head the walk reads to hash them is already their + whole content, so a plaintext body is extracted in the same `read` and + stored complete. Every run ends — whether it completed or was stopped — with an optimize pass on its own connection: checkpoint, VACUUM if the file has at least 10% slack to reclaim, `PRAGMA optimize`, checkpoint again. Progress streams through a polled @@ -377,5 +383,6 @@ pagination: the table is virtualized, so a single scroll list capped at - `QSB_SNIPPET_PERF=1 cargo test --release -p quicksearch-core --test snippet_perf -- --nocapture`: snippet pipeline benchmark. - New extractors: implement `extract::Extractor` and register it in - `Registry::default_set()`. New cascade behavior: `search/cascade.rs` + `Registry::default_set()` — order matters, the first extractor whose + `supports` accepts a MIME wins. New cascade behavior: `search/cascade.rs` documents the rank invariants that keep streamed results append-only. diff --git a/config_example.toml b/config_example.toml index 34f469b..538c469 100644 --- a/config_example.toml +++ b/config_example.toml @@ -46,6 +46,11 @@ include_hidden = false # Empty = extract text from every supported format. Non-empty = content # indexing only for these extensions; other files are still listed for # filename search. Entries are case-insensitive, leading dot optional. +# The reserved entry "(none)" whitelists files that have no extension at +# all (Makefile, README, .bashrc); a non-empty list without it skips them. +# Inside an entry, "#" starts a comment that runs to its end, so entries may +# be annotated or commented out: +# content_extensions = ["txt", "md # docs", "# pdf — too slow", "(none)"] content_extensions = [] # Excluded from the index entirely. A pattern without a separator matches # any single path component (so ".git" prunes whole subtrees); patterns @@ -74,7 +79,10 @@ ignore_patterns = [".git", "node_modules", "*.tmp", ".venv", "venv"] # Bytes read from the start of each file for its content hash, which is # `sha256(size || first hash_length bytes)` and backs duplicate detection. # Only the head is read: seeking to the end for a second block costs an -# extra round trip per file on network shares. +# extra round trip per file on network shares. The same bytes are also the +# detection window that decides what a file is: magic-byte matching, and +# the text sniff that lets extensionless or unknown-extension files be +# indexed as text — so shrinking this judges files on less evidence. # # Known limitation: files of identical size whose heads match will be # reported as duplicates. In practice that means pre-allocated VM disk diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index af1c7aa..5f62a9f 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -33,6 +33,17 @@ serde = { version = "1.0", features = ["derive"] } toml = "0.8" mime_guess = "2.0" infer = "0.15" +# Charset decoding for non-UTF-8 text (UTF-16 .reg exports, legacy +# single-byte and CJK encodings). Already in the lockfile transitively via +# pdf-extract, so naming it directly compiles nothing new. +encoding_rs = "0.8" +# Statistical charset detection (Firefox's detector) for text that is neither +# UTF-8 nor BOM-marked. Its mandatory deps beyond encoding_rs are tiny +# (cfg-if, memchr, detone). +chardetng = "1.0" +# RTF text extraction. Pure Rust; with the default `jsbindings` feature off +# (it exists for the crate's WASM build) it depends only on serde. +rtf-parser = { version = "0.4", default-features = false } pdf-extract = "0.12" lopdf = "0.32" lofty = "0.19" diff --git a/crates/quicksearch-core/src/config.rs b/crates/quicksearch-core/src/config.rs index dfddbe7..57e64c0 100644 --- a/crates/quicksearch-core/src/config.rs +++ b/crates/quicksearch-core/src/config.rs @@ -61,8 +61,12 @@ pub struct IndexingConfig { /// supports. Non-empty = only files with these extensions get content /// extraction/FTS; everything else is still listed for filename search /// (`content_state = NA`). Entries are case-insensitive, with or - /// without a leading dot. Applied at walk time, when the row is written — - /// which is why changing this forces a rebuild (see [`diff_actions`]). + /// without a leading dot. The reserved entry [`EXTENSIONLESS`] whitelists + /// files that have no extension at all (`Makefile`, `README`, `.bashrc`), + /// which are otherwise excluded by any non-empty filter. `#` starts a + /// comment — whole-entry or trailing — see [`content_filter_entries`]. + /// Applied at walk time, when the row is written — which is why changing + /// what it matches forces a rebuild (see [`diff_actions`]). pub content_extensions: Vec, /// Excluded from the index entirely — never even listed. A pattern /// without `/` matches any single path component (so `.git` prunes @@ -82,14 +86,17 @@ pub struct IndexingConfig { #[serde(default)] pub struct ProcessingConfig { /// Bytes read from the head of each new or changed file. Those bytes do - /// three jobs, so this one number sets more than the hash: + /// four jobs, so this one number sets more than the hash: /// /// 1. with the size, they identify the file (see `get_file_hash`); /// 2. they are the magic-byte window for MIME detection — `infer` reads /// 8 KiB from a path and its longest matcher needs 262 bytes, so the /// default is exactly as good as opening the file, and a value under /// 262 makes some formats undetectable except by extension; - /// 3. any plaintext file no larger than this is extracted during the + /// 3. they are the text-sniff window (`textenc::looks_like_text`) that + /// decides whether an unknown-extension or extensionless file is + /// text — small values judge files on less evidence; + /// 4. any plaintext file no larger than this is extracted during the /// walk, sparing the content pass an open/read/close. /// /// Changing it invalidates stored hashes and forces a rebuild. @@ -504,21 +511,43 @@ impl Config { } } +/// Reserved `content_extensions` entry standing for "files with no +/// extension". Matched case-insensitively, and it cannot collide with a real +/// extension because the parentheses are not part of one. +pub const EXTENSIONLESS: &str = "(none)"; + +/// The `content_extensions` entries that actually filter: `#` starts a +/// comment and runs to the end of the entry, so a whole-line comment drops +/// out entirely and `md # notes` filters on `md`. Surrounding space is +/// trimmed; what is left of a `#` never is. +pub fn content_filter_entries(list: &[String]) -> impl Iterator { + list.iter().filter_map(|raw| { + let entry = raw.split('#').next().unwrap_or_default().trim(); + (!entry.is_empty()).then_some(entry) + }) +} + /// Whether a file's content (text extraction + FTS) should be indexed under /// the `content_extensions` filter. Files that fail this are still listed -/// for filename search. Empty filter = everything allowed. +/// for filename search. No entries (empty, or nothing but comments) = +/// everything allowed. pub fn content_allowed(path: &Path, cfg: &Config) -> bool { - if cfg.indexing.content_extensions.is_empty() { + let list = &cfg.indexing.content_extensions; + if content_filter_entries(list).next().is_none() { return true; } - let ext = match path.extension().and_then(|e| e.to_str()) { - Some(e) => e.to_ascii_lowercase(), - None => return false, - }; - cfg.indexing - .content_extensions - .iter() - .any(|allowed| allowed.trim_start_matches('.').eq_ignore_ascii_case(&ext)) + // `Path::extension` is None for `Makefile` and for dot-only names like + // `.bashrc`, so without the sentinel a non-empty filter always skips them. + match path.extension().and_then(|e| e.to_str()) { + // The sentinel is reserved: it never doubles as an extension, so a + // file named `x.(none)` is not whitelisted by it. + Some(ext) => content_filter_entries(list) + .filter(|allowed| !allowed.eq_ignore_ascii_case(EXTENSIONLESS)) + .any(|allowed| allowed.trim_start_matches('.').eq_ignore_ascii_case(ext)), + None => { + content_filter_entries(list).any(|allowed| allowed.eq_ignore_ascii_case(EXTENSIONLESS)) + } + } } /// Compiled ignore patterns, split by matching scope: patterns without a @@ -678,7 +707,10 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions { // outside every root, which no sweep will ever reach. || old.indexing.follow_symlinks != new.indexing.follow_symlinks || old.indexing.ignore_patterns != new.indexing.ignore_patterns - || old.indexing.content_extensions != new.indexing.content_extensions + // Comments are not part of the filter, so annotating the list is not + // a reason to rebuild — only a change to what it actually matches is. + || !content_filter_entries(&old.indexing.content_extensions) + .eq(content_filter_entries(&new.indexing.content_extensions)) || old.security.password_protected != new.security.password_protected || old.security.salt != new.security.salt || roots_changed; @@ -815,6 +847,81 @@ mod tests { assert!(content_allowed(Path::new("/a/readme.md"), &cfg), "leading dot + case in filter"); assert!(!content_allowed(Path::new("/a/b.pdf"), &cfg)); assert!(!content_allowed(Path::new("/a/noext"), &cfg)); + assert!(!content_allowed(Path::new("/a/.bashrc"), &cfg), "dot-only name has no ext"); + } + + #[test] + fn content_allowed_extensionless_sentinel() { + let mut cfg = Config::default(); + cfg.indexing.content_extensions = vec!["txt".into(), " (NonE) ".into()]; + assert!(content_allowed(Path::new("/a/Makefile"), &cfg)); + assert!(content_allowed(Path::new("/a/.bashrc"), &cfg), "dot-only name"); + assert!(content_allowed(Path::new("/a/b.txt"), &cfg), "real extensions still work"); + assert!(!content_allowed(Path::new("/a/b.pdf"), &cfg), "sentinel is not a wildcard"); + // The sentinel is not itself an extension: a file literally named + // `x.none` is not whitelisted by it. + assert!(!content_allowed(Path::new("/a/x.none"), &cfg)); + assert!(!content_allowed(Path::new("/a/x.(none)"), &cfg)); + + // Every capitalisation of the word means the same thing. + for spelling in ["(none)", "(NONE)", "(NonE)", "(nOnE)"] { + let mut c = Config::default(); + c.indexing.content_extensions = vec![spelling.to_string()]; + assert!(content_allowed(Path::new("/a/README"), &c), "{spelling}"); + } + + // A leading dot is stripped for extensions but must not turn some + // other entry into the sentinel. + let mut only_txt = Config::default(); + only_txt.indexing.content_extensions = vec!["txt".into()]; + assert!(!content_allowed(Path::new("/a/Makefile"), &only_txt)); + } + + #[test] + fn content_allowed_comments() { + let mut cfg = Config::default(); + cfg.indexing.content_extensions = vec![ + "# source files only".into(), + "rs # rust".into(), + " .MD\t# docs ".into(), + " # indented whole-line comment".into(), + "(none) # Makefile, LICENSE, ...".into(), + ]; + assert!(content_allowed(Path::new("/a/b.rs"), &cfg)); + assert!(content_allowed(Path::new("/a/b.md"), &cfg), "dot + trailing comment"); + assert!(content_allowed(Path::new("/a/Makefile"), &cfg), "sentinel + comment"); + assert!(!content_allowed(Path::new("/a/b.pdf"), &cfg)); + // Comment text is not itself a filter entry. + assert!(!content_allowed(Path::new("/a/b.rust"), &cfg)); + assert!(!content_allowed(Path::new("/a/b.only"), &cfg)); + assert!(!content_allowed(Path::new("/a/b.docs"), &cfg)); + + // Nothing but comments filters nothing — same as an empty list. + let mut all_comments = Config::default(); + all_comments.indexing.content_extensions = + vec!["# nothing enabled yet".into(), " ".into(), "#".into()]; + assert!(content_allowed(Path::new("/a/b.pdf"), &all_comments)); + assert!(content_allowed(Path::new("/a/Makefile"), &all_comments)); + } + + #[test] + fn comment_only_edit_does_not_force_rebuild() { + let mut old = Config::default(); + old.indexing.content_extensions = vec!["txt".into(), "md".into()]; + let mut new = old.clone(); + new.indexing.content_extensions = + vec!["# my notes".into(), "txt".into(), "md # markdown".into()]; + assert!(!diff_actions(&old, &new).requires_rebuild); + + // Changing what the list matches still does. + let mut changed = old.clone(); + changed.indexing.content_extensions = vec!["txt".into(), "md".into(), "(none)".into()]; + assert!(diff_actions(&old, &changed).requires_rebuild); + + // ... including commenting an entry out. + let mut disabled = old.clone(); + disabled.indexing.content_extensions = vec!["txt".into(), "# md".into()]; + assert!(diff_actions(&old, &disabled).requires_rebuild); } #[test] diff --git a/crates/quicksearch-core/src/coordinator.rs b/crates/quicksearch-core/src/coordinator.rs index cb8b4f9..7025cb1 100644 --- a/crates/quicksearch-core/src/coordinator.rs +++ b/crates/quicksearch-core/src/coordinator.rs @@ -821,6 +821,12 @@ impl Inner { let idle = matches!(status, IndexingStatus::Idle | IndexingStatus::Error(_)); if !idle { self.indexing.request_stop(); + // Dropping the service joins its worker, and a VACUUM answers to + // nothing but `sqlite3_interrupt` — without this, closing the + // window during an optimize pass would wait out a rewrite of the + // whole index. The interrupted VACUUM rolls back, and the next + // run's checkpoints land the log. + self.indexing.cancel_optimizing(); } if let Some(conn) = self.write_conn.take() { if idle { diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs index 69a309c..dc79aa4 100644 --- a/crates/quicksearch-core/src/db/open.rs +++ b/crates/quicksearch-core/src/db/open.rs @@ -30,10 +30,14 @@ use crate::security::IndexKey; pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: "; /// Bump this whenever [`SCHEMA_CURRENT`] or [`fts_create_sql`] changes in -/// a way that makes an old DB unreadable by new code. Any such bump -/// causes existing indexes to be wiped on next open — there's no -/// migration path by design. -pub const CURRENT_SCHEMA_VERSION: u32 = 4; +/// a way that makes an old DB unreadable by new code — or when stored, +/// classifier-derived values go stale: `files.mime`, `files.type` and +/// `content_state` are computed at walk time and never re-derived for +/// unchanged files, so a classification change (v5: text sniffing, charset +/// decoding, RTF) needs the wipe to apply everywhere. Any such bump causes +/// existing indexes to be wiped on next open — there's no migration path +/// by design. +pub const CURRENT_SCHEMA_VERSION: u32 = 5; /// Open `db_path`, applying fast-path pragmas, and ensure the on-disk /// schema matches what this build expects. If it doesn't, delete the diff --git a/crates/quicksearch-core/src/extract/mod.rs b/crates/quicksearch-core/src/extract/mod.rs index 07ee7ca..55de551 100644 --- a/crates/quicksearch-core/src/extract/mod.rs +++ b/crates/quicksearch-core/src/extract/mod.rs @@ -20,6 +20,7 @@ pub mod image; pub mod office; pub mod pdf; pub mod plaintext; +pub mod rtf; /// Result of a successful extraction. `text` feeds the FTS5 `text` column; /// `properties` feeds both the `properties` FTS5 column (as `key:value` @@ -169,10 +170,18 @@ impl Registry { self.find(mime).and_then(|e| e.extract_from_head(path, head)) } - /// The default set wired up for Set A: plaintext, office docs, PDF, - /// audio tags, image EXIF. + /// The default set: RTF, plaintext, office docs, PDF, audio tags, + /// image EXIF. + /// + /// Order matters — the first extractor whose `supports` accepts a MIME + /// wins. RTF precedes plaintext because plaintext claims every `text/*` + /// and would swallow `text/rtf` as raw control words. Plaintext + /// precedes audio and image because it deliberately claims playlist + /// (`audio/x-mpegurl`, `audio/scpls`) and SVG MIMEs whose text is worth + /// more than their tags. pub fn default_set() -> Self { Self::new() + .with(rtf::RtfExtractor) .with(plaintext::PlaintextExtractor) .with(office::OfficeExtractor) .with(pdf::PdfExtractor) @@ -230,6 +239,26 @@ mod tests { "{} should extract from a head", mime ); } + for mime in ["application/rtf", "text/rtf"] { + assert!( + r.extract_complete_head(p, mime, br"{\rtf1 x}").is_some(), + "{} should extract from a head", mime + ); + } + } + + /// `text/rtf` must dispatch to the RTF extractor, not to plaintext's + /// `text/*` claim — i.e. the registration order does its job. The RTF + /// parser strips control words; plaintext would keep them. + #[test] + fn text_rtf_reaches_the_rtf_extractor_not_plaintext() { + let r = Registry::default_set(); + let p = Path::new("/tmp/whatever.rtf"); + let out = r + .extract_complete_head(p, "text/rtf", br"{\rtf1\ansi Hello {\b World}}") + .expect("claimed") + .expect("parsed"); + assert_eq!(out.text, "Hello World"); } #[test] diff --git a/crates/quicksearch-core/src/extract/plaintext.rs b/crates/quicksearch-core/src/extract/plaintext.rs index 4a721f0..553ca75 100644 --- a/crates/quicksearch-core/src/extract/plaintext.rs +++ b/crates/quicksearch-core/src/extract/plaintext.rs @@ -1,5 +1,7 @@ -//! Read the file as UTF-8 text. Handles text/plain, text/x-*, application/json -//! and most source-code MIMEs. +//! Read the file as text, decoding UTF-8, BOM-marked UTF-16, and detected +//! legacy charsets to UTF-8 for storage (see [`crate::textenc`]). Handles +//! text/plain, text/x-*, and the non-`text/*` formats in +//! [`EXTRA_TEXT_MIMES`]. use std::fs::File; use std::io::Read; @@ -7,36 +9,57 @@ use std::path::Path; use super::{ExtractError, ExtractedContent, Extractor}; +/// Non-`text/*` MIMEs the plaintext extractor claims. Every entry must be +/// reachable — emitted by [`crate::mime::guess_mime_from_head`] via the +/// override table, `mime_guess`, `infer`, or the text sniff — and must map +/// to a [`crate::mime::FileType`] containing TEXT; the cross-check tests in +/// `mime.rs` enforce both. +/// +/// The `audio/*` and `image/*` entries (playlists, SVG) rely on this +/// extractor being registered before the audio and image extractors in +/// [`super::Registry::default_set`] — first match wins, and their text +/// content is worth more than their tags. `.svgz` also resolves to +/// `image/svg+xml`; its gzip body fails the binary guard and is recorded as +/// a failure rather than silently skipped. +pub(crate) const EXTRA_TEXT_MIMES: &[&str] = &[ + "application/geo+json", + "application/javascript", + "application/json", + "application/json5", + "application/mbox", + "application/vnd.dart", + "application/x-csh", + "application/x-httpd-php", + "application/x-perl", + "application/x-sh", + // `.sql` resolves here rather than to `text/*`, so without it schema + // dumps are listed by name but never full-text indexed. + "application/x-sql", + "application/x-subrip", + "application/x-tcl", + "application/x-tex", + "application/x-texinfo", + "application/x-troff", + "application/x-troff-man", + "application/xhtml+xml", + "application/xml", + "audio/scpls", + "audio/x-mpegurl", + "image/svg+xml", + "message/rfc822", +]; + /// Decode bytes that are known to be a complete file. Shared by both entry /// points so on-disk and already-in-memory extraction cannot drift apart. fn decode(bytes: Vec, path: &Path) -> Result { - match String::from_utf8(bytes) { - Ok(text) => Ok(ExtractedContent::with_text(text)), - Err(e) => Err(format!("plaintext read {}: {}", path.display(), e.utf8_error())), - } + crate::textenc::decode_text(bytes, path).map(ExtractedContent::with_text) } pub struct PlaintextExtractor; impl Extractor for PlaintextExtractor { fn supports(&self, mime: &str) -> bool { - if mime.starts_with("text/") { - return true; - } - matches!( - mime, - "application/json" - | "application/xml" - | "application/javascript" - | "application/x-shellscript" - | "application/x-python" - | "application/toml" - | "application/yaml" - | "application/x-yaml" - // `.sql` resolves here rather than to `text/*`, so without it - // schema dumps are listed by name but never full-text indexed. - | "application/x-sql" - ) + mime.starts_with("text/") || EXTRA_TEXT_MIMES.contains(&mime) } /// Read the whole file, sized from the handle we just opened. @@ -135,15 +158,41 @@ mod tests { } #[test] - fn both_paths_reject_invalid_utf8_and_name_the_file() { - let p = tmp("badutf8", &[0x68, 0x69, 0xff, 0xfe]); + fn both_paths_reject_binary_and_name_the_file() { + // A NUL keeps this undecodable now that legacy charsets decode. + let body = [0x68, 0x69, 0x00, 0xff]; + let p = tmp("binary", &body); let disk_err = PlaintextExtractor.extract(&p).unwrap_err(); let head_err = PlaintextExtractor - .extract_from_head(&p, &[0x68, 0x69, 0xff, 0xfe]) + .extract_from_head(&p, &body) .unwrap() .unwrap_err(); assert_eq!(disk_err, head_err, "one decode path, one message"); - assert!(disk_err.contains("badutf8"), "the failure names the file: {}", disk_err); + assert!(disk_err.contains("binary"), "the failure names the file: {}", disk_err); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn latin1_decodes_via_both_paths() { + let body = b"une journ\xe9e agr\xe9able pr\xe8s de la rivi\xe8re"; + let p = tmp("latin1", body); + let from_disk = PlaintextExtractor.extract(&p).unwrap(); + let from_head = PlaintextExtractor.extract_from_head(&p, body).unwrap().unwrap(); + assert_eq!(from_disk.text, from_head.text); + assert_eq!(from_disk.text, "une journée agréable près de la rivière"); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn utf16le_bom_decodes_via_both_paths() { + let src = "Windows Registry Editor Version 5.00\r\n[HKEY_CURRENT_USER\\Software]\r\n"; + let mut body = vec![0xFF, 0xFE]; + body.extend(src.encode_utf16().flat_map(|u| u.to_le_bytes())); + let p = tmp("utf16", &body); + let from_disk = PlaintextExtractor.extract(&p).unwrap(); + let from_head = PlaintextExtractor.extract_from_head(&p, &body).unwrap().unwrap(); + assert_eq!(from_disk.text, from_head.text); + assert_eq!(from_disk.text, src, "stored text is the UTF-8 decode, BOM stripped"); std::fs::remove_file(&p).ok(); } @@ -215,6 +264,14 @@ mod tests { assert!(e.supports("text/plain")); assert!(e.supports("text/x-rust")); assert!(e.supports("application/json")); + assert!(e.supports("application/x-sh")); + assert!(e.supports("image/svg+xml")); + assert!(e.supports("audio/x-mpegurl")); + assert!(e.supports("message/rfc822")); + // Never emitted by any MIME source; removed as dead. + assert!(!e.supports("application/x-shellscript")); + // RTF belongs to the RTF extractor, which registers first. + assert!(!e.supports("application/rtf")); assert!(!e.supports("application/pdf")); assert!(!e.supports("image/png")); } diff --git a/crates/quicksearch-core/src/extract/rtf.rs b/crates/quicksearch-core/src/extract/rtf.rs new file mode 100644 index 0000000..511439b --- /dev/null +++ b/crates/quicksearch-core/src/extract/rtf.rs @@ -0,0 +1,113 @@ +//! RTF text extraction via the `rtf-parser` crate. +//! +//! Claims `application/rtf` (what both `mime_guess` and `infer`'s magic +//! matcher emit) and `text/rtf` (a common alias). Registered *before* the +//! plaintext extractor in [`super::Registry::default_set`], because +//! plaintext claims every `text/*` and would otherwise swallow `text/rtf` +//! and index the control-word noise raw. + +use std::path::Path; + +use rtf_parser::document::RtfDocument; + +use super::{ExtractError, ExtractedContent, Extractor}; + +/// Parse a complete RTF file's bytes. Shared by both entry points so +/// on-disk and already-in-memory extraction cannot drift apart. +/// +/// RTF is 7-bit ASCII by design — non-ASCII characters travel as `\'hh` and +/// `\uN` escapes — so a lossy UTF-8 view loses nothing from a well-formed +/// document, and a malformed one fails in the parser with a real reason +/// rather than in the decode. +fn parse(bytes: Vec, path: &Path) -> Result { + let source = String::from_utf8_lossy(&bytes); + match RtfDocument::try_from(source.as_ref()) { + Ok(doc) => Ok(ExtractedContent::with_text(doc.get_text())), + Err(e) => Err(format!("rtf parse {}: {}", path.display(), e)), + } +} + +pub struct RtfExtractor; + +impl Extractor for RtfExtractor { + fn supports(&self, mime: &str) -> bool { + mime == "application/rtf" || mime == "text/rtf" + } + + fn extract(&self, path: &Path) -> Result { + // Plain read: RTF files are rare and small enough that plaintext's + // sized-read syscall trimming would be tuning without a workload. + let bytes = std::fs::read(path) + .map_err(|e| format!("rtf read {}: {}", path.display(), e))?; + parse(bytes, path) + } + + /// RTF has no trailer and needs no seeking, so a head that is the whole + /// file parses exactly like the on-disk path. + fn extract_from_head( + &self, + path: &Path, + head: &[u8], + ) -> Option> { + Some(parse(head.to_vec(), path)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp(tag: &str, body: &[u8]) -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "qs-rtf-{}-{}-{}.rtf", + tag, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&p, body).unwrap(); + p + } + + #[test] + fn extracts_text_without_control_words() { + let body = br"{\rtf1\ansi Hello {\b World}!}"; + let p = tmp("basic", body); + let c = RtfExtractor.extract(&p).unwrap(); + assert_eq!(c.text, "Hello World!"); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn head_extraction_matches_reading_the_file() { + // `\'e9` is the RTF hex escape for an e-acute: the literal itself + // stays 7-bit ASCII while the extracted text does not. + let body = br"{\rtf1\ansi caf\'e9 at noon}"; + let p = tmp("agree", body); + let from_disk = RtfExtractor.extract(&p).unwrap(); + let from_head = RtfExtractor.extract_from_head(&p, body).unwrap().unwrap(); + assert_eq!(from_disk.text, from_head.text); + assert!(from_disk.text.contains("café"), "{:?}", from_disk.text); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn malformed_input_errors_and_names_the_file() { + let p = tmp("broken", br"{\rtf1 truncated"); + let err = RtfExtractor.extract(&p).unwrap_err(); + assert!(err.contains("qs-rtf-broken"), "must name the file: {}", err); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn supports_rtf_mimes_only() { + let e = RtfExtractor; + assert!(e.supports("application/rtf")); + assert!(e.supports("text/rtf")); + assert!(!e.supports("text/plain")); + assert!(!e.supports("application/pdf")); + } +} diff --git a/crates/quicksearch-core/src/file_handling.rs b/crates/quicksearch-core/src/file_handling.rs index 8ed0a2e..9d51b48 100644 --- a/crates/quicksearch-core/src/file_handling.rs +++ b/crates/quicksearch-core/src/file_handling.rs @@ -1629,31 +1629,34 @@ mod count_and_extract_tests { let registry = Registry::default_set(); // Real files, because `decide_content` runs the extractor for anything - // it claims and its answer must be a genuine one. - let cases = [ - "notes.txt", - "data.json", - "schema.sql", - "song.mp3", - "photo.jpg", - "movie.mp4", - "archive.zip", - "blob.bin", - "noextension", + // it claims and its answer must be a genuine one. A text body sniffs + // as text for the extension-less names; the NUL body keeps the + // unclaimed side of the invariant exercised too. + let cases: [(&str, &[u8]); 10] = [ + ("notes.txt", b"plain bytes with no magic"), + ("data.json", b"plain bytes with no magic"), + ("schema.sql", b"plain bytes with no magic"), + ("song.mp3", b"plain bytes with no magic"), + ("photo.jpg", b"plain bytes with no magic"), + ("movie.mp4", b"plain bytes with no magic"), + ("archive.zip", b"plain bytes with no magic"), + ("blob.bin", b"plain bytes with no magic"), + ("noextension", b"plain bytes with no magic"), + ("real.bin", b"\x00\x01\x02\x03"), ]; - for name in cases { + for (name, body) in cases { let p = root.join(name); - std::fs::write(&p, b"plain bytes with no magic").unwrap(); + std::fs::write(&p, body).unwrap(); } // Once with the filter off (the default: everything the registry // claims), once with it narrowed to `.txt`. for filter in [Vec::new(), vec!["txt".to_string()]] { cfg.indexing.content_extensions = filter.clone(); - for name in cases { + for (name, body) in cases { let p = root.join(name); let path = p.to_str().unwrap(); - let mime = guess_mime_from_head(&p, b"plain bytes with no magic"); + let mime = guess_mime_from_head(&p, body); let claimed = content_extractable(&p, mime.as_deref(), &cfg, ®istry); let outcome = decide_content(path, mime.as_deref(), ®istry, &cfg); assert_eq!( @@ -1685,17 +1688,20 @@ mod count_and_extract_tests { .needs_content }; - for name in ["notes.txt", "song.mp3", "movie.mp4", "blob.bin"] { + for name in ["notes.txt", "song.mp3", "movie.mp4", "README"] { std::fs::write(root.join(name), b"body").unwrap(); } + // NUL bytes: the text sniff must not rescue a genuinely binary blob. + std::fs::write(root.join("blob.bin"), b"\x00\x01\x02\x03").unwrap(); let big = root.join("huge.txt"); std::fs::write(&big, vec![b'x'; 4096]).unwrap(); let cfg = Config::default(); assert!(needs(&cfg, "notes.txt"), "plaintext is claimed"); assert!(needs(&cfg, "song.mp3"), "audio tags are content too"); + assert!(needs(&cfg, "README"), "an extensionless text head sniffs as text/plain"); assert!(!needs(&cfg, "movie.mp4"), "no extractor claims video"); - assert!(!needs(&cfg, "blob.bin"), "unsniffable: no MIME, no extractor"); + assert!(!needs(&cfg, "blob.bin"), "binary content: no MIME, no extractor"); // Over `maximum_text_file_size`, so the content pass would never read // it even though plaintext claims the MIME. @@ -1722,13 +1728,19 @@ mod count_and_extract_tests { let mut db = root.clone(); db.set_extension("sqlite"); - // 3 files an extractor claims, 5 it never will. Big enough that the - // old behaviour (every row pending) can't coincide with the new one. - let claimed = ["a.txt", "b.json", "c.mp3"]; + // 4 files an extractor claims (README via the extensionless text + // sniff), 5 it never will — the unclaimed set gets NUL bodies so + // neither the extension tables nor the sniff have anything to say. + // Big enough that the old behaviour (every row pending) can't + // coincide with the new one. + let claimed = ["a.txt", "b.json", "c.mp3", "README"]; let unclaimed = ["d.mp4", "e.zip", "f.bin", "g.exe", "h"]; - for name in claimed.iter().chain(unclaimed.iter()) { + for name in claimed.iter() { std::fs::write(root.join(name), b"body bytes, no magic").unwrap(); } + for name in unclaimed.iter() { + std::fs::write(root.join(name), b"\x00\x01body\x00").unwrap(); + } let config = Config::default(); let registry = Registry::default_set(); @@ -1752,13 +1764,13 @@ mod count_and_extract_tests { let cursor = ExtractCursor::for_root(root.to_str().unwrap()); let scope = extract_scope_prepare(&conn_mutex, &cursor, &config).unwrap(); - // `extract_total` in the GUI. The two small text-ish files were + // `extract_total` in the GUI. The three small text-ish files were // finished inline by the walk so they land in `already_done`; the mp3 // needs the disk pass. Either way the denominator is the claimed set — - // before this was decided at walk time it read 8, every indexed file. + // before this was decided at walk time it read 9, every indexed file. assert_eq!( (scope.pending, scope.already_done), - (1, 2), + (1, 3), "denominator must be the files needing text, not every indexed file" ); assert_eq!(scope.pending + scope.already_done, claimed.len()); diff --git a/crates/quicksearch-core/src/lib.rs b/crates/quicksearch-core/src/lib.rs index c6b6032..a022b07 100644 --- a/crates/quicksearch-core/src/lib.rs +++ b/crates/quicksearch-core/src/lib.rs @@ -16,5 +16,6 @@ pub mod search; pub mod security; pub mod shutdown; pub mod snippet; +pub mod textenc; pub mod walk; pub mod watcher; diff --git a/crates/quicksearch-core/src/mime.rs b/crates/quicksearch-core/src/mime.rs index a998039..3be96e4 100644 --- a/crates/quicksearch-core/src/mime.rs +++ b/crates/quicksearch-core/src/mime.rs @@ -1,16 +1,23 @@ //! MIME type guessing and `FileType` bitmask classification. //! -//! Two stages: -//! 1. [`guess_mime_from_head`] infers a MIME type — extension first via -//! `mime_guess`, falling back to magic-byte sniffing via `infer` for files -//! whose extension is missing or ambiguous. -//! 2. [`mime_to_type`] maps a MIME string to a [`FileType`] bitmask so a single -//! file can belong to multiple categories (e.g. a `.docx` is Document|Text). +//! [`guess_mime_from_head`] infers a MIME type in three stages: extension +//! first (an override table, then `mime_guess`), magic-byte sniffing via +//! `infer` next, and finally a text sniff ([`crate::textenc`]) that answers +//! `text/plain` for anything whose head reads as text — which is how +//! extensionless files (README, Makefile) and source extensions no MIME +//! table knows (`.go`, `.zig`) get their contents indexed. Extensions in +//! [`AMBIGUOUS_EXTENSIONS`] invert the order: content decides, and the +//! extension's MIME is only a fallback. //! -//! The magic bytes are always ones the caller already holds. Indexing reads +//! [`mime_to_type`] then maps a MIME string to a [`FileType`] bitmask so a +//! single file can belong to multiple categories (e.g. a `.docx` is +//! Document|Text). +//! +//! The head bytes are always ones the caller already holds. Indexing reads //! the head of every new or changed file to hash it, and those are the same -//! bytes `infer` wants, so there is no path-based variant that goes back to -//! disk for them — that was a second open/read/close per undetectable file. +//! bytes `infer` and the text sniff want, so there is no path-based variant +//! that goes back to disk for them — that was a second open/read/close per +//! undetectable file. use std::path::Path; @@ -70,22 +77,21 @@ impl std::ops::BitOrAssign for FileType { } } -/// Extensions `mime_guess` gets wrong or does not know, and what they really -/// are. +/// Extensions whose MIME is pinned regardless of what `mime_guess` or the +/// file's bytes say. /// -/// Consulted *before* `mime_guess`, because for these the table is not a -/// fallback but a correction. Everything here is plain text that would -/// otherwise get no content indexing at all: +/// Consulted *before* everything else, because for these the table is not a +/// fallback but a correction or a guarantee: /// -/// - `.ps1`/`.psm1`/`.psd1` and `.url` are simply absent from `mime_guess`, -/// and `infer` only knows binary magic, so they end up with no MIME — and -/// [`crate::extract::Registry`] has no extractor to offer, so the file is -/// marked "not applicable". PowerShell is the most common script type on a -/// Windows machine. -/// - `.bat` maps to `application/x-msdownload`, i.e. an executable. It is a -/// text file, and the plaintext extractor rightly refuses the executable -/// type. (`.cmd` already resolves to `text/plain`; it is listed so the pair +/// - `.bat` maps in `mime_guess` to `application/x-msdownload`, i.e. an +/// executable, and a non-empty `mime_guess` answer would preempt the text +/// sniff — so without this entry batch files are never content-indexed. +/// (`.cmd` already resolves to `text/plain`; it is listed so the pair /// cannot drift.) +/// - `.ps1`/`.psm1`/`.psd1`, `.inf` and `.url` are absent from `mime_guess`. +/// The text sniff would usually catch them, but pinning them costs +/// nothing and classifies them deterministically, whatever their head +/// bytes happen to look like. /// /// Platform-neutral on purpose: a `.ps1` copied to a Linux box should classify /// the same way. @@ -99,6 +105,27 @@ const EXTENSION_OVERRIDES: &[(&str, &str)] = &[ ("url", "text/plain"), ]; +/// Extensions `mime_guess` maps to a binary format that is, on a modern +/// disk, at least as often a text file: `.ts`/`.mts` TypeScript vs MPEG +/// transport stream, `.mod` go.mod vs `video/mpeg`, `.org` Org-mode vs +/// Lotus Organizer, `.scm` Scheme vs Lotus ScreenCam, `.pot` gettext +/// template vs PowerPoint template, `.vhd` VHDL source vs VirtualBox disk +/// image. +/// +/// For these the content decides: magic bytes first, then the text sniff, +/// and only if both decline does `mime_guess`'s extension answer stand — so +/// a real MPEG-TS recording still classifies as video. +const AMBIGUOUS_EXTENSIONS: &[&str] = &["mod", "mts", "org", "pot", "scm", "ts", "vhd"]; + +/// Whether `path`'s extension is in [`AMBIGUOUS_EXTENSIONS`]. +/// ASCII-case-insensitive, like [`extension_override`]. +fn extension_is_ambiguous(path: &Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_ascii_lowercase()) + .is_some_and(|e| AMBIGUOUS_EXTENSIONS.contains(&e.as_str())) +} + /// Look up [`EXTENSION_OVERRIDES`] for `path`. Extension comparison is /// ASCII-case-insensitive, which matters more on Windows where `REPORT.BAT` is /// as common as the lowercase spelling. @@ -112,15 +139,21 @@ fn extension_override(path: &Path) -> Option<&'static str> { /// Infer a MIME type from a path plus the file's leading bytes. /// -/// Extension first — an override table, then `mime_guess` — and magic bytes -/// only when those come up empty or say `application/octet-stream`. +/// Extension first — an override table, then `mime_guess` — then magic +/// bytes when those come up empty or say `application/octet-stream`, and +/// finally a text sniff that answers `text/plain` for any head that reads +/// as text ([`crate::textenc::looks_like_text`]). For +/// [`AMBIGUOUS_EXTENSIONS`] the `mime_guess` answer is demoted to a last +/// resort behind both content checks. /// /// `head` is whatever the caller already read; indexing passes the same buffer /// it hashes. It bounds magic-byte detection, so a caller that supplies fewer /// than 262 bytes (`infer`'s longest signature) can get `None` where a longer /// head would have matched. The indexer's `hash_length` defaults to 8 KiB — /// exactly what `infer` itself reads from a path — so at default config this -/// is as good as opening the file, and strictly cheaper. +/// is as good as opening the file, and strictly cheaper. The same buffer +/// bounds the text sniff, which tolerates a multibyte character cut off at +/// the buffer's end. /// /// A `None` result is a real answer, not a "don't know": the content pass /// stores it and does not re-derive it (see @@ -129,13 +162,29 @@ pub fn guess_mime_from_head(path: &Path, head: &[u8]) -> Option { if let Some(m) = extension_override(path) { return Some(m.to_string()); } - if let Some(g) = mime_guess::from_path(path).first() { + let by_extension = mime_guess::from_path(path).first().and_then(|g| { let s = g.essence_str(); - if !s.is_empty() && s != "application/octet-stream" { - return Some(s.to_string()); - } + (!s.is_empty() && s != "application/octet-stream").then(|| s.to_string()) + }); + if !extension_is_ambiguous(path) && by_extension.is_some() { + return by_extension; } - infer::get(head).map(|t| t.mime_type().to_string()) + if let Some(t) = infer::get(head) { + let magic = t.mime_type(); + // For an ambiguous extension, `infer`'s generic OLE-container answer + // is less specific than the extension's: a real PowerPoint `.pot` + // template must resolve to vnd.ms-powerpoint (which the office + // extractor claims), not to a container MIME nothing claims. + if magic == "application/x-ole-storage" && by_extension.is_some() { + return by_extension; + } + return Some(magic.to_string()); + } + if crate::textenc::looks_like_text(head) { + return Some("text/plain".to_string()); + } + // Only an ambiguous extension still has an answer left to fall back on. + by_extension } /// Map a MIME string to a [`FileType`] bitmask. Ported from Baloo's @@ -153,8 +202,9 @@ pub fn mime_to_type(mime: &str) -> FileType { "video" => t |= FileType::VIDEO, "text" => { t |= FileType::TEXT; - // HTML counts as a document too in Baloo. - if sub == "html" || sub == "xhtml+xml" { + // HTML counts as a document too in Baloo. (xhtml+xml is handled + // in the subtype match below, whatever its top level.) + if sub == "html" { t |= FileType::DOCUMENT; } } @@ -204,8 +254,19 @@ pub fn mime_to_type(mime: &str) -> FileType { | "x-msi" => { t |= FileType::ARCHIVE; } - // application/xml is structured text - "xml" | "json" | "javascript" | "x-shellscript" | "x-python" => { + // XHTML is text and, like HTML above, a document in Baloo's model — + // whichever top level it arrives under. + "xhtml+xml" => { + t |= FileType::TEXT | FileType::DOCUMENT; + } + // Structured text: everything the plaintext extractor claims beyond + // `text/*` (see `extract::plaintext::EXTRA_TEXT_MIMES` and the + // cross-check test below). Keyed on the subtype alone, so playlists + // stay AUDIO|TEXT and SVG stays IMAGE|TEXT. + "xml" | "json" | "json5" | "geo+json" | "javascript" | "mbox" | "rfc822" + | "vnd.dart" | "x-csh" | "x-httpd-php" | "x-perl" | "x-sh" | "x-sql" + | "x-subrip" | "x-tcl" | "x-tex" | "x-texinfo" | "x-troff" | "x-troff-man" + | "x-mpegurl" | "scpls" | "svg+xml" => { t |= FileType::TEXT; } _ => {} @@ -396,15 +457,19 @@ mod tests { } /// The other side of that bound: starve the head below `infer`'s longest - /// signature and detection legitimately degrades. Documented behaviour of - /// a non-default `hash_length`, not a bug — but it must stay a `None` - /// rather than a wrong guess. + /// signature and magic detection legitimately degrades. Documented + /// behaviour of a non-default `hash_length`, not a bug — but a binary + /// head must stay a `None` rather than become a wrong guess. (A head + /// that *reads as text* is a different case: the text sniff answers for + /// it, however short.) #[test] fn a_head_shorter_than_the_signature_declines_rather_than_guessing() { use std::path::PathBuf; let path = PathBuf::from("/tmp/qs-sniff-truncated"); assert_eq!(guess_mime_from_head(&path, b"").as_deref(), None); - assert_eq!(guess_mime_from_head(&path, &[0x89]).as_deref(), None); + // A PNG magic truncated to two bytes: not a magic match, and the + // NUL fails the binary guard, so no text guess either. + assert_eq!(guess_mime_from_head(&path, &[0x89, 0x00]).as_deref(), None); // Enough bytes, and it resolves. assert_eq!( guess_mime_from_head(&path, &[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a]) @@ -420,4 +485,155 @@ mod tests { assert!(mime_to_type("application/vnd.ms-outlook").contains(FileType::DOCUMENT)); assert!(mime_to_type("application/vnd.ms-htmlhelp").contains(FileType::DOCUMENT)); } + + /// Every extension fixed by this round of coverage work must reach the + /// plaintext extractor through real dispatch — `extract_complete_head` + /// rather than `supports` — so the svg/m3u/pls cases prove the + /// plaintext-first registration *order*, not just the MIME claim. + #[test] + fn newly_claimed_extensions_reach_the_plaintext_extractor() { + use crate::extract::Registry; + use std::path::PathBuf; + + let registry = Registry::default_set(); + let samples: &[(&str, &[u8])] = &[ + ("deploy.sh", b"echo hi"), + ("env.csh", b"setenv X 1"), + ("script.pl", b"print 1;"), + ("Module.pm", b"package M;"), + ("index.php", b""), + ("notes.json5", b"{a: 1}"), + ("map.geojson", b"{}"), + ("subs.srt", b"1\n00:00:01 --> 00:00:02\nhi\n"), + ("run.tcl", b"puts hi"), + ("main.dart", b"void main() {}"), + ("page.man", b".TH TEST 1"), + ("test.t", b"use Test::More;"), + ("doc.texi", b"@node Top"), + ("mail.eml", b"Subject: hi\n\nbody"), + ("inbox.mbox", b"From a@b\n\nbody"), + ("icon.svg", b""), + ("list.m3u", b"#EXTM3U\ntrack.mp3"), + ("radio.pls", b"[playlist]"), + ]; + for (name, head) in samples { + let path = PathBuf::from(name); + let mime = guess_mime_from_head(&path, head) + .unwrap_or_else(|| panic!("{} has no MIME", name)); + let extracted = registry + .extract_complete_head(&path, &mime, head) + .unwrap_or_else(|| panic!("{} -> {} not claimed by a head-capable extractor", name, mime)) + .unwrap_or_else(|e| panic!("{} -> {} failed to extract: {}", name, mime, e)); + assert!( + !extracted.text.is_empty(), + "{} -> {} extracted no text", + name, + mime + ); + } + } + + /// Extensionless files are decided by their bytes: text heads index, + /// binary heads stay unclassified. + #[test] + fn extensionless_files_sniff_by_content() { + use std::path::PathBuf; + let readme = PathBuf::from("README"); + assert_eq!( + guess_mime_from_head(&readme, b"QuickSearch indexes your files.\n").as_deref(), + Some("text/plain") + ); + let makefile = PathBuf::from("Makefile"); + assert_eq!( + guess_mime_from_head(&makefile, b"all:\n\tcargo build\n").as_deref(), + Some("text/plain") + ); + let blob = PathBuf::from("blob"); + assert_eq!(guess_mime_from_head(&blob, &[0x00, 0x01, 0x02, 0xFF]).as_deref(), None); + } + + /// Ambiguous extensions resolve by content in both directions: source + /// code beats the extension table, real binary keeps the extension's + /// MIME as the fallback. + #[test] + fn ambiguous_extensions_resolve_by_content_both_ways() { + use std::path::PathBuf; + + let ts_source = b"export function hi(): string { return 'hi'; }\n"; + assert_eq!( + guess_mime_from_head(&PathBuf::from("app.ts"), ts_source).as_deref(), + Some("text/plain") + ); + // Uppercase, as Windows likes it. + assert_eq!( + guess_mime_from_head(&PathBuf::from("APP.TS"), ts_source).as_deref(), + Some("text/plain") + ); + // An MPEG transport stream: 0x47 sync bytes with NUL-heavy payloads. + // No magic matcher, fails the text sniff, so the extension answers. + let mut ts_video = vec![0u8; 376]; + ts_video[0] = 0x47; + ts_video[188] = 0x47; + assert_eq!( + guess_mime_from_head(&PathBuf::from("clip.ts"), &ts_video).as_deref(), + Some("video/vnd.dlna.mpeg-tts") + ); + + assert_eq!( + guess_mime_from_head(&PathBuf::from("go.mod"), b"module example.com/x\n\ngo 1.22\n") + .as_deref(), + Some("text/plain") + ); + + // gettext template vs PowerPoint template: text decides one way, + // binary bytes fall back to the extension's office MIME (whether + // infer's OLE matcher fires or the guard rejects, the answer agrees). + assert_eq!( + guess_mime_from_head(&PathBuf::from("app.pot"), b"msgid \"hello\"\nmsgstr \"\"\n") + .as_deref(), + Some("text/plain") + ); + let ole = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1, 0x00, 0x00]; + assert_eq!( + guess_mime_from_head(&PathBuf::from("slides.pot"), &ole).as_deref(), + Some("application/vnd.ms-powerpoint") + ); + + assert_eq!( + guess_mime_from_head(&PathBuf::from("cpu.vhd"), b"entity cpu is\nend cpu;\n") + .as_deref(), + Some("text/plain") + ); + assert_eq!( + guess_mime_from_head(&PathBuf::from("disk.vhd"), &[0x00, 0x01, 0x02, 0x03]) + .as_deref(), + Some("application/x-virtualbox-vhd") + ); + } + + /// Everything the plaintext extractor claims must carry the TEXT bit, + /// or `type:Text` silently misses content-indexed files (the pre-fix + /// state of `.sql`). Iterates the actual claim list so the two can + /// never drift apart. + #[test] + fn every_plaintext_claim_carries_the_text_bit() { + for mime in crate::extract::plaintext::EXTRA_TEXT_MIMES { + assert!( + mime_to_type(mime).contains(FileType::TEXT), + "{} is extractable as text but lacks FileType::TEXT", + mime + ); + } + // The multi-category cases keep their native category too. + let svg = mime_to_type("image/svg+xml"); + assert!(svg.contains(FileType::IMAGE) && svg.contains(FileType::TEXT)); + let m3u = mime_to_type("audio/x-mpegurl"); + assert!(m3u.contains(FileType::AUDIO) && m3u.contains(FileType::TEXT)); + let xhtml = mime_to_type("application/xhtml+xml"); + assert!(xhtml.contains(FileType::TEXT) && xhtml.contains(FileType::DOCUMENT)); + // And the `text/` prefix arm still covers the rest. + assert!(mime_to_type("text/x-toml").contains(FileType::TEXT)); + } } diff --git a/crates/quicksearch-core/src/textenc.rs b/crates/quicksearch-core/src/textenc.rs new file mode 100644 index 0000000..882fa27 --- /dev/null +++ b/crates/quicksearch-core/src/textenc.rs @@ -0,0 +1,282 @@ +//! Text detection and charset decoding, shared by the MIME sniff and the +//! plaintext extractor. +//! +//! Both callers route through one classifier so they cannot drift: a head +//! that [`looks_like_text`] accepts is guaranteed to decode via +//! [`decode_text`] — every class except `Binary` decodes unconditionally +//! (strict UTF-8 after validation, BOM'd and legacy decodes are +//! lossy-with-replacement and never fail). +//! +//! Classification order is load-bearing: +//! +//! 1. **BOM** ([`encoding_rs::Encoding::for_bom`]) — before the binary +//! guard, because UTF-16 text is full of NUL bytes the guard would +//! reject. +//! 2. **Binary guard** — any NUL byte, or control bytes (outside +//! `\t \n \r`, with ESC tolerated for ANSI-colored logs) above 10% of +//! the buffer. UTF-16 without a BOM fails here by design. +//! 3. **Strict UTF-8** — with one tolerance for the sniff: a head is a +//! prefix of the file (indexing hashes the first `hash_length` bytes and +//! sniffs the same buffer), so a multibyte sequence cut off by the end +//! of the buffer does not disqualify it. +//! 4. **Legacy** — everything else. [`decode_text`] runs charset detection +//! (chardetng, windows-1252 floor) and decodes with replacement. +//! +//! The sniff sees only the head, so a file with a text head and a binary +//! tail classifies as text and then fails the whole-file decode; that lands +//! as a FAILED row with a reason, the normal shape of head-based +//! classification. + +use std::path::Path; + +enum TextClass { + /// Strict UTF-8 (modulo the truncated-tail tolerance). + Utf8, + /// Starts with a BOM; decode with this encoding. + Bom(&'static encoding_rs::Encoding), + /// Not UTF-8 but passes the binary guard: charset detection will decode. + Legacy, + /// Fails the binary guard. + Binary, +} + +/// Control bytes tolerated in text: ordinary whitespace, plus ESC because +/// ANSI-colored logs are text worth indexing. +fn is_benign_control(b: u8) -> bool { + matches!(b, b'\t' | b'\n' | b'\r' | 0x1B) +} + +/// Classify `bytes` as text or binary. `truncated` marks a buffer that may +/// be a prefix of the file rather than its entirety. +fn classify(bytes: &[u8], truncated: bool) -> TextClass { + if let Some((enc, _bom_len)) = encoding_rs::Encoding::for_bom(bytes) { + return TextClass::Bom(enc); + } + + // Binary guard. NUL never appears in text of any supported encoding + // (UTF-16 was handled above, by BOM or not at all); a run of other + // control bytes marks compressed or machine data that merely lacks NULs. + let mut suspect = 0usize; + for &b in bytes { + if b == 0 { + return TextClass::Binary; + } + if (b < 0x20 && !is_benign_control(b)) || b == 0x7F { + suspect += 1; + } + } + if suspect * 10 > bytes.len() { + return TextClass::Binary; + } + + match std::str::from_utf8(bytes) { + Ok(_) => TextClass::Utf8, + // `error_len() == None` means the only defect is a multibyte + // sequence running off the end of the buffer — for a truncated head + // that is the file boundary's fault, not the file's. + Err(e) if truncated && e.error_len().is_none() => TextClass::Utf8, + Err(_) => TextClass::Legacy, + } +} + +/// Whether `head` — a possibly-truncated prefix of a file — reads as text. +/// +/// Cheap: a byte scan plus UTF-8 validation, no charset detection. An empty +/// head proves nothing and answers `false`; that keeps zero-size +/// extensionless files (procfs included) unclassified rather than blanket +/// `text/plain`. +pub fn looks_like_text(head: &[u8]) -> bool { + !head.is_empty() && !matches!(classify(head, true), TextClass::Binary) +} + +/// Decode a complete file's bytes to UTF-8 for storage. +/// +/// Takes ownership so the dominant valid-UTF-8 case is a zero-copy move. +/// `path` is used only to name the file in the error, matching the +/// extractor error convention. +pub fn decode_text(bytes: Vec, path: &Path) -> Result { + if bytes.is_empty() { + return Ok(String::new()); + } + match classify(&bytes, false) { + // Cannot fail: classify ran strict validation with truncated=false. + TextClass::Utf8 => Ok(String::from_utf8(bytes).expect("classified as UTF-8")), + TextClass::Bom(enc) => { + // BOM-aware decode: strips the BOM, replaces malformed + // sequences (e.g. a truncated trailing code unit) with U+FFFD. + let (text, _, _) = enc.decode(&bytes); + Ok(text.into_owned()) + } + TextClass::Legacy => { + // ISO-2022-JP detection is safe here: the browser caveat about + // it concerns script-running web content, not indexed files. + let mut det = + chardetng::EncodingDetector::new(chardetng::Iso2022JpDetection::Allow); + det.feed(&bytes, true); + // Deny UTF-8: strict UTF-8 was already ruled out, so a UTF-8 + // guess could only mean malformed UTF-8. + let enc = det.guess(None, chardetng::Utf8Detection::Deny); + let (text, _, _) = enc.decode(&bytes); + Ok(text.into_owned()) + } + TextClass::Binary => Err(format!( + "plaintext read {}: binary content", + path.display() + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn p() -> PathBuf { + PathBuf::from("/tmp/textenc-test-file") + } + + #[test] + fn utf8_decodes_unchanged() { + let body = "plain ascii and café über 日本語".as_bytes().to_vec(); + assert!(looks_like_text(&body)); + assert_eq!(decode_text(body, &p()).unwrap(), "plain ascii and café über 日本語"); + } + + #[test] + fn utf8_bom_is_stripped() { + let mut body = vec![0xEF, 0xBB, 0xBF]; + body.extend_from_slice("hello".as_bytes()); + assert!(looks_like_text(&body)); + let text = decode_text(body, &p()).unwrap(); + assert_eq!(text, "hello", "BOM must not survive into stored text"); + } + + /// The shape of a Windows registry export: UTF-16LE with BOM. + #[test] + fn utf16le_bom_decodes() { + let src = "Windows Registry Editor Version 5.00\r\n"; + let mut body = vec![0xFF, 0xFE]; + for unit in src.encode_utf16() { + body.extend_from_slice(&unit.to_le_bytes()); + } + assert!(looks_like_text(&body)); + assert_eq!(decode_text(body, &p()).unwrap(), src); + } + + #[test] + fn utf16be_bom_decodes() { + let src = "big endian text"; + let mut body = vec![0xFE, 0xFF]; + for unit in src.encode_utf16() { + body.extend_from_slice(&unit.to_be_bytes()); + } + assert!(looks_like_text(&body)); + assert_eq!(decode_text(body, &p()).unwrap(), src); + } + + #[test] + fn windows_1252_decodes() { + // A sentence long enough for chardetng to settle on a Western + // single-byte encoding. + let body = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.".to_vec(); + assert!(looks_like_text(&body)); + assert_eq!( + decode_text(body, &p()).unwrap(), + "Le café près de la fenêtre est agréable en été." + ); + } + + #[test] + fn shift_jis_decodes() { + // "日本語のテキストです。これはシフトJISでエンコードされています。" + let src = "日本語のテキストです。これはシフトJISでエンコードされています。"; + let (encoded, _, had_errors) = encoding_rs::SHIFT_JIS.encode(src); + assert!(!had_errors); + let body = encoded.into_owned(); + assert!(looks_like_text(&body)); + assert_eq!(decode_text(body, &p()).unwrap(), src); + } + + #[test] + fn nul_bytes_are_binary() { + let body = b"looks like text until\x00it does not".to_vec(); + assert!(!looks_like_text(&body)); + let err = decode_text(body, &p()).unwrap_err(); + assert!(err.contains("binary content"), "{err}"); + assert!(err.contains("textenc-test-file"), "error must name the file: {err}"); + } + + #[test] + fn control_density_is_binary() { + // 4 control bytes in 24 total = 16% > 10%. + let body = b"abcdefghijklmnopqrst\x01\x02\x03\x04".to_vec(); + assert!(!looks_like_text(&body)); + assert!(decode_text(body, &p()).is_err()); + } + + #[test] + fn ansi_log_is_text() { + // ESC-heavy colored log output stays text. + let body = b"\x1b[31mERROR\x1b[0m something failed\n\x1b[33mWARN\x1b[0m retrying\n".to_vec(); + assert!(looks_like_text(&body)); + assert!(decode_text(body, &p()).is_ok()); + } + + #[test] + fn truncated_utf8_tail_still_sniffs_as_text() { + let mut head = "ends mid-char: caf".as_bytes().to_vec(); + head.push(0xC3); // first byte of a two-byte sequence, cut off + assert!(looks_like_text(&head)); + // The same bytes as a *complete* file are not valid UTF-8, so the + // decoder treats them as legacy-encoded — still Ok, never a panic. + assert!(decode_text(head, &p()).is_ok()); + } + + #[test] + fn empty_head_is_not_text_but_empty_file_decodes() { + assert!(!looks_like_text(b"")); + assert_eq!(decode_text(Vec::new(), &p()).unwrap(), ""); + } + + /// UTF-16 without a BOM is out of scope: its NULs trip the binary + /// guard. This test documents the decision rather than a limitation we + /// intend to lift. + #[test] + fn utf16_without_bom_is_rejected() { + let mut body = Vec::new(); + for unit in "no bom here".encode_utf16() { + body.extend_from_slice(&unit.to_le_bytes()); + } + assert!(!looks_like_text(&body)); + assert!(decode_text(body, &p()).is_err()); + } + + /// Every head the sniff accepts must decode — the invariant that makes + /// "sniffed as text/plain" safe to act on. + #[test] + fn sniffed_text_is_guaranteed_decodable() { + let heads: Vec> = vec![ + b"ordinary ascii".to_vec(), + "utf-8 caf\u{e9}".as_bytes().to_vec(), + b"latin-1 caf\xe9 body".to_vec(), + { + let mut v = vec![0xFF, 0xFE]; + v.extend("utf16".encode_utf16().flat_map(|u| u.to_le_bytes())); + v + }, + { + let mut v = "truncated tail caf".as_bytes().to_vec(); + v.push(0xC3); + v + }, + ]; + for head in heads { + if looks_like_text(&head) { + assert!( + decode_text(head.clone(), &p()).is_ok(), + "sniffed-as-text head failed to decode: {head:?}" + ); + } + } + } +} diff --git a/crates/quicksearch-core/tests/full_index.rs b/crates/quicksearch-core/tests/full_index.rs index 2ddf486..7a39e49 100644 --- a/crates/quicksearch-core/tests/full_index.rs +++ b/crates/quicksearch-core/tests/full_index.rs @@ -777,10 +777,13 @@ fn seed_mixed_tree(root: &Path) { touch(&root.join("small.txt"), b"a small plaintext body with xylophone in it"); touch(&root.join("large.txt"), big.as_bytes()); touch(&root.join("empty.txt"), b""); - // Invalid UTF-8 with a .txt extension: claimed by the plaintext extractor, - // but not decodable, so it must be reported as a failure either way. + // Binary bytes with a .txt extension: claimed by the plaintext + // extractor, but the NUL fails the binary guard (and the FF FE pair is + // not at offset 0, so it is no BOM), so it must be reported as a + // failure either way. touch(&root.join("bad.txt"), &[0x68, 0x69, 0xff, 0xfe, 0x00, 0x41]); - // No extension `infer` or `mime_guess` recognises: no extractor claims it. + // No extension table, magic, or text sniff has an answer for NUL soup: + // no MIME, no extractor. touch(&root.join("blob.bin"), &[0x00, 0x01, 0x02, 0xfd, 0xfe, 0xff]); touch(&root.join("nested/deep/note.md"), b"# heading\n\nquagmire body text\n"); } @@ -859,7 +862,9 @@ fn undecodable_small_files_are_reported_as_failures_not_silently_skipped() { let db_dir = tmp_dir("inline-badutf8-db"); let db = db_dir.join("index.sqlite"); - touch(&root.join("bad.txt"), &[0x68, 0x69, 0xff, 0xfe]); + // The NUL keeps this undecodable: without it these bytes would now + // decode as windows-1252 and the test would assert nothing. + touch(&root.join("bad.txt"), &[0x68, 0x00, 0x69, 0xff]); index_once(&root, &db, &Config::default()); let conn = rusqlite::Connection::open(&db).unwrap(); @@ -882,6 +887,114 @@ fn undecodable_small_files_are_reported_as_failures_not_silently_skipped() { std::fs::remove_dir_all(&db_dir).ok(); } +/// The text sniff end-to-end: extensionless text files (README, Makefile, +/// go.sum) are content-indexed off their head bytes, while an extensionless +/// binary blob stays NA. +#[test] +fn extensionless_text_files_are_indexed() { + let root = tmp_dir("extless"); + let db_dir = tmp_dir("extless-db"); + let db = db_dir.join("index.sqlite"); + + touch(&root.join("README"), b"QuickSearch indexes zanzibar contents.\n"); + touch(&root.join("Makefile"), b"all:\n\tcargo build --release\n"); + touch(&root.join("go.sum"), b"example.com/x v1.0.0 h1:abcdef=\n"); + touch(&root.join("blob"), &[0x00, 0x01, 0xfe, 0xff]); + index_once(&root, &db, &Config::default()); + + let conn = rusqlite::Connection::open(&db).unwrap(); + let state_of = |name: &str| -> i64 { + conn.query_row( + "SELECT content_state FROM files WHERE path LIKE '%' || ?1", + [name], + |r| r.get(0), + ) + .unwrap() + }; + for name in ["README", "Makefile", "go.sum"] { + assert_eq!(state_of(name), 1, "{} should be content-indexed", name); + } + assert_eq!(state_of("blob"), 3, "binary blob stays not-applicable"); + drop(conn); + + assert_eq!( + stored_text(&db, "README").as_deref(), + Some("QuickSearch indexes zanzibar contents.\n"), + "the stored body round-trips" + ); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// Charset decoding end-to-end: UTF-16LE files (the shape of a Windows +/// registry export) and legacy single-byte text are stored as UTF-8 — +/// `stored_text` decodes the zstd sidecar with `String::from_utf8`, so a +/// `Some` result *is* the storage-is-UTF-8 assertion. +#[test] +fn utf16_files_are_stored_as_utf8() { + let root = tmp_dir("charset"); + let db_dir = tmp_dir("charset-db"); + let db = db_dir.join("index.sqlite"); + + let reg_src = "Windows Registry Editor Version 5.00\r\n\r\n[HKEY_CURRENT_USER\\Software\\Xylograph]\r\n"; + let mut reg_body = vec![0xFF, 0xFE]; + reg_body.extend(reg_src.encode_utf16().flat_map(|u| u.to_le_bytes())); + touch(&root.join("export.reg"), ®_body); + + // The same encoding behind no extension at all: BOM first, sniff after. + let mut extless = vec![0xFF, 0xFE]; + extless.extend("utf16 notes about quokkas".encode_utf16().flat_map(|u| u.to_le_bytes())); + touch(&root.join("NOTES16"), &extless); + + touch(&root.join("legacy.txt"), b"un caf\xe9 tr\xe8s agr\xe9able pr\xe8s du mus\xe9e"); + index_once(&root, &db, &Config::default()); + + assert_eq!(stored_text(&db, "export.reg").as_deref(), Some(reg_src)); + assert_eq!( + stored_text(&db, "NOTES16").as_deref(), + Some("utf16 notes about quokkas") + ); + assert_eq!( + stored_text(&db, "legacy.txt").as_deref(), + Some("un café très agréable près du musée") + ); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// RTF end-to-end through both extraction paths: a small file the walk +/// finishes inline, and one past `hash_length` that the content pass opens. +/// Stored text is the parsed prose, not RTF control words. +#[test] +fn rtf_files_are_extracted() { + let root = tmp_dir("rtf"); + let db_dir = tmp_dir("rtf-db"); + let db = db_dir.join("index.sqlite"); + + touch( + &root.join("small.rtf"), + br"{\rtf1\ansi Meeting notes about the pangolin budget.}", + ); + let big_body = format!( + r"{{\rtf1\ansi {}}}", + r"paragraphs about the pangolin budget \par ".repeat(400) + ); + assert!(big_body.len() > 8192, "must exceed the default head"); + touch(&root.join("big.rtf"), big_body.as_bytes()); + index_once(&root, &db, &Config::default()); + + for name in ["small.rtf", "big.rtf"] { + let text = stored_text(&db, name).unwrap_or_else(|| panic!("{} has no stored text", name)); + assert!(text.contains("pangolin budget"), "{}: {:?}", name, &text[..text.len().min(80)]); + assert!(!text.contains(r"\rtf"), "{} stored control words", name); + } + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + /// End-to-end version of the fix: the extraction denominator the manage-index /// tab renders is `extract_total`, and it must count files that need text — /// not every indexed file. Asserted through a real `IndexingService` run so it @@ -894,13 +1007,15 @@ fn the_extraction_denominator_counts_only_files_that_need_text() { // Three files an extractor claims, seven it never will. `big.txt` is the // interesting one: larger than `hash_length`, so the walk cannot finish it - // inline and it is the only row the content pass actually opens. + // inline and it is the only row the content pass actually opens. The + // unclaimed seven get NUL-bearing bodies so neither the extension tables + // nor the text sniff have anything to say about them. for name in ["a.txt", "b.json"] { touch(&root.join(name), b"body bytes with no magic"); } touch(&root.join("big.txt"), &vec![b'z'; 32 * 1024]); for name in ["d.mp4", "e.zip", "f.bin", "g.exe", "h.iso", "i.so", "j"] { - touch(&root.join(name), b"body bytes with no magic"); + touch(&root.join(name), b"\x00\x01body bytes\x00"); } let config = Config::default(); diff --git a/crates/quicksearch-gui/src/manage_tab.rs b/crates/quicksearch-gui/src/manage_tab.rs index 802ac97..4509321 100644 --- a/crates/quicksearch-gui/src/manage_tab.rs +++ b/crates/quicksearch-gui/src/manage_tab.rs @@ -308,12 +308,21 @@ impl ManageTab { ui.heading(egui::RichText::new("Content filters").strong()); ui.columns(2, |cols| { cols[0].label("Full-text extensions whitelist (empty = all supported):"); - cols[0].add( - egui::TextEdit::multiline(&mut self.ext_filter_text) - .desired_rows(4) - .desired_width(f32::INFINITY) - .hint_text("txt\nmd\npdf"), - ); + cols[0] + .add( + egui::TextEdit::multiline(&mut self.ext_filter_text) + .desired_rows(4) + .desired_width(f32::INFINITY) + .hint_text("txt\nmd\npdf # comments allowed\n(none)"), + ) + .on_hover_text( + "One extension per line, leading dot optional. A non-empty \ + list also excludes files that have no extension at all \ + (Makefile, README, .bashrc) — add the line \"(none)\" to \ + keep extracting text from those.\n\n\ + \"#\" starts a comment, either on its own line or after an \ + entry, so a type can be commented out without losing it.", + ); cols[1].label("Ignore patterns (excluded entirely):"); let mut remove_pat: Option = None; // The list grows and shrinks — including from outside