//! Content extractors: the searchable text of a file. //! //! An [`Extractor`] decides whether it can handle a given MIME type and, if //! so, produces [`ExtractedContent`] for the file. The [`Registry`] picks the //! first registered extractor that accepts the MIME and runs it. //! //! Dispatch is by MIME only — a file with no detected type is recorded as //! "not applicable" rather than guessed at again here. "What is this file" //! is decided once, upstream in [`crate::mime::guess_mime_from_head`]; //! nothing downstream reopens the file to ask again. use std::path::Path; pub mod audio; // pub mod image; // parked — see `ExtractedContent` below pub mod office; pub mod ole; pub mod pdf; pub mod plaintext; pub mod rtf; /// Result of a successful extraction: `text` feeds the FTS5 `text` column. /// /// Extractors may return an empty `text` when the file has no narrative /// content (an audio file whose tags are all empty, say). Filename search /// still works in that case. /// /// # Structured properties are parked /// /// Extractors used to return a `properties: HashMap` beside /// the text — EXIF, audio tags, the PDF `Info` dictionary — stored in a /// `properties` table *and* concatenated into a `properties` FTS column. /// Nothing ever read either back: no query, no result row, no UI. So the /// storage is gone and the extraction is commented out rather than deleted. /// /// Reviving it means restoring, together: this field and /// `properties_sorted`, the blocks marked "properties (parked)" in /// `image.rs` / `audio.rs` / `pdf.rs`, the `image` module registration in /// [`Registry::default_set`], the `properties` table and FTS column in /// [`crate::db::schema`], the `properties` argument to /// [`crate::db::repo::set_content_done`] — and a consumer that shows them. #[derive(Debug, Default, Clone)] pub struct ExtractedContent { pub text: String, // pub properties: HashMap, } impl ExtractedContent { pub fn with_text(text: impl Into) -> Self { Self { text: text.into() } } // /// Convert properties into the `Vec<(String, String)>` shape expected by // /// [`crate::db::repo::set_content_done`]. Keys are sorted for determinism // /// in tests and snapshots. // pub fn properties_sorted(&self) -> Vec<(String, String)> { // let mut v: Vec<(String, String)> = self // .properties // .iter() // .map(|(k, v)| (k.clone(), v.clone())) // .collect(); // v.sort_by(|a, b| a.0.cmp(&b.0)); // v // } } /// Boxed error type for extractor failures. A string reason is stored on the /// file row (see [`crate::db::repo::set_content_failed`]), so extractors /// should surface human-readable messages. pub type ExtractError = String; /// A pluggable content extractor. Stateless; implementors should not hold /// file handles across calls. pub trait Extractor: Send + Sync { /// Whether this extractor can handle the given MIME type. `mime` is /// normalized to lowercase before dispatch. fn supports(&self, mime: &str) -> bool; /// Read the file at `path` and return its extracted content. Return an /// [`ExtractError`] to mark the file's content state as failed (so it /// won't be retried every run). fn extract(&self, path: &Path) -> Result; /// Extract from bytes the caller already holds, when those bytes are the /// file's *entire* contents. For anything no larger than `hash_length` /// the whole file is already in memory at walk time, so working from the /// buffer saves the content pass an open/read/close and keeps the text /// consistent with the size, mtime and hash read alongside it. /// /// The default is `None`: "I need the file on disk." Formats that seek, /// or read a central directory at the end of the file, must keep it. /// `Some(Err(_))` is a real extraction failure; `None` defers to /// [`Extractor::extract`]. `path` is passed only so failures name the /// file — nothing here may open it. fn extract_from_head( &self, _path: &Path, _head: &[u8], ) -> Option> { None } } /// An ordered dispatch table of extractors. The first extractor whose /// [`supports`](Extractor::supports) returns true for the MIME is used. pub struct Registry { extractors: Vec>, } impl Registry { pub fn new() -> Self { Self { extractors: Vec::new(), } } pub fn with(mut self, e: impl Extractor + 'static) -> Self { self.extractors.push(Box::new(e)); self } /// The extractor that claims `mime`, if any — the one place dispatch /// happens. fn find(&self, mime: &str) -> Option<&dyn Extractor> { let lower = mime.to_ascii_lowercase(); self.extractors .iter() .find(|e| e.supports(&lower)) .map(|e| &**e) } /// Whether any extractor claims `mime`, without touching the file — what /// lets the walk decide a row's `content_state` up front (see /// [`crate::file_handling::content_extractable`]). pub fn supports(&self, mime: &str) -> bool { self.find(mime).is_some() } /// Look up a handler for `mime` and run it against `path`. Returns /// `Ok(None)` if no extractor claims the MIME — the caller should then /// decide whether the file is "not applicable" (text state NA). pub fn extract( &self, path: &Path, mime: &str, ) -> Result, ExtractError> { self.find(mime).map(|e| e.extract(path)).transpose() } /// [`Registry::extract`] for a file whose complete contents the caller /// already holds. `None` when no extractor claims the MIME or the one /// that does needs the file on disk — both mean "leave this to the /// content pass". pub fn extract_complete_head( &self, path: &Path, mime: &str, head: &[u8], ) -> Option> { self.find(mime) .and_then(|e| e.extract_from_head(path, head)) } /// The default set: RTF, plaintext, office docs, PDF, audio tags. /// /// Order matters — the first extractor whose `supports` accepts a MIME /// wins. RTF precedes plaintext because plaintext claims every `text/*` /// and would swallow `text/rtf` as raw control words. Plaintext /// precedes audio because it deliberately claims playlist /// (`audio/x-mpegurl`, `audio/scpls`) and SVG MIMEs whose text is worth /// more than their tags. /// /// No image extractor: it produced EXIF properties and never any text, /// so with properties parked it would open and parse every image on /// disk to return nothing. Leaving `image/*` unclaimed is what makes /// [`crate::file_handling::content_extractable`] record images as /// `STATE_NA` at walk time, so the content pass never opens them. /// Filenames are indexed exactly as before. pub fn default_set() -> Self { Self::new() .with(rtf::RtfExtractor) .with(plaintext::PlaintextExtractor) .with(office::OfficeExtractor) .with(pdf::PdfExtractor) .with(audio::AudioExtractor) // .with(image::ImageExtractor) // parked with `ExtractedContent` } } impl Default for Registry { fn default() -> Self { Self::default_set() } } #[cfg(test)] mod tests { use super::*; #[test] fn empty_registry_returns_none() { let r = Registry::new(); let out = r .extract(Path::new("/tmp/x"), "text/plain") .expect("no error"); assert!(out.is_none()); } #[test] fn complete_head_extraction_dispatches_only_to_extractors_that_opt_in() { let r = Registry::default_set(); let p = Path::new("/tmp/whatever"); // Plaintext opts in, so a small text file never reaches the disk pass. let out = r.extract_complete_head(p, "text/plain", b"hello"); assert!(matches!(out, Some(Ok(ref c)) if c.text == "hello")); // A format that seeks or reads a trailer must not be handed a buffer. // `None` here is what routes it back to the on-disk extractor. assert!(r .extract_complete_head(p, "application/pdf", b"%PDF-1.4") .is_none()); // No extractor claims images at all now — the head path must agree. assert!(r .extract_complete_head(p, "image/png", b"\x89PNG") .is_none()); // No extractor claims the MIME at all. assert!(r .extract_complete_head(p, "application/x-nonesuch", b"..") .is_none()); } #[test] fn complete_head_extraction_matches_the_on_disk_dispatch() { // Both entry points must pick the same extractor for a MIME, or a // file's text would depend on which pass happened to handle it. let r = Registry::default_set(); let p = Path::new("/tmp/whatever"); for mime in [ "text/plain", "TEXT/PLAIN", "application/json", "application/x-sql", ] { assert!( r.extract_complete_head(p, mime, b"x").is_some(), "{} should extract from a head", mime ); } for mime in ["application/rtf", "text/rtf"] { assert!( r.extract_complete_head(p, mime, br"{\rtf1 x}").is_some(), "{} should extract from a head", mime ); } } /// `text/rtf` must dispatch to the RTF extractor, not to plaintext's /// `text/*` claim — i.e. the registration order does its job. The RTF /// parser strips control words; plaintext would keep them. #[test] fn text_rtf_reaches_the_rtf_extractor_not_plaintext() { let r = Registry::default_set(); let p = Path::new("/tmp/whatever.rtf"); let out = r .extract_complete_head(p, "text/rtf", br"{\rtf1\ansi Hello {\b World}}") .expect("claimed") .expect("parsed"); assert_eq!(out.text, "Hello World"); } #[test] fn supports_agrees_with_extract_dispatch() { // `supports` is the cheap form of the question `extract` answers with // `Ok(None)`. They must agree for every MIME, or the walk would write // a content state the content pass then contradicts. The path does not // exist, so a claimed MIME surfaces as `Err`, not `Ok(None)` — which is // exactly the distinction under test. let r = Registry::default_set(); let missing = Path::new("/nonexistent/quicksearch-supports-probe"); for mime in [ "text/plain", "TEXT/PLAIN", "text/x-rust", "application/json", "APPLICATION/PDF", "application/pdf", "audio/mpeg", "Image/JPEG", "application/msword", "application/vnd.oasis.opendocument.text", // Real MIMEs with no extractor: the population the fix is about. "video/mp4", "application/zip", "application/x-executable", "application/octet-stream", "", ] { let claimed = !matches!(r.extract(missing, mime), Ok(None)); assert_eq!( r.supports(mime), claimed, "supports and extract disagree about {:?}", mime ); } } /// Images are claimed by nothing, so the walk records them `NA` and the /// content pass never opens them. Pins the parked image extractor. #[test] fn images_are_not_claimed_by_any_extractor() { let r = Registry::default_set(); for mime in ["image/jpeg", "image/png", "Image/JPEG", "image/tiff"] { assert!(!r.supports(mime), "{} should be unclaimed", mime); } } }