//! Read the file as UTF-8 text. Handles text/plain, text/x-*, application/json //! and most source-code MIMEs. use std::fs::File; use std::io::Read; use std::path::Path; use super::{ExtractError, ExtractedContent, Extractor}; /// Decode bytes that are known to be a complete file. Shared by both entry /// points so on-disk and already-in-memory extraction cannot drift apart. fn decode(bytes: Vec, path: &Path) -> Result { match String::from_utf8(bytes) { Ok(text) => Ok(ExtractedContent::with_text(text)), Err(e) => Err(format!("plaintext read {}: {}", path.display(), e.utf8_error())), } } pub struct PlaintextExtractor; impl Extractor for PlaintextExtractor { fn supports(&self, mime: &str) -> bool { if mime.starts_with("text/") { return true; } matches!( mime, "application/json" | "application/xml" | "application/javascript" | "application/x-shellscript" | "application/x-python" | "application/toml" | "application/yaml" | "application/x-yaml" // `.sql` resolves here rather than to `text/*`, so without it // schema dumps are listed by name but never full-text indexed. | "application/x-sql" ) } /// Read the whole file, sized from the handle we just opened. /// /// `std::fs::read_to_string` would cost two extra syscalls here: a /// path-based `statx` to size its buffer, and a second `read` returning 0, /// because "read to EOF" can only observe EOF that way — `read_to_end` /// terminates on `Ok(0)` alone, so a short read does not end it. Sizing /// the buffer ourselves lets the loop finish on `filled == size` and issue /// exactly one `read` for a file that fits. /// /// A file that shrank between the `fstat` and the `read` keeps its prefix /// rather than failing. A file that grew is read up to the size we saw; /// its mtime moved, so the next run reclassifies it as changed and /// re-extracts (see [`crate::file_handling::classify_for_indexing`]). /// Neither case was ever atomic — a concurrent writer can tear a file /// across any read sequence, including `read_to_string`'s. fn extract(&self, path: &Path) -> Result { let mut f = File::open(path) .map_err(|e| format!("plaintext read {}: {}", path.display(), e))?; let size = f .metadata() .map_err(|e| format!("plaintext read {}: {}", path.display(), e))? .len() as usize; // procfs, sysfs and some FUSE mounts report zero for files that do // have content, so a sized read would store nothing. Only these pay // the read-to-EOF probe — which is what a genuinely empty file cost // before anyway. if size == 0 { let mut buf = Vec::new(); f.read_to_end(&mut buf) .map_err(|e| format!("plaintext read {}: {}", path.display(), e))?; return decode(buf, path); } let mut buf = vec![0u8; size]; let mut filled = 0; while filled < size { match f.read(&mut buf[filled..]) { Ok(0) => break, Ok(n) => filled += n, Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {} Err(e) => return Err(format!("plaintext read {}: {}", path.display(), e)), } } buf.truncate(filled); decode(buf, path) } fn extract_from_head( &self, path: &Path, head: &[u8], ) -> Option> { Some(decode(head.to_vec(), path)) } } #[cfg(test)] mod tests { use super::*; fn tmp(tag: &str, body: &[u8]) -> std::path::PathBuf { let mut p = std::env::temp_dir(); p.push(format!( "qs-plaintext-{}-{}-{}.txt", tag, std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos() )); std::fs::write(&p, body).unwrap(); p } #[test] fn reads_utf8_file() { let p = tmp("basic", b"hello world"); let c = PlaintextExtractor.extract(&p).unwrap(); assert_eq!(c.text, "hello world"); std::fs::remove_file(&p).ok(); } #[test] fn head_extraction_matches_reading_the_file() { let p = tmp("agree", b"shared body with unicode: caf\xc3\xa9 \xe2\x9c\x93"); let from_disk = PlaintextExtractor.extract(&p).unwrap(); let bytes = std::fs::read(&p).unwrap(); let from_head = PlaintextExtractor.extract_from_head(&p, &bytes).unwrap().unwrap(); assert_eq!(from_disk.text, from_head.text); assert_eq!(from_disk.properties, from_head.properties); std::fs::remove_file(&p).ok(); } #[test] fn both_paths_reject_invalid_utf8_and_name_the_file() { let p = tmp("badutf8", &[0x68, 0x69, 0xff, 0xfe]); let disk_err = PlaintextExtractor.extract(&p).unwrap_err(); let head_err = PlaintextExtractor .extract_from_head(&p, &[0x68, 0x69, 0xff, 0xfe]) .unwrap() .unwrap_err(); assert_eq!(disk_err, head_err, "one decode path, one message"); assert!(disk_err.contains("badutf8"), "the failure names the file: {}", disk_err); std::fs::remove_file(&p).ok(); } #[test] fn reads_a_file_larger_than_one_buffer_completely() { // Past any plausible head window, so the read loop has to iterate if // the kernel returns a short read. let body = "abcdefgh".repeat(200 * 1024 / 8); let p = tmp("large", body.as_bytes()); let c = PlaintextExtractor.extract(&p).unwrap(); assert_eq!(c.text.len(), body.len()); assert_eq!(c.text, body); std::fs::remove_file(&p).ok(); } #[test] fn an_empty_file_extracts_to_empty_text() { let p = tmp("empty", b""); assert_eq!(PlaintextExtractor.extract(&p).unwrap().text, ""); assert_eq!( PlaintextExtractor.extract_from_head(&p, &[]).unwrap().unwrap().text, "" ); std::fs::remove_file(&p).ok(); } /// A file whose reported size is a lie in the "there is more than this" /// direction — the shape procfs and sysfs have. Sizing the buffer from /// `st_size` alone would store nothing, so `extract` must fall back to /// reading until EOF. #[test] fn a_file_reporting_zero_size_is_still_read_to_eof() { let p = Path::new("/proc/self/status"); if !p.exists() { return; // not Linux; the guard is only reachable there } assert_eq!( std::fs::metadata(p).unwrap().len(), 0, "precondition: procfs reports zero size" ); let c = PlaintextExtractor.extract(p).unwrap(); assert!( c.text.contains("Name:"), "content must survive a zero st_size, got {} bytes", c.text.len() ); } /// The same lie in the other direction, which the sized read handles by /// keeping whatever was actually there. #[test] fn a_file_that_shrank_after_sizing_keeps_its_prefix() { let p = tmp("shrink", &vec![b'x'; 4096]); let f = File::options().write(true).open(&p).unwrap(); // Truncate behind `extract`'s back is not reproducible, so assert the // property directly: a buffer sized larger than the file yields the // file, not an error. f.set_len(10).unwrap(); drop(f); let c = PlaintextExtractor.extract(&p).unwrap(); assert_eq!(c.text, "xxxxxxxxxx", "a shrunk file reads short, not fatal"); std::fs::remove_file(&p).ok(); } #[test] fn supports_text_mimes() { let e = PlaintextExtractor; assert!(e.supports("text/plain")); assert!(e.supports("text/x-rust")); assert!(e.supports("application/json")); assert!(!e.supports("application/pdf")); assert!(!e.supports("image/png")); } }