quick_search/crates/quicksearch-core/src/extract/plaintext.rs

//! Read the file as UTF-8 text. Handles text/plain, text/x-*, application/json
//! and most source-code MIMEs.

use std::fs::File;
use std::io::Read;
use std::path::Path;

use super::{ExtractError, ExtractedContent, Extractor};

/// Decode bytes that are known to be a complete file. Shared by both entry
/// points so on-disk and already-in-memory extraction cannot drift apart.
fn decode(bytes: Vec<u8>, path: &Path) -> Result<ExtractedContent, ExtractError> {
    match String::from_utf8(bytes) {
        Ok(text) => Ok(ExtractedContent::with_text(text)),
        Err(e) => Err(format!("plaintext read {}: {}", path.display(), e.utf8_error())),
    }
}

pub struct PlaintextExtractor;

impl Extractor for PlaintextExtractor {
    fn supports(&self, mime: &str) -> bool {
        if mime.starts_with("text/") {
            return true;
        }
        matches!(
            mime,
            "application/json"
                | "application/xml"
                | "application/javascript"
                | "application/x-shellscript"
                | "application/x-python"
                | "application/toml"
                | "application/yaml"
                | "application/x-yaml"
                // `.sql` resolves here rather than to `text/*`, so without it
                // schema dumps are listed by name but never full-text indexed.
                | "application/x-sql"
        )
    }

    /// Read the whole file, sized from the handle we just opened.
    ///
    /// `std::fs::read_to_string` would cost two extra syscalls here: a
    /// path-based `statx` to size its buffer, and a second `read` returning 0,
    /// because "read to EOF" can only observe EOF that way — `read_to_end`
    /// terminates on `Ok(0)` alone, so a short read does not end it. Sizing
    /// the buffer ourselves lets the loop finish on `filled == size` and issue
    /// exactly one `read` for a file that fits.
    ///
    /// A file that shrank between the `fstat` and the `read` keeps its prefix
    /// rather than failing. A file that grew is read up to the size we saw;
    /// its mtime moved, so the next run reclassifies it as changed and
    /// re-extracts (see [`crate::file_handling::classify_for_indexing`]).
    /// Neither case was ever atomic — a concurrent writer can tear a file
    /// across any read sequence, including `read_to_string`'s.
    fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
        let mut f = File::open(path)
            .map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
        let size = f
            .metadata()
            .map_err(|e| format!("plaintext read {}: {}", path.display(), e))?
            .len() as usize;

        // procfs, sysfs and some FUSE mounts report zero for files that do
        // have content, so a sized read would store nothing. Only these pay
        // the read-to-EOF probe — which is what a genuinely empty file cost
        // before anyway.
        if size == 0 {
            let mut buf = Vec::new();
            f.read_to_end(&mut buf)
                .map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
            return decode(buf, path);
        }

        let mut buf = vec![0u8; size];
        let mut filled = 0;
        while filled < size {
            match f.read(&mut buf[filled..]) {
                Ok(0) => break,
                Ok(n) => filled += n,
                Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
                Err(e) => return Err(format!("plaintext read {}: {}", path.display(), e)),
            }
        }
        buf.truncate(filled);
        decode(buf, path)
    }

    fn extract_from_head(
        &self,
        path: &Path,
        head: &[u8],
    ) -> Option<Result<ExtractedContent, ExtractError>> {
        Some(decode(head.to_vec(), path))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn tmp(tag: &str, body: &[u8]) -> std::path::PathBuf {
        let mut p = std::env::temp_dir();
        p.push(format!(
            "qs-plaintext-{}-{}-{}.txt",
            tag,
            std::process::id(),
            std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        ));
        std::fs::write(&p, body).unwrap();
        p
    }

    #[test]
    fn reads_utf8_file() {
        let p = tmp("basic", b"hello world");
        let c = PlaintextExtractor.extract(&p).unwrap();
        assert_eq!(c.text, "hello world");
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn head_extraction_matches_reading_the_file() {
        let p = tmp("agree", b"shared body with unicode: caf\xc3\xa9 \xe2\x9c\x93");
        let from_disk = PlaintextExtractor.extract(&p).unwrap();
        let bytes = std::fs::read(&p).unwrap();
        let from_head = PlaintextExtractor.extract_from_head(&p, &bytes).unwrap().unwrap();
        assert_eq!(from_disk.text, from_head.text);
        assert_eq!(from_disk.properties, from_head.properties);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn both_paths_reject_invalid_utf8_and_name_the_file() {
        let p = tmp("badutf8", &[0x68, 0x69, 0xff, 0xfe]);
        let disk_err = PlaintextExtractor.extract(&p).unwrap_err();
        let head_err = PlaintextExtractor
            .extract_from_head(&p, &[0x68, 0x69, 0xff, 0xfe])
            .unwrap()
            .unwrap_err();
        assert_eq!(disk_err, head_err, "one decode path, one message");
        assert!(disk_err.contains("badutf8"), "the failure names the file: {}", disk_err);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn reads_a_file_larger_than_one_buffer_completely() {
        // Past any plausible head window, so the read loop has to iterate if
        // the kernel returns a short read.
        let body = "abcdefgh".repeat(200 * 1024 / 8);
        let p = tmp("large", body.as_bytes());
        let c = PlaintextExtractor.extract(&p).unwrap();
        assert_eq!(c.text.len(), body.len());
        assert_eq!(c.text, body);
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn an_empty_file_extracts_to_empty_text() {
        let p = tmp("empty", b"");
        assert_eq!(PlaintextExtractor.extract(&p).unwrap().text, "");
        assert_eq!(
            PlaintextExtractor.extract_from_head(&p, &[]).unwrap().unwrap().text,
            ""
        );
        std::fs::remove_file(&p).ok();
    }

    /// A file whose reported size is a lie in the "there is more than this"
    /// direction — the shape procfs and sysfs have. Sizing the buffer from
    /// `st_size` alone would store nothing, so `extract` must fall back to
    /// reading until EOF.
    #[test]
    fn a_file_reporting_zero_size_is_still_read_to_eof() {
        let p = Path::new("/proc/self/status");
        if !p.exists() {
            return; // not Linux; the guard is only reachable there
        }
        assert_eq!(
            std::fs::metadata(p).unwrap().len(),
            0,
            "precondition: procfs reports zero size"
        );
        let c = PlaintextExtractor.extract(p).unwrap();
        assert!(
            c.text.contains("Name:"),
            "content must survive a zero st_size, got {} bytes",
            c.text.len()
        );
    }

    /// The same lie in the other direction, which the sized read handles by
    /// keeping whatever was actually there.
    #[test]
    fn a_file_that_shrank_after_sizing_keeps_its_prefix() {
        let p = tmp("shrink", &vec![b'x'; 4096]);
        let f = File::options().write(true).open(&p).unwrap();
        // Truncate behind `extract`'s back is not reproducible, so assert the
        // property directly: a buffer sized larger than the file yields the
        // file, not an error.
        f.set_len(10).unwrap();
        drop(f);
        let c = PlaintextExtractor.extract(&p).unwrap();
        assert_eq!(c.text, "xxxxxxxxxx", "a shrunk file reads short, not fatal");
        std::fs::remove_file(&p).ok();
    }

    #[test]
    fn supports_text_mimes() {
        let e = PlaintextExtractor;
        assert!(e.supports("text/plain"));
        assert!(e.supports("text/x-rust"));
        assert!(e.supports("application/json"));
        assert!(!e.supports("application/pdf"));
        assert!(!e.supports("image/png"));
    }
}