2026-04-21 23:00:47 -04:00
|
|
|
//! MIME type guessing and `FileType` bitmask classification.
|
|
|
|
|
//!
|
2026-08-03 12:33:47 -04:00
|
|
|
//! [`guess_mime_from_head`] infers a MIME type in three stages: extension
|
|
|
|
|
//! first (an override table, then `mime_guess`), magic-byte sniffing via
|
|
|
|
|
//! `infer` next, and finally a text sniff ([`crate::textenc`]) that answers
|
|
|
|
|
//! `text/plain` for anything whose head reads as text — which is how
|
|
|
|
|
//! extensionless files (README, Makefile) and source extensions no MIME
|
|
|
|
|
//! table knows (`.go`, `.zig`) get their contents indexed. Extensions in
|
|
|
|
|
//! [`AMBIGUOUS_EXTENSIONS`] invert the order: content decides, and the
|
|
|
|
|
//! extension's MIME is only a fallback.
|
2026-08-02 19:04:30 -04:00
|
|
|
//!
|
2026-08-03 12:33:47 -04:00
|
|
|
//! [`mime_to_type`] then maps a MIME string to a [`FileType`] bitmask so a
|
|
|
|
|
//! single file can belong to multiple categories (e.g. a `.docx` is
|
|
|
|
|
//! Document|Text).
|
|
|
|
|
//!
|
|
|
|
|
//! The head bytes are always ones the caller already holds. Indexing reads
|
2026-08-02 19:04:30 -04:00
|
|
|
//! the head of every new or changed file to hash it, and those are the same
|
2026-08-03 12:33:47 -04:00
|
|
|
//! bytes `infer` and the text sniff want, so there is no path-based variant
|
|
|
|
|
//! that goes back to disk for them — that was a second open/read/close per
|
|
|
|
|
//! undetectable file.
|
2026-04-21 23:00:47 -04:00
|
|
|
|
|
|
|
|
use std::path::Path;
|
|
|
|
|
|
|
|
|
|
/// Bit-flag category for a file. Unlike the MIME string this is designed for
|
|
|
|
|
/// cheap bitmask queries like `type & FileType::AUDIO != 0`.
|
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
|
|
|
pub struct FileType(pub u32);
|
|
|
|
|
|
|
|
|
|
impl FileType {
|
|
|
|
|
pub const EMPTY: FileType = FileType(0);
|
|
|
|
|
pub const AUDIO: FileType = FileType(1 << 0);
|
|
|
|
|
pub const IMAGE: FileType = FileType(1 << 1);
|
|
|
|
|
pub const VIDEO: FileType = FileType(1 << 2);
|
|
|
|
|
pub const DOCUMENT: FileType = FileType(1 << 3);
|
|
|
|
|
pub const TEXT: FileType = FileType(1 << 4);
|
|
|
|
|
pub const ARCHIVE: FileType = FileType(1 << 5);
|
|
|
|
|
pub const PRESENTATION: FileType = FileType(1 << 6);
|
|
|
|
|
pub const SPREADSHEET: FileType = FileType(1 << 7);
|
|
|
|
|
pub const FOLDER: FileType = FileType(1 << 8);
|
|
|
|
|
|
|
|
|
|
pub const fn bits(self) -> u32 {
|
|
|
|
|
self.0
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub const fn contains(self, other: FileType) -> bool {
|
|
|
|
|
(self.0 & other.0) == other.0
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Parse a single Baloo-style category name (`Audio`, `Image`, ...).
|
|
|
|
|
/// Case-insensitive. Returns `EMPTY` for unknown names.
|
|
|
|
|
pub fn from_name(s: &str) -> FileType {
|
|
|
|
|
match s.to_ascii_lowercase().as_str() {
|
|
|
|
|
"audio" => FileType::AUDIO,
|
|
|
|
|
"image" => FileType::IMAGE,
|
|
|
|
|
"video" => FileType::VIDEO,
|
|
|
|
|
"document" => FileType::DOCUMENT,
|
|
|
|
|
"text" => FileType::TEXT,
|
|
|
|
|
"archive" => FileType::ARCHIVE,
|
|
|
|
|
"presentation" => FileType::PRESENTATION,
|
|
|
|
|
"spreadsheet" => FileType::SPREADSHEET,
|
|
|
|
|
"folder" => FileType::FOLDER,
|
|
|
|
|
_ => FileType::EMPTY,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl std::ops::BitOr for FileType {
|
|
|
|
|
type Output = FileType;
|
|
|
|
|
fn bitor(self, rhs: FileType) -> FileType {
|
|
|
|
|
FileType(self.0 | rhs.0)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl std::ops::BitOrAssign for FileType {
|
|
|
|
|
fn bitor_assign(&mut self, rhs: FileType) {
|
|
|
|
|
self.0 |= rhs.0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-03 12:33:47 -04:00
|
|
|
/// Extensions whose MIME is pinned regardless of what `mime_guess` or the
|
|
|
|
|
/// file's bytes say.
|
2026-08-02 19:04:30 -04:00
|
|
|
///
|
2026-08-03 12:33:47 -04:00
|
|
|
/// Consulted *before* everything else, because for these the table is not a
|
|
|
|
|
/// fallback but a correction or a guarantee:
|
2026-08-02 19:04:30 -04:00
|
|
|
///
|
2026-08-03 12:33:47 -04:00
|
|
|
/// - `.bat` maps in `mime_guess` to `application/x-msdownload`, i.e. an
|
|
|
|
|
/// executable, and a non-empty `mime_guess` answer would preempt the text
|
|
|
|
|
/// sniff — so without this entry batch files are never content-indexed.
|
|
|
|
|
/// (`.cmd` already resolves to `text/plain`; it is listed so the pair
|
2026-08-02 19:04:30 -04:00
|
|
|
/// cannot drift.)
|
2026-08-03 12:33:47 -04:00
|
|
|
/// - `.ps1`/`.psm1`/`.psd1`, `.inf` and `.url` are absent from `mime_guess`.
|
|
|
|
|
/// The text sniff would usually catch them, but pinning them costs
|
|
|
|
|
/// nothing and classifies them deterministically, whatever their head
|
|
|
|
|
/// bytes happen to look like.
|
2026-08-02 19:04:30 -04:00
|
|
|
///
|
|
|
|
|
/// Platform-neutral on purpose: a `.ps1` copied to a Linux box should classify
|
|
|
|
|
/// the same way.
|
|
|
|
|
const EXTENSION_OVERRIDES: &[(&str, &str)] = &[
|
|
|
|
|
("bat", "text/plain"),
|
|
|
|
|
("cmd", "text/plain"),
|
|
|
|
|
("inf", "text/plain"),
|
|
|
|
|
("ps1", "text/plain"),
|
|
|
|
|
("psd1", "text/plain"),
|
|
|
|
|
("psm1", "text/plain"),
|
|
|
|
|
("url", "text/plain"),
|
|
|
|
|
];
|
|
|
|
|
|
2026-08-03 12:33:47 -04:00
|
|
|
/// Extensions `mime_guess` maps to a binary format that is, on a modern
|
|
|
|
|
/// disk, at least as often a text file: `.ts`/`.mts` TypeScript vs MPEG
|
|
|
|
|
/// transport stream, `.mod` go.mod vs `video/mpeg`, `.org` Org-mode vs
|
|
|
|
|
/// Lotus Organizer, `.scm` Scheme vs Lotus ScreenCam, `.pot` gettext
|
|
|
|
|
/// template vs PowerPoint template, `.vhd` VHDL source vs VirtualBox disk
|
|
|
|
|
/// image.
|
|
|
|
|
///
|
|
|
|
|
/// For these the content decides: magic bytes first, then the text sniff,
|
|
|
|
|
/// and only if both decline does `mime_guess`'s extension answer stand — so
|
|
|
|
|
/// a real MPEG-TS recording still classifies as video.
|
|
|
|
|
const AMBIGUOUS_EXTENSIONS: &[&str] = &["mod", "mts", "org", "pot", "scm", "ts", "vhd"];
|
|
|
|
|
|
|
|
|
|
/// Whether `path`'s extension is in [`AMBIGUOUS_EXTENSIONS`].
|
|
|
|
|
/// ASCII-case-insensitive, like [`extension_override`].
|
|
|
|
|
fn extension_is_ambiguous(path: &Path) -> bool {
|
|
|
|
|
path.extension()
|
|
|
|
|
.and_then(|e| e.to_str())
|
|
|
|
|
.map(|e| e.to_ascii_lowercase())
|
|
|
|
|
.is_some_and(|e| AMBIGUOUS_EXTENSIONS.contains(&e.as_str()))
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-02 19:04:30 -04:00
|
|
|
/// Look up [`EXTENSION_OVERRIDES`] for `path`. Extension comparison is
|
|
|
|
|
/// ASCII-case-insensitive, which matters more on Windows where `REPORT.BAT` is
|
|
|
|
|
/// as common as the lowercase spelling.
|
|
|
|
|
fn extension_override(path: &Path) -> Option<&'static str> {
|
|
|
|
|
let ext = path.extension()?.to_str()?.to_ascii_lowercase();
|
|
|
|
|
EXTENSION_OVERRIDES
|
|
|
|
|
.iter()
|
|
|
|
|
.find(|(e, _)| *e == ext)
|
|
|
|
|
.map(|(_, mime)| *mime)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Infer a MIME type from a path plus the file's leading bytes.
|
2026-04-21 23:00:47 -04:00
|
|
|
///
|
2026-08-03 12:33:47 -04:00
|
|
|
/// Extension first — an override table, then `mime_guess` — then magic
|
|
|
|
|
/// bytes when those come up empty or say `application/octet-stream`, and
|
|
|
|
|
/// finally a text sniff that answers `text/plain` for any head that reads
|
|
|
|
|
/// as text ([`crate::textenc::looks_like_text`]). For
|
|
|
|
|
/// [`AMBIGUOUS_EXTENSIONS`] the `mime_guess` answer is demoted to a last
|
|
|
|
|
/// resort behind both content checks.
|
2026-04-21 23:00:47 -04:00
|
|
|
///
|
2026-08-02 19:04:30 -04:00
|
|
|
/// `head` is whatever the caller already read; indexing passes the same buffer
|
|
|
|
|
/// it hashes. It bounds magic-byte detection, so a caller that supplies fewer
|
|
|
|
|
/// than 262 bytes (`infer`'s longest signature) can get `None` where a longer
|
|
|
|
|
/// head would have matched. The indexer's `hash_length` defaults to 8 KiB —
|
|
|
|
|
/// exactly what `infer` itself reads from a path — so at default config this
|
2026-08-03 12:33:47 -04:00
|
|
|
/// is as good as opening the file, and strictly cheaper. The same buffer
|
|
|
|
|
/// bounds the text sniff, which tolerates a multibyte character cut off at
|
|
|
|
|
/// the buffer's end.
|
2026-08-02 19:04:30 -04:00
|
|
|
///
|
|
|
|
|
/// A `None` result is a real answer, not a "don't know": the content pass
|
|
|
|
|
/// stores it and does not re-derive it (see
|
|
|
|
|
/// [`crate::file_handling::extract_and_store`]).
|
|
|
|
|
pub fn guess_mime_from_head(path: &Path, head: &[u8]) -> Option<String> {
|
|
|
|
|
if let Some(m) = extension_override(path) {
|
|
|
|
|
return Some(m.to_string());
|
|
|
|
|
}
|
2026-08-03 12:33:47 -04:00
|
|
|
let by_extension = mime_guess::from_path(path).first().and_then(|g| {
|
2026-04-21 23:00:47 -04:00
|
|
|
let s = g.essence_str();
|
2026-08-03 12:33:47 -04:00
|
|
|
(!s.is_empty() && s != "application/octet-stream").then(|| s.to_string())
|
|
|
|
|
});
|
|
|
|
|
if !extension_is_ambiguous(path) && by_extension.is_some() {
|
|
|
|
|
return by_extension;
|
|
|
|
|
}
|
|
|
|
|
if let Some(t) = infer::get(head) {
|
|
|
|
|
let magic = t.mime_type();
|
|
|
|
|
// For an ambiguous extension, `infer`'s generic OLE-container answer
|
|
|
|
|
// is less specific than the extension's: a real PowerPoint `.pot`
|
|
|
|
|
// template must resolve to vnd.ms-powerpoint (which the office
|
|
|
|
|
// extractor claims), not to a container MIME nothing claims.
|
|
|
|
|
if magic == "application/x-ole-storage" && by_extension.is_some() {
|
|
|
|
|
return by_extension;
|
2026-04-21 23:00:47 -04:00
|
|
|
}
|
2026-08-03 12:33:47 -04:00
|
|
|
return Some(magic.to_string());
|
2026-04-21 23:00:47 -04:00
|
|
|
}
|
2026-08-03 12:33:47 -04:00
|
|
|
if crate::textenc::looks_like_text(head) {
|
|
|
|
|
return Some("text/plain".to_string());
|
|
|
|
|
}
|
|
|
|
|
// Only an ambiguous extension still has an answer left to fall back on.
|
|
|
|
|
by_extension
|
2026-04-21 23:00:47 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Map a MIME string to a [`FileType`] bitmask. Ported from Baloo's
|
|
|
|
|
/// `basicindexingjob.cpp:typesForMimeType`.
|
|
|
|
|
pub fn mime_to_type(mime: &str) -> FileType {
|
|
|
|
|
let lower = mime.to_ascii_lowercase();
|
|
|
|
|
let (top, sub) = match lower.split_once('/') {
|
|
|
|
|
Some(pair) => pair,
|
|
|
|
|
None => return FileType::EMPTY,
|
|
|
|
|
};
|
|
|
|
|
let mut t = FileType::EMPTY;
|
|
|
|
|
match top {
|
|
|
|
|
"audio" => t |= FileType::AUDIO,
|
|
|
|
|
"image" => t |= FileType::IMAGE,
|
|
|
|
|
"video" => t |= FileType::VIDEO,
|
|
|
|
|
"text" => {
|
|
|
|
|
t |= FileType::TEXT;
|
2026-08-03 12:33:47 -04:00
|
|
|
// HTML counts as a document too in Baloo. (xhtml+xml is handled
|
|
|
|
|
// in the subtype match below, whatever its top level.)
|
|
|
|
|
if sub == "html" {
|
2026-04-21 23:00:47 -04:00
|
|
|
t |= FileType::DOCUMENT;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
_ => {}
|
|
|
|
|
}
|
|
|
|
|
// Subtype-based classification for the `application/*` grab bag.
|
|
|
|
|
match sub {
|
|
|
|
|
// Office formats
|
|
|
|
|
"msword"
|
|
|
|
|
| "vnd.openxmlformats-officedocument.wordprocessingml.document"
|
|
|
|
|
| "vnd.oasis.opendocument.text"
|
|
|
|
|
| "rtf"
|
|
|
|
|
| "pdf"
|
|
|
|
|
| "epub+zip"
|
|
|
|
|
| "x-mobipocket-ebook" => {
|
|
|
|
|
t |= FileType::DOCUMENT;
|
|
|
|
|
}
|
|
|
|
|
"vnd.ms-excel"
|
|
|
|
|
| "vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
|
|
|
| "vnd.oasis.opendocument.spreadsheet" => {
|
|
|
|
|
t |= FileType::DOCUMENT | FileType::SPREADSHEET;
|
|
|
|
|
}
|
|
|
|
|
"vnd.ms-powerpoint"
|
|
|
|
|
| "vnd.openxmlformats-officedocument.presentationml.presentation"
|
|
|
|
|
| "vnd.oasis.opendocument.presentation" => {
|
|
|
|
|
t |= FileType::DOCUMENT | FileType::PRESENTATION;
|
|
|
|
|
}
|
2026-08-02 19:04:30 -04:00
|
|
|
// Outlook saved messages and compiled HTML help are documents; both
|
|
|
|
|
// are ordinary things to find in a Windows home directory.
|
|
|
|
|
"vnd.ms-outlook" | "vnd.ms-htmlhelp" => {
|
|
|
|
|
t |= FileType::DOCUMENT;
|
|
|
|
|
}
|
|
|
|
|
// Archives. The Windows installer/cabinet formats are containers in
|
|
|
|
|
// exactly the same sense as the rest of this list.
|
2026-04-21 23:00:47 -04:00
|
|
|
"zip"
|
|
|
|
|
| "x-tar"
|
|
|
|
|
| "x-7z-compressed"
|
|
|
|
|
| "x-rar"
|
|
|
|
|
| "x-rar-compressed"
|
|
|
|
|
| "gzip"
|
|
|
|
|
| "x-bzip"
|
|
|
|
|
| "x-bzip2"
|
|
|
|
|
| "x-xz"
|
|
|
|
|
| "vnd.debian.binary-package"
|
2026-08-02 19:04:30 -04:00
|
|
|
| "x-rpm"
|
|
|
|
|
| "vnd.ms-cab-compressed"
|
|
|
|
|
| "x-msi" => {
|
2026-04-21 23:00:47 -04:00
|
|
|
t |= FileType::ARCHIVE;
|
|
|
|
|
}
|
2026-08-03 12:33:47 -04:00
|
|
|
// XHTML is text and, like HTML above, a document in Baloo's model —
|
|
|
|
|
// whichever top level it arrives under.
|
|
|
|
|
"xhtml+xml" => {
|
|
|
|
|
t |= FileType::TEXT | FileType::DOCUMENT;
|
|
|
|
|
}
|
|
|
|
|
// Structured text: everything the plaintext extractor claims beyond
|
|
|
|
|
// `text/*` (see `extract::plaintext::EXTRA_TEXT_MIMES` and the
|
|
|
|
|
// cross-check test below). Keyed on the subtype alone, so playlists
|
|
|
|
|
// stay AUDIO|TEXT and SVG stays IMAGE|TEXT.
|
|
|
|
|
"xml" | "json" | "json5" | "geo+json" | "javascript" | "mbox" | "rfc822"
|
|
|
|
|
| "vnd.dart" | "x-csh" | "x-httpd-php" | "x-perl" | "x-sh" | "x-sql"
|
|
|
|
|
| "x-subrip" | "x-tcl" | "x-tex" | "x-texinfo" | "x-troff" | "x-troff-man"
|
|
|
|
|
| "x-mpegurl" | "scpls" | "svg+xml" => {
|
2026-04-21 23:00:47 -04:00
|
|
|
t |= FileType::TEXT;
|
|
|
|
|
}
|
|
|
|
|
_ => {}
|
|
|
|
|
}
|
|
|
|
|
t
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
mod tests {
|
|
|
|
|
use super::*;
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn audio_mime() {
|
|
|
|
|
assert!(mime_to_type("audio/mpeg").contains(FileType::AUDIO));
|
|
|
|
|
assert!(mime_to_type("audio/flac").contains(FileType::AUDIO));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn image_mime() {
|
|
|
|
|
assert!(mime_to_type("image/jpeg").contains(FileType::IMAGE));
|
|
|
|
|
assert!(mime_to_type("image/png").contains(FileType::IMAGE));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn docx_is_document_and_office() {
|
|
|
|
|
let t = mime_to_type(
|
|
|
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
|
|
|
);
|
|
|
|
|
assert!(t.contains(FileType::DOCUMENT));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn xlsx_is_spreadsheet_and_document() {
|
|
|
|
|
let t = mime_to_type(
|
|
|
|
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
|
|
|
);
|
|
|
|
|
assert!(t.contains(FileType::DOCUMENT));
|
|
|
|
|
assert!(t.contains(FileType::SPREADSHEET));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn pptx_is_presentation() {
|
|
|
|
|
let t = mime_to_type(
|
|
|
|
|
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
|
|
|
);
|
|
|
|
|
assert!(t.contains(FileType::PRESENTATION));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn html_is_text_and_document() {
|
|
|
|
|
let t = mime_to_type("text/html");
|
|
|
|
|
assert!(t.contains(FileType::TEXT));
|
|
|
|
|
assert!(t.contains(FileType::DOCUMENT));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn plain_text() {
|
|
|
|
|
let t = mime_to_type("text/plain");
|
|
|
|
|
assert!(t.contains(FileType::TEXT));
|
|
|
|
|
assert!(!t.contains(FileType::DOCUMENT));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn zip_is_archive() {
|
|
|
|
|
assert!(mime_to_type("application/zip").contains(FileType::ARCHIVE));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn unknown_mime_is_empty() {
|
|
|
|
|
assert_eq!(mime_to_type("weird/blob"), FileType::EMPTY);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn from_name_round_trip() {
|
|
|
|
|
for n in ["Audio", "Image", "Video", "Document", "Text", "Archive",
|
|
|
|
|
"Spreadsheet", "Presentation", "Folder"] {
|
|
|
|
|
assert_ne!(FileType::from_name(n), FileType::EMPTY, "{}", n);
|
|
|
|
|
}
|
|
|
|
|
assert_eq!(FileType::from_name("Weird"), FileType::EMPTY);
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-02 19:04:30 -04:00
|
|
|
/// Extension resolution happens before magic bytes are consulted, so an
|
|
|
|
|
/// empty head is enough to exercise it.
|
2026-04-21 23:00:47 -04:00
|
|
|
#[test]
|
|
|
|
|
fn guess_mime_by_extension() {
|
|
|
|
|
use std::path::PathBuf;
|
2026-08-02 19:04:30 -04:00
|
|
|
let by_ext = |n: &str| guess_mime_from_head(&PathBuf::from(n), b"").unwrap_or_default();
|
|
|
|
|
assert_eq!(by_ext("a.txt"), "text/plain");
|
|
|
|
|
assert_eq!(by_ext("a.png"), "image/png");
|
|
|
|
|
assert_eq!(by_ext("a.mp3"), "audio/mpeg");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Every override must land on a type the plaintext extractor accepts —
|
|
|
|
|
/// the point of the table is that these files get their contents indexed.
|
|
|
|
|
#[test]
|
|
|
|
|
fn windows_script_types_reach_the_plaintext_extractor() {
|
|
|
|
|
use crate::extract::{plaintext::PlaintextExtractor, Extractor};
|
|
|
|
|
use std::path::PathBuf;
|
|
|
|
|
|
|
|
|
|
for name in [
|
|
|
|
|
"deploy.ps1",
|
|
|
|
|
"Module.psm1",
|
|
|
|
|
"Module.psd1",
|
|
|
|
|
"build.bat",
|
|
|
|
|
"build.cmd",
|
|
|
|
|
"driver.inf",
|
|
|
|
|
"bookmark.url",
|
|
|
|
|
] {
|
|
|
|
|
let mime = guess_mime_from_head(&PathBuf::from(name), b"")
|
|
|
|
|
.unwrap_or_else(|| panic!("{} has no MIME", name));
|
|
|
|
|
assert!(
|
|
|
|
|
PlaintextExtractor.supports(&mime),
|
|
|
|
|
"{} -> {} is not extractable as text",
|
|
|
|
|
name,
|
|
|
|
|
mime
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn extension_overrides_are_case_insensitive() {
|
|
|
|
|
use std::path::PathBuf;
|
|
|
|
|
// Uppercase extensions are ordinary on Windows.
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("DEPLOY.PS1"), b"").as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("Build.Bat"), b"").as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The override table must win over the file's actual content: a `.ps1`
|
|
|
|
|
/// holding something `infer` would recognise is still a script.
|
|
|
|
|
#[test]
|
|
|
|
|
fn extension_overrides_beat_magic_bytes() {
|
|
|
|
|
use std::path::PathBuf;
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("a.ps1"), b"Write-Host hi").as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("a.ps1"), b"%PDF-1.7").as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn sql_dumps_are_extractable() {
|
|
|
|
|
use crate::extract::{plaintext::PlaintextExtractor, Extractor};
|
|
|
|
|
use std::path::PathBuf;
|
|
|
|
|
let mime = guess_mime_from_head(&PathBuf::from("schema.sql"), b"").unwrap();
|
|
|
|
|
assert!(PlaintextExtractor.supports(&mime), "{}", mime);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The content pass trusts the MIME the walk stored, including `None`, and
|
|
|
|
|
/// never reopens the file to second-guess it. That is only sound if a
|
|
|
|
|
/// `hash_length`-sized head is enough to recognise a format from its magic
|
|
|
|
|
/// bytes — `infer`'s longest signature is 262 bytes and the default head is
|
|
|
|
|
/// 8 KiB, so it is by a wide margin. This pins that for extensionless
|
|
|
|
|
/// files, where magic bytes are the only signal there is.
|
|
|
|
|
#[test]
|
|
|
|
|
fn a_default_sized_head_is_enough_for_magic_byte_detection() {
|
|
|
|
|
use std::path::PathBuf;
|
|
|
|
|
let head_bytes = crate::config::ProcessingConfig::default().hash_length;
|
|
|
|
|
|
|
|
|
|
let samples: &[(&str, &[u8], &str)] = &[
|
|
|
|
|
("png", &[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a], "image/png"),
|
|
|
|
|
("gif", b"GIF89a", "image/gif"),
|
|
|
|
|
("pdf", b"%PDF-1.7", "application/pdf"),
|
|
|
|
|
("zip", &[0x50, 0x4b, 0x03, 0x04], "application/zip"),
|
|
|
|
|
("gz", &[0x1f, 0x8b, 0x08], "application/gzip"),
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
for (tag, magic, expected) in samples {
|
|
|
|
|
// No extension at all, so nothing but the bytes can answer.
|
|
|
|
|
let path = PathBuf::from(format!("/tmp/qs-sniff-{}", tag));
|
|
|
|
|
let mut body = magic.to_vec();
|
|
|
|
|
body.resize(head_bytes, 0);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&path, &body).as_deref(),
|
|
|
|
|
Some(*expected),
|
|
|
|
|
"{} must be detectable from a default-sized head",
|
|
|
|
|
tag
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The other side of that bound: starve the head below `infer`'s longest
|
2026-08-03 12:33:47 -04:00
|
|
|
/// signature and magic detection legitimately degrades. Documented
|
|
|
|
|
/// behaviour of a non-default `hash_length`, not a bug — but a binary
|
|
|
|
|
/// head must stay a `None` rather than become a wrong guess. (A head
|
|
|
|
|
/// that *reads as text* is a different case: the text sniff answers for
|
|
|
|
|
/// it, however short.)
|
2026-08-02 19:04:30 -04:00
|
|
|
#[test]
|
|
|
|
|
fn a_head_shorter_than_the_signature_declines_rather_than_guessing() {
|
|
|
|
|
use std::path::PathBuf;
|
|
|
|
|
let path = PathBuf::from("/tmp/qs-sniff-truncated");
|
|
|
|
|
assert_eq!(guess_mime_from_head(&path, b"").as_deref(), None);
|
2026-08-03 12:33:47 -04:00
|
|
|
// A PNG magic truncated to two bytes: not a magic match, and the
|
|
|
|
|
// NUL fails the binary guard, so no text guess either.
|
|
|
|
|
assert_eq!(guess_mime_from_head(&path, &[0x89, 0x00]).as_deref(), None);
|
2026-08-02 19:04:30 -04:00
|
|
|
// Enough bytes, and it resolves.
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&path, &[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a])
|
|
|
|
|
.as_deref(),
|
|
|
|
|
Some("image/png")
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn windows_container_and_document_types_classify() {
|
|
|
|
|
assert!(mime_to_type("application/vnd.ms-cab-compressed").contains(FileType::ARCHIVE));
|
|
|
|
|
assert!(mime_to_type("application/x-msi").contains(FileType::ARCHIVE));
|
|
|
|
|
assert!(mime_to_type("application/vnd.ms-outlook").contains(FileType::DOCUMENT));
|
|
|
|
|
assert!(mime_to_type("application/vnd.ms-htmlhelp").contains(FileType::DOCUMENT));
|
2026-04-21 23:00:47 -04:00
|
|
|
}
|
2026-08-03 12:33:47 -04:00
|
|
|
|
|
|
|
|
/// Every extension fixed by this round of coverage work must reach the
|
|
|
|
|
/// plaintext extractor through real dispatch — `extract_complete_head`
|
|
|
|
|
/// rather than `supports` — so the svg/m3u/pls cases prove the
|
|
|
|
|
/// plaintext-first registration *order*, not just the MIME claim.
|
|
|
|
|
#[test]
|
|
|
|
|
fn newly_claimed_extensions_reach_the_plaintext_extractor() {
|
|
|
|
|
use crate::extract::Registry;
|
|
|
|
|
use std::path::PathBuf;
|
|
|
|
|
|
|
|
|
|
let registry = Registry::default_set();
|
|
|
|
|
let samples: &[(&str, &[u8])] = &[
|
|
|
|
|
("deploy.sh", b"echo hi"),
|
|
|
|
|
("env.csh", b"setenv X 1"),
|
|
|
|
|
("script.pl", b"print 1;"),
|
|
|
|
|
("Module.pm", b"package M;"),
|
|
|
|
|
("index.php", b"<?php echo 1;"),
|
|
|
|
|
("paper.tex", b"\\documentclass{article}"),
|
|
|
|
|
("page.xhtml", b"<html/>"),
|
|
|
|
|
("notes.json5", b"{a: 1}"),
|
|
|
|
|
("map.geojson", b"{}"),
|
|
|
|
|
("subs.srt", b"1\n00:00:01 --> 00:00:02\nhi\n"),
|
|
|
|
|
("run.tcl", b"puts hi"),
|
|
|
|
|
("main.dart", b"void main() {}"),
|
|
|
|
|
("page.man", b".TH TEST 1"),
|
|
|
|
|
("test.t", b"use Test::More;"),
|
|
|
|
|
("doc.texi", b"@node Top"),
|
|
|
|
|
("mail.eml", b"Subject: hi\n\nbody"),
|
|
|
|
|
("inbox.mbox", b"From a@b\n\nbody"),
|
|
|
|
|
("icon.svg", b"<svg xmlns='x'/>"),
|
|
|
|
|
("list.m3u", b"#EXTM3U\ntrack.mp3"),
|
|
|
|
|
("radio.pls", b"[playlist]"),
|
|
|
|
|
];
|
|
|
|
|
for (name, head) in samples {
|
|
|
|
|
let path = PathBuf::from(name);
|
|
|
|
|
let mime = guess_mime_from_head(&path, head)
|
|
|
|
|
.unwrap_or_else(|| panic!("{} has no MIME", name));
|
|
|
|
|
let extracted = registry
|
|
|
|
|
.extract_complete_head(&path, &mime, head)
|
|
|
|
|
.unwrap_or_else(|| panic!("{} -> {} not claimed by a head-capable extractor", name, mime))
|
|
|
|
|
.unwrap_or_else(|e| panic!("{} -> {} failed to extract: {}", name, mime, e));
|
|
|
|
|
assert!(
|
|
|
|
|
!extracted.text.is_empty(),
|
|
|
|
|
"{} -> {} extracted no text",
|
|
|
|
|
name,
|
|
|
|
|
mime
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Extensionless files are decided by their bytes: text heads index,
|
|
|
|
|
/// binary heads stay unclassified.
|
|
|
|
|
#[test]
|
|
|
|
|
fn extensionless_files_sniff_by_content() {
|
|
|
|
|
use std::path::PathBuf;
|
|
|
|
|
let readme = PathBuf::from("README");
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&readme, b"QuickSearch indexes your files.\n").as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
let makefile = PathBuf::from("Makefile");
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&makefile, b"all:\n\tcargo build\n").as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
let blob = PathBuf::from("blob");
|
|
|
|
|
assert_eq!(guess_mime_from_head(&blob, &[0x00, 0x01, 0x02, 0xFF]).as_deref(), None);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Ambiguous extensions resolve by content in both directions: source
|
|
|
|
|
/// code beats the extension table, real binary keeps the extension's
|
|
|
|
|
/// MIME as the fallback.
|
|
|
|
|
#[test]
|
|
|
|
|
fn ambiguous_extensions_resolve_by_content_both_ways() {
|
|
|
|
|
use std::path::PathBuf;
|
|
|
|
|
|
|
|
|
|
let ts_source = b"export function hi(): string { return 'hi'; }\n";
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("app.ts"), ts_source).as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
// Uppercase, as Windows likes it.
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("APP.TS"), ts_source).as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
// An MPEG transport stream: 0x47 sync bytes with NUL-heavy payloads.
|
|
|
|
|
// No magic matcher, fails the text sniff, so the extension answers.
|
|
|
|
|
let mut ts_video = vec![0u8; 376];
|
|
|
|
|
ts_video[0] = 0x47;
|
|
|
|
|
ts_video[188] = 0x47;
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("clip.ts"), &ts_video).as_deref(),
|
|
|
|
|
Some("video/vnd.dlna.mpeg-tts")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("go.mod"), b"module example.com/x\n\ngo 1.22\n")
|
|
|
|
|
.as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// gettext template vs PowerPoint template: text decides one way,
|
|
|
|
|
// binary bytes fall back to the extension's office MIME (whether
|
|
|
|
|
// infer's OLE matcher fires or the guard rejects, the answer agrees).
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("app.pot"), b"msgid \"hello\"\nmsgstr \"\"\n")
|
|
|
|
|
.as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
let ole = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1, 0x00, 0x00];
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("slides.pot"), &ole).as_deref(),
|
|
|
|
|
Some("application/vnd.ms-powerpoint")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("cpu.vhd"), b"entity cpu is\nend cpu;\n")
|
|
|
|
|
.as_deref(),
|
|
|
|
|
Some("text/plain")
|
|
|
|
|
);
|
|
|
|
|
assert_eq!(
|
|
|
|
|
guess_mime_from_head(&PathBuf::from("disk.vhd"), &[0x00, 0x01, 0x02, 0x03])
|
|
|
|
|
.as_deref(),
|
|
|
|
|
Some("application/x-virtualbox-vhd")
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Everything the plaintext extractor claims must carry the TEXT bit,
|
|
|
|
|
/// or `type:Text` silently misses content-indexed files (the pre-fix
|
|
|
|
|
/// state of `.sql`). Iterates the actual claim list so the two can
|
|
|
|
|
/// never drift apart.
|
|
|
|
|
#[test]
|
|
|
|
|
fn every_plaintext_claim_carries_the_text_bit() {
|
|
|
|
|
for mime in crate::extract::plaintext::EXTRA_TEXT_MIMES {
|
|
|
|
|
assert!(
|
|
|
|
|
mime_to_type(mime).contains(FileType::TEXT),
|
|
|
|
|
"{} is extractable as text but lacks FileType::TEXT",
|
|
|
|
|
mime
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
// The multi-category cases keep their native category too.
|
|
|
|
|
let svg = mime_to_type("image/svg+xml");
|
|
|
|
|
assert!(svg.contains(FileType::IMAGE) && svg.contains(FileType::TEXT));
|
|
|
|
|
let m3u = mime_to_type("audio/x-mpegurl");
|
|
|
|
|
assert!(m3u.contains(FileType::AUDIO) && m3u.contains(FileType::TEXT));
|
|
|
|
|
let xhtml = mime_to_type("application/xhtml+xml");
|
|
|
|
|
assert!(xhtml.contains(FileType::TEXT) && xhtml.contains(FileType::DOCUMENT));
|
|
|
|
|
// And the `text/` prefix arm still covers the rest.
|
|
|
|
|
assert!(mime_to_type("text/x-toml").contains(FileType::TEXT));
|
|
|
|
|
}
|
2026-04-21 23:00:47 -04:00
|
|
|
}
|