diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index b3f573d..83722cc 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -47,6 +47,15 @@ jobs: runs-on: self-hosted container: image: catthehacker/ubuntu:act-22.04 + # Jobs run as root, and root ignores permission bits: CAP_DAC_OVERRIDE and + # CAP_DAC_READ_SEARCH let uid 0 read a mode-000 file or directory anyway. + # Several tests build an unreadable directory with platform::deny_read and + # assert the walk reports it rather than reporting an empty directory - the + # distinction matters because an empty listing deletes index rows. Under + # root those tests see a perfectly readable tree and fail. Dropping the two + # DAC capabilities makes root obey the mode bits, which is exactly the + # environment the tests assume and get on a developer machine. + options: --cap-drop=DAC_OVERRIDE --cap-drop=DAC_READ_SEARCH env: # crates/quicksearch-core/src/config.rs has a test that expects a home # directory and panics without one. diff --git a/Cargo.toml b/Cargo.toml index 4738cbc..87f125a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ ] [workspace.package] -version = "0.9.0" +version = "0.9.1" edition = "2021" license = "GPL-3.0-or-later" authors = ["Jeremy "] diff --git a/README.md b/README.md index 3cdd8bf..d525eab 100644 --- a/README.md +++ b/README.md @@ -294,11 +294,13 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. sweep stale rows, then extract content (plaintext, RTF, Office, PDF, audio tags, EXIF; see `extract/`) for FTS. Files whose extension no MIME table knows — including extensionless ones like `README` or `Makefile` — - are sniffed from their head bytes and indexed as text when they read as - text (`mime.rs`, `textenc.rs`); non-UTF-8 text (UTF-16 with BOM, legacy - charsets via chardetng) is decoded and stored as UTF-8. More claimed - files means a bigger index — `indexing.content_extensions` remains the - throttle. Files no larger than `processing.hash_length` skip that second + are sniffed from their head bytes and indexed as text only when that head + is provably text: valid UTF-8, or BOM-marked (`mime.rs`, `textenc.rs`). + Legacy charsets are decoded via chardetng and stored as UTF-8, but only + for files something *else* typed as text, normally their extension — + chardetng's windows-1252 floor never fails, so accepting it on a bare + sniff would adopt any binary lacking NUL bytes. More claimed files means a + bigger index — `indexing.content_extensions` remains the throttle. Files no larger than `processing.hash_length` skip that second pass entirely: the head the walk reads to hash them is already their whole content, so a plaintext body is extracted in the same `read` and stored complete. Every run ends — whether diff --git a/crates/quicksearch-core/examples/memprobe.rs b/crates/quicksearch-core/examples/memprobe.rs index dacf2ec..2f5e97e 100644 --- a/crates/quicksearch-core/examples/memprobe.rs +++ b/crates/quicksearch-core/examples/memprobe.rs @@ -11,10 +11,14 @@ //! ./target/release/examples/memprobe cold /media/shared /var/tmp/qs-mem/index.db //! ./target/release/examples/memprobe warm /media/shared /var/tmp/qs-mem/index.db //! ./target/release/examples/memprobe cold /media/shared /var/tmp/qs-mem/index.db 10 +//! ./target/release/examples/memprobe cold ~ /var/tmp/qs-mem/index.db 250 probe.toml //! ``` //! //! The optional trailing number is the sampling interval in milliseconds -//! (default 100). Finer sampling resolves the *shape* of a spike, not its +//! (default 100), and the one after it a config file to load instead of the +//! defaults — the only way to probe a tree that the shipped `include_hidden = +//! false` would walk past, such as a home directory that is nearly all dotdirs. +//! Finer sampling resolves the *shape* of a spike, not its //! cause: the file column is only as good as `RootProgress::current_file`, //! which [`crate::indexing`] publishes once per extraction batch, holding the //! last file of the batch that just finished. During a batch it therefore @@ -24,10 +28,20 @@ //! largest single consumer on this tree. //! //! `cold` deletes the database first: every file is new, so the walk hashes -//! and extracts all of them and `existing_files` starts empty. `warm` re-runs -//! against the finished database, which is the case that loads one -//! `existing_files` entry per indexed path up front — the allocation that -//! scales with tree size rather than with in-flight work. +//! and extracts all of them. `warm` re-runs against the finished database, +//! where most files classify as unchanged and the run is dominated by +//! reconciliation rather than extraction. Warm peaks *below* cold on the same +//! tree — the walk reads one directory's rows at a time (`repo::dir_rows`, +//! held in an `Arc` only while that directory is in flight), so there is no +//! up-front load that scales with tree size. +//! +//! **`growth per file` is a ratio, not a per-file cost.** It divides a peak +//! that is essentially constant by the file count, so it *falls* as the tree +//! grows: measured 2052 B/file over 99,477 files and 644 B/file over 279,936, +//! with the larger tree peaking *lower* in absolute terms. Read the peak, not +//! the quotient. What actually scales with tree size is `seen_paths` +//! (`indexing.rs`), a `HashSet` of path digests — ~17 bytes per file +//! including hashbrown's control bytes and load factor. //! //! Two peaks are reported and they measure different things: //! @@ -94,11 +108,11 @@ fn main() { let mut args = std::env::args().skip(1); let mode = args.next().unwrap_or_default(); let (Some(root), Some(db)) = (args.next(), args.next()) else { - eprintln!("usage: memprobe [sample_ms]"); + eprintln!("usage: memprobe [sample_ms] [config.toml]"); std::process::exit(2); }; if mode != "cold" && mode != "warm" { - eprintln!("usage: memprobe [sample_ms]"); + eprintln!("usage: memprobe [sample_ms] [config.toml]"); std::process::exit(2); } let interval = Duration::from_millis( @@ -107,6 +121,7 @@ fn main() { .unwrap_or(DEFAULT_SAMPLE_MS) .max(1), ); + let config_path = args.next().map(PathBuf::from); let db = PathBuf::from(db); if let Some(parent) = db.parent() { @@ -118,11 +133,18 @@ fn main() { } } - run(&mode, &root, &db, interval); + run(&mode, &root, &db, interval, config_path.as_deref()); } -fn run(mode: &str, root: &str, db: &Path, interval: Duration) { - let config = Config::default(); +fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Option<&Path>) { + // The root and database always come from argv; a config file only supplies + // the knobs that change *what* indexing does — `include_hidden`, + // `ignore_patterns`, `maximum_text_size` and so on. Without one the probe + // measures the shipped defaults, which is what makes two runs comparable. + let config = match config_path { + Some(p) => Config::load_from(p).expect("load probe config"), + None => Config::default(), + }; // Cleared for the same reason indexprobe clears it: the marker is the // only unambiguous completion signal, and a stale one from the previous diff --git a/crates/quicksearch-core/src/config.rs b/crates/quicksearch-core/src/config.rs index a17819e..818c06e 100644 --- a/crates/quicksearch-core/src/config.rs +++ b/crates/quicksearch-core/src/config.rs @@ -95,7 +95,9 @@ pub struct ProcessingConfig { /// 262 makes some formats undetectable except by extension; /// 3. they are the text-sniff window (`textenc::looks_like_text`) that /// decides whether an unknown-extension or extensionless file is - /// text — small values judge files on less evidence; + /// valid UTF-8 and so worth extracting — small values judge files on + /// less evidence, and a binary whose first bytes happen to be valid + /// UTF-8 is likelier to slip through a short window than a long one; /// 4. any plaintext file no larger than this is extracted during the /// walk, sparing the content pass an open/read/close. /// diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs index b79640b..5f59bfe 100644 --- a/crates/quicksearch-core/src/db/open.rs +++ b/crates/quicksearch-core/src/db/open.rs @@ -34,10 +34,11 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: "; /// classifier-derived values go stale: `files.mime`, `files.type` and /// `content_state` are computed at walk time and never re-derived for /// unchanged files, so a classification change (v5: text sniffing, charset -/// decoding, RTF) needs the wipe to apply everywhere. Any such bump causes -/// existing indexes to be wiped on next open — there's no migration path -/// by design. -pub const CURRENT_SCHEMA_VERSION: u32 = 5; +/// decoding, RTF; v6: the text sniff now requires valid UTF-8, so binaries +/// previously stored as mojibake must be reclassified) needs the wipe to +/// apply everywhere. Any such bump causes existing indexes to be wiped on +/// next open — there's no migration path by design. +pub const CURRENT_SCHEMA_VERSION: u32 = 6; /// Open `db_path`, applying fast-path pragmas, and ensure the on-disk /// schema matches what this build expects. If it doesn't, delete the diff --git a/crates/quicksearch-core/src/file_handling.rs b/crates/quicksearch-core/src/file_handling.rs index ee14f33..955299a 100644 --- a/crates/quicksearch-core/src/file_handling.rs +++ b/crates/quicksearch-core/src/file_handling.rs @@ -1665,7 +1665,14 @@ mod count_and_extract_tests { // it claims and its answer must be a genuine one. A text body sniffs // as text for the extension-less names; the NUL body keeps the // unclaimed side of the invariant exercised too. - let cases: [(&str, &[u8]); 10] = [ + // + // The last two are the high-byte pair: identical non-UTF-8 bytes, + // once behind an extension the MIME table knows (claimed, decoded as + // legacy) and once behind one it doesn't (unclaimed, since the sniff + // takes only provable text). Both sides must agree with the + // predicate, whichever way they land. + let legacy = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9."; + let cases: [(&str, &[u8]); 12] = [ ("notes.txt", b"plain bytes with no magic"), ("data.json", b"plain bytes with no magic"), ("schema.sql", b"plain bytes with no magic"), @@ -1676,6 +1683,8 @@ mod count_and_extract_tests { ("blob.bin", b"plain bytes with no magic"), ("noextension", b"plain bytes with no magic"), ("real.bin", b"\x00\x01\x02\x03"), + ("legacy.txt", legacy), + ("legacy.unknownext", legacy), ]; for (name, body) in cases { let p = root.join(name); diff --git a/crates/quicksearch-core/src/mime.rs b/crates/quicksearch-core/src/mime.rs index 124d38d..3ce6c63 100644 --- a/crates/quicksearch-core/src/mime.rs +++ b/crates/quicksearch-core/src/mime.rs @@ -3,11 +3,17 @@ //! [`guess_mime_from_head`] infers a MIME type in three stages: extension //! first (an override table, then `mime_guess`), magic-byte sniffing via //! `infer` next, and finally a text sniff ([`crate::textenc`]) that answers -//! `text/plain` for anything whose head reads as text — which is how -//! extensionless files (README, Makefile) and source extensions no MIME -//! table knows (`.go`, `.zig`) get their contents indexed. Extensions in -//! [`AMBIGUOUS_EXTENSIONS`] invert the order: content decides, and the -//! extension's MIME is only a fallback. +//! `text/plain` for a head that is *provably* text — valid UTF-8 or +//! BOM-marked — which is how extensionless files (README, Makefile) and +//! source extensions no MIME table knows (`.go`, `.zig`) get their contents +//! indexed. Extensions in [`AMBIGUOUS_EXTENSIONS`] invert the order: content +//! decides, and the extension's MIME is only a fallback. +//! +//! That last stage is the only one with no corroborating evidence behind it, +//! so it demands the most from the bytes. Merely lacking NUL bytes does not +//! qualify — protobuf and similar `0x80-0xFF` formats clear that bar and +//! were being stored as mojibake full text. See [`crate::textenc`] for the +//! measurements. //! //! [`mime_to_type`] then maps a MIME string to a [`FileType`] bitmask so a //! single file can belong to multiple categories (e.g. a `.docx` is @@ -141,8 +147,8 @@ fn extension_override(path: &Path) -> Option<&'static str> { /// /// Extension first — an override table, then `mime_guess` — then magic /// bytes when those come up empty or say `application/octet-stream`, and -/// finally a text sniff that answers `text/plain` for any head that reads -/// as text ([`crate::textenc::looks_like_text`]). For +/// finally a text sniff that answers `text/plain` for a head that is valid +/// UTF-8 or BOM-marked ([`crate::textenc::looks_like_text`]). For /// [`AMBIGUOUS_EXTENSIONS`] the `mime_guess` answer is demoted to a last /// resort behind both content checks. /// @@ -571,6 +577,38 @@ mod tests { ); } + /// The catch-all is the one stage with no corroborating evidence, so it + /// demands valid UTF-8. Formats made of high bytes clear the binary + /// guard (no NUL, no control bytes) yet are not text, and before this + /// they were adopted as `text/plain` and stored as mojibake. + #[test] + fn high_byte_binary_is_not_sniffed_as_text() { + use std::path::PathBuf; + + // Head of a real protobuf-framed GPS log: varint record framing + // wrapping ASCII NMEA sentences. `mime_guess` has no `.pb`, `infer` + // has no protobuf matcher, so this reaches the sniff. + let mut pb = b"\x10\n\x02v1\x10\x01\x18\xe2\xe3\xfc\xd3\x9d\xca\x97\xe4\x189\x08".to_vec(); + pb.extend_from_slice(b"\x12*$GNGGA,181558.00,,,,,0,00,99.99,,,,,,*78\r\n"); + assert_eq!(guess_mime_from_head(&PathBuf::from("rtk.pb"), &pb), None); + + // The other half of the contract: an extension the MIME table knows + // never reaches the sniff, so legacy-encoded documents still type as + // text and still get their charset decoded downstream. + let latin1 = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9."; + assert_eq!( + guess_mime_from_head(&PathBuf::from("notes.txt"), latin1).as_deref(), + Some("text/plain") + ); + + // And an unknown extension is not itself disqualifying — the bytes + // decide, so a `.pb` that really is UTF-8 text still indexes. + assert_eq!( + guess_mime_from_head(&PathBuf::from("notes.pb"), b"just some words\n").as_deref(), + Some("text/plain") + ); + } + /// Ambiguous extensions resolve by content in both directions: source /// code beats the extension table, real binary keeps the extension's /// MIME as the fallback. diff --git a/crates/quicksearch-core/src/textenc.rs b/crates/quicksearch-core/src/textenc.rs index 26c4133..65e68e3 100644 --- a/crates/quicksearch-core/src/textenc.rs +++ b/crates/quicksearch-core/src/textenc.rs @@ -1,11 +1,17 @@ //! Text detection and charset decoding, shared by the MIME sniff and the //! plaintext extractor. //! -//! Both callers route through one classifier so they cannot drift: a head -//! that [`looks_like_text`] accepts is guaranteed to decode via -//! [`decode_text`] — every class except `Binary` decodes unconditionally -//! (strict UTF-8 after validation, BOM'd and legacy decodes are -//! lossy-with-replacement and never fail). +//! Both callers route through one classifier so they cannot drift, but they +//! accept different amounts of it, because they are answering different +//! questions: +//! +//! - [`decode_text`] is asked "this file is text — render it". Something +//! else already established that, usually the extension. Every class but +//! `Binary` decodes unconditionally. +//! - [`looks_like_text`] is asked "is this text at all?", by a caller that +//! has *no other evidence*: no known extension, no magic bytes. It +//! accepts only `Utf8` and `Bom`, the two classes that carry positive +//! proof. //! //! Classification order is load-bearing: //! @@ -20,7 +26,18 @@ //! sniffs the same buffer), so a multibyte sequence cut off by the end //! of the buffer does not disqualify it. //! 4. **Legacy** — everything else. [`decode_text`] runs charset detection -//! (chardetng, windows-1252 floor) and decodes with replacement. +//! (chardetng, windows-1252 floor) and decodes with replacement. The +//! sniff **rejects** this class, and that asymmetry is the whole point: +//! chardetng's windows-1252 floor means it never fails, so treating +//! `Legacy` as proof of text makes the sniff unfalsifiable. The binary +//! guard only rejects NUL and control bytes, so any format built out of +//! `0x80-0xFF` — protobuf varints, packed binary telemetry — walks +//! straight through it and gets stored as mojibake. Measured on a +//! 99k-file tree, that one leak was 93% of all extracted text; the +//! legitimate `Legacy`-decoding files it costs us are the ones with no +//! extension *and* no magic bytes, which measured 5 files and 0.1 MB. +//! A file with a known text extension is unaffected: it is typed by +//! `mime_guess`, never reaches the sniff, and still decodes as legacy. //! //! The sniff sees only the head, so a file with a text head and a binary //! tail classifies as text and then fails the whole-file decode; that lands @@ -79,18 +96,30 @@ fn classify(bytes: &[u8], truncated: bool) -> TextClass { } } -/// Whether `head` — a possibly-truncated prefix of a file — reads as text. +/// Whether `head` — a possibly-truncated prefix of a file — is *provably* +/// text: valid UTF-8, or BOM-marked. +/// +/// This is the sniff behind [`crate::mime::guess_mime_from_head`]'s +/// `text/plain` catch-all, which fires only when nothing else identified the +/// file. With no extension and no magic bytes to corroborate it, "not +/// obviously binary" is too weak a test — see the module docs on why +/// `TextClass::Legacy` is rejected here but accepted by [`decode_text`]. /// /// Cheap: a byte scan plus UTF-8 validation, no charset detection. An empty -/// head proves nothing and answers `false`; that keeps zero-size -/// extensionless files (procfs included) unclassified rather than blanket -/// `text/plain`. +/// head proves nothing and answers `false`; without that guard `classify` +/// would call it valid UTF-8 (the byte loop never runs, `from_utf8(b"")` +/// succeeds) and every zero-size procfs file would become `text/plain`. pub fn looks_like_text(head: &[u8]) -> bool { - !head.is_empty() && !matches!(classify(head, true), TextClass::Binary) + !head.is_empty() && matches!(classify(head, true), TextClass::Utf8 | TextClass::Bom(_)) } /// Decode a complete file's bytes to UTF-8 for storage. /// +/// Accepts every class [`looks_like_text`] does and `Legacy` besides: by the +/// time this runs, something has already decided the file is text — usually +/// its extension, which is evidence the sniff does not have — so a +/// windows-1252 `.txt` or a Shift-JIS `.csv` still decodes here. +/// /// Takes ownership so the dominant valid-UTF-8 case is a zero-copy move. /// `path` is used only to name the file in the error, matching the /// extractor error convention. @@ -173,12 +202,19 @@ mod tests { assert_eq!(decode_text(body, &p()).unwrap(), src); } + /// Legacy charsets decode, but do not *sniff*: a `.txt` extension routes + /// these bytes to `decode_text` and they render correctly, while the + /// same bytes with no extension and no magic are not text enough to + /// adopt on their own. #[test] - fn windows_1252_decodes() { + fn windows_1252_decodes_but_does_not_sniff() { // A sentence long enough for chardetng to settle on a Western // single-byte encoding. let body = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.".to_vec(); - assert!(looks_like_text(&body)); + assert!( + !looks_like_text(&body), + "the sniff must not adopt a non-UTF-8 head on its own" + ); assert_eq!( decode_text(body, &p()).unwrap(), "Le café près de la fenêtre est agréable en été." @@ -186,13 +222,13 @@ mod tests { } #[test] - fn shift_jis_decodes() { + fn shift_jis_decodes_but_does_not_sniff() { // "日本語のテキストです。これはシフトJISでエンコードされています。" let src = "日本語のテキストです。これはシフトJISでエンコードされています。"; let (encoded, _, had_errors) = encoding_rs::SHIFT_JIS.encode(src); assert!(!had_errors); let body = encoded.into_owned(); - assert!(looks_like_text(&body)); + assert!(!looks_like_text(&body)); assert_eq!(decode_text(body, &p()).unwrap(), src); } @@ -255,31 +291,69 @@ mod tests { } /// Every head the sniff accepts must decode — the invariant that makes - /// "sniffed as text/plain" safe to act on. + /// "sniffed as text/plain" safe to act on. The `expect_sniff` column + /// pins which side of the UTF-8 line each head falls on, so a head that + /// silently stops being sniffed can't quietly weaken this test. #[test] fn sniffed_text_is_guaranteed_decodable() { - let heads: Vec> = vec![ - b"ordinary ascii".to_vec(), - "utf-8 caf\u{e9}".as_bytes().to_vec(), - b"latin-1 caf\xe9 body".to_vec(), - { + let heads: Vec<(bool, Vec)> = vec![ + (true, b"ordinary ascii".to_vec()), + (true, "utf-8 caf\u{e9}".as_bytes().to_vec()), + // Decodes, but only for a caller that already knows it's text. + (false, b"latin-1 caf\xe9 body".to_vec()), + (true, { let mut v = vec![0xFF, 0xFE]; v.extend("utf16".encode_utf16().flat_map(|u| u.to_le_bytes())); v - }, - { + }), + (true, { let mut v = "truncated tail caf".as_bytes().to_vec(); v.push(0xC3); v - }, + }), ]; - for head in heads { - if looks_like_text(&head) { - assert!( - decode_text(head.clone(), &p()).is_ok(), - "sniffed-as-text head failed to decode: {head:?}" - ); - } + for (expect_sniff, head) in heads { + assert_eq!( + looks_like_text(&head), + expect_sniff, + "sniff verdict changed for {head:?}" + ); + assert!( + decode_text(head.clone(), &p()).is_ok(), + "head failed to decode: {head:?}" + ); } } + + /// The regression this guard exists for. Protobuf wire format is varint + /// field tags and lengths — bytes in `0x80-0xFF`, which are neither NUL + /// nor control bytes, so the binary guard passes them and chardetng's + /// windows-1252 floor then "decodes" them into mojibake that never + /// fails. On a real 99k-file tree this single hole was 93% of all + /// extracted text. Bytes below are the head of an actual `.pb` GPS log: + /// varint-framed records wrapping ASCII NMEA sentences. + #[test] + fn protobuf_head_is_not_text() { + let mut body = b"\x10\n\x02v1\x10\x01\x18\xe2\xe3\xfc\xd3\x9d\xca\x97\xe4\x189\x08\ + \xbc\xf3\xf8\xd2\x9e\xca\x97\xe4\x18\x12*" + .to_vec(); + body.extend_from_slice(b"$GNGGA,181558.00,,,,,0,00,99.99,,,,,,*78\r\n"); + body.extend_from_slice(b"\x18\xe3e>\x08\xeb\xac\xfb\xd2\x9e\xca\x97\xe4\x18\x12/"); + body.extend_from_slice(b"$GNGSA,M,1,,,,,,,,,,,,,99.99,99.99,99.99,1*3F\r\n"); + + // It clears the binary guard — that is exactly why the guard alone + // was not enough — but it is not valid UTF-8, so the sniff declines. + assert!( + !looks_like_text(&body), + "protobuf must not be adopted as text/plain" + ); + assert!( + std::str::from_utf8(&body).is_err(), + "test fixture must be invalid UTF-8 or it proves nothing" + ); + assert!( + !body.contains(&0u8), + "test fixture must have no NUL, or the old guard would have caught it" + ); + } } diff --git a/crates/quicksearch-core/tests/full_index.rs b/crates/quicksearch-core/tests/full_index.rs index 6926c52..863fa2a 100644 --- a/crates/quicksearch-core/tests/full_index.rs +++ b/crates/quicksearch-core/tests/full_index.rs @@ -1501,3 +1501,75 @@ fn a_stopped_run_is_still_optimized() { std::fs::remove_dir_all(&root).ok(); std::fs::remove_dir_all(&db_dir).ok(); } + +/// High-byte binaries are listed but never full-text extracted. +/// +/// The whole reason the text sniff demands valid UTF-8. Protobuf and friends +/// carry no NUL and no control bytes, so the binary guard passes them; before +/// the guard was tightened they were adopted as `text/plain`, read in full, +/// run through chardetng's never-failing windows-1252 floor and stored as +/// mojibake. On a real 99k-file tree that was 93% of every byte of extracted +/// text. +/// +/// End-to-end because the interesting part is the *combination*: the row must +/// survive in `files` (the file is still findable by name) while acquiring no +/// `documents_text` sidecar and no `failed_files` entry — it is not a failure, +/// it is a file with no text in it. The `.txt` alongside it holds the same +/// bytes and must still extract, which is what proves the fix cost nothing for +/// files an extension already identified. +#[test] +fn high_byte_binaries_are_listed_but_not_text_extracted() { + let root = tmp_dir("sniff-binary"); + let db_dir = tmp_dir("sniff-binary-db"); + let db = db_dir.join("index.sqlite"); + + // Head of a real protobuf-framed GPS log: varint framing around ASCII + // NMEA sentences. No NUL, no control-byte density — it clears the binary + // guard on its own. + let mut pb = b"\x10\n\x02v1\x10\x01\x18\xe2\xe3\xfc\xd3\x9d\xca\x97\xe4\x189\x08".to_vec(); + pb.extend_from_slice(b"\x12*$GNGGA,181558.00,,,,,0,00,99.99,,,,,,*78\r\n"); + assert!(!pb.contains(&0u8), "fixture must not trip the NUL guard"); + + let legacy = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9."; + + touch(&root.join("rtk.pb"), &pb); + touch(&root.join("legacy.txt"), legacy); + touch(&root.join("notes.md"), b"ordinary utf-8 prose"); + index_once(&root, &db, &Config::default()); + + let conn = rusqlite::Connection::open(&db).unwrap(); + let probe = |suffix: &str| -> (i64, i64, i64) { + conn.query_row( + "SELECT f.content_state, + (SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id), + (SELECT COUNT(*) FROM failed_files x WHERE x.file_id = f.id) + FROM files f WHERE f.path LIKE '%' || ?1", + [suffix], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .unwrap_or_else(|e| panic!("{suffix} must be indexed: {e}")) + }; + + // 3 = not applicable. Present in `files`, so filename search still finds + // it; no sidecar, so none of its bytes reached the index. + assert_eq!( + probe("rtk.pb"), + (3, 0, 0), + "a high-byte binary must be listed, not extracted, and not a failure" + ); + + // Same bytes, known extension: typed by mime_guess, never sniffed, still + // decoded through chardetng and stored. + let (state, sidecars, failures) = probe("legacy.txt"); + assert_eq!( + (state, failures), + (1, 0), + "a legacy-charset .txt must still extract" + ); + assert_eq!(sidecars, 1, "and must still store its text"); + + assert_eq!(probe("notes.md"), (1, 1, 0), "ordinary UTF-8 is unaffected"); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +}