Improved rejection of non-text files, improved CI.

.forgejo/workflows/ci.yml

@@ -47,6 +47,15 @@ jobs:
     runs-on: self-hosted
     container:
       image: catthehacker/ubuntu:act-22.04
+      # Jobs run as root, and root ignores permission bits: CAP_DAC_OVERRIDE and
+      # CAP_DAC_READ_SEARCH let uid 0 read a mode-000 file or directory anyway.
+      # Several tests build an unreadable directory with platform::deny_read and
+      # assert the walk reports it rather than reporting an empty directory - the
+      # distinction matters because an empty listing deletes index rows. Under
+      # root those tests see a perfectly readable tree and fail. Dropping the two
+      # DAC capabilities makes root obey the mode bits, which is exactly the
+      # environment the tests assume and get on a developer machine.
+      options: --cap-drop=DAC_OVERRIDE --cap-drop=DAC_READ_SEARCH
     env:
       # crates/quicksearch-core/src/config.rs has a test that expects a home
       # directory and panics without one.

Cargo.toml

@@ -6,7 +6,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.9.0"
+version = "0.9.1"
 edition = "2021"
 license = "GPL-3.0-or-later"
 authors = ["Jeremy <jeremy@karsttech.com>"]

README.md

@@ -294,11 +294,13 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
   sweep stale rows, then extract content (plaintext, RTF, Office, PDF,
   audio tags, EXIF; see `extract/`) for FTS. Files whose extension no MIME
   table knows — including extensionless ones like `README` or `Makefile` —
-  are sniffed from their head bytes and indexed as text when they read as
-  text (`mime.rs`, `textenc.rs`); non-UTF-8 text (UTF-16 with BOM, legacy
-  charsets via chardetng) is decoded and stored as UTF-8. More claimed
-  files means a bigger index — `indexing.content_extensions` remains the
-  throttle. Files no larger than `processing.hash_length` skip that second
+  are sniffed from their head bytes and indexed as text only when that head
+  is provably text: valid UTF-8, or BOM-marked (`mime.rs`, `textenc.rs`).
+  Legacy charsets are decoded via chardetng and stored as UTF-8, but only
+  for files something *else* typed as text, normally their extension —
+  chardetng's windows-1252 floor never fails, so accepting it on a bare
+  sniff would adopt any binary lacking NUL bytes. More claimed files means a
+  bigger index — `indexing.content_extensions` remains the throttle. Files no larger than `processing.hash_length` skip that second
   pass entirely: the head the walk reads to hash them is already their
   whole content, so a plaintext body is extracted in the same `read` and
   stored complete. Every run ends — whether

crates/quicksearch-core/examples/memprobe.rs

@@ -11,10 +11,14 @@
 //! ./target/release/examples/memprobe cold /media/shared /var/tmp/qs-mem/index.db
 //! ./target/release/examples/memprobe warm /media/shared /var/tmp/qs-mem/index.db
 //! ./target/release/examples/memprobe cold /media/shared /var/tmp/qs-mem/index.db 10
+//! ./target/release/examples/memprobe cold ~ /var/tmp/qs-mem/index.db 250 probe.toml
 //! ```
 //!
 //! The optional trailing number is the sampling interval in milliseconds
-//! (default 100). Finer sampling resolves the *shape* of a spike, not its
+//! (default 100), and the one after it a config file to load instead of the
+//! defaults — the only way to probe a tree that the shipped `include_hidden =
+//! false` would walk past, such as a home directory that is nearly all dotdirs.
+//! Finer sampling resolves the *shape* of a spike, not its
 //! cause: the file column is only as good as `RootProgress::current_file`,
 //! which [`crate::indexing`] publishes once per extraction batch, holding the
 //! last file of the batch that just finished. During a batch it therefore
@@ -24,10 +28,20 @@
 //! largest single consumer on this tree.
 //!
 //! `cold` deletes the database first: every file is new, so the walk hashes
-//! and extracts all of them and `existing_files` starts empty. `warm` re-runs
-//! against the finished database, which is the case that loads one
-//! `existing_files` entry per indexed path up front — the allocation that
-//! scales with tree size rather than with in-flight work.
+//! and extracts all of them. `warm` re-runs against the finished database,
+//! where most files classify as unchanged and the run is dominated by
+//! reconciliation rather than extraction. Warm peaks *below* cold on the same
+//! tree — the walk reads one directory's rows at a time (`repo::dir_rows`,
+//! held in an `Arc` only while that directory is in flight), so there is no
+//! up-front load that scales with tree size.
+//!
+//! **`growth per file` is a ratio, not a per-file cost.** It divides a peak
+//! that is essentially constant by the file count, so it *falls* as the tree
+//! grows: measured 2052 B/file over 99,477 files and 644 B/file over 279,936,
+//! with the larger tree peaking *lower* in absolute terms. Read the peak, not
+//! the quotient. What actually scales with tree size is `seen_paths`
+//! (`indexing.rs`), a `HashSet<u128>` of path digests — ~17 bytes per file
+//! including hashbrown's control bytes and load factor.
 //!
 //! Two peaks are reported and they measure different things:
 //!
@@ -94,11 +108,11 @@ fn main() {
     let mut args = std::env::args().skip(1);
     let mode = args.next().unwrap_or_default();
     let (Some(root), Some(db)) = (args.next(), args.next()) else {
-        eprintln!("usage: memprobe <cold|warm> <root> <db> [sample_ms]");
+        eprintln!("usage: memprobe <cold|warm> <root> <db> [sample_ms] [config.toml]");
         std::process::exit(2);
     };
     if mode != "cold" && mode != "warm" {
-        eprintln!("usage: memprobe <cold|warm> <root> <db> [sample_ms]");
+        eprintln!("usage: memprobe <cold|warm> <root> <db> [sample_ms] [config.toml]");
         std::process::exit(2);
     }
     let interval = Duration::from_millis(
@@ -107,6 +121,7 @@ fn main() {
             .unwrap_or(DEFAULT_SAMPLE_MS)
             .max(1),
     );
+    let config_path = args.next().map(PathBuf::from);
     let db = PathBuf::from(db);
 
     if let Some(parent) = db.parent() {
@@ -118,11 +133,18 @@ fn main() {
         }
     }
 
-    run(&mode, &root, &db, interval);
+    run(&mode, &root, &db, interval, config_path.as_deref());
 }
 
-fn run(mode: &str, root: &str, db: &Path, interval: Duration) {
-    let config = Config::default();
+fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Option<&Path>) {
+    // The root and database always come from argv; a config file only supplies
+    // the knobs that change *what* indexing does — `include_hidden`,
+    // `ignore_patterns`, `maximum_text_size` and so on. Without one the probe
+    // measures the shipped defaults, which is what makes two runs comparable.
+    let config = match config_path {
+        Some(p) => Config::load_from(p).expect("load probe config"),
+        None => Config::default(),
+    };
 
     // Cleared for the same reason indexprobe clears it: the marker is the
     // only unambiguous completion signal, and a stale one from the previous

crates/quicksearch-core/src/config.rs

@@ -95,7 +95,9 @@ pub struct ProcessingConfig {
     ///    262 makes some formats undetectable except by extension;
     /// 3. they are the text-sniff window (`textenc::looks_like_text`) that
     ///    decides whether an unknown-extension or extensionless file is
-    ///    text — small values judge files on less evidence;
+    ///    valid UTF-8 and so worth extracting — small values judge files on
+    ///    less evidence, and a binary whose first bytes happen to be valid
+    ///    UTF-8 is likelier to slip through a short window than a long one;
     /// 4. any plaintext file no larger than this is extracted during the
     ///    walk, sparing the content pass an open/read/close.
     ///

crates/quicksearch-core/src/db/open.rs

@@ -34,10 +34,11 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: ";
 /// classifier-derived values go stale: `files.mime`, `files.type` and
 /// `content_state` are computed at walk time and never re-derived for
 /// unchanged files, so a classification change (v5: text sniffing, charset
-/// decoding, RTF) needs the wipe to apply everywhere. Any such bump causes
-/// existing indexes to be wiped on next open — there's no migration path
-/// by design.
-pub const CURRENT_SCHEMA_VERSION: u32 = 5;
+/// decoding, RTF; v6: the text sniff now requires valid UTF-8, so binaries
+/// previously stored as mojibake must be reclassified) needs the wipe to
+/// apply everywhere. Any such bump causes existing indexes to be wiped on
+/// next open — there's no migration path by design.
+pub const CURRENT_SCHEMA_VERSION: u32 = 6;
 
 /// Open `db_path`, applying fast-path pragmas, and ensure the on-disk
 /// schema matches what this build expects. If it doesn't, delete the

crates/quicksearch-core/src/file_handling.rs

@@ -1665,7 +1665,14 @@ mod count_and_extract_tests {
         // it claims and its answer must be a genuine one. A text body sniffs
         // as text for the extension-less names; the NUL body keeps the
         // unclaimed side of the invariant exercised too.
-        let cases: [(&str, &[u8]); 10] = [
+        //
+        // The last two are the high-byte pair: identical non-UTF-8 bytes,
+        // once behind an extension the MIME table knows (claimed, decoded as
+        // legacy) and once behind one it doesn't (unclaimed, since the sniff
+        // takes only provable text). Both sides must agree with the
+        // predicate, whichever way they land.
+        let legacy = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.";
+        let cases: [(&str, &[u8]); 12] = [
             ("notes.txt", b"plain bytes with no magic"),
             ("data.json", b"plain bytes with no magic"),
             ("schema.sql", b"plain bytes with no magic"),
@@ -1676,6 +1683,8 @@ mod count_and_extract_tests {
             ("blob.bin", b"plain bytes with no magic"),
             ("noextension", b"plain bytes with no magic"),
             ("real.bin", b"\x00\x01\x02\x03"),
+            ("legacy.txt", legacy),
+            ("legacy.unknownext", legacy),
         ];
         for (name, body) in cases {
             let p = root.join(name);

crates/quicksearch-core/src/mime.rs

@@ -3,11 +3,17 @@
 //! [`guess_mime_from_head`] infers a MIME type in three stages: extension
 //! first (an override table, then `mime_guess`), magic-byte sniffing via
 //! `infer` next, and finally a text sniff ([`crate::textenc`]) that answers
-//! `text/plain` for anything whose head reads as text — which is how
-//! extensionless files (README, Makefile) and source extensions no MIME
-//! table knows (`.go`, `.zig`) get their contents indexed. Extensions in
-//! [`AMBIGUOUS_EXTENSIONS`] invert the order: content decides, and the
-//! extension's MIME is only a fallback.
+//! `text/plain` for a head that is *provably* text — valid UTF-8 or
+//! BOM-marked — which is how extensionless files (README, Makefile) and
+//! source extensions no MIME table knows (`.go`, `.zig`) get their contents
+//! indexed. Extensions in [`AMBIGUOUS_EXTENSIONS`] invert the order: content
+//! decides, and the extension's MIME is only a fallback.
+//!
+//! That last stage is the only one with no corroborating evidence behind it,
+//! so it demands the most from the bytes. Merely lacking NUL bytes does not
+//! qualify — protobuf and similar `0x80-0xFF` formats clear that bar and
+//! were being stored as mojibake full text. See [`crate::textenc`] for the
+//! measurements.
 //!
 //! [`mime_to_type`] then maps a MIME string to a [`FileType`] bitmask so a
 //! single file can belong to multiple categories (e.g. a `.docx` is
@@ -141,8 +147,8 @@ fn extension_override(path: &Path) -> Option<&'static str> {
 ///
 /// Extension first — an override table, then `mime_guess` — then magic
 /// bytes when those come up empty or say `application/octet-stream`, and
-/// finally a text sniff that answers `text/plain` for any head that reads
-/// as text ([`crate::textenc::looks_like_text`]). For
+/// finally a text sniff that answers `text/plain` for a head that is valid
+/// UTF-8 or BOM-marked ([`crate::textenc::looks_like_text`]). For
 /// [`AMBIGUOUS_EXTENSIONS`] the `mime_guess` answer is demoted to a last
 /// resort behind both content checks.
 ///
@@ -571,6 +577,38 @@ mod tests {
         );
     }
 
+    /// The catch-all is the one stage with no corroborating evidence, so it
+    /// demands valid UTF-8. Formats made of high bytes clear the binary
+    /// guard (no NUL, no control bytes) yet are not text, and before this
+    /// they were adopted as `text/plain` and stored as mojibake.
+    #[test]
+    fn high_byte_binary_is_not_sniffed_as_text() {
+        use std::path::PathBuf;
+
+        // Head of a real protobuf-framed GPS log: varint record framing
+        // wrapping ASCII NMEA sentences. `mime_guess` has no `.pb`, `infer`
+        // has no protobuf matcher, so this reaches the sniff.
+        let mut pb = b"\x10\n\x02v1\x10\x01\x18\xe2\xe3\xfc\xd3\x9d\xca\x97\xe4\x189\x08".to_vec();
+        pb.extend_from_slice(b"\x12*$GNGGA,181558.00,,,,,0,00,99.99,,,,,,*78\r\n");
+        assert_eq!(guess_mime_from_head(&PathBuf::from("rtk.pb"), &pb), None);
+
+        // The other half of the contract: an extension the MIME table knows
+        // never reaches the sniff, so legacy-encoded documents still type as
+        // text and still get their charset decoded downstream.
+        let latin1 = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.";
+        assert_eq!(
+            guess_mime_from_head(&PathBuf::from("notes.txt"), latin1).as_deref(),
+            Some("text/plain")
+        );
+
+        // And an unknown extension is not itself disqualifying — the bytes
+        // decide, so a `.pb` that really is UTF-8 text still indexes.
+        assert_eq!(
+            guess_mime_from_head(&PathBuf::from("notes.pb"), b"just some words\n").as_deref(),
+            Some("text/plain")
+        );
+    }
+
     /// Ambiguous extensions resolve by content in both directions: source
     /// code beats the extension table, real binary keeps the extension's
     /// MIME as the fallback.

crates/quicksearch-core/src/textenc.rs

@@ -1,11 +1,17 @@
 //! Text detection and charset decoding, shared by the MIME sniff and the
 //! plaintext extractor.
 //!
-//! Both callers route through one classifier so they cannot drift: a head
-//! that [`looks_like_text`] accepts is guaranteed to decode via
-//! [`decode_text`] — every class except `Binary` decodes unconditionally
-//! (strict UTF-8 after validation, BOM'd and legacy decodes are
-//! lossy-with-replacement and never fail).
+//! Both callers route through one classifier so they cannot drift, but they
+//! accept different amounts of it, because they are answering different
+//! questions:
+//!
+//! - [`decode_text`] is asked "this file is text — render it". Something
+//!   else already established that, usually the extension. Every class but
+//!   `Binary` decodes unconditionally.
+//! - [`looks_like_text`] is asked "is this text at all?", by a caller that
+//!   has *no other evidence*: no known extension, no magic bytes. It
+//!   accepts only `Utf8` and `Bom`, the two classes that carry positive
+//!   proof.
 //!
 //! Classification order is load-bearing:
 //!
@@ -20,7 +26,18 @@
 //!    sniffs the same buffer), so a multibyte sequence cut off by the end
 //!    of the buffer does not disqualify it.
 //! 4. **Legacy** — everything else. [`decode_text`] runs charset detection
-//!    (chardetng, windows-1252 floor) and decodes with replacement.
+//!    (chardetng, windows-1252 floor) and decodes with replacement. The
+//!    sniff **rejects** this class, and that asymmetry is the whole point:
+//!    chardetng's windows-1252 floor means it never fails, so treating
+//!    `Legacy` as proof of text makes the sniff unfalsifiable. The binary
+//!    guard only rejects NUL and control bytes, so any format built out of
+//!    `0x80-0xFF` — protobuf varints, packed binary telemetry — walks
+//!    straight through it and gets stored as mojibake. Measured on a
+//!    99k-file tree, that one leak was 93% of all extracted text; the
+//!    legitimate `Legacy`-decoding files it costs us are the ones with no
+//!    extension *and* no magic bytes, which measured 5 files and 0.1 MB.
+//!    A file with a known text extension is unaffected: it is typed by
+//!    `mime_guess`, never reaches the sniff, and still decodes as legacy.
 //!
 //! The sniff sees only the head, so a file with a text head and a binary
 //! tail classifies as text and then fails the whole-file decode; that lands
@@ -79,18 +96,30 @@ fn classify(bytes: &[u8], truncated: bool) -> TextClass {
     }
 }
 
-/// Whether `head` — a possibly-truncated prefix of a file — reads as text.
+/// Whether `head` — a possibly-truncated prefix of a file — is *provably*
+/// text: valid UTF-8, or BOM-marked.
+///
+/// This is the sniff behind [`crate::mime::guess_mime_from_head`]'s
+/// `text/plain` catch-all, which fires only when nothing else identified the
+/// file. With no extension and no magic bytes to corroborate it, "not
+/// obviously binary" is too weak a test — see the module docs on why
+/// `TextClass::Legacy` is rejected here but accepted by [`decode_text`].
 ///
 /// Cheap: a byte scan plus UTF-8 validation, no charset detection. An empty
-/// head proves nothing and answers `false`; that keeps zero-size
-/// extensionless files (procfs included) unclassified rather than blanket
-/// `text/plain`.
+/// head proves nothing and answers `false`; without that guard `classify`
+/// would call it valid UTF-8 (the byte loop never runs, `from_utf8(b"")`
+/// succeeds) and every zero-size procfs file would become `text/plain`.
 pub fn looks_like_text(head: &[u8]) -> bool {
-    !head.is_empty() && !matches!(classify(head, true), TextClass::Binary)
+    !head.is_empty() && matches!(classify(head, true), TextClass::Utf8 | TextClass::Bom(_))
 }
 
 /// Decode a complete file's bytes to UTF-8 for storage.
 ///
+/// Accepts every class [`looks_like_text`] does and `Legacy` besides: by the
+/// time this runs, something has already decided the file is text — usually
+/// its extension, which is evidence the sniff does not have — so a
+/// windows-1252 `.txt` or a Shift-JIS `.csv` still decodes here.
+///
 /// Takes ownership so the dominant valid-UTF-8 case is a zero-copy move.
 /// `path` is used only to name the file in the error, matching the
 /// extractor error convention.
@@ -173,12 +202,19 @@ mod tests {
         assert_eq!(decode_text(body, &p()).unwrap(), src);
     }
 
+    /// Legacy charsets decode, but do not *sniff*: a `.txt` extension routes
+    /// these bytes to `decode_text` and they render correctly, while the
+    /// same bytes with no extension and no magic are not text enough to
+    /// adopt on their own.
     #[test]
-    fn windows_1252_decodes() {
+    fn windows_1252_decodes_but_does_not_sniff() {
         // A sentence long enough for chardetng to settle on a Western
         // single-byte encoding.
         let body = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.".to_vec();
-        assert!(looks_like_text(&body));
+        assert!(
+            !looks_like_text(&body),
+            "the sniff must not adopt a non-UTF-8 head on its own"
+        );
         assert_eq!(
             decode_text(body, &p()).unwrap(),
             "Le café près de la fenêtre est agréable en été."
@@ -186,13 +222,13 @@ mod tests {
     }
 
     #[test]
-    fn shift_jis_decodes() {
+    fn shift_jis_decodes_but_does_not_sniff() {
         // "日本語のテキストです。これはシフトJISでエンコードされています。"
         let src = "日本語のテキストです。これはシフトJISでエンコードされています。";
         let (encoded, _, had_errors) = encoding_rs::SHIFT_JIS.encode(src);
         assert!(!had_errors);
         let body = encoded.into_owned();
-        assert!(looks_like_text(&body));
+        assert!(!looks_like_text(&body));
         assert_eq!(decode_text(body, &p()).unwrap(), src);
     }
 
@@ -255,31 +291,69 @@ mod tests {
     }
 
     /// Every head the sniff accepts must decode — the invariant that makes
-    /// "sniffed as text/plain" safe to act on.
+    /// "sniffed as text/plain" safe to act on. The `expect_sniff` column
+    /// pins which side of the UTF-8 line each head falls on, so a head that
+    /// silently stops being sniffed can't quietly weaken this test.
     #[test]
     fn sniffed_text_is_guaranteed_decodable() {
-        let heads: Vec<Vec<u8>> = vec![
-            b"ordinary ascii".to_vec(),
-            "utf-8 caf\u{e9}".as_bytes().to_vec(),
-            b"latin-1 caf\xe9 body".to_vec(),
-            {
+        let heads: Vec<(bool, Vec<u8>)> = vec![
+            (true, b"ordinary ascii".to_vec()),
+            (true, "utf-8 caf\u{e9}".as_bytes().to_vec()),
+            // Decodes, but only for a caller that already knows it's text.
+            (false, b"latin-1 caf\xe9 body".to_vec()),
+            (true, {
                 let mut v = vec![0xFF, 0xFE];
                 v.extend("utf16".encode_utf16().flat_map(|u| u.to_le_bytes()));
                 v
-            },
-            {
+            }),
+            (true, {
                 let mut v = "truncated tail caf".as_bytes().to_vec();
                 v.push(0xC3);
                 v
-            },
+            }),
         ];
-        for head in heads {
-            if looks_like_text(&head) {
+        for (expect_sniff, head) in heads {
+            assert_eq!(
+                looks_like_text(&head),
+                expect_sniff,
+                "sniff verdict changed for {head:?}"
+            );
             assert!(
                 decode_text(head.clone(), &p()).is_ok(),
-                    "sniffed-as-text head failed to decode: {head:?}"
+                "head failed to decode: {head:?}"
             );
         }
     }
+
+    /// The regression this guard exists for. Protobuf wire format is varint
+    /// field tags and lengths — bytes in `0x80-0xFF`, which are neither NUL
+    /// nor control bytes, so the binary guard passes them and chardetng's
+    /// windows-1252 floor then "decodes" them into mojibake that never
+    /// fails. On a real 99k-file tree this single hole was 93% of all
+    /// extracted text. Bytes below are the head of an actual `.pb` GPS log:
+    /// varint-framed records wrapping ASCII NMEA sentences.
+    #[test]
+    fn protobuf_head_is_not_text() {
+        let mut body = b"\x10\n\x02v1\x10\x01\x18\xe2\xe3\xfc\xd3\x9d\xca\x97\xe4\x189\x08\
+                         \xbc\xf3\xf8\xd2\x9e\xca\x97\xe4\x18\x12*"
+            .to_vec();
+        body.extend_from_slice(b"$GNGGA,181558.00,,,,,0,00,99.99,,,,,,*78\r\n");
+        body.extend_from_slice(b"\x18\xe3e>\x08\xeb\xac\xfb\xd2\x9e\xca\x97\xe4\x18\x12/");
+        body.extend_from_slice(b"$GNGSA,M,1,,,,,,,,,,,,,99.99,99.99,99.99,1*3F\r\n");
+
+        // It clears the binary guard — that is exactly why the guard alone
+        // was not enough — but it is not valid UTF-8, so the sniff declines.
+        assert!(
+            !looks_like_text(&body),
+            "protobuf must not be adopted as text/plain"
+        );
+        assert!(
+            std::str::from_utf8(&body).is_err(),
+            "test fixture must be invalid UTF-8 or it proves nothing"
+        );
+        assert!(
+            !body.contains(&0u8),
+            "test fixture must have no NUL, or the old guard would have caught it"
+        );
     }
 }

crates/quicksearch-core/tests/full_index.rs

@@ -1501,3 +1501,75 @@ fn a_stopped_run_is_still_optimized() {
     std::fs::remove_dir_all(&root).ok();
     std::fs::remove_dir_all(&db_dir).ok();
 }
+
+/// High-byte binaries are listed but never full-text extracted.
+///
+/// The whole reason the text sniff demands valid UTF-8. Protobuf and friends
+/// carry no NUL and no control bytes, so the binary guard passes them; before
+/// the guard was tightened they were adopted as `text/plain`, read in full,
+/// run through chardetng's never-failing windows-1252 floor and stored as
+/// mojibake. On a real 99k-file tree that was 93% of every byte of extracted
+/// text.
+///
+/// End-to-end because the interesting part is the *combination*: the row must
+/// survive in `files` (the file is still findable by name) while acquiring no
+/// `documents_text` sidecar and no `failed_files` entry — it is not a failure,
+/// it is a file with no text in it. The `.txt` alongside it holds the same
+/// bytes and must still extract, which is what proves the fix cost nothing for
+/// files an extension already identified.
+#[test]
+fn high_byte_binaries_are_listed_but_not_text_extracted() {
+    let root = tmp_dir("sniff-binary");
+    let db_dir = tmp_dir("sniff-binary-db");
+    let db = db_dir.join("index.sqlite");
+
+    // Head of a real protobuf-framed GPS log: varint framing around ASCII
+    // NMEA sentences. No NUL, no control-byte density — it clears the binary
+    // guard on its own.
+    let mut pb = b"\x10\n\x02v1\x10\x01\x18\xe2\xe3\xfc\xd3\x9d\xca\x97\xe4\x189\x08".to_vec();
+    pb.extend_from_slice(b"\x12*$GNGGA,181558.00,,,,,0,00,99.99,,,,,,*78\r\n");
+    assert!(!pb.contains(&0u8), "fixture must not trip the NUL guard");
+
+    let legacy = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.";
+
+    touch(&root.join("rtk.pb"), &pb);
+    touch(&root.join("legacy.txt"), legacy);
+    touch(&root.join("notes.md"), b"ordinary utf-8 prose");
+    index_once(&root, &db, &Config::default());
+
+    let conn = rusqlite::Connection::open(&db).unwrap();
+    let probe = |suffix: &str| -> (i64, i64, i64) {
+        conn.query_row(
+            "SELECT f.content_state,
+                    (SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id),
+                    (SELECT COUNT(*) FROM failed_files x WHERE x.file_id = f.id)
+               FROM files f WHERE f.path LIKE '%' || ?1",
+            [suffix],
+            |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
+        )
+        .unwrap_or_else(|e| panic!("{suffix} must be indexed: {e}"))
+    };
+
+    // 3 = not applicable. Present in `files`, so filename search still finds
+    // it; no sidecar, so none of its bytes reached the index.
+    assert_eq!(
+        probe("rtk.pb"),
+        (3, 0, 0),
+        "a high-byte binary must be listed, not extracted, and not a failure"
+    );
+
+    // Same bytes, known extension: typed by mime_guess, never sniffed, still
+    // decoded through chardetng and stored.
+    let (state, sidecars, failures) = probe("legacy.txt");
+    assert_eq!(
+        (state, failures),
+        (1, 0),
+        "a legacy-charset .txt must still extract"
+    );
+    assert_eq!(sidecars, 1, "and must still store its text");
+
+    assert_eq!(probe("notes.md"), (1, 1, 0), "ordinary UTF-8 is unaffected");
+
+    std::fs::remove_dir_all(&root).ok();
+    std::fs::remove_dir_all(&db_dir).ok();
+}