Improved PPTX extraction and added comma separators to UI large numbers.
All checks were successful
CI / linux (push) Successful in 15m13s
CI / windows-cross (push) Successful in 8m14s
CI / release (push) Successful in 11s

This commit is contained in:
Jeremy Karst 2026-09-18 18:09:05 -04:00
parent eb1500324f
commit a4dad44a50
17 changed files with 820 additions and 47 deletions

4
Cargo.lock generated
View file

@ -3108,7 +3108,7 @@ dependencies = [
[[package]] [[package]]
name = "quicksearch-core" name = "quicksearch-core"
version = "1.1.8" version = "1.1.9"
dependencies = [ dependencies = [
"argon2", "argon2",
"cfb", "cfb",
@ -3150,7 +3150,7 @@ dependencies = [
[[package]] [[package]]
name = "quicksearch-gui" name = "quicksearch-gui"
version = "1.1.8" version = "1.1.9"
dependencies = [ dependencies = [
"chrono", "chrono",
"eframe", "eframe",

View file

@ -12,7 +12,7 @@ pdf-extract = { path = "vendor/pdf-extract" } # Patched unbounded reads which co
rtf-parser = { path = "vendor/rtf-parser" } # Patched a parsing error which occurs on UTF-16 characters rtf-parser = { path = "vendor/rtf-parser" } # Patched a parsing error which occurs on UTF-16 characters
[workspace.package] [workspace.package]
version = "1.1.8" version = "1.1.9"
edition = "2021" edition = "2021"
license = "GPL-3.0-or-later" license = "GPL-3.0-or-later"
authors = ["Jeremy <jeremy@karsttech.com>"] authors = ["Jeremy <jeremy@karsttech.com>"]

View file

@ -86,7 +86,7 @@ hash_length = 8192
maximum_text_size = 262144 maximum_text_size = 262144
# Files larger than this skip text extraction entirely (bytes). Clamped to # Files larger than this skip text extraction entirely (bytes). Clamped to
# 1..4294967296 on load. # 1..4294967296 on load.
maximum_text_file_size = 2097152 maximum_text_file_size = 52428800
# Files per batch during walks / inserts / extraction. # Files per batch during walks / inserts / extraction.
batch_size = 500 batch_size = 500
# Writer time one indexing root's turn may take before the round-robin # Writer time one indexing root's turn may take before the round-robin

View file

@ -24,9 +24,12 @@ pub struct IndexWork {
/// no walk and no root range would ever reach it again, so every row /// no walk and no root range would ever reach it again, so every row
/// outside the roots goes. /// outside the roots goes.
pub drop_aliases: bool, pub drop_aliases: bool,
/// `content_extensions` changed. Re-tested both ways: newly-included /// `content_extensions` or `maximum_text_file_size` changed. Re-tested
/// files go back to pending, newly-excluded ones give up text and FTS /// both ways: newly-included files go back to pending, newly-excluded
/// but keep the name/path row filename search needs. /// ones give up text and FTS but keep the name/path row filename search
/// needs. Both settings decide the same thing — whether a file's text is
/// read — and [`crate::scope::advance`] re-runs the whole predicate,
/// size included, so one flag covers them.
pub reconcile_content: bool, pub reconcile_content: bool,
/// `store_text_for_snippets` turned on; rows extracted under the old /// `store_text_for_snippets` turned on; rows extracted under the old
/// setting kept no text, so they must run again. /// setting kept no text, so they must run again.
@ -91,7 +94,9 @@ impl IndexWork {
parts.push("symlinks no longer followed".into()); parts.push("symlinks no longer followed".into());
} }
if self.reconcile_content { if self.reconcile_content {
parts.push("changed content extensions".into()); // Either setting that decides it; the flag does not say which,
// and the effect on the index is the same line of work.
parts.push("changed which files have their text read".into());
} }
if self.restore_text { if self.restore_text {
parts.push("snippet text turned on".into()); parts.push("snippet text turned on".into());
@ -213,7 +218,15 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
let old_content = content_filter_set(&old.indexing.content_extensions); let old_content = content_filter_set(&old.indexing.content_extensions);
let new_content = content_filter_set(&new.indexing.content_extensions); let new_content = content_filter_set(&new.indexing.content_extensions);
let content_widened = filter_widened(&old_content, &new_content); let content_widened = filter_widened(&old_content, &new_content);
work.reconcile_content = old_content != new_content;
// The size cap decides the same thing the extension filter does —
// whether a file's text is read — so changing it has to be applied
// the same way. Without this a raised cap did nothing to an existing
// index: every file already written off as too large stays `NA`,
// because nothing re-asks until that file itself changes.
let old_cap = old.processing.maximum_text_file_size;
let new_cap = new.processing.maximum_text_file_size;
work.reconcile_content = old_content != new_content || old_cap != new_cap;
let old_store = old.processing.store_text_for_snippets; let old_store = old.processing.store_text_for_snippets;
let new_store = new.processing.store_text_for_snippets; let new_store = new.processing.store_text_for_snippets;
@ -228,6 +241,10 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
|| (!old.indexing.include_hidden && new.indexing.include_hidden) || (!old.indexing.include_hidden && new.indexing.include_hidden)
|| (!old.indexing.follow_symlinks && new.indexing.follow_symlinks) || (!old.indexing.follow_symlinks && new.indexing.follow_symlinks)
|| content_widened || content_widened
// A raised cap widens the same way an added extension does: the
// reconcile puts those rows back to pending, and the content
// pass that reads them runs as part of an indexing run.
|| new_cap > old_cap
|| work.restore_text; || work.restore_text;
} }

View file

@ -236,7 +236,7 @@ impl Default for ProcessingConfig {
ProcessingConfig { ProcessingConfig {
hash_length: 1024 * 8, hash_length: 1024 * 8,
maximum_text_size: 1024 * 256, maximum_text_size: 1024 * 256,
maximum_text_file_size: 1024 * 1024 * 2, maximum_text_file_size: 1024 * 1024 * 50,
batch_size: 500, batch_size: 500,
writer_turn_slice_ms: 100, writer_turn_slice_ms: 100,
maximum_wal_size: 1024 * 1024 * 1024 * 2, maximum_wal_size: 1024 * 1024 * 1024 * 2,

View file

@ -220,6 +220,50 @@ fn comment_only_edit_is_no_work_at_all() {
assert!(a.work.reconcile_content && !a.work.reindex); assert!(a.work.reconcile_content && !a.work.reindex);
} }
/// The size cap decides what gets read exactly as the extension filter
/// does, so it has to be applied the same way. It was not: a raised cap left
/// every file already written off as too large sitting at `NA`, and nothing
/// re-asks about a file until the file itself changes — so the setting
/// appeared to do nothing at all on an existing index.
#[test]
fn changing_the_text_file_size_cap_re_decides_the_stored_rows() {
let old = Config::default();
let mut raised = old.clone();
raised.processing.maximum_text_file_size = old.processing.maximum_text_file_size * 2;
let a = diff_actions(&old, &raised);
assert!(
!a.requires_rebuild,
"no rebuild: the rows are all still right"
);
assert!(
a.work.reconcile_content,
"rows written off as too large must be re-decided"
);
assert!(
a.work.reindex,
"widening needs a run: the content pass is what reads the newly eligible files"
);
let mut lowered = old.clone();
lowered.processing.maximum_text_file_size = old.processing.maximum_text_file_size / 2;
let a = diff_actions(&old, &lowered);
assert!(
a.work.reconcile_content,
"rows now over the cap must give up their text"
);
assert!(
!a.work.reindex,
"narrowing finds nothing new on disk, so there is nothing to walk for"
);
assert_eq!(
diff_actions(&old, &old.clone()),
ConfigActions::default(),
"an unchanged cap is not work"
);
}
/// An empty list means "everything allowed" — a superset of every other /// An empty list means "everything allowed" — a superset of every other
/// list, the case plain set arithmetic reads backwards. /// list, the case plain set arithmetic reads backwards.
#[test] #[test]

View file

@ -23,7 +23,7 @@ pub type ExtractError = String;
/// ///
/// Every extractor used to carry its own 64 MiB constants, chosen /// Every extractor used to carry its own 64 MiB constants, chosen
/// independently of the settings that actually bound the work: the content /// independently of the settings that actually bound the work: the content
/// pass never offers a file above `maximum_text_file_size` (2 MiB by /// pass never offers a file above `maximum_text_file_size` (50 MiB by
/// default), and everything an extractor produces past `maximum_text_size` /// default), and everything an extractor produces past `maximum_text_size`
/// (256 KiB) is discarded by the caller moments later. A ceiling 256× above /// (256 KiB) is discarded by the caller moments later. A ceiling 256× above
/// the largest result that can be kept is not a safety margin, it is the /// the largest result that can be kept is not a safety margin, it is the

View file

@ -2,8 +2,19 @@
//! containers holding XML. Five share one event loop ([`collect_xml_text`], //! containers holding XML. Five share one event loop ([`collect_xml_text`],
//! driven by a per-format [`TextSpec`]); XLSX keeps its own two loops because //! driven by a per-format [`TextSpec`]); XLSX keeps its own two loops because
//! its text is not in the sheet at all but in a shared-string table the cells //! its text is not in the sheet at all but in a shared-string table the cells
//! index into. Dispatch is by file extension, not MIME: `.docm` carries the //! index into.
//! same MIME as `.docx`, and the extension is what distinguishes them. //!
//! Each of those formats has template and macro-enabled siblings holding the
//! same parts under the same names — `.pptx`, `.ppsx`, `.pptm`, `.potx` are
//! one parser — so every extension an office suite writes is routed here
//! rather than only the six default ones.
//!
//! Dispatch is by file extension, not MIME, because the pre-2007 formats
//! give a document and its template one MIME between them: `.doc` and `.dot`
//! are both `application/msword`, and only the name says which parser to
//! run. [`is_office_mime`] decides *whether* to run, the extension decides
//! *what*, and the two tables have to stay in step — the test at the bottom
//! of this file is what holds them there.
use std::error::Error; use std::error::Error;
use std::fs::File; use std::fs::File;
@ -18,19 +29,43 @@ use super::{ExtractError, Extractor, Scratch};
pub struct OfficeExtractor; pub struct OfficeExtractor;
fn mime_to_ext(mime: &str) -> Option<&'static str> { /// Every MIME this extractor claims, spelled lowercase because
match mime { /// [`super::Registry::find`] folds case before dispatch — the registered
"application/msword" => Some("doc"), /// spelling of the macro-enabled types is `macroEnabled`.
"application/vnd.openxmlformats-officedocument.wordprocessingml.document" => Some("docx"), ///
"application/vnd.ms-excel" => Some("xls"), /// Membership only: it answers "is this an office document", never which
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => Some("xlsx"), /// parser runs. See the module docs for why that is the extension's job.
"application/vnd.ms-powerpoint" => Some("ppt"), fn is_office_mime(mime: &str) -> bool {
"application/vnd.openxmlformats-officedocument.presentationml.presentation" => Some("pptx"), matches!(
"application/vnd.oasis.opendocument.text" => Some("odt"), mime,
"application/vnd.oasis.opendocument.spreadsheet" => Some("ods"), // Word: .doc/.dot, .docx/.dotx, .docm/.dotm
"application/vnd.oasis.opendocument.presentation" => Some("odp"), "application/msword"
_ => None, | "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
} | "application/vnd.openxmlformats-officedocument.wordprocessingml.template"
| "application/vnd.ms-word.document.macroenabled.12"
| "application/vnd.ms-word.template.macroenabled.12"
// Excel: .xls/.xlt, .xlsx/.xltx, .xlsm/.xltm
| "application/vnd.ms-excel"
| "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
| "application/vnd.openxmlformats-officedocument.spreadsheetml.template"
| "application/vnd.ms-excel.sheet.macroenabled.12"
| "application/vnd.ms-excel.template.macroenabled.12"
// PowerPoint: .ppt/.pps/.pot, .pptx/.ppsx/.potx, .pptm/.ppsm/.potm
| "application/vnd.ms-powerpoint"
| "application/vnd.openxmlformats-officedocument.presentationml.presentation"
| "application/vnd.openxmlformats-officedocument.presentationml.slideshow"
| "application/vnd.openxmlformats-officedocument.presentationml.template"
| "application/vnd.ms-powerpoint.presentation.macroenabled.12"
| "application/vnd.ms-powerpoint.slideshow.macroenabled.12"
| "application/vnd.ms-powerpoint.template.macroenabled.12"
// OpenDocument: .odt/.ott, .ods/.ots, .odp/.otp
| "application/vnd.oasis.opendocument.text"
| "application/vnd.oasis.opendocument.text-template"
| "application/vnd.oasis.opendocument.spreadsheet"
| "application/vnd.oasis.opendocument.spreadsheet-template"
| "application/vnd.oasis.opendocument.presentation"
| "application/vnd.oasis.opendocument.presentation-template"
)
} }
// The shared XML text walk // The shared XML text walk
@ -286,6 +321,19 @@ fn collect_members(
Ok(()) Ok(())
} }
/// A deck's slides, then its speaker notes.
///
/// Notes live in parts of their own (`ppt/notesSlides/`) that the slide text
/// never mentions, so a deck whose argument is in the notes was previously
/// indexed with none of it.
///
/// They are appended in a block rather than placed under the slide each
/// belongs to. A note is bound to its slide by that slide's `.rels` and
/// **not** by its number — PowerPoint numbers notes parts in creation order,
/// so a deck whose only note is on slide 5 stores it as `notesSlide1.xml`.
/// Reading a relationship per slide to get the pairing right buys a search
/// nothing, and guessing it from the number would file notes under the wrong
/// slide in a snippet, which is worse than not claiming to know.
fn extract_pptx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<(), Box<dyn Error>> { fn extract_pptx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<(), Box<dyn Error>> {
let limits = scratch.limits(); let limits = scratch.limits();
let mut archive = open_container(path)?; let mut archive = open_container(path)?;
@ -301,6 +349,19 @@ fn extract_pptx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<
out.push_str("\n--- New Slide ---\n"); out.push_str("\n--- New Slide ---\n");
Ok(()) Ok(())
}, },
)?;
// The marker leads here, where the slides' trails: it introduces the note
// that follows it, rather than separating one from the next.
collect_members(
&mut archive,
"ppt/notesSlides/notesSlide",
out,
limits.text,
|archive, name, out| {
member_bytes(archive, name, limits.inflate, bytes)?;
out.push_str("\n--- Speaker Notes ---\n");
collect_xml_text(xml_over(bytes), &PPTX, out, limits.text, events)
},
) )
} }
@ -475,6 +536,13 @@ fn extract_xlsx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<
/// Extract text from an office document, chosen by lowercase extension. An /// Extract text from an office document, chosen by lowercase extension. An
/// unhandled extension yields empty text: the MIME was claimed, so the file /// unhandled extension yields empty text: the MIME was claimed, so the file
/// was simply named unlike its type. /// was simply named unlike its type.
///
/// A template (`.dotx`, `.xltx`, `.potx`) and a macro-enabled document
/// (`.docm`, `.xlsm`, `.pptm`) hold their text in the same parts under the
/// same names as the plain form, so each is an alias rather than a format.
/// Leaving them off this list did not make them fail loudly — a claimed MIME
/// falling through to `_` is a file recorded as *successfully* holding no
/// text, which nothing later retries.
fn extract_document_text( fn extract_document_text(
path: &Path, path: &Path,
extension: &str, extension: &str,
@ -482,20 +550,26 @@ fn extract_document_text(
scratch: &mut Scratch, scratch: &mut Scratch,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
match extension { match extension {
"docx" => single_member(path, "word/document.xml", &DOCX, out, scratch), "docx" | "docm" | "dotx" | "dotm" => {
"xlsx" => extract_xlsx(path, out, scratch), single_member(path, "word/document.xml", &DOCX, out, scratch)
"pptx" => extract_pptx(path, out, scratch), }
"odt" | "odp" => single_member(path, "content.xml", &ODF_TEXT, out, scratch), "xlsx" | "xlsm" | "xltx" | "xltm" => extract_xlsx(path, out, scratch),
"ods" => single_member(path, "content.xml", &ODF_SHEET, out, scratch), "pptx" | "pptm" | "ppsx" | "ppsm" | "potx" | "potm" => extract_pptx(path, out, scratch),
"odt" | "ott" | "odp" | "otp" => {
single_member(path, "content.xml", &ODF_TEXT, out, scratch)
}
"ods" | "ots" => single_member(path, "content.xml", &ODF_SHEET, out, scratch),
// Pre-2007 binary formats: a different container entirely. // Pre-2007 binary formats: a different container entirely.
"doc" | "xls" | "ppt" => super::ole::extract_ole_text(path, extension, out, scratch), "doc" | "dot" | "xls" | "xlt" | "ppt" | "pps" | "pot" => {
super::ole::extract_ole_text(path, extension, out, scratch)
}
_ => Ok(()), _ => Ok(()),
} }
} }
impl Extractor for OfficeExtractor { impl Extractor for OfficeExtractor {
fn supports(&self, mime: &str) -> bool { fn supports(&self, mime: &str) -> bool {
mime_to_ext(mime).is_some() is_office_mime(mime)
} }
fn extract( fn extract(
@ -732,6 +806,14 @@ mod tests {
<a:p><a:r><a:t>Body</a:t></a:r></a:p>\ <a:p><a:r><a:t>Body</a:t></a:r></a:p>\
</p:txBody></p:sp></p:spTree></p:cSld></p:sld>"; </p:txBody></p:sp></p:spTree></p:cSld></p:sld>";
/// A notes part: the note itself, then the slide-number field every
/// notes slide carries. The field's text is indexed like any other —
/// suppressing it would mean suppressing dates and footers too.
const PPTX_NOTES: &str = "<p:notes><p:cSld><p:spTree>\
<p:sp><p:txBody><a:p><a:r><a:t>Mention the budget</a:t></a:r></a:p></p:txBody></p:sp>\
<p:sp><p:txBody><a:p><a:fld><a:t>3</a:t></a:fld></a:p></p:txBody></p:sp>\
</p:spTree></p:cSld></p:notes>";
const ODT_BODY: &str = "<office:document-content><office:body><office:text>\ const ODT_BODY: &str = "<office:document-content><office:body><office:text>\
<text:h>Heading</text:h>\ <text:h>Heading</text:h>\
<text:p>Para<text:span>span</text:span></text:p>\ <text:p>Para<text:span>span</text:span></text:p>\
@ -785,6 +867,127 @@ mod tests {
); );
} }
/// Speaker notes are in parts of their own that the slide text never
/// mentions, so a deck arguing its case in the notes used to be indexed
/// with none of it.
#[test]
fn pptx_extracts_speaker_notes() {
let p = container(
"pptx-notes",
"pptx",
&[
("ppt/slides/slide1.xml", PPTX_SLIDE),
("ppt/notesSlides/notesSlide1.xml", PPTX_NOTES),
],
);
assert_eq!(
extract_document_text(&p, "pptx").unwrap(),
"Title\nBody\n\n--- New Slide ---\n\n--- Speaker Notes ---\nMention the budget\n3\n"
);
}
/// The notes prefix must select notes *slides* and nothing that sits
/// beside them: the master is boilerplate on every deck, and a `.rels`
/// is a manifest whose text is filenames.
#[test]
fn the_notes_master_and_rels_are_not_notes() {
let p = container(
"pptx-notes-neighbours",
"pptx",
&[
("ppt/slides/slide1.xml", PPTX_SLIDE),
(
"ppt/notesMasters/notesMaster1.xml",
"<p:notesMaster><a:p><a:r><a:t>MASTERTEXT</a:t></a:r></a:p></p:notesMaster>",
),
(
"ppt/notesSlides/_rels/notesSlide1.xml.rels",
"<Relationships><Relationship Target=\"RELSTEXT\"/></Relationships>",
),
("ppt/notesSlides/notesSlide1.xml", PPTX_NOTES),
],
);
let out = extract_document_text(&p, "pptx").unwrap();
assert!(out.contains("Mention the budget"), "{:?}", out);
assert!(
!out.contains("MASTERTEXT"),
"the notes master leaked: {:?}",
out
);
assert!(
!out.contains("RELSTEXT"),
"a rels manifest leaked: {:?}",
out
);
}
/// A deck with no notes at all must read exactly as it did before notes
/// were collected — no stray marker, no trailing blank run.
#[test]
fn a_deck_without_notes_is_unchanged() {
let p = container(
"pptx-no-notes",
"pptx",
&[("ppt/slides/slide1.xml", PPTX_SLIDE)],
);
assert_eq!(
extract_document_text(&p, "pptx").unwrap(),
"Title\nBody\n\n--- New Slide ---\n"
);
}
/// Templates and macro-enabled documents hold the same parts under the
/// same names, so each name is an alias for one parser. Leaving one off
/// the dispatch did not fail loudly: the MIME was still claimed, so the
/// file extracted to nothing and was recorded as *successfully* holding
/// no text, which nothing later retries.
#[test]
fn template_and_macro_enabled_names_reach_the_same_parser() {
/// One parser's aliases: the names, the members of a container to
/// try them against, and the text every one of them must produce.
type AliasCase = (
&'static [&'static str],
&'static [(&'static str, &'static str)],
&'static str,
);
let cases: &[AliasCase] = &[
(
&["docx", "docm", "dotx", "dotm"],
&[("word/document.xml", DOCX_BODY)],
"Helloworld\nSecond\n",
),
(
&["pptx", "pptm", "ppsx", "ppsm", "potx", "potm"],
&[("ppt/slides/slide1.xml", PPTX_SLIDE)],
"Title\nBody\n\n--- New Slide ---\n",
),
(
&["xlsx", "xlsm", "xltx", "xltm"],
&[
("xl/sharedStrings.xml", XLSX_SHARED),
("xl/worksheets/sheet1.xml", XLSX_SHEET),
],
"Shared 42 \nSecond \n",
),
(
&["odt", "ott", "odp", "otp"],
&[("content.xml", ODT_BODY)],
"Heading\nParaspan\n",
),
(&["ods", "ots"], &[("content.xml", ODS_BODY)], "A1 \nB1 \n"),
];
for (extensions, members, expected) in cases {
for ext in *extensions {
let p = container(&format!("alias-{ext}"), ext, members);
assert_eq!(
&extract_document_text(&p, ext).unwrap(),
expected,
".{ext} did not reach the same parser as its siblings"
);
}
}
}
#[test] #[test]
fn odt_and_odp_are_the_same_extraction() { fn odt_and_odp_are_the_same_extraction() {
let odt = container("odt", "odt", &[("content.xml", ODT_BODY)]); let odt = container("odt", "odt", &[("content.xml", ODT_BODY)]);
@ -980,6 +1183,54 @@ mod tests {
); );
} }
/// Every name this extractor handles, walked end to end: the sniffer
/// names the file, [`is_office_mime`] decides whether to run, and the
/// dispatch in [`extract_document_text`] decides what runs. All three
/// have to agree for one document to be indexed, and disagreeing costs
/// nothing visible at either end — an unclaimed MIME is never offered to
/// an extractor at all, and a claimed MIME whose extension the dispatch
/// does not know yields empty text on a file recorded as done.
const CLAIMED: &[&str] = &[
"docx", "docm", "dotx", "dotm", "xlsx", "xlsm", "xltx", "xltm", "pptx", "pptm", "ppsx",
"ppsm", "potx", "potm", "odt", "ott", "ods", "ots", "odp", "otp", "doc", "dot", "xls",
"xlt", "ppt", "pps",
];
#[test]
fn every_claimed_name_is_sniffed_claimed_and_parsed() {
// Nothing opens it: a dispatched extension fails on a missing file,
// while the fall-through arm reports success with no text. That is
// what separates "no parser" from "parser found nothing".
let missing = Path::new("/nonexistent/quicksearch-office-probe");
for ext in CLAIMED {
let named = std::path::PathBuf::from(format!("probe.{ext}"));
let guessed = crate::mime::guess_mime_from_head(&named, b"")
.unwrap_or_else(|| panic!(".{ext} is sniffed as nothing at all"));
// Lowercased the way the registry lowercases before dispatch:
// the registered spelling is `macroEnabled`.
assert!(
OfficeExtractor.supports(&guessed.to_ascii_lowercase()),
".{ext} is sniffed as {guessed}, which no extractor claims"
);
assert!(
extract_document_text(missing, ext).is_err(),
".{ext} is claimed but falls through to the no-op arm, so a real \
one would be stored as holding no text"
);
}
}
/// `.pot` is the one name shared with an unrelated text format (gettext
/// templates), so it is sniffed from content rather than the name — the
/// PowerPoint one has to survive that.
#[test]
fn a_gettext_template_is_not_a_powerpoint_template() {
let text = b"msgid \"\"\nmsgstr \"\"\n\"Project-Id-Version: x\\n\"\n";
let mime = crate::mime::guess_mime_from_head(Path::new("messages.pot"), text);
assert_eq!(mime, Some("text/plain"));
assert!(!OfficeExtractor.supports("text/plain"));
}
/// A self-closed `<text:p/>` has no `End` to hang the break on. /// A self-closed `<text:p/>` has no `End` to hang the break on.
#[test] #[test]
fn a_self_closed_paragraph_still_breaks_the_line() { fn a_self_closed_paragraph_still_breaks_the_line() {

View file

@ -1,5 +1,6 @@
//! Legacy binary Office formats: `.doc`, `.xls`, `.ppt` — OLE2 compound //! Legacy binary Office formats: `.doc`, `.xls`, `.ppt` and the template and
//! files; three parsers sharing a reader, wanting only the *text*. Every //! slideshow names for the same three containers — OLE2 compound files; three
//! parsers sharing a reader, wanting only the *text*. Every
//! offset and count in these formats is attacker-controlled, so every read is //! offset and count in these formats is attacker-controlled, so every read is
//! bounds-checked, declared lengths are clamped, nothing is preallocated from //! bounds-checked, declared lengths are clamped, nothing is preallocated from
//! a declared count, and a malformed file yields `Err` — never a partial //! a declared count, and a malformed file yields `Err` — never a partial
@ -26,10 +27,14 @@ pub fn extract_ole_text(
let budget = scratch.limits().text; let budget = scratch.limits().text;
let mut cfb = cfb::CompoundFile::open(File::open(path)?) let mut cfb = cfb::CompoundFile::open(File::open(path)?)
.map_err(|e| format!("not a readable OLE2 compound file: {}", e))?; .map_err(|e| format!("not a readable OLE2 compound file: {}", e))?;
// A template holds the same streams as the document written from it, and
// a `.pps` is a `.ppt` that opens in the slideshow: one parser each, three
// names apiece. They share a MIME with the plain form, so leaving them out
// meant a claimed file extracting to nothing at all.
let text = match extension { let text = match extension {
"doc" => doc::extract(&mut cfb, budget), "doc" | "dot" => doc::extract(&mut cfb, budget),
"xls" => xls::extract(&mut cfb, budget), "xls" | "xlt" => xls::extract(&mut cfb, budget),
"ppt" => ppt::extract(&mut cfb, budget), "ppt" | "pps" | "pot" => ppt::extract(&mut cfb, budget),
other => Err(format!("no OLE2 parser for .{}", other).into()), other => Err(format!("no OLE2 parser for .{}", other).into()),
}?; }?;
if out.is_empty() { if out.is_empty() {

View file

@ -361,6 +361,69 @@ fn ppt_reads_both_atom_widths() {
assert!(text.contains("Ωmega body"), "{text}"); assert!(text.contains("Ωmega body"), "{text}");
} }
/// A template and a slideshow are the same container under another name, and
/// they share the plain form's MIME — so a name the dispatch did not know was
/// claimed, opened, and extracted to nothing rather than failing.
#[test]
fn legacy_alias_names_reach_the_same_parser() {
let bytes = encoding_rs::WINDOWS_1252
.encode("Slide title")
.0
.into_owned();
let deck = container(
"ole-alias-ppt",
"ppt",
&[(
"PowerPoint Document",
ppt_record(0x0000, ppt::TEXT_BYTES_ATOM, &bytes),
)],
);
for ext in ["ppt", "pps", "pot"] {
assert!(
extract_ole_text(&deck, ext)
.unwrap()
.contains("Slide title"),
".{ext} did not reach the PowerPoint parser"
);
}
// The trailing `\r` is Word's paragraph mark, which extracts as `\n`.
let prose = container(
"ole-alias-doc",
"doc",
&word_doc(&[("Hello from Word\r", true)]),
);
for ext in ["doc", "dot"] {
assert_eq!(
extract_ole_text(&prose, ext).unwrap(),
"Hello from Word\n",
".{ext} did not reach the Word parser"
);
}
let mut sst = Vec::new();
sst.extend_from_slice(&le32(1));
sst.extend_from_slice(&le32(1));
sst.extend_from_slice(&sst_string("Revenue", false));
let mut book = biff(xls::REC_SST, &sst);
book.extend_from_slice(&biff(xls::REC_LABELSST, &labelsst(0)));
let sheet = container("ole-alias-xls", "xls", &[("Workbook", book)]);
for ext in ["xls", "xlt"] {
assert!(
extract_ole_text(&sheet, ext).unwrap().contains("Revenue"),
".{ext} did not reach the Excel parser"
);
}
}
/// An extension no parser handles must say so, not come back empty.
#[test]
fn an_unknown_extension_is_an_error() {
let deck = container("ole-unknown", "ppt", &[("PowerPoint Document", vec![])]);
let err = extract_ole_text(&deck, "zzz").unwrap_err().to_string();
assert!(err.contains("no OLE2 parser"), "{err}");
}
#[test] #[test]
fn ppt_descends_into_containers() { fn ppt_descends_into_containers() {
let bytes = encoding_rs::WINDOWS_1252 let bytes = encoding_rs::WINDOWS_1252

View file

@ -218,9 +218,16 @@ pub fn mime_to_type(mime: &str) -> FileType {
// Subtype-based classification for the `application/*` grab bag. // Subtype-based classification for the `application/*` grab bag.
match sub { match sub {
// Office formats // Office formats
// Each of the three families covers its template and macro-enabled
// names too: they are the same format, and a `.pptm` the Presentation
// filter hid would be a deck the user cannot find by type.
"msword" "msword"
| "vnd.openxmlformats-officedocument.wordprocessingml.document" | "vnd.openxmlformats-officedocument.wordprocessingml.document"
| "vnd.openxmlformats-officedocument.wordprocessingml.template"
| "vnd.ms-word.document.macroenabled.12"
| "vnd.ms-word.template.macroenabled.12"
| "vnd.oasis.opendocument.text" | "vnd.oasis.opendocument.text"
| "vnd.oasis.opendocument.text-template"
| "rtf" | "rtf"
| "pdf" | "pdf"
| "epub+zip" | "epub+zip"
@ -229,12 +236,22 @@ pub fn mime_to_type(mime: &str) -> FileType {
} }
"vnd.ms-excel" "vnd.ms-excel"
| "vnd.openxmlformats-officedocument.spreadsheetml.sheet" | "vnd.openxmlformats-officedocument.spreadsheetml.sheet"
| "vnd.oasis.opendocument.spreadsheet" => { | "vnd.openxmlformats-officedocument.spreadsheetml.template"
| "vnd.ms-excel.sheet.macroenabled.12"
| "vnd.ms-excel.template.macroenabled.12"
| "vnd.oasis.opendocument.spreadsheet"
| "vnd.oasis.opendocument.spreadsheet-template" => {
t |= FileType::DOCUMENT | FileType::SPREADSHEET; t |= FileType::DOCUMENT | FileType::SPREADSHEET;
} }
"vnd.ms-powerpoint" "vnd.ms-powerpoint"
| "vnd.openxmlformats-officedocument.presentationml.presentation" | "vnd.openxmlformats-officedocument.presentationml.presentation"
| "vnd.oasis.opendocument.presentation" => { | "vnd.openxmlformats-officedocument.presentationml.slideshow"
| "vnd.openxmlformats-officedocument.presentationml.template"
| "vnd.ms-powerpoint.presentation.macroenabled.12"
| "vnd.ms-powerpoint.slideshow.macroenabled.12"
| "vnd.ms-powerpoint.template.macroenabled.12"
| "vnd.oasis.opendocument.presentation"
| "vnd.oasis.opendocument.presentation-template" => {
t |= FileType::DOCUMENT | FileType::PRESENTATION; t |= FileType::DOCUMENT | FileType::PRESENTATION;
} }
// Outlook saved messages and compiled HTML help are documents. // Outlook saved messages and compiled HTML help are documents.
@ -312,6 +329,79 @@ mod tests {
assert!(t.contains(FileType::PRESENTATION)); assert!(t.contains(FileType::PRESENTATION));
} }
/// Every name an office suite writes classifies as the format it is, or
/// the type filter hides a deck the extractor happily read. Case is the
/// registered spelling — `macroEnabled` — because that is what reaches
/// the classifier from [`guess_mime_from_head`].
#[test]
fn template_and_macro_enabled_names_classify_as_their_format() {
let cases: &[(&str, FileType)] = &[
(
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
FileType::PRESENTATION,
),
(
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
FileType::PRESENTATION,
),
(
"application/vnd.openxmlformats-officedocument.presentationml.template",
FileType::PRESENTATION,
),
(
"application/vnd.ms-powerpoint.slideshow.macroEnabled.12",
FileType::PRESENTATION,
),
(
"application/vnd.ms-powerpoint.template.macroEnabled.12",
FileType::PRESENTATION,
),
(
"application/vnd.oasis.opendocument.presentation-template",
FileType::PRESENTATION,
),
(
"application/vnd.ms-excel.sheet.macroEnabled.12",
FileType::SPREADSHEET,
),
(
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
FileType::SPREADSHEET,
),
(
"application/vnd.ms-excel.template.macroEnabled.12",
FileType::SPREADSHEET,
),
(
"application/vnd.oasis.opendocument.spreadsheet-template",
FileType::SPREADSHEET,
),
(
"application/vnd.ms-word.document.macroEnabled.12",
FileType::DOCUMENT,
),
(
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
FileType::DOCUMENT,
),
(
"application/vnd.ms-word.template.macroEnabled.12",
FileType::DOCUMENT,
),
(
"application/vnd.oasis.opendocument.text-template",
FileType::DOCUMENT,
),
];
for (mime, expected) in cases {
let t = mime_to_type(mime);
assert!(t.contains(*expected), "{} is not classified", mime);
// Presentations and spreadsheets are documents as well, the way
// Baloo counts them.
assert!(t.contains(FileType::DOCUMENT), "{} is not a document", mime);
}
}
#[test] #[test]
fn html_is_text_and_document() { fn html_is_text_and_document() {
let t = mime_to_type("text/html"); let t = mime_to_type("text/html");

View file

@ -55,7 +55,7 @@ files are still right.
`sample.ppt` is ~460 KB because the PowerPoint 97 export filter embeds the `sample.ppt` is ~460 KB because the PowerPoint 97 export filter embeds the
master slide. There is no filter option that trims it, and it is well inside master slide. There is no filter option that trims it, and it is well inside
the 2 MiB `maximum_text_file_size` the end-to-end run indexes with. the 50 MiB `maximum_text_file_size` the end-to-end run indexes with.
Unlike the generated half of the corpus, the text in these three is fixed: Unlike the generated half of the corpus, the text in these three is fixed:
`QUICKSEARCH_CORPUS_SEED` does not affect them. `QUICKSEARCH_CORPUS_SEED` does not affect them.

View file

@ -1914,3 +1914,143 @@ fn repeated_warm_reindexes_do_not_grow_the_fts_index() {
sizes sizes
); );
} }
// ---------------------------------------------------------------------------
// Container size against extraction cost
// ---------------------------------------------------------------------------
/// A presentation carrying media: one slide of text inside megabytes of
/// picture, which is the shape every real deck has.
fn media_heavy_deck(path: &Path, media_bytes: usize) {
let file = std::fs::File::create(path).unwrap();
let mut zip = zip::ZipWriter::new(file);
zip.start_file("ppt/slides/slide1.xml", zip::write::FileOptions::default())
.unwrap();
std::io::Write::write_all(
&mut zip,
b"<p:sld><p:cSld><p:spTree><p:sp><p:txBody>\
<a:p><a:r><a:t>Quarterly chalcedony forecast</a:t></a:r></a:p>\
</p:txBody></p:sp></p:spTree></p:cSld></p:sld>",
)
.unwrap();
// Stored, not deflated: the file on disk has to really be this large, or
// the size gate under test never sees it.
zip.start_file(
"ppt/media/image1.png",
zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored),
)
.unwrap();
std::io::Write::write_all(&mut zip, &vec![0u8; media_bytes]).unwrap();
zip.finish().unwrap();
}
/// The content pass gates on a file's *size*, which for a container is its
/// media and not its text: a 5 MiB deck holds about 10 KiB of words, and
/// reading them costs a seek through the central directory plus the one
/// member named. A cap tuned to prose skipped every deck in the tree —
/// indexed by name, with nothing inside findable and no failure to show for
/// it. This pins the default cap against the shape that exposed it.
#[test]
fn a_media_heavy_presentation_is_indexed_under_the_default_cap() {
let root = Scratch::dir("deck-media");
let db_dir = Scratch::dir("deck-media-db");
let db = db_dir.join("index.sqlite");
let deck = root.join("quarterly.pptx");
media_heavy_deck(&deck, 5 * 1024 * 1024);
let config = Config::default();
let size = std::fs::metadata(&deck).unwrap().len();
assert!(
size > 2 * 1024 * 1024,
"the fixture is only a test of the cap if it is large: {} bytes",
size
);
assert!(
size <= config.processing.maximum_text_file_size,
"a {} byte deck is over the {} byte default cap, so its text is skipped",
size,
config.processing.maximum_text_file_size
);
index_once(&root, &db, &config);
let text = stored_text(&db, "quarterly.pptx").expect("the deck stored no text at all");
assert!(
text.contains("Quarterly chalcedony forecast"),
"the slide's text is not what was stored: {:?}",
text
);
let conn = rusqlite::Connection::open(&db).unwrap();
let hits: i64 = conn
.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
["\"chalcedony\""],
|r| r.get(0),
)
.unwrap();
assert_eq!(hits, 1, "a word inside the deck is not searchable");
}
/// Raising the cap has to reach an index that already wrote a file off. The
/// row is `NA` and nothing re-asks about a file until the file itself
/// changes, so without a reconcile the setting looks like it does nothing:
/// the deck stays findable by name only, with no failure to explain it.
#[test]
fn raising_the_size_cap_reaches_files_already_written_off() {
use std::sync::atomic::AtomicBool;
let root = Scratch::dir("cap-raise");
let db_dir = Scratch::dir("cap-raise-db");
let db = db_dir.join("index.sqlite");
media_heavy_deck(&root.join("quarterly.pptx"), 5 * 1024 * 1024);
// A cap under the deck's size: its text is never read. The root goes in
// the config too — `WorkCursor` scans the roots the config names, not
// the ones an indexing run was handed.
let mut tight = Config::default();
tight.paths.indexing_paths = vec![root.to_string_lossy().into_owned()];
tight.processing.maximum_text_file_size = 1024 * 1024;
index_once(&root, &db, &tight);
assert!(
stored_text(&db, "quarterly.pptx").is_none(),
"the fixture has to start out skipped, or this proves nothing"
);
// Raised, the way Settings raises it: one field, nothing else.
let mut raised = tight.clone();
raised.processing.maximum_text_file_size = Config::default().processing.maximum_text_file_size;
let work = quicksearch_core::config::diff_actions(&tight, &raised).work;
assert!(
work.reconcile_content && work.reindex,
"a raised cap must ask for both halves: {:?}",
work
);
let mut conn = rusqlite::Connection::open(&db).unwrap();
let registry = quicksearch_core::extract::Registry::default_set();
let mut cursor = quicksearch_core::scope::WorkCursor::new(work, &raised).unwrap();
let cancel = AtomicBool::new(false);
while !cursor.done() {
quicksearch_core::scope::advance(
&mut conn,
&raised,
&registry,
&mut cursor,
Instant::now(),
&cancel,
)
.unwrap();
}
drop(conn);
index_once(&root, &db, &raised);
let text = stored_text(&db, "quarterly.pptx")
.expect("the deck is still skipped after the cap was raised");
assert!(
text.contains("Quarterly chalcedony forecast"),
"the deck's text is not what was stored: {:?}",
text
);
}

View file

@ -65,6 +65,9 @@ pub fn fmt_interval(minutes: u64) -> String {
} }
/// `1,234,567`. /// `1,234,567`.
///
/// Read as well as written: the byte fields in Settings are *edited* in this
/// form, so [`parse_grouped`] has to take back whatever this puts out.
pub fn group_thousands(n: u64) -> String { pub fn group_thousands(n: u64) -> String {
let digits = n.to_string(); let digits = n.to_string();
let mut out = String::with_capacity(digits.len() + digits.len() / 3); let mut out = String::with_capacity(digits.len() + digits.len() / 3);
@ -77,6 +80,30 @@ pub fn group_thousands(n: u64) -> String {
out out
} }
/// [`group_thousands`] read back, whether or not the separators were typed:
/// `52,428,800`, `52428800` and `52 428 800` are one number.
///
/// `None` for anything that is not a whole number — including the empty
/// string a half-cleared field holds. A `DragValue` whose parser declines
/// keeps the value it had, so a number being retyped cannot land in the
/// config as a zero on its way through.
pub fn parse_grouped(text: &str) -> Option<u64> {
let mut digits = String::with_capacity(text.len());
for c in text.chars() {
// Every separator someone might type or paste in, including the
// narrow no-break space a spreadsheet copies out.
if c == ',' || c == '_' || c == '\u{202f}' || c.is_whitespace() {
continue;
}
if !c.is_ascii_digit() {
return None;
}
digits.push(c);
}
// `parse` turns down the empty string and anything past `u64::MAX`.
digits.parse().ok()
}
/// Files/sec. A nonzero rate never renders as "0.0": slow rates go per-minute. /// Files/sec. A nonzero rate never renders as "0.0": slow rates go per-minute.
pub fn fmt_rate(files_per_sec: f64) -> String { pub fn fmt_rate(files_per_sec: f64) -> String {
if files_per_sec <= 0.0 { if files_per_sec <= 0.0 {
@ -205,6 +232,66 @@ mod tests {
assert_eq!(group_thousands(1_234_567), "1,234,567"); assert_eq!(group_thousands(1_234_567), "1,234,567");
} }
/// A typed number goes in whether or not it is punctuated — the field
/// shows separators, and someone retyping the value will not.
#[test]
fn grouped_numbers_parse_with_or_without_their_separators() {
assert_eq!(parse_grouped("52,428,800"), Some(52_428_800));
assert_eq!(parse_grouped("52428800"), Some(52_428_800));
assert_eq!(parse_grouped("52 428 800"), Some(52_428_800));
assert_eq!(parse_grouped("52\u{202f}428\u{202f}800"), Some(52_428_800));
assert_eq!(parse_grouped("52_428_800"), Some(52_428_800));
// Separators are stripped, not validated: the field is for entering
// a number, not for spelling one correctly.
assert_eq!(parse_grouped("5,2,4,2,8,8,0,0"), Some(52_428_800));
assert_eq!(parse_grouped(" 1024 "), Some(1024));
assert_eq!(parse_grouped("0"), Some(0));
}
/// Anything that is not a whole number leaves the field's value alone.
#[test]
fn a_partial_or_bogus_entry_declines() {
// Empty is what a cleared field holds mid-edit; taking it as zero
// would write a zero to the config between keystrokes.
assert_eq!(parse_grouped(""), None);
assert_eq!(parse_grouped(" "), None);
assert_eq!(parse_grouped(","), None);
assert_eq!(parse_grouped("1.5"), None);
assert_eq!(parse_grouped("-1"), None);
assert_eq!(parse_grouped("2 MB"), None);
assert_eq!(parse_grouped("1e6"), None);
assert_eq!(parse_grouped("٤٢"), None);
// Past `u64::MAX`, rather than wrapping to something plausible.
assert_eq!(parse_grouped("18,446,744,073,709,551,616"), None);
}
/// What the byte fields do every frame: render the stored number, then
/// read back exactly it. Every default and clamp bound, plus the widest
/// value any of those fields accepts.
#[test]
fn every_byte_field_value_survives_the_round_trip() {
for n in [
0,
1,
512,
1024,
8192,
262_144,
2_097_152,
52_428_800,
1_073_741_824,
8_589_934_592,
u64::MAX,
] {
assert_eq!(
parse_grouped(&group_thousands(n)),
Some(n),
"{} did not survive being displayed",
n
);
}
}
#[test] #[test]
fn rates_never_show_zero_for_nonzero() { fn rates_never_show_zero_for_nonzero() {
assert_eq!(fmt_rate(0.0), "0 files/s"); assert_eq!(fmt_rate(0.0), "0 files/s");

View file

@ -1,6 +1,7 @@
//! The Settings tab. Edits happen on a draft; Apply validates, saves, and //! The Settings tab. Edits happen on a draft; Apply validates, saves, and
//! hands the new config to the app. //! hands the new config to the app.
use crate::format;
use crate::keychain; use crate::keychain;
use crate::tips::{self, tip_row, Tipped}; use crate::tips::{self, tip_row, Tipped};
use crate::ui_util::hint; use crate::ui_util::hint;
@ -61,6 +62,39 @@ impl Form {
ui.add(egui::DragValue::new(value).range(range)) ui.add(egui::DragValue::new(value).range(range))
}); });
} }
/// [`Form::drag`] for a byte count, grouped as `52,428,800`.
///
/// Only the byte fields get this. They are the ones whose values run to
/// eight and ten digits, where a misplaced zero is invisible; the rest
/// of the form counts files, minutes and milliseconds and reads better
/// plain.
///
/// The number still goes in either way — [`format::parse_grouped`]
/// takes it with the separators or without — so a pasted `52,428,800`
/// and a typed `52428800` are the same edit.
fn drag_bytes<N: egui::emath::Numeric>(
self,
level: Level,
ui: &mut egui::Ui,
label: impl Into<egui::WidgetText>,
tip: &'static tips::Tip,
value: &mut N,
range: std::ops::RangeInclusive<N>,
) {
self.row(level, ui, label, tip, |ui| {
ui.add(
egui::DragValue::new(value)
.range(range)
// `DragValue` renders and parses through `f64`. Byte
// counts here are bounded by the config's own clamps (8
// GiB at the largest), far inside the 2^53 an `f64`
// holds exactly, so the round trip cannot lose a digit.
.custom_formatter(|n, _| format::group_thousands(n.max(0.0) as u64))
.custom_parser(|s| format::parse_grouped(s).map(|n| n as f64)),
)
});
}
} }
/// A label ruled underneath in the palette's orange. /// A label ruled underneath in the palette's orange.
@ -884,7 +918,7 @@ fn config_editor_ui(
ui.end_row(); ui.end_row();
} }
form.drag( form.drag_bytes(
Level::Advanced, Level::Advanced,
ui, ui,
"Hash sample size (bytes)", "Hash sample size (bytes)",
@ -893,7 +927,7 @@ fn config_editor_ui(
512..=1_048_576, 512..=1_048_576,
); );
form.drag( form.drag_bytes(
Level::Advanced, Level::Advanced,
ui, ui,
"Max stored text (bytes)", "Max stored text (bytes)",
@ -902,7 +936,7 @@ fn config_editor_ui(
1024..=16_777_216, 1024..=16_777_216,
); );
form.drag( form.drag_bytes(
Level::Advanced, Level::Advanced,
ui, ui,
"Max text file size (bytes)", "Max text file size (bytes)",
@ -920,7 +954,7 @@ fn config_editor_ui(
10..=100_000, 10..=100_000,
); );
form.drag( form.drag_bytes(
Level::Advanced, Level::Advanced,
ui, ui,
"Max WAL size (bytes)", "Max WAL size (bytes)",

View file

@ -706,6 +706,46 @@ fn advanced_rows_are_hidden_until_asked_for() {
); );
} }
/// The byte fields are shown grouped: their values run to eight and ten
/// digits, where a misplaced zero is invisible. Everything else in the form
/// counts files or milliseconds and stays plain, which is the other half of
/// the assertion — the grouping is scoped to the fields that needed it.
#[test]
fn only_the_byte_fields_are_shown_with_thousands_separators() {
let ctx = crate::test_ui::ctx();
let mut cfg = Config::default();
// The top of its range, so the plain rendering is long enough that a
// separator would show up if one were being inserted.
cfg.processing.batch_size = 100_000;
let out = ctx.run(
crate::test_ui::raw_input(egui::vec2(600.0, 800.0), vec![]),
|ctx| {
egui::CentralPanel::default().show(ctx, |ui| {
config_editor_ui(
ui,
&mut cfg,
Section::Processing,
None,
Form { advanced: true },
)
});
},
);
let text = painted_text(&out).join("\n");
// Hash sample size, max stored text, max text file size, max WAL size.
for grouped in ["8,192", "262,144", "52,428,800", "2,147,483,648"] {
assert!(
text.contains(grouped),
"{grouped} is not on screen; the byte fields read: {text}"
);
}
assert!(
text.contains("100000") && !text.contains("100,000"),
"batch size counts files, not bytes, and must stay plain: {text}"
);
}
/// Revealing a setting is not editing one: the toggle writes through /// Revealing a setting is not editing one: the toggle writes through
/// `SettingsOutput` and must never make the tab read as dirty, or looking at /// `SettingsOutput` and must never make the tab read as dirty, or looking at
/// an advanced setting would demand an Apply. /// an advanced setting would demand an Apply.

View file

@ -211,8 +211,10 @@ pub static MAX_TEXT_FILE_SIZE: Tip = Tip {
Those files still appear in results, found by their name, size or \ Those files still appear in results, found by their name, size or \
date.", date.",
examples: &[ examples: &[
"2097152, 2 MB, skips very few ordinary documents.", "52428800, 50 MB, the default, reads inside large log files, scanned PDFs \
"52428800, 50 MB, when you search inside large log files or scanned PDFs.", and presentations whose size is mostly pictures.",
"2097152, 2 MB, to open ordinary documents only and leave anything larger \
to be found by name.",
], ],
caution: None, caution: None,
}; };