940 lines
38 KiB
Rust
940 lines
38 KiB
Rust
//! Office document extraction: DOCX, XLSX, PPTX, ODT, ODP, ODS.
|
||
//!
|
||
//! All six are zip containers holding XML, and five of the six want the same
|
||
//! thing from it: the character data of a few named elements, with a newline
|
||
//! where a paragraph closes. That shape lives in [`collect_xml_text`], driven
|
||
//! by a per-format [`TextSpec`], so there is one event loop rather than one
|
||
//! per format.
|
||
//!
|
||
//! XLSX is the exception and keeps its own two loops: its text is not in the
|
||
//! sheet at all but in a shared-string table the cells index into, which is a
|
||
//! different machine, not a different table of element names.
|
||
//!
|
||
//! Dispatch is by file extension rather than MIME. `.docm` carries the same
|
||
//! MIME as `.docx` but needs the same reader, and the extension is what
|
||
//! distinguishes them.
|
||
|
||
use std::error::Error;
|
||
use std::fs::File;
|
||
use std::io::{BufReader, Read, Seek};
|
||
use std::path::Path;
|
||
|
||
use quick_xml::events::Event;
|
||
use quick_xml::Reader;
|
||
use zip::ZipArchive;
|
||
|
||
use super::{ExtractError, ExtractedContent, Extractor};
|
||
|
||
pub struct OfficeExtractor;
|
||
|
||
fn mime_to_ext(mime: &str) -> Option<&'static str> {
|
||
match mime {
|
||
"application/msword" => Some("doc"),
|
||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document" => Some("docx"),
|
||
"application/vnd.ms-excel" => Some("xls"),
|
||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => Some("xlsx"),
|
||
"application/vnd.ms-powerpoint" => Some("ppt"),
|
||
"application/vnd.openxmlformats-officedocument.presentationml.presentation" => Some("pptx"),
|
||
"application/vnd.oasis.opendocument.text" => Some("odt"),
|
||
"application/vnd.oasis.opendocument.spreadsheet" => Some("ods"),
|
||
"application/vnd.oasis.opendocument.presentation" => Some("odp"),
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
// The shared XML text walk
|
||
|
||
/// Which elements of a format's XML carry text, and where paragraphs end.
|
||
///
|
||
/// `text` and `breaks` are matched independently on a closing tag: an element
|
||
/// can be in both (ODF's `text:p` both holds text and ends a paragraph), in
|
||
/// only one (`w:p` breaks but holds nothing directly), or in `text` alone
|
||
/// (`text:span`, which ends a run without ending the line).
|
||
struct TextSpec {
|
||
/// Elements whose character data is body text.
|
||
text: &'static [&'static [u8]],
|
||
/// Elements that close a paragraph, emitting `'\n'`.
|
||
breaks: &'static [&'static [u8]],
|
||
/// Emitted after each text run. Spreadsheets separate cells with it;
|
||
/// prose formats leave it `None` so runs within a paragraph stay joined.
|
||
separator: Option<char>,
|
||
}
|
||
|
||
const DOCX: TextSpec = TextSpec {
|
||
text: &[b"w:t"],
|
||
breaks: &[b"w:p"],
|
||
separator: None,
|
||
};
|
||
|
||
const PPTX: TextSpec = TextSpec {
|
||
text: &[b"a:t"],
|
||
breaks: &[b"a:p"],
|
||
separator: None,
|
||
};
|
||
|
||
/// ODT and ODP are the same format as far as text extraction is concerned —
|
||
/// both are ODF prose with headings, paragraphs and spans.
|
||
const ODF_TEXT: TextSpec = TextSpec {
|
||
text: &[b"text:p", b"text:h", b"text:span"],
|
||
breaks: &[b"text:p", b"text:h"],
|
||
separator: None,
|
||
};
|
||
|
||
const ODF_SHEET: TextSpec = TextSpec {
|
||
text: &[b"text:p", b"text:span"],
|
||
breaks: &[b"text:p"],
|
||
separator: Some(' '),
|
||
};
|
||
|
||
/// The text an `&entity;` or `Ӓ` reference stands for.
|
||
///
|
||
/// quick-xml 0.41 reports a reference as its own event instead of resolving
|
||
/// it inside the surrounding `Text`, so a reader that ignores this event
|
||
/// silently drops every `&`, `<` and `’` from the document —
|
||
/// no error, just missing characters in the index. Only the five predefined
|
||
/// entities and numeric references are resolvable without a DTD; anything
|
||
/// else is a document-defined entity we cannot expand, and is skipped.
|
||
fn entity_text(raw: &str) -> Option<String> {
|
||
if let Some(digits) = raw.strip_prefix('#') {
|
||
let code = match digits.strip_prefix(['x', 'X']) {
|
||
Some(hex) => u32::from_str_radix(hex, 16).ok()?,
|
||
None => digits.parse::<u32>().ok()?,
|
||
};
|
||
let c = char::from_u32(code)?;
|
||
// `char::from_u32` accepts far more than XML's character production
|
||
// does: every C0 control but tab, newline and carriage return is
|
||
// forbidden, and `�` in particular would put a literal NUL into the
|
||
// indexed text and from there into an FTS5 column. `None` here reaches
|
||
// the callers as the same "unknown entity" error an unexpandable name
|
||
// gets — a `failed_files` row naming the file, which is the visible
|
||
// outcome this extractor prefers to a quietly mangled document.
|
||
let legal = !c.is_control() || matches!(c, '\t' | '\n' | '\r');
|
||
return legal.then(|| String::from(c));
|
||
}
|
||
quick_xml::escape::resolve_predefined_entity(raw).map(String::from)
|
||
}
|
||
|
||
/// Append the text `spec` selects out of `xml` to `out`.
|
||
///
|
||
/// Text-bearing elements are counted, not flagged. ODF nests them — a
|
||
/// `<text:span>` inside a `<text:p>` — and a flag made the span's own close
|
||
/// end the run, dropping every character between it and the paragraph's
|
||
/// close. Counting also gives the separator somewhere honest to go: it
|
||
/// belongs after a *run*, and since quick-xml 0.41 a run arrives as several
|
||
/// events, so emitting one per event put a space in the middle of every cell
|
||
/// containing an entity.
|
||
fn collect_xml_text(xml: &str, spec: &TextSpec, out: &mut String) -> Result<(), Box<dyn Error>> {
|
||
let mut reader = Reader::from_str(xml);
|
||
// Deliberately no `trim_text`: it trims each *event*, and since 0.41 an
|
||
// entity reference splits the character data around it into separate
|
||
// events — so `Jack & Jill` would come back as `Jack&Jill`, with the
|
||
// spaces trimmed off the ends of the two fragments. Nothing needs it
|
||
// either: whitespace between elements arrives while the `in_text`/`in_cell`
|
||
// flag is false and is ignored there, and whitespace *inside* a
|
||
// text-bearing element is content.
|
||
let mut buf = Vec::new();
|
||
// How many text-bearing elements are open. The run ends when it returns
|
||
// to zero, not when the innermost one closes.
|
||
let mut depth = 0usize;
|
||
|
||
loop {
|
||
match reader.read_event_into(&mut buf) {
|
||
Ok(Event::Start(ref e)) => {
|
||
if spec.text.contains(&e.name().as_ref()) {
|
||
depth += 1;
|
||
}
|
||
}
|
||
Ok(Event::Text(e)) if depth > 0 => {
|
||
out.push_str(&e.decode()?);
|
||
}
|
||
// An entity reference is its own event in 0.41; without this arm
|
||
// every `&` in a document would vanish from the index.
|
||
Ok(Event::GeneralRef(e)) if depth > 0 => {
|
||
let raw = e.decode()?;
|
||
// An entity nothing can expand is an error, as it was when
|
||
// `unescape` resolved these inline: dropping it would take
|
||
// characters out of the indexed text with nothing to show for
|
||
// it, and this reader has no DTD to define one with.
|
||
let text = entity_text(&raw)
|
||
.ok_or_else(|| format!("Error parsing XML: unknown entity &{};", raw))?;
|
||
out.push_str(&text);
|
||
}
|
||
Ok(Event::End(ref e)) => {
|
||
let name = e.name();
|
||
if spec.text.contains(&name.as_ref()) {
|
||
depth = depth.saturating_sub(1);
|
||
// Closing the outermost one closes the run.
|
||
if depth == 0 {
|
||
if let Some(sep) = spec.separator {
|
||
out.push(sep);
|
||
}
|
||
}
|
||
}
|
||
if spec.breaks.contains(&name.as_ref()) {
|
||
out.push('\n');
|
||
}
|
||
}
|
||
// A self-closed element gets no `Start` and no `End` of its own,
|
||
// so `<text:p/>` — ODF's blank line — would otherwise lose both
|
||
// its separator and its paragraph break. It carries no text, so
|
||
// the run it opens is empty and closes immediately.
|
||
Ok(Event::Empty(ref e)) => {
|
||
let name = e.name();
|
||
if depth == 0 && spec.text.contains(&name.as_ref()) {
|
||
if let Some(sep) = spec.separator {
|
||
out.push(sep);
|
||
}
|
||
}
|
||
if spec.breaks.contains(&name.as_ref()) {
|
||
out.push('\n');
|
||
}
|
||
}
|
||
Ok(Event::Eof) => break,
|
||
// Propagated rather than ignored: the reader re-reports the same
|
||
// error without advancing, so ignoring it is a hang.
|
||
Err(e) => return Err(format!("Error parsing XML: {}", e).into()),
|
||
_ => {}
|
||
}
|
||
buf.clear();
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
// Container access
|
||
|
||
type Archive = ZipArchive<BufReader<File>>;
|
||
|
||
fn open_container(path: &Path) -> Result<Archive, Box<dyn Error>> {
|
||
Ok(ZipArchive::new(BufReader::new(File::open(path)?))?)
|
||
}
|
||
|
||
/// Cap on one decompressed member, mirroring `ole::MAX_TEXT_BYTES`: the zip
|
||
/// header declares sizes, but the deflate stream is what we actually read, so
|
||
/// a tiny archive can inflate without bound.
|
||
const MAX_XML_BYTES: usize = 64 * 1024 * 1024;
|
||
|
||
/// Cap on the text taken from one *container*, mirroring [`ole::MAX_TEXT_BYTES`].
|
||
///
|
||
/// [`MAX_XML_BYTES`] bounds each member on its own, which is not the same
|
||
/// thing: a workbook or a deck holds one member per sheet or per slide, and
|
||
/// nothing stops a small archive from carrying dozens that each inflate to
|
||
/// that cap. The truncation to `maximum_text_size` happens only after the
|
||
/// whole string is built and handed back, so without a running total the peak
|
||
/// is members × 64 MiB — gigabytes from a file measured in megabytes, on every
|
||
/// extraction worker at once, and an allocation failure aborts rather than
|
||
/// unwinding.
|
||
const MAX_TEXT_BYTES: usize = 64 * 1024 * 1024;
|
||
|
||
/// One member's bytes as a string. An over-cap member keeps its prefix.
|
||
fn member_text<R: Read + Seek>(
|
||
archive: &mut ZipArchive<R>,
|
||
name: &str,
|
||
) -> Result<String, Box<dyn Error>> {
|
||
let mut body = Vec::new();
|
||
archive
|
||
.by_name(name)?
|
||
.take(MAX_XML_BYTES as u64 + 1)
|
||
.read_to_end(&mut body)?;
|
||
let truncated = body.len() > MAX_XML_BYTES;
|
||
body.truncate(MAX_XML_BYTES);
|
||
match String::from_utf8(body) {
|
||
Ok(text) => Ok(text),
|
||
// Only a cut at the cap may split a character; invalid UTF-8 anywhere
|
||
// else still fails the extraction, as `read_to_string` always did.
|
||
Err(e) if truncated && e.utf8_error().valid_up_to() >= MAX_XML_BYTES - 3 => {
|
||
let valid = e.utf8_error().valid_up_to();
|
||
let mut bytes = e.into_bytes();
|
||
bytes.truncate(valid);
|
||
Ok(String::from_utf8(bytes)?)
|
||
}
|
||
Err(e) => Err(e.into()),
|
||
}
|
||
}
|
||
|
||
/// Names of the `.xml` members under `prefix`, in archive order.
|
||
///
|
||
/// Indexed rather than taken from `file_names()`, which iterates a hash map:
|
||
/// slide order is the archive's order, and hashing it would shuffle the
|
||
/// slides of every presentation.
|
||
fn xml_members_under<R: Read + Seek>(
|
||
archive: &mut ZipArchive<R>,
|
||
prefix: &str,
|
||
) -> Result<Vec<String>, Box<dyn Error>> {
|
||
let mut names = Vec::new();
|
||
for i in 0..archive.len() {
|
||
let name = archive.by_index(i)?.name().to_string();
|
||
if name.starts_with(prefix) && name.ends_with(".xml") {
|
||
names.push(name);
|
||
}
|
||
}
|
||
Ok(names)
|
||
}
|
||
|
||
/// A format whose whole text lives in one member under one spec.
|
||
fn single_member(path: &Path, member: &str, spec: &TextSpec) -> Result<String, Box<dyn Error>> {
|
||
let mut archive = open_container(path)?;
|
||
let xml = member_text(&mut archive, member)?;
|
||
let mut out = String::new();
|
||
collect_xml_text(&xml, spec, &mut out)?;
|
||
Ok(out)
|
||
}
|
||
|
||
fn extract_pptx(path: &Path) -> Result<String, Box<dyn Error>> {
|
||
let mut archive = open_container(path)?;
|
||
let mut out = String::new();
|
||
for name in xml_members_under(&mut archive, "ppt/slides/slide")? {
|
||
// Per-container budget: see `MAX_TEXT_BYTES`. Whole slides are kept or
|
||
// dropped rather than cut mid-way, which is why the test is here
|
||
// rather than inside the collector.
|
||
if out.len() >= MAX_TEXT_BYTES {
|
||
break;
|
||
}
|
||
let xml = member_text(&mut archive, &name)?;
|
||
collect_xml_text(&xml, &PPTX, &mut out)?;
|
||
out.push_str("\n--- New Slide ---\n");
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
// XLSX: shared strings plus cells
|
||
|
||
/// The workbook's shared-string table, in index order. Absent or unreadable
|
||
/// is not an error: a sheet of nothing but numbers has no table at all.
|
||
fn shared_strings<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Vec<String> {
|
||
let Ok(xml) = member_text(archive, "xl/sharedStrings.xml") else {
|
||
return Vec::new();
|
||
};
|
||
let mut reader = Reader::from_str(&xml);
|
||
// Deliberately no `trim_text`: it trims each *event*, and since 0.41 an
|
||
// entity reference splits the character data around it into separate
|
||
// events — so `Jack & Jill` would come back as `Jack&Jill`, with the
|
||
// spaces trimmed off the ends of the two fragments. Nothing needs it
|
||
// either: whitespace between elements arrives while the `in_text`/`in_cell`
|
||
// flag is false and is ignored there, and whitespace *inside* a
|
||
// text-bearing element is content.
|
||
let mut buf = Vec::new();
|
||
let mut strings = Vec::new();
|
||
let mut in_text = false;
|
||
// One `<t>` is one shared string, but it is not one event: an entity
|
||
// reference inside it arrives separately and splits the character data
|
||
// around it. Accumulated here and pushed on the closing tag, or a cell
|
||
// containing `&` would become three table entries and every later
|
||
// index would point at the wrong one.
|
||
let mut current = String::new();
|
||
loop {
|
||
match reader.read_event_into(&mut buf) {
|
||
Ok(Event::Start(ref e)) if e.name().as_ref() == b"t" => {
|
||
in_text = true;
|
||
current.clear();
|
||
}
|
||
// `<t/>` — an empty cell. quick-xml reports a self-closed element
|
||
// as its own event with no `Start` and no `End`, so without this
|
||
// arm the entry is never pushed and **every later index is off by
|
||
// one**: `collect_sheet` then renders a real string for the wrong
|
||
// cell, with nothing anywhere reporting a problem. LibreOffice,
|
||
// openpyxl and POI all write `<si><t/></si>` for a blank, so this
|
||
// is ordinary output rather than a crafted file.
|
||
Ok(Event::Empty(ref e)) if e.name().as_ref() == b"t" => {
|
||
strings.push(String::new());
|
||
}
|
||
Ok(Event::Text(e)) if in_text => match e.decode() {
|
||
Ok(s) => current.push_str(&s),
|
||
Err(_) => return strings,
|
||
},
|
||
Ok(Event::GeneralRef(e)) if in_text => {
|
||
// Unlike the other two readers this one cannot fail — a
|
||
// missing table is not an error here — so an entity nothing
|
||
// can expand is simply left out.
|
||
if let Ok(raw) = e.decode() {
|
||
if let Some(text) = entity_text(&raw) {
|
||
current.push_str(&text);
|
||
}
|
||
}
|
||
}
|
||
Ok(Event::End(ref e)) if e.name().as_ref() == b"t" => {
|
||
in_text = false;
|
||
strings.push(std::mem::take(&mut current));
|
||
}
|
||
Ok(Event::Eof) | Err(_) => break,
|
||
_ => {}
|
||
}
|
||
buf.clear();
|
||
}
|
||
strings
|
||
}
|
||
|
||
/// One worksheet's cells. A `t="s"` cell holds an index into `strings`
|
||
/// rather than text of its own; every other type holds its value inline.
|
||
fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(), Box<dyn Error>> {
|
||
let mut reader = Reader::from_str(xml);
|
||
// Deliberately no `trim_text`: it trims each *event*, and since 0.41 an
|
||
// entity reference splits the character data around it into separate
|
||
// events — so `Jack & Jill` would come back as `Jack&Jill`, with the
|
||
// spaces trimmed off the ends of the two fragments. Nothing needs it
|
||
// either: whitespace between elements arrives while the `in_text`/`in_cell`
|
||
// flag is false and is ignored there, and whitespace *inside* a
|
||
// text-bearing element is content.
|
||
let mut buf = Vec::new();
|
||
let mut in_cell = false;
|
||
let mut cell_type = String::new();
|
||
|
||
loop {
|
||
match reader.read_event_into(&mut buf) {
|
||
Ok(Event::Start(ref e)) if e.name().as_ref() == b"c" => {
|
||
in_cell = true;
|
||
cell_type.clear();
|
||
// `with_checks(false)`: the default duplicate-attribute-name
|
||
// check compares each name against every name already seen on
|
||
// the tag, which is quadratic in the count and has no bound
|
||
// but the tag's own size (RUSTSEC-2026-0194). A member may be
|
||
// 64 MiB of inflated XML, so one crafted `<c>` can hold
|
||
// millions of attributes and hold this worker for hours —
|
||
// uncancellably, since the stop flag is only read between
|
||
// files. Rejecting duplicate names was never this extractor's
|
||
// job; it wants one attribute and stops at it.
|
||
for attr in e.attributes().with_checks(false) {
|
||
let attr = attr?;
|
||
if attr.key.as_ref() == b"t" {
|
||
cell_type = String::from_utf8_lossy(&attr.value).to_string();
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
Ok(Event::Text(e)) if in_cell => {
|
||
let text = e.decode()?;
|
||
if cell_type == "s" {
|
||
// A shared-string reference. An index past the end of the
|
||
// table is a corrupt workbook, not something to guess at.
|
||
//
|
||
// `trim` because this reader no longer sets `trim_text`
|
||
// (see the comment above): a generator that indents its
|
||
// XML hands `<v>` over as "\n 0\n", and an untrimmed
|
||
// parse would fail and drop the string with nothing to
|
||
// show for it. Whitespace around an integer index is not
|
||
// content, unlike whitespace inside a `<t>`.
|
||
if let Some(s) = text
|
||
.trim()
|
||
.parse::<usize>()
|
||
.ok()
|
||
.and_then(|i| strings.get(i))
|
||
{
|
||
out.push_str(s);
|
||
out.push(' ');
|
||
}
|
||
} else if !text.trim().is_empty() {
|
||
// Whitespace-only fragments are the indentation *between*
|
||
// a cell's child elements, which reaches this arm now that
|
||
// the reader no longer sets `trim_text`. Skipped rather
|
||
// than pushed: `in_cell` is a flag, so it cannot tell an
|
||
// indent from a value, and a cell whose entire content is
|
||
// whitespace contributes nothing to a search index either
|
||
// way. The value itself is pushed whole — no `trim` — so a
|
||
// deliberate `xml:space="preserve"` inline string keeps
|
||
// its shape.
|
||
out.push_str(&text);
|
||
out.push(' ');
|
||
}
|
||
}
|
||
// See `entity_text`. Only inline values can carry one: a `t="s"`
|
||
// cell's text is an integer index, and an entity inside it would
|
||
// be a corrupt workbook rather than a character to recover.
|
||
Ok(Event::GeneralRef(e)) if in_cell && cell_type != "s" => {
|
||
let raw = e.decode()?;
|
||
let text = entity_text(&raw)
|
||
.ok_or_else(|| format!("Error parsing XML: unknown entity &{};", raw))?;
|
||
out.push_str(&text);
|
||
}
|
||
Ok(Event::End(ref e)) => {
|
||
let name = e.name();
|
||
if name.as_ref() == b"c" {
|
||
in_cell = false;
|
||
} else if name.as_ref() == b"row" {
|
||
out.push('\n');
|
||
}
|
||
}
|
||
Ok(Event::Eof) => break,
|
||
Err(e) => return Err(format!("Error parsing XML: {}", e).into()),
|
||
_ => {}
|
||
}
|
||
buf.clear();
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn extract_xlsx(path: &Path) -> Result<String, Box<dyn Error>> {
|
||
let mut archive = open_container(path)?;
|
||
let strings = shared_strings(&mut archive);
|
||
let mut out = String::new();
|
||
for name in xml_members_under(&mut archive, "xl/worksheets/sheet")? {
|
||
// Per-container budget: see `MAX_TEXT_BYTES`.
|
||
if out.len() >= MAX_TEXT_BYTES {
|
||
break;
|
||
}
|
||
let xml = member_text(&mut archive, &name)?;
|
||
collect_sheet(&xml, &strings, &mut out)?;
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
// Dispatch
|
||
|
||
/// Extract text from an office document, chosen by lowercase extension.
|
||
///
|
||
/// An extension nothing here handles yields empty text rather than an error:
|
||
/// the caller reaches this only for a MIME [`mime_to_ext`] claimed, so an
|
||
/// unrecognized extension means the file was named unlike its type.
|
||
fn extract_document_text(path: &Path, extension: &str) -> Result<String, Box<dyn Error>> {
|
||
match extension {
|
||
"docx" => single_member(path, "word/document.xml", &DOCX),
|
||
"xlsx" => extract_xlsx(path),
|
||
"pptx" => extract_pptx(path),
|
||
"odt" | "odp" => single_member(path, "content.xml", &ODF_TEXT),
|
||
"ods" => single_member(path, "content.xml", &ODF_SHEET),
|
||
// Pre-2007 binary formats: a different container entirely.
|
||
"doc" | "xls" | "ppt" => super::ole::extract_ole_text(path, extension),
|
||
_ => Ok(String::new()),
|
||
}
|
||
}
|
||
|
||
impl Extractor for OfficeExtractor {
|
||
fn supports(&self, mime: &str) -> bool {
|
||
mime_to_ext(mime).is_some()
|
||
}
|
||
|
||
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
|
||
// The extension from the path, not from the MIME: `.docm` and `.docx`
|
||
// share a MIME but the dispatch above is by extension.
|
||
let ext = path
|
||
.extension()
|
||
.and_then(|s| s.to_str())
|
||
.map(|s| s.to_ascii_lowercase())
|
||
.unwrap_or_default();
|
||
let text = extract_document_text(path, &ext)
|
||
.map_err(|e| format!("office extractor {}: {}", path.display(), e))?;
|
||
Ok(ExtractedContent::with_text(text))
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use std::io::Write;
|
||
|
||
/// Entity references must survive extraction.
|
||
///
|
||
/// quick-xml 0.41 reports `&` as its own `GeneralRef` event instead of
|
||
/// resolving it into the surrounding text, so a reader that only handles
|
||
/// `Event::Text` loses the character with no error to show for it. This is
|
||
/// the test that makes that visible: it fails by producing "Blake Co"
|
||
/// rather than by failing to compile.
|
||
#[test]
|
||
fn entity_references_survive_extraction() {
|
||
let body = "<w:document><w:body><w:p><w:r>\
|
||
<w:t>Blake & Co <tags> ’24 ’25</w:t>\
|
||
</w:r></w:p></w:body></w:document>";
|
||
let path = container("docx-entities", "docx", &[("word/document.xml", body)]);
|
||
let out = OfficeExtractor.extract(&path).expect("extract");
|
||
assert!(
|
||
out.text.contains("Blake & Co"),
|
||
"predefined entity lost: {:?}",
|
||
out.text
|
||
);
|
||
assert!(
|
||
out.text.contains("<tags>"),
|
||
"angle-bracket entities lost: {:?}",
|
||
out.text
|
||
);
|
||
assert!(
|
||
out.text.contains('\u{2019}'),
|
||
"numeric entities lost: {:?}",
|
||
out.text
|
||
);
|
||
assert!(
|
||
!out.text.contains("&") && !out.text.contains("&#"),
|
||
"entities left unresolved: {:?}",
|
||
out.text
|
||
);
|
||
}
|
||
|
||
/// The same, through the shared-string table an `.xlsx` cell indexes into
|
||
/// — a separate reader, and so a separate chance to drop the character.
|
||
#[test]
|
||
fn entity_references_survive_shared_strings() {
|
||
let shared = "<sst><si><t>Jack & Jill</t></si></sst>";
|
||
let sheet = "<worksheet><sheetData><row>\
|
||
<c t=\"s\"><v>0</v></c></row></sheetData></worksheet>";
|
||
let path = container(
|
||
"xlsx-entities",
|
||
"xlsx",
|
||
&[
|
||
("xl/sharedStrings.xml", shared),
|
||
("xl/worksheets/sheet1.xml", sheet),
|
||
],
|
||
);
|
||
let out = OfficeExtractor.extract(&path).expect("extract");
|
||
assert!(
|
||
out.text.contains("Jack & Jill"),
|
||
"entity lost through the shared-string table: {:?}",
|
||
out.text
|
||
);
|
||
}
|
||
|
||
/// A shared-string reference must survive an indented `<v>`.
|
||
///
|
||
/// The reader deliberately does not set `trim_text` (an entity reference
|
||
/// splits the character data around it, and trimming each fragment would
|
||
/// eat the spaces at the split). A `t="s"` cell's `<v>` is an integer
|
||
/// index, though, so a generator that pretty-prints its sheet XML hands
|
||
/// this reader `"\n 0\n "` — and an untrimmed `parse::<usize>()`
|
||
/// fails, dropping the cell's text with no error and no `failed_files`
|
||
/// row. Whitespace-only fragments between a cell's children must not
|
||
/// reach the output either.
|
||
#[test]
|
||
fn an_indented_shared_string_reference_still_resolves() {
|
||
let shared = "<sst><si><t>Marmalade</t></si></sst>";
|
||
let sheet = "<worksheet>\n <sheetData>\n <row>\n \
|
||
<c t=\"s\">\n <v>\n 0\n </v>\n </c>\n \
|
||
<c t=\"n\">\n <v>17</v>\n </c>\n \
|
||
</row>\n </sheetData>\n</worksheet>";
|
||
let path = container(
|
||
"xlsx-indented",
|
||
"xlsx",
|
||
&[
|
||
("xl/sharedStrings.xml", shared),
|
||
("xl/worksheets/sheet1.xml", sheet),
|
||
],
|
||
);
|
||
let out = OfficeExtractor.extract(&path).expect("extract");
|
||
assert!(
|
||
out.text.contains("Marmalade"),
|
||
"the shared string was dropped by an indented index: {:?}",
|
||
out.text
|
||
);
|
||
assert!(
|
||
out.text.contains("17"),
|
||
"the inline value was dropped: {:?}",
|
||
out.text
|
||
);
|
||
// The indentation itself is not content: every run of whitespace in
|
||
// the output should be a separator this reader put there, never a
|
||
// line of the source XML's own layout.
|
||
assert!(
|
||
!out.text.contains("\n "),
|
||
"sheet indentation reached the indexed text: {:?}",
|
||
out.text
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn supports_docx_and_friends() {
|
||
let e = OfficeExtractor;
|
||
for m in [
|
||
"application/msword",
|
||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||
"application/vnd.oasis.opendocument.text",
|
||
] {
|
||
assert!(e.supports(m), "should support {}", m);
|
||
}
|
||
assert!(!e.supports("image/png"));
|
||
}
|
||
|
||
/// A zip container holding `members`, written to a scratch file.
|
||
fn container(tag: &str, ext: &str, members: &[(&str, &str)]) -> std::path::PathBuf {
|
||
let path = crate::testutil::scratch_dir(tag).join(format!("doc.{ext}"));
|
||
let file = File::create(&path).unwrap();
|
||
let mut zip = zip::ZipWriter::new(file);
|
||
for (name, body) in members {
|
||
zip.start_file(*name, zip::write::FileOptions::default())
|
||
.unwrap();
|
||
zip.write_all(body.as_bytes()).unwrap();
|
||
}
|
||
zip.finish().unwrap();
|
||
path
|
||
}
|
||
|
||
const DOCX_BODY: &str = "<w:document><w:body>\
|
||
<w:p><w:r><w:t>Hello</w:t></w:r><w:r><w:t>world</w:t></w:r></w:p>\
|
||
<w:p><w:r><w:t>Second</w:t></w:r></w:p>\
|
||
</w:body></w:document>";
|
||
|
||
const PPTX_SLIDE: &str = "<p:sld><p:cSld><p:spTree><p:sp><p:txBody>\
|
||
<a:p><a:r><a:t>Title</a:t></a:r></a:p>\
|
||
<a:p><a:r><a:t>Body</a:t></a:r></a:p>\
|
||
</p:txBody></p:sp></p:spTree></p:cSld></p:sld>";
|
||
|
||
const ODT_BODY: &str = "<office:document-content><office:body><office:text>\
|
||
<text:h>Heading</text:h>\
|
||
<text:p>Para<text:span>span</text:span></text:p>\
|
||
</office:text></office:body></office:document-content>";
|
||
|
||
const ODS_BODY: &str = "<office:document-content><office:body><office:spreadsheet>\
|
||
<table:table><table:table-row>\
|
||
<table:table-cell><text:p>A1</text:p></table:table-cell>\
|
||
<table:table-cell><text:p>B1</text:p></table:table-cell>\
|
||
</table:table-row></table:table>\
|
||
</office:spreadsheet></office:body></office:document-content>";
|
||
|
||
const XLSX_SHARED: &str = "<sst><si><t>Shared</t></si><si><t>Second</t></si></sst>";
|
||
|
||
/// The first entry is `<si><t/></si>` — a blank cell, which every
|
||
/// spreadsheet writer emits. It still occupies index 0.
|
||
const XLSX_SHARED_WITH_BLANK: &str = "<sst><si><t/></si><si><t>Second</t></si></sst>";
|
||
|
||
/// One cell, referring to shared string index 1.
|
||
const XLSX_SHEET_INDEX_1: &str = "<worksheet><sheetData>\
|
||
<row><c t=\"s\"><v>1</v></c></row>\
|
||
</sheetData></worksheet>";
|
||
|
||
const XLSX_SHEET: &str = "<worksheet><sheetData>\
|
||
<row><c t=\"s\"><v>0</v></c><c t=\"n\"><v>42</v></c></row>\
|
||
<row><c t=\"s\"><v>1</v></c></row>\
|
||
</sheetData></worksheet>";
|
||
|
||
// The golden set: pins the exact extraction shapes.
|
||
|
||
#[test]
|
||
fn docx() {
|
||
let p = container("docx", "docx", &[("word/document.xml", DOCX_BODY)]);
|
||
assert_eq!(
|
||
extract_document_text(&p, "docx").unwrap(),
|
||
"Helloworld\nSecond\n"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn pptx_marks_each_slide_and_keeps_archive_order() {
|
||
let p = container(
|
||
"pptx",
|
||
"pptx",
|
||
&[
|
||
("ppt/slides/slide1.xml", PPTX_SLIDE),
|
||
("ppt/slides/slide2.xml", PPTX_SLIDE),
|
||
],
|
||
);
|
||
assert_eq!(
|
||
extract_document_text(&p, "pptx").unwrap(),
|
||
"Title\nBody\n\n--- New Slide ---\nTitle\nBody\n\n--- New Slide ---\n"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn odt_and_odp_are_the_same_extraction() {
|
||
let odt = container("odt", "odt", &[("content.xml", ODT_BODY)]);
|
||
let odp = container("odp", "odp", &[("content.xml", ODT_BODY)]);
|
||
assert_eq!(
|
||
extract_document_text(&odt, "odt").unwrap(),
|
||
"Heading\nParaspan\n"
|
||
);
|
||
assert_eq!(
|
||
extract_document_text(&odt, "odt").unwrap(),
|
||
extract_document_text(&odp, "odp").unwrap(),
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn ods_separates_cells_with_a_space() {
|
||
let p = container("ods", "ods", &[("content.xml", ODS_BODY)]);
|
||
assert_eq!(extract_document_text(&p, "ods").unwrap(), "A1 \nB1 \n");
|
||
}
|
||
|
||
#[test]
|
||
fn xlsx_resolves_shared_strings() {
|
||
let p = container(
|
||
"xlsx",
|
||
"xlsx",
|
||
&[
|
||
("xl/sharedStrings.xml", XLSX_SHARED),
|
||
("xl/worksheets/sheet1.xml", XLSX_SHEET),
|
||
],
|
||
);
|
||
assert_eq!(
|
||
extract_document_text(&p, "xlsx").unwrap(),
|
||
"Shared 42 \nSecond \n"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn an_extension_nothing_handles_is_empty() {
|
||
let p = container("none", "bin", &[("whatever", "x")]);
|
||
assert_eq!(extract_document_text(&p, "zzz").unwrap(), "");
|
||
}
|
||
|
||
/// A workbook of pure numbers has no shared-string table. Its absence is
|
||
/// normal, not a failure.
|
||
#[test]
|
||
fn xlsx_without_a_shared_string_table_still_reads_its_cells() {
|
||
let p = container(
|
||
"xlsx-nosst",
|
||
"xlsx",
|
||
&[(
|
||
"xl/worksheets/sheet1.xml",
|
||
"<worksheet><sheetData>\
|
||
<row><c t=\"n\"><v>7</v></c></row></sheetData></worksheet>",
|
||
)],
|
||
);
|
||
assert_eq!(extract_document_text(&p, "xlsx").unwrap(), "7 \n");
|
||
}
|
||
|
||
/// A shared-string index past the end of the table is dropped rather than
|
||
/// panicking on the slice.
|
||
#[test]
|
||
fn an_out_of_range_shared_string_index_is_dropped() {
|
||
let p = container(
|
||
"xlsx-oob",
|
||
"xlsx",
|
||
&[
|
||
("xl/sharedStrings.xml", "<sst><si><t>only</t></si></sst>"),
|
||
(
|
||
"xl/worksheets/sheet1.xml",
|
||
"<worksheet><sheetData><row>\
|
||
<c t=\"s\"><v>0</v></c><c t=\"s\"><v>99</v></c>\
|
||
</row></sheetData></worksheet>",
|
||
),
|
||
],
|
||
);
|
||
assert_eq!(extract_document_text(&p, "xlsx").unwrap(), "only \n");
|
||
}
|
||
|
||
/// Malformed XML is an error, not a hang. An undefined entity inside a
|
||
/// text run is the cheapest way to reach the error arm, and it is a real
|
||
/// shape: tools do emit HTML entities into OOXML.
|
||
#[test]
|
||
fn malformed_xml_returns_an_error_rather_than_looping() {
|
||
for (ext, member, body) in [
|
||
(
|
||
"docx",
|
||
"word/document.xml",
|
||
"<w:t>bad &nonsuch; entity</w:t>",
|
||
),
|
||
(
|
||
"odt",
|
||
"content.xml",
|
||
"<text:p>bad &nonsuch; entity</text:p>",
|
||
),
|
||
(
|
||
"ods",
|
||
"content.xml",
|
||
"<text:p>bad &nonsuch; entity</text:p>",
|
||
),
|
||
(
|
||
"pptx",
|
||
"ppt/slides/slide1.xml",
|
||
"<a:t>bad &nonsuch; entity</a:t>",
|
||
),
|
||
] {
|
||
let p = container(&format!("bad-{ext}"), ext, &[(member, body)]);
|
||
assert!(
|
||
extract_document_text(&p, ext).is_err(),
|
||
"{ext} should report malformed XML"
|
||
);
|
||
}
|
||
}
|
||
|
||
/// Mismatched tags are caught too — quick_xml checks closing names.
|
||
#[test]
|
||
fn mismatched_tags_are_an_error() {
|
||
let p = container(
|
||
"mismatch",
|
||
"docx",
|
||
&[("word/document.xml", "<w:body><w:t>x</w:body>")],
|
||
);
|
||
assert!(extract_document_text(&p, "docx").is_err());
|
||
}
|
||
|
||
/// A container missing the member the format is defined by.
|
||
#[test]
|
||
fn a_missing_member_is_an_error() {
|
||
let p = container("empty", "docx", &[("unrelated.xml", "<x/>")]);
|
||
assert!(extract_document_text(&p, "docx").is_err());
|
||
}
|
||
|
||
/// Not a zip file at all — the shape a truncated download or a
|
||
/// misidentified file arrives in.
|
||
#[test]
|
||
fn a_non_container_is_an_error() {
|
||
let dir = crate::testutil::scratch_dir("notzip");
|
||
let p = dir.join("doc.docx");
|
||
crate::testutil::touch(&p, b"this is not a zip archive");
|
||
assert!(extract_document_text(&p, "docx").is_err());
|
||
}
|
||
|
||
/// A self-closed `<t/>` is a whole shared-string entry. Skip it and every
|
||
/// later index slides by one, so the sheet renders a real string for the
|
||
/// wrong cell — clean extraction, no error, wrong content. This is the
|
||
/// regression that made the batch worth doing.
|
||
#[test]
|
||
fn a_blank_shared_string_still_occupies_its_index() {
|
||
let p = container(
|
||
"xlsx-blank-si",
|
||
"xlsx",
|
||
&[
|
||
("xl/sharedStrings.xml", XLSX_SHARED_WITH_BLANK),
|
||
("xl/worksheets/sheet1.xml", XLSX_SHEET_INDEX_1),
|
||
],
|
||
);
|
||
assert_eq!(
|
||
extract_document_text(&p, "xlsx").unwrap(),
|
||
"Second \n",
|
||
"index 1 must still be the second entry"
|
||
);
|
||
}
|
||
|
||
/// The same shape one level up: a `<si>` holding nothing at all.
|
||
#[test]
|
||
fn an_empty_si_still_occupies_its_index() {
|
||
let p = container(
|
||
"xlsx-empty-si",
|
||
"xlsx",
|
||
&[
|
||
(
|
||
"xl/sharedStrings.xml",
|
||
"<sst><si><t></t></si><si><t>Second</t></si></sst>",
|
||
),
|
||
("xl/worksheets/sheet1.xml", XLSX_SHEET_INDEX_1),
|
||
],
|
||
);
|
||
assert_eq!(extract_document_text(&p, "xlsx").unwrap(), "Second \n");
|
||
}
|
||
|
||
/// An entity splits its run into three events. The separator belongs to
|
||
/// the run, so the cell must read `A&B` — not `A &B`, and not `A & B`.
|
||
#[test]
|
||
fn an_entity_does_not_split_an_ods_cell() {
|
||
let body = "<office:document-content><office:body><office:spreadsheet>\
|
||
<table:table><table:table-row>\
|
||
<table:table-cell><text:p>A&B</text:p></table:table-cell>\
|
||
</table:table-row></table:table>\
|
||
</office:spreadsheet></office:body></office:document-content>";
|
||
let p = container("ods-entity", "ods", &[("content.xml", body)]);
|
||
assert_eq!(extract_document_text(&p, "ods").unwrap(), "A&B \n");
|
||
}
|
||
|
||
/// A span closing inside a paragraph ends the span, not the paragraph:
|
||
/// the text after it is body text and must be indexed.
|
||
#[test]
|
||
fn text_after_a_nested_span_is_not_dropped() {
|
||
let body = "<office:document-content><office:body><office:text>\
|
||
<text:p>before<text:span>inside</text:span>after</text:p>\
|
||
</office:text></office:body></office:document-content>";
|
||
let p = container("odt-span-tail", "odt", &[("content.xml", body)]);
|
||
assert_eq!(
|
||
extract_document_text(&p, "odt").unwrap(),
|
||
"beforeinsideafter\n"
|
||
);
|
||
}
|
||
|
||
/// ODF writes a blank line as a self-closed `<text:p/>`, which has no
|
||
/// `End` to hang the paragraph break on.
|
||
#[test]
|
||
fn a_self_closed_paragraph_still_breaks_the_line() {
|
||
let body = "<office:document-content><office:body><office:text>\
|
||
<text:p>first</text:p><text:p/><text:p>third</text:p>\
|
||
</office:text></office:body></office:document-content>";
|
||
let p = container("odt-empty-p", "odt", &[("content.xml", body)]);
|
||
assert_eq!(
|
||
extract_document_text(&p, "odt").unwrap(),
|
||
"first\n\nthird\n"
|
||
);
|
||
}
|
||
}
|