jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 7 Packages Wiki Activity Actions

Performance optimization and hardening against malformed files.
Some checks failed
CI / linux (push) Successful in 9m21s
Details
CI / windows-cross (push) Successful in 4m21s
Details
CI / windows-cross (pull_request) Has been cancelled
Details
CI / release (pull_request) Has been cancelled
Details
CI / linux (pull_request) Has been cancelled
Details
CI / release (push) Successful in 12s
Details

Browse source
This commit is contained in:
= 2026-08-20 00:13:10 -04:00
parent 84900f4335
commit 3772eeb6a5
33 changed files with 14672 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Cargo.lock generated
View file

@ -2920,8 +2920,6 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "pdf-extract"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "417e8fdc940f1d5bc62c5f89864c3a2255f74f69aa353c98509213d67df61e73"
dependencies = [
"adobe-cmap-parser",
"cff-parser",

27
Cargo.toml
View file

@ -4,9 +4,34 @@ members = [
"crates/quicksearch-core",
"crates/quicksearch-gui",
]
# `vendor/pdf-extract` is deliberately NOT a member: it is a third-party crate
# carried here for one patch, not part of this workspace's lints, tests or
# release profile. `[patch.crates-io]` below is what makes the dependency graph
# resolve to it.
exclude = ["vendor/pdf-extract"]
# pdf-extract 0.12.0, with the two unbounded recursions in it bounded.
#
# `get_inherited` follows `/Parent` and `process_stream`'s `Do` arm follows
# Form XObjects, neither with a depth counter or a visited set. A page whose
# `/Parent` is itself, or an XObject whose content stream draws itself, walks
# the stack until it hits the guard page — and a stack overflow is not a panic
# that `catch_unwind` can contain (`extract/pdf.rs` has one, for the parser's
# ordinary panics): Rust's handler calls `abort()`, so a ~600-byte file kills
# the process. It recurs on every run, because the row keeps
# `content_state = 0` and the feeder selects exactly those; and the live
# watcher re-extracts on-screen rows on the GUI thread, so such a file crashes
# the app when it merely appears in a result list.
#
# Vendored rather than forked-by-URL so the build stays offline, `--locked`
# keeps meaning what it means, and the cross-compile job needs no new host.
# The patch is marked LOCAL PATCH in the source and is upstreamable; the crate
# is MIT and the copy is recorded in `packaging/copyright`.
[patch.crates-io]
pdf-extract = { path = "vendor/pdf-extract" }
[workspace.package]
version = "1.1.0"
version = "1.1.1"
edition = "2021"
license = "GPL-3.0-or-later"
authors = ["Jeremy <jeremy@karsttech.com>"]

47
crates/quicksearch-core/src/config/mod.rs
View file

@ -549,16 +549,31 @@ impl Config {
/// Write back to the file this config was loaded from (or the default
/// location), creating parent directories as needed. Raw values are
/// written verbatim — relative paths in a portable config stay relative.
///
/// Atomic: see the comment on the rename below.
pub fn save(&self) -> Result<(), String> {
let path = self.source.clone().unwrap_or_else(Self::config_path);
if let Some(dir) = path.parent() {
fs::create_dir_all(dir)
crate::platform::create_dir_private(dir)
.map_err(|e| format!("Failed to create config dir {}: {}", dir.display(), e))?;
}
let content = toml::to_string_pretty(self)
.map_err(|e| format!("Failed to serialize config: {}", e))?;
fs::write(&path, content)
.map_err(|e| format!("Failed to write config file {}: {}", path.display(), e))?;
// Written beside the target and renamed over it, rather than
// truncate-then-write. `[security].salt` exists *only* in this file:
// it is not derivable from the index and not stored anywhere else, so
// a config truncated by a crash, a full disk or a power cut in the
// middle of `write` is an encrypted index that no password can ever
// open again. `rename` is atomic on both platforms, and the `sync_all`
// before it means the bytes are on the disk before the name points at
// them.
let tmp = path.with_extension("toml.tmp");
write_private(&tmp, content.as_bytes())
.map_err(|e| format!("Failed to write config file {}: {}", tmp.display(), e))?;
fs::rename(&tmp, &path).map_err(|e| {
let _ = fs::remove_file(&tmp);
format!("Failed to replace config file {}: {}", path.display(), e)
})?;
Ok(())
}
@ -616,6 +631,32 @@ impl Config {
}
}
/// Write `bytes` to `path`, owner-readable only, and flush them to the disk
/// before returning.
///
/// `O_NOFOLLOW` on Unix: the config directory is not always somewhere only
/// this user can write — a portable install can sit in a shared or removable
/// directory — and a symlink left at the config's name would otherwise
/// redirect this write onto whatever it points at.
fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
use std::io::Write;
let mut opts = fs::OpenOptions::new();
opts.write(true).create(true).truncate(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.custom_flags(libc::O_NOFOLLOW);
opts.mode(0o600);
}
let mut f = opts.open(path)?;
f.write_all(bytes)?;
// The rename that follows is atomic with respect to the *directory*, not
// to the file's contents: without this, a crash can leave the new name
// pointing at a block of zeroes.
f.sync_all()
}
/// Reserved `content_extensions` entry standing for "files with no
/// extension". Matched case-insensitively, and it cannot collide with a real
/// extension because the parentheses are not part of one.

32
crates/quicksearch-core/src/coordinator.rs
View file

@ -36,7 +36,7 @@ use rusqlite::Connection;
use crate::config::{diff_actions, Config, IgnoreSet, IndexWork};
use crate::db;
use crate::extract::Registry;
use crate::incremental::apply_fs_event;
use crate::incremental::{apply_fs_event, Applied, Budget};
use crate::indexing::{ConfigChange, IndexingService, IndexingStatus, PrepStep, ReconcileProgress};
use crate::scope::WorkCursor;
use crate::watcher::{FsEvent, WatchError, WatchFilters, Watcher, WatcherConfig};
@ -230,6 +230,7 @@ impl IndexCoordinator {
watcher_gen: 0,
pending: HashMap::new(),
targeted: HashMap::new(),
resume_from: HashMap::new(),
last_event_at: None,
pending_since: None,
needs_full_run: false,
@ -378,6 +379,35 @@ impl Drop for IndexCoordinator {
}
}
/// The verb a path submitted through [`IndexCoordinator::update_paths`]
/// deserves, or `None` to leave the index alone.
///
/// The caller knows a file changed, not what it changed into. `is_file()` is
/// the fast answer and almost always the right one — one `stat`, and this runs
/// while results are on screen — but it folds every stat error into `false`,
/// and a `Remove` costs the row *and everything beneath it*. So the negative
/// answer, and only it, is confirmed with a second `stat` that can tell "gone"
/// from "cannot see it just now": a share that dropped, a drive pulled while
/// its rows were displayed, a parent another process chmod'd.
///
/// In doubt the index wins. A stale row is a wrong line on screen until the
/// next full run; a deleted live one is data no run brings back until the file
/// is walked again — and if the reason it could not be read was that its whole
/// tree went away, that walk will not reach it either.
fn verb_for(path: PathBuf) -> Option<FsEvent> {
if path.is_file() {
return Some(FsEvent::Modify(path));
}
// `metadata`, not `symlink_metadata`: it has to agree with `is_file()`
// above about following links, or the two can disagree about the verb.
match std::fs::metadata(&path) {
// There, but no longer something the walk would index.
Ok(_) => Some(FsEvent::Remove(path)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Some(FsEvent::Remove(path)),
Err(_) => None,
}
}
/// Fold `event` into the last-event-wins pending map. Renames split into
/// their halves so downstream application never needs pair handling.
fn enqueue(pending: &mut HashMap<PathBuf, FsEvent>, event: FsEvent) {

83
crates/quicksearch-core/src/coordinator/inner.rs
View file

@ -28,6 +28,11 @@ pub(super) struct Inner {
/// in manual mode, because it exists to keep the rows a user is *reading*
/// in step with the disk however the indexer is configured.
pub(super) targeted: HashMap<PathBuf, FsEvent>,
/// How far a directory event got before its turn's budget ran out, so the
/// next turn resumes rather than re-walking what it already applied. Keyed
/// by the same path as the queue the event went back into, and removed
/// when the event completes or is dropped.
pub(super) resume_from: HashMap<PathBuf, usize>,
/// When the most recent event arrived; the burst is over once this is
/// `pending_settle` old.
pub(super) last_event_at: Option<Instant>,
@ -181,14 +186,10 @@ impl Inner {
if !prefixes.iter().any(|lo| spelled.starts_with(lo.as_str())) {
continue;
}
// Existence decides the verb. The caller knows a file
// changed, not what it changed into, and a `Modify` for a
// path that is gone would be silently skipped rather than
// removing the row.
let event = if path.is_file() {
FsEvent::Modify(path)
} else {
FsEvent::Remove(path)
// Existence decides the verb; `verb_for` also decides when
// it cannot be decided at all, and says so with `None`.
let Some(event) = verb_for(path) else {
continue;
};
enqueue(&mut self.targeted, event);
}
@ -421,6 +422,11 @@ impl Inner {
// `clear` keeps the map's capacity — up to 100k slots after a storm;
// shrinking is the point.
self.pending.shrink_to_fit();
// Resume points describe events that no longer exist. Entries for
// `targeted` events survive, which is why this filters rather than
// clearing: that queue deliberately outlives this call.
self.resume_from
.retain(|p, _| self.targeted.contains_key(p));
self.last_event_at = None;
self.pending_since = None;
}
@ -490,13 +496,35 @@ impl Inner {
let Some(ev) = self.pending.remove(&path) else {
continue;
};
if let Err(e) =
apply_fs_event(&mut conn, &ev, &self.config, &self.ignore, &self.registry)
{
// As above: the event is out of `pending`, so only a full
// run still picks the file up.
crate::log_warn!("coordinator: apply {:?}: {}; scheduling full run", ev, e);
self.needs_full_run = true;
match apply_fs_event(
&mut conn,
&ev,
&self.config,
&self.ignore,
&self.registry,
&Budget {
deadline,
cancel: &self.reconcile_stop.cancel,
resume_from: self.resume_from.remove(&path).unwrap_or(0),
},
) {
// A directory event can cover a whole moved-in tree. Put
// it back, with a note of how far it got, and let the next
// tick continue it — so one `mv` cannot hold this loop, or
// the shutdown queued behind it, for as long as the tree
// takes.
Ok(Applied::Unfinished { done }) => {
self.resume_from.insert(path.clone(), done);
self.pending.insert(path, ev);
break;
}
Ok(Applied::Done) => {}
Err(e) => {
// As above: the event is out of `pending`, so only a
// full run still picks the file up.
crate::log_warn!("coordinator: apply {:?}: {}; scheduling full run", ev, e);
self.needs_full_run = true;
}
}
if Instant::now() >= deadline {
break;
@ -564,10 +592,27 @@ impl Inner {
let Some(ev) = self.targeted.remove(&path) else {
continue;
};
if let Err(e) =
apply_fs_event(&mut conn, &ev, &self.config, &self.ignore, &self.registry)
{
crate::log_warn!("coordinator: targeted apply {:?}: {}", ev, e);
match apply_fs_event(
&mut conn,
&ev,
&self.config,
&self.ignore,
&self.registry,
&Budget {
deadline,
cancel: &self.reconcile_stop.cancel,
resume_from: self.resume_from.remove(&path).unwrap_or(0),
},
) {
Ok(Applied::Unfinished { done }) => {
self.resume_from.insert(path.clone(), done);
self.targeted.insert(path, ev);
break;
}
Ok(Applied::Done) => {}
Err(e) => {
crate::log_warn!("coordinator: targeted apply {:?}: {}", ev, e);
}
}
if Instant::now() >= deadline {
break;

47
crates/quicksearch-core/src/coordinator/tests.rs
View file

@ -948,6 +948,53 @@ fn a_deletion_during_a_full_run_is_queued_then_applied() {
coord.shutdown();
}
/// A path that cannot be read is not a path that is gone.
///
/// `update_paths` is fed by the frontend from the rows it is displaying, and
/// its `Remove` verb takes the row *and its whole subtree*. `is_file()` cannot
/// tell "not a regular file" from "I could not look", so an unreadable file —
/// a network share that dropped, a removable drive unplugged with its results
/// on screen, a directory another process just chmod'd — used to read as a
/// deletion. The next full run cannot undo it: an unreachable root is recorded
/// unreadable rather than re-walked.
#[test]
fn an_unreadable_path_is_not_treated_as_deleted() {
let dir = crate::testutil::scratch_dir("verb");
let present = dir.join("here.txt");
std::fs::write(&present, b"x").unwrap();
assert!(
matches!(verb_for(present.clone()), Some(FsEvent::Modify(_))),
"a readable file is a Modify"
);
let gone = dir.join("never-existed.txt");
assert!(
matches!(verb_for(gone), Some(FsEvent::Remove(_))),
"a genuinely absent file is a Remove"
);
// A directory is not something the walk indexes, so it is still a Remove.
assert!(matches!(verb_for(dir.clone()), Some(FsEvent::Remove(_))));
// The case that matters: the file is there, and unreadable.
let locked = dir.join("locked");
std::fs::create_dir_all(&locked).unwrap();
let hidden = locked.join("file.txt");
std::fs::write(&hidden, b"x").unwrap();
if crate::platform::deny_read(&locked).is_ok() {
// Skipped when the test runs with rights that ignore the mode — CI
// drops CAP_DAC_OVERRIDE with capsh for exactly this reason.
if std::fs::metadata(&hidden).is_err() {
assert!(
verb_for(hidden).is_none(),
"an unreadable file must leave the index alone"
);
}
let _ = crate::platform::restore_read(&locked);
}
std::fs::remove_file(&present).ok();
}
#[test]
fn enqueue_last_wins_and_rename_splits() {
let mut pending = HashMap::new();

11
crates/quicksearch-core/src/db/open.rs
View file

@ -42,12 +42,18 @@ pub(crate) fn open_or_recreate_keyed(
let path = Path::new(db_path).to_path_buf();
if let Some(dir) = path.parent() {
if !dir.as_os_str().is_empty() {
std::fs::create_dir_all(dir)
crate::platform::create_dir_private(dir)
.map_err(|e| format!("Failed to create database dir {}: {}", dir.display(), e))?;
}
}
let conn = Connection::open(db_path)
.map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?;
// Before a single row is written. SQLite creates the file 0644 and hands
// that mode on to `-wal` and `-shm`, so on a default umask every other
// user on the machine could read the index — which holds the names and
// full text of everything under the configured roots, including files
// whose own permissions are 0600.
crate::platform::restrict_to_owner(&path);
key_and_probe(&conn, db_path, key)?;
conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas: {}", e))?;
@ -345,6 +351,9 @@ fn wipe_and_reopen(
}
let conn = Connection::open(path)
.map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?;
// A rebuild creates the file afresh, so it needs narrowing again for the
// same reason the first open does.
crate::platform::restrict_to_owner(path);
key_and_probe(&conn, &path.to_string_lossy(), key)?;
conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?;

52
crates/quicksearch-core/src/db/open_tests.rs
View file

@ -546,3 +546,55 @@ fn open_existing_rw_allows_delete() {
drop(conn);
std::fs::remove_file(&p).ok();
}
/// The index, and the WAL and SHM it hands its mode to, must not be readable
/// by other users on the machine.
///
/// SQLite creates its database file 0644 and copies that mode to `-wal` and
/// `-shm`; with the near-universal umask 022 that leaves the names and full
/// text of everything under the configured roots — including documents whose
/// own files are 0600 — readable by every account on a shared machine. The
/// README's carve-out is about other *processes of the same user*, not other
/// users, so nothing else covers this.
#[cfg(unix)]
#[test]
fn a_fresh_index_and_its_sidecars_are_owner_only() {
use std::os::unix::fs::PermissionsExt;
// A directory that does not exist yet, so the creation path is the one
// under test: an existing directory keeps whatever mode its owner chose.
let p = tmp_db_path()
.parent()
.unwrap()
.join("data")
.join("index.sqlite");
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
// A write, so the WAL and SHM exist to be checked.
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('a', '/perm-a', '/', 0, 0)",
[],
)
.unwrap();
let mode_of = |path: &std::path::Path| {
std::fs::metadata(path)
.unwrap_or_else(|e| panic!("stat {}: {e}", path.display()))
.permissions()
.mode()
& 0o777
};
assert_eq!(mode_of(&p), 0o600, "index at {}", p.display());
for suffix in ["-wal", "-shm"] {
let sidecar = std::path::PathBuf::from(format!("{}{}", p.display(), suffix));
if sidecar.exists() {
assert_eq!(mode_of(&sidecar), 0o600, "sidecar {}", sidecar.display());
}
}
// And the directory created for it, which would otherwise take the umask
// and let any account list what is indexed.
assert_eq!(mode_of(p.parent().unwrap()), 0o700);
drop(conn);
std::fs::remove_file(&p).ok();
}

17
crates/quicksearch-core/src/db/repo.rs
View file

@ -524,7 +524,22 @@ pub fn pending_content_page(
) -> Result<Vec<PendingContentRow>, String> {
let mut stmt = conn
.prepare_cached(
"SELECT id, name, path, mime FROM files
// `INDEXED BY` rather than a hint, because the planner gets this
// one wrong exactly when it costs most. Left to itself it takes
// `UNIQUE(path)` for the range and then sorts the survivors into a
// temp b-tree to satisfy `ORDER BY id` — which means every page
// walks the whole root's range and fetches each row's heap entry
// to test `content_state`. At `FEED_PAGE` rows per page that is
// quadratic over a run. The partial index below is already
// id-ordered, so it answers `id > ?` and the ORDER BY together and
// holds only pending rows. Measured on 500k rows with everything
// pending — the first index of a tree, i.e. the case that matters:
// 127 ms/page against 0.1 ms/page.
//
// The planner only prefers it once pending rows are a small
// minority, and never before ANALYZE has run at all, which is why
// this cannot be left to statistics.
"SELECT id, name, path, mime FROM files INDEXED BY idx_files_content_pending
WHERE content_state = 0 AND size <= ?1 AND id > ?2
AND path >= ?3 AND path < ?4
ORDER BY id

45
crates/quicksearch-core/src/extract/mod.rs
View file

@ -69,6 +69,17 @@ impl ExtractedContent {
/// should surface human-readable messages.
pub type ExtractError = String;
/// Run `f`, turning a panic into an [`ExtractError`] naming the file.
///
/// The extractors drive third-party parsers — `pdf-extract`, `rtf-parser`,
/// `cfb`, `lofty`, `quick-xml` — over bytes chosen by whoever wrote the file,
/// and several of them are documented to panic on malformed input. See
/// [`Registry::extract`] for what each caller stands to lose.
fn contain_panic<T>(path: &Path, f: impl FnOnce() -> T) -> Result<T, ExtractError> {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(f))
.map_err(|_| format!("extractor panicked on {}", path.display()))
}
/// A pluggable content extractor. Stateless; implementors should not hold
/// file handles across calls.
pub trait Extractor: Send + Sync {
@ -139,26 +150,54 @@ impl Registry {
/// Look up a handler for `mime` and run it against `path`. Returns
/// `Ok(None)` if no extractor claims the MIME — the caller should then
/// decide whether the file is "not applicable" (text state NA).
///
/// A panicking parser becomes an `Err`, here rather than at each call
/// site: this and [`Registry::extract_complete_head`] are the two places
/// third-party code is handed a file nobody vouched for, and every caller
/// has more than one file to lose. A content worker's panic silently
/// drops the row it claimed; a *walk* worker's costs the root its whole
/// content pass and disables stale cleanup run-wide; the live watcher's
/// costs every displayed row for the rest of the session. Containing it
/// at the boundary means a new caller cannot forget.
///
/// This cannot help with a stack overflow, which aborts rather than
/// unwinding — see `vendor/pdf-extract`, which bounds the recursion that
/// made that reachable.
pub fn extract(
&self,
path: &Path,
mime: &str,
) -> Result<Option<ExtractedContent>, ExtractError> {
self.find(mime).map(|e| e.extract(path)).transpose()
let Some(extractor) = self.find(mime) else {
return Ok(None);
};
contain_panic(path, || extractor.extract(path))
.and_then(|r| r)
.map(Some)
}
/// [`Registry::extract`] for a file whose complete contents the caller
/// already holds. `None` when no extractor claims the MIME or the one
/// that does needs the file on disk — both mean "leave this to the
/// content pass".
/// Contained the same way [`Registry::extract`] is, and this is the one
/// that runs on a walk worker.
pub fn extract_complete_head(
&self,
path: &Path,
mime: &str,
head: &[u8],
) -> Option<Result<ExtractedContent, ExtractError>> {
self.find(mime)
.and_then(|e| e.extract_from_head(path, head))
let extractor = self.find(mime)?;
// `extract_from_head` returning `None` means "needs the file on
// disk", which is not a failure and must stay distinguishable from
// one — so the guard wraps the whole `Option` and a panic becomes
// `Some(Err(..))`, i.e. a failure this file is charged with rather
// than a deferral to the content pass that would meet the same panic.
match contain_panic(path, || extractor.extract_from_head(path, head)) {
Ok(outcome) => outcome,
Err(e) => Some(Err(e)),
}
}
/// The default set: RTF, plaintext, office docs, PDF, audio tags.

34
crates/quicksearch-core/src/extract/office.rs
View file

@ -143,6 +143,18 @@ fn open_container(path: &Path) -> Result<Archive, Box<dyn Error>> {
/// a tiny archive can inflate without bound.
const MAX_XML_BYTES: usize = 64 * 1024 * 1024;
/// Cap on the text taken from one *container*, mirroring [`ole::MAX_TEXT_BYTES`].
///
/// [`MAX_XML_BYTES`] bounds each member on its own, which is not the same
/// thing: a workbook or a deck holds one member per sheet or per slide, and
/// nothing stops a small archive from carrying dozens that each inflate to
/// that cap. The truncation to `maximum_text_size` happens only after the
/// whole string is built and handed back, so without a running total the peak
/// is members × 64 MiB — gigabytes from a file measured in megabytes, on every
/// extraction worker at once, and an allocation failure aborts rather than
/// unwinding.
const MAX_TEXT_BYTES: usize = 64 * 1024 * 1024;
/// One member's bytes as a string. An over-cap member keeps its prefix.
fn member_text<R: Read + Seek>(
archive: &mut ZipArchive<R>,
@ -201,6 +213,12 @@ fn extract_pptx(path: &Path) -> Result<String, Box<dyn Error>> {
let mut archive = open_container(path)?;
let mut out = String::new();
for name in xml_members_under(&mut archive, "ppt/slides/slide")? {
// Per-container budget: see `MAX_TEXT_BYTES`. Whole slides are kept or
// dropped rather than cut mid-way, which is why the test is here
// rather than inside the collector.
if out.len() >= MAX_TEXT_BYTES {
break;
}
let xml = member_text(&mut archive, &name)?;
collect_xml_text(&xml, &PPTX, &mut out)?;
out.push_str("\n--- New Slide ---\n");
@ -251,10 +269,20 @@ fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(),
Ok(Event::Start(ref e)) if e.name().as_ref() == b"c" => {
in_cell = true;
cell_type.clear();
for attr in e.attributes() {
// `with_checks(false)`: the default duplicate-attribute-name
// check compares each name against every name already seen on
// the tag, which is quadratic in the count and has no bound
// but the tag's own size (RUSTSEC-2026-0194). A member may be
// 64 MiB of inflated XML, so one crafted `<c>` can hold
// millions of attributes and hold this worker for hours —
// uncancellably, since the stop flag is only read between
// files. Rejecting duplicate names was never this extractor's
// job; it wants one attribute and stops at it.
for attr in e.attributes().with_checks(false) {
let attr = attr?;
if attr.key.as_ref() == b"t" {
cell_type = String::from_utf8_lossy(&attr.value).to_string();
break;
}
}
}
@ -294,6 +322,10 @@ fn extract_xlsx(path: &Path) -> Result<String, Box<dyn Error>> {
let strings = shared_strings(&mut archive);
let mut out = String::new();
for name in xml_members_under(&mut archive, "xl/worksheets/sheet")? {
// Per-container budget: see `MAX_TEXT_BYTES`.
if out.len() >= MAX_TEXT_BYTES {
break;
}
let xml = member_text(&mut archive, &name)?;
collect_sheet(&xml, &strings, &mut out)?;
}

96
crates/quicksearch-core/src/extract/pdf.rs
View file

@ -266,4 +266,100 @@ mod tests {
err
);
}
/// A page whose `/Parent` is itself must be skipped, not fatal.
///
/// `get_inherited` walks `/Parent` looking for `Resources` and `MediaBox`,
/// and upstream did so without a depth bound — a cycle recursed until the
/// guard page. That is *not* a panic `catch_unwind` can hold: Rust's
/// handler aborts, so this test could not even be written before
/// `vendor/pdf-extract` bounded the walk; it would have taken the test
/// binary down with it. What arrives now is an ordinary contained failure.
#[test]
fn a_self_referential_page_parent_is_contained() {
let mut doc = Document::with_version("1.5");
let contents = doc.add_object(Stream::new(dictionary! {}, b"BT ET".to_vec()));
let page_id = doc.new_object_id();
// Neither `Resources` nor `MediaBox` here, so both lookups have to
// follow `/Parent` — which points back at this same dictionary.
doc.objects.insert(
page_id,
Object::Dictionary(dictionary! {
"Type" => "Page",
"Parent" => page_id,
"Contents" => contents,
}),
);
let pages_id = doc.add_object(dictionary! {
"Type" => "Pages",
"Kids" => vec![page_id.into()],
"Count" => 1,
});
let catalog = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => pages_id,
});
doc.trailer.set("Root", catalog);
let path = crate::testutil::scratch_dir("pdf-parent-cycle").join("cycle.pdf");
doc.save(&path).expect("write fixture pdf");
// The verdict that matters is that we reach this line at all.
let _ = PdfExtractor.extract(&path);
}
/// A Form XObject whose content stream draws itself must be skipped, not
/// fatal — the second unbounded recursion, in `process_stream`'s `Do` arm.
/// The same bound also caps the branching shape, where each level draws
/// the next twice and a shallow document costs 2^depth calls.
#[test]
fn a_self_drawing_form_xobject_is_contained() {
let mut doc = Document::with_version("1.5");
let form_id = doc.new_object_id();
// Its own resources name it, so `/X0 Do` inside it re-enters itself.
doc.objects.insert(
form_id,
Object::Stream(Stream::new(
dictionary! {
"Type" => "XObject",
"Subtype" => "Form",
"BBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
"Resources" => dictionary! {
"XObject" => dictionary! { "X0" => form_id },
},
},
b"/X0 Do".to_vec(),
)),
);
let resources = doc.add_object(dictionary! {
"XObject" => dictionary! { "X0" => form_id },
});
let contents = doc.add_object(Stream::new(dictionary! {}, b"/X0 Do".to_vec()));
let pages_id = doc.new_object_id();
let page = doc.add_object(dictionary! {
"Type" => "Page",
"Parent" => pages_id,
"Contents" => contents,
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
});
doc.objects.insert(
pages_id,
Object::Dictionary(dictionary! {
"Type" => "Pages",
"Kids" => vec![page.into()],
"Count" => 1,
"Resources" => resources,
}),
);
let catalog = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => pages_id,
});
doc.trailer.set("Root", catalog);
let path = crate::testutil::scratch_dir("pdf-xobject-cycle").join("cycle.pdf");
doc.save(&path).expect("write fixture pdf");
let _ = PdfExtractor.extract(&path);
}
}

47
crates/quicksearch-core/src/extract/rtf.rs
View file

@ -83,6 +83,53 @@ mod tests {
std::fs::remove_file(&p).ok();
}
/// A `\\u` escape naming a lone UTF-16 surrogate must fail the file, not
/// the thread.
///
/// `rtf-parser` reaches `String::from_utf16(..).unwrap()` with whatever
/// `\\uN` supplied, and screens nothing for the surrogate range. RTF is one
/// of the two extractors that also run at *walk* time, off
/// `extract_from_head`, where a panicking worker costs the root its entire
/// content pass and disables stale cleanup run-wide — so this is contained
/// in `decide_content` and `prepare_file_record` rather than left to the
/// parser. Both entry points are exercised here.
#[test]
fn a_lone_surrogate_escape_is_contained() {
let body = br"{\rtf1\u55296 }";
let p = tmp("surrogate", body);
// The on-disk path, as the content pass reaches it.
let outcome = crate::file_handling::decide_content(
p.to_str().unwrap(),
Some("application/rtf"),
&crate::extract::Registry::default_set(),
&crate::config::Config::default(),
);
assert!(
matches!(outcome, crate::file_handling::ContentOutcome::Failed(_)),
"a panicking parser must record a failure, not unwind: {:?}",
outcome
);
// And the head path, as a walk worker reaches it: through the
// registry, which is where the containment lives. The raw
// `RtfExtractor::extract_from_head` below it still panics — that is
// third-party code doing what it does, and the point is that no
// caller in this crate is exposed to it.
let head = crate::extract::Registry::default_set().extract_complete_head(
&p,
"application/rtf",
body,
);
assert!(
matches!(head, Some(Err(_))),
"a panicking parser must be charged to the file, not the worker: {:?}",
head.map(|r| r.map(|c| c.text))
);
std::fs::remove_file(&p).ok();
}
#[test]
fn malformed_input_errors_and_names_the_file() {
let p = tmp("broken", br"{\rtf1 truncated");

12
crates/quicksearch-core/src/file_handling/records.rs
View file

@ -78,7 +78,10 @@ pub(super) fn get_file_hash(
path: &Path,
hash_length: usize,
) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> {
let mut f: File = File::open(path)?;
// The caller's `is_file()` came from a `stat` taken before this open; a
// FIFO renamed over the name in between would block this walk worker
// forever and park the pool behind it. See `platform::open_regular_file`.
let mut f: File = crate::platform::open_regular_file(path)?;
// Files shorter than the window hash whole; `min` keeps the cast sound
// for large files on 32-bit targets.
let mut head = vec![0u8; size.min(hash_length as u64) as usize];
@ -233,6 +236,9 @@ pub fn prepare_file_record(
if size == 0 || size > config.processing.hash_length as u64 {
return None;
}
// A panicking parser arrives here as `Some(Err(..))` — contained by
// the registry, which is what keeps a walk worker alive. See
// `Registry::extract`.
match registry.extract_complete_head(Path::new(path), m, &head) {
Some(Ok(content)) => {
let mut text = content.text;
@ -344,7 +350,9 @@ pub fn decide_content(
return ContentOutcome::NotApplicable;
}
// `content_extractable` established that an extractor claims this MIME,
// so the `Ok(None)` arm below is unreachable.
// so the `Ok(None)` arm below is unreachable. A panicking parser is
// contained by the registry and arrives as `Err`, which becomes this
// row's recorded failure reason rather than a dead worker.
let result = match mime {
Some(m) => registry.extract(p, m),
None => Ok(None),

141
crates/quicksearch-core/src/incremental.rs
View file

@ -11,6 +11,8 @@
//! containment check is repeated here.
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant;
use rusqlite::{Connection, OptionalExtension};
@ -24,22 +26,59 @@ use crate::file_handling::{
use crate::platform::path_has_hidden_component_under;
use crate::watcher::FsEvent;
/// How much of one event may be applied in this turn, and where the last turn
/// stopped.
///
/// One event is not always one file: a directory moved into a watched tree
/// arrives as a single `Create` covering everything beneath it. Shaped like
/// [`crate::scope::advance`]'s arguments and there for the same reason — this
/// runs on the coordinator's own thread, so an unbounded call is a command
/// loop that reads no commands, including the shutdown a closing window is
/// waiting on.
pub struct Budget<'a> {
pub deadline: Instant,
pub cancel: &'a AtomicBool,
/// Entries an earlier turn already applied for this event.
pub resume_from: usize,
}
impl Budget<'_> {
fn spent(&self) -> bool {
self.cancel.load(Ordering::Relaxed) || Instant::now() >= self.deadline
}
}
/// Whether an event was applied in full, or ran out of budget partway.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Applied {
/// Everything the event implied is in the index.
Done,
/// Budget spent. What was written is committed; the caller should re-queue
/// the same event with `resume_from` set to `done`.
Unfinished { done: usize },
}
/// Apply one filesystem event to the index. Missing files are treated as
/// no-ops (a Create followed by a quick delete resolves via the Remove
/// event); unchanged mtimes short-circuit without touching the DB.
///
/// See [`Budget`] for the one event that is not small.
pub fn apply_fs_event(
conn: &mut Connection,
event: &FsEvent,
config: &Config,
ignore: &IgnoreSet,
registry: &Registry,
) -> Result<(), String> {
budget: &Budget<'_>,
) -> Result<Applied, String> {
match event {
FsEvent::Create(p) | FsEvent::Modify(p) => upsert_path(conn, p, config, ignore, registry),
FsEvent::Remove(p) => remove_path(conn, p),
FsEvent::Create(p) | FsEvent::Modify(p) => {
upsert_path(conn, p, config, ignore, registry, budget)
}
FsEvent::Remove(p) => remove_path(conn, p).map(|()| Applied::Done),
FsEvent::Rename { from, to } => {
remove_path(conn, from)?;
upsert_path(conn, to, config, ignore, registry)
upsert_path(conn, to, config, ignore, registry, budget)
}
}
}
@ -50,9 +89,10 @@ fn upsert_path(
config: &Config,
ignore: &IgnoreSet,
registry: &Registry,
) -> Result<(), String> {
budget: &Budget<'_>,
) -> Result<Applied, String> {
if ignore.matches_path(path) {
return Ok(());
return Ok(Applied::Done);
}
// Measured from the innermost configured root: the walk never filters
// the root it was handed, so a root that is itself hidden must not be
@ -60,11 +100,11 @@ fn upsert_path(
if !config.indexing.include_hidden
&& path_has_hidden_component_under(path, &config.resolved_indexing_paths())
{
return Ok(());
return Ok(Applied::Done);
}
let Ok(meta) = std::fs::metadata(path) else {
// Already gone again — the pending Remove event handles it.
return Ok(());
return Ok(Applied::Done);
};
if meta.is_dir() {
// A moved-in tree surfaces as one directory event; walk it with
@ -76,20 +116,37 @@ fn upsert_path(
let Some(root) = path.to_str() else {
return Err(format!("directory path is not valid UTF-8: {:?}", path));
};
let entries: Vec<_> = filtered_walk(
// Streamed, not collected: `mv` of a large tree is one event, and
// materialising its entries first is a `DirEntry` per file resident
// before a single row is written. Each file is its own transaction,
// so stopping between two of them leaves the index consistent and the
// remainder for the next turn.
//
// `skip` rather than re-testing every entry: `upsert_file` on an
// unchanged file is cheap but not free, and paying it again for
// everything already done would make a large tree quadratic in the
// number of turns it takes. The walk order is deterministic for an
// unchanged tree; if the tree does change under us the count is only
// an optimisation, and the next full run is what makes it exact.
let mut done = budget.resume_from;
for entry in filtered_walk(
root,
config.indexing.follow_symlinks,
config.indexing.include_hidden,
ignore,
&UnreadableDirs::default(),
)
.collect();
for entry in entries {
.skip(budget.resume_from)
{
if budget.spent() {
return Ok(Applied::Unfinished { done });
}
upsert_file(conn, entry.path(), config, registry)?;
done += 1;
}
Ok(())
Ok(Applied::Done)
} else {
upsert_file(conn, path, config, registry)
upsert_file(conn, path, config, registry).map(|()| Applied::Done)
}
}
@ -195,6 +252,7 @@ pub fn remove_paths(
mod tests {
use super::*;
use crate::db::open_or_recreate;
use std::time::Duration;
struct Fixture {
conn: Connection,
@ -224,15 +282,32 @@ mod tests {
}
}
/// Applies with an effectively unlimited budget: these tests are about
/// what lands in the index, not about the slicing. See
/// [`Fixture::apply_within`] for the budget itself.
fn apply(&mut self, event: &FsEvent) {
let done = self.apply_within(event, Duration::from_secs(3600));
assert_eq!(done, Applied::Done, "unexpectedly ran out of budget");
}
fn apply_within(&mut self, event: &FsEvent, budget: Duration) -> Applied {
self.apply_resuming(event, budget, 0)
}
fn apply_resuming(&mut self, event: &FsEvent, budget: Duration, from: usize) -> Applied {
apply_fs_event(
&mut self.conn,
event,
&self.config,
&self.ignore,
&self.registry,
&Budget {
deadline: Instant::now() + budget,
cancel: &AtomicBool::new(false),
resume_from: from,
},
)
.unwrap();
.unwrap()
}
fn write(&self, name: &str, content: &str) -> std::path::PathBuf {
@ -296,6 +371,44 @@ mod tests {
/// The collapse must not change what ends up deleted — only how much work
/// it takes to get there.
/// A directory event is applied in slices, and a slice resumes where the
/// last one stopped instead of re-walking what it already did.
///
/// `mv` of a large tree is one `Create`. Applying it in one go held the
/// coordinator's command loop — and the shutdown queued behind it — for as
/// long as the whole tree took; applying it in slices that each restarted
/// from the top would be quadratic instead. Asserted by resume point
/// rather than by clock, because a timing-based assertion says nothing
/// reliable on a loaded CI runner.
#[test]
fn a_directory_event_resumes_where_its_budget_ran_out() {
let mut f = Fixture::new();
for i in 0..4 {
f.write(&format!("sub/f{i}.txt"), "body");
}
let sub = f.dir.join("sub");
// Nothing may be spent, so nothing is applied and the resume point is
// where it started.
let outcome = f.apply_within(&FsEvent::Create(sub.clone()), Duration::ZERO);
assert_eq!(outcome, Applied::Unfinished { done: 0 });
assert_eq!(f.counts().0, 0, "a spent budget must write nothing");
// Resuming past the first two entries applies only what is left, which
// is what makes slicing linear rather than quadratic.
let outcome = f.apply_resuming(&FsEvent::Create(sub.clone()), Duration::from_secs(3600), 2);
assert_eq!(outcome, Applied::Done);
assert_eq!(
f.counts().0,
2,
"entries before the resume point must be skipped, not re-applied"
);
// And from the start, the rest arrive.
f.apply(&FsEvent::Create(sub));
assert_eq!(f.counts().0, 4);
}
#[test]
fn collapsed_removal_deletes_the_same_rows_as_the_full_set() {
let mut f = Fixture::new();

5
crates/quicksearch-core/src/live.rs
View file

@ -643,7 +643,10 @@ impl Loop {
/// The first `limit` bytes of a file, for MIME sniffing. A short read is the
/// whole file and is not an error; an unreadable file simply has no MIME.
fn read_head(path: &Path, limit: usize) -> Option<Vec<u8>> {
let file = std::fs::File::open(path).ok()?;
// The row's own `stat` decided this was a file; by now it may be a FIFO,
// and a blocking open on this thread costs every live row for the rest of
// the session — and hangs exit, since `LiveWatcher::stop` joins here.
let file = crate::platform::open_regular_file(path).ok()?;
let mut head = Vec::new();
file.take(limit as u64).read_to_end(&mut head).ok()?;
Some(head)

83
crates/quicksearch-core/src/platform.rs
View file

@ -490,6 +490,89 @@ pub fn release_free_heap() {
// spans to the kernel on free.
}
/// Create `dir` and its parents, readable only by their owner.
///
/// `create_dir_all` leaves the mode to the umask, which is 022 nearly
/// everywhere and so grants the world `+rx`. The two directories this is
/// called for hold the index and the config, i.e. the names and text of
/// everything under the configured roots, plus the salt.
///
/// Only directories *created here* are narrowed: an existing one keeps its
/// mode, because a user who chose `~/Documents` as their data directory did
/// not ask for it to be locked down.
pub fn create_dir_private(dir: &Path) -> std::io::Result<()> {
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(dir)
}
#[cfg(not(unix))]
{
// Windows: a file created under the user's profile inherits an ACL
// that already excludes other users.
std::fs::create_dir_all(dir)
}
}
/// Narrow `path` to owner-only access, best effort.
///
/// SQLite creates its database 0644 (`SQLITE_DEFAULT_FILE_PERMISSIONS`, which
/// the bundled build does not override) and then copies that mode to the
/// `-wal` and `-shm` it derives from it, so narrowing the main file before
/// anything else is opened covers all three. The index is a strictly larger
/// secret than any single file it was read from: it holds the full text of
/// documents whose own permissions may be far tighter.
///
/// Failure is ignored: on a filesystem with no Unix permissions (a FAT stick,
/// a network share) there is nothing to set and nothing to report.
pub fn restrict_to_owner(path: &Path) {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600));
}
#[cfg(not(unix))]
{
let _ = path;
}
}
/// Open `path` for reading, refusing anything that is not a regular file.
///
/// Every caller has already decided the path *was* a regular file, from a
/// `stat` taken earlier — during the walk, or when a result row was put on
/// screen. A rename can put a FIFO, a tty or a character device at that name
/// in between, and `open` on one of those blocks in the kernel until a writer
/// or a carrier appears: uninterruptibly, past any stop flag, and for as long
/// as the process lives. One such open strands a walk worker while it still
/// holds its job slot, which parks the whole pool behind it.
///
/// `O_NONBLOCK` makes the open itself return, and the handle is then asked
/// what it actually is — `fstat` on the descriptor, so nothing can swap the
/// name again underneath the answer. Both are free on a regular file, which
/// is the only case that matters for speed: the flag is ignored for ordinary
/// files and the `fstat` hits the inode already in hand.
pub fn open_regular_file(path: &Path) -> std::io::Result<std::fs::File> {
let mut opts = std::fs::OpenOptions::new();
opts.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.custom_flags(libc::O_NONBLOCK);
}
let file = opts.open(path)?;
if !file.metadata()?.file_type().is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"not a regular file",
));
}
Ok(file)
}
/// How long to keep retrying a delete that fails because something else holds
/// the file open.
#[cfg(windows)]

12
crates/quicksearch-core/src/search/cascade/passes.rs
View file

@ -67,6 +67,18 @@ impl<'a> Cx<'a> {
if scanned.is_multiple_of(cancel_every) && self.cancelled() {
return Ok(false);
}
// The display limit is already full, and holding a row proves at
// least one more match exists than will be shown — so `limited` is
// exactly true here, and everything below is work whose result
// `flush_pass` would throw away. That work is not small: the
// full-text passes decompress the document, fold a copy of it, and
// cut a snippet, per row. `cascade::run` makes the same test
// between passes; without this one a single pass over a common
// term runs to the end of the candidate set.
if self.remaining() == 0 {
self.limited = true;
break;
}
let file_id: i64 = col(row, 0)?;
// Borrowed from the statement rather than `col::<String>`: this
// runs for every *scanned* row — a full-table scan on the filename

7
crates/quicksearch-core/src/verify.rs
View file

@ -109,7 +109,12 @@ pub fn verify_identical(paths: &[PathBuf], cancel: &AtomicBool, on: &mut dyn FnM
let mut reference: Option<(usize, File, u64)> = None;
let mut rest: Vec<(usize, File)> = Vec::new();
for (i, path) in paths.iter().enumerate() {
let file = match File::open(path) {
// These paths come from the index, which records what each file was
// when it was walked. A member replaced by a FIFO since then needs no
// race at all to be sitting here — the walk keeps the old row when a
// regular file turns into something else — and a blocking open would
// strand this worker before it reported a single verdict.
let file = match crate::platform::open_regular_file(path) {
Ok(f) => f,
Err(e) => {
verdicts[i] = MemberVerdict::Unreadable(describe(path, &e));

9
crates/quicksearch-gui/src/app/security.rs
View file

@ -349,9 +349,18 @@ impl QuickSearchApp {
.resolved_database_path()
.to_string_lossy()
.into_owned();
let previous_security = self.cfg.security.clone();
self.cfg.security = new_security;
if let Err(e) = self.cfg.save() {
// Fail closed, exactly as `SecurityAction::SetKeychain` does: the
// salt this change depends on lives only in that file. Carrying on
// would install the new key and rebuild the index under it while
// the config on disk still describes the old state — an index
// encrypted with a salt that reached no disk, which no password
// can open afterwards. Put the config back and stop.
self.cfg.security = previous_security;
self.config_error = Some(e);
return;
}
match (&new_key, self.cfg.security.use_keychain) {
(Some(key), true) => {

18
crates/quicksearch-gui/src/unlock.rs
View file

@ -211,10 +211,20 @@ impl UnlockScreen {
match result {
Ok(key) => return self.unlocked(ctx, key),
Err(e) => {
self.error = Some(if e.starts_with(db::KEY_MISMATCH_PREFIX) {
"Wrong password.".to_string()
} else {
e
// A tagged mismatch has three distinct causes and
// `key_mismatch_message` already told them apart. Only one
// of them is a wrong password; the others are "the config
// says protected but the index on disk is not" (a crash
// between saving the config and rebuilding) and "this
// index wants a password at all". Collapsing them all into
// "Wrong password." tells a user with the right password
// that it is wrong, and the only button on this screen
// deletes their index and turns protection off. The detail
// names the database path and nothing secret.
self.error = Some(match e.strip_prefix(db::KEY_MISMATCH_PREFIX) {
Some(_) if e.contains("wrong password") => "Wrong password.".to_string(),
Some(detail) => detail.to_string(),
None => e,
});
// The field was cleared on submit; put the caret back
// for the retry.

6
packaging/copyright
View file

@ -78,5 +78,11 @@ Comment:
font data rather than as linked program code, which is the same basis on
which Debian ships these fonts and other egui-based applications.
.
One dependency is carried in-tree rather than fetched: vendor/pdf-extract
is pdf-extract 0.12.0 (MIT, Jeff Muizelaar,
https://github.com/jrmuizel/pdf-extract) with two unbounded recursions
given a depth limit, marked "LOCAL PATCH" in the source. Its licence is
unchanged and is the MIT stanza below.
.
Run `cargo metadata --all-features` against the source tree to reproduce
the per-crate licence list.

1
vendor/pdf-extract/.cargo-ok vendored Normal file
View file

@ -0,0 +1 @@
{"v":1}

6
vendor/pdf-extract/.cargo_vcs_info.json vendored Normal file
View file

@ -0,0 +1,6 @@
{
"git": {
"sha1": "b95bf9f6268772d5088f09b0034e488e64294835"
},
"path_in_vcs": ""
}

73
vendor/pdf-extract/Cargo.toml vendored Normal file
View file

@ -0,0 +1,73 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2018"
name = "pdf-extract"
version = "0.12.0"
authors = ["Jeff Muizelaar <jmuizelaar@mozilla.com>"]
build = false
include = [
"src/**/*",
"README.md",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "A library to extract content from pdfs"
documentation = "https://docs.rs/crate/pdf-extract/"
readme = "README.md"
keywords = [
"pdf2text",
"text",
"pdf",
"pdf2txt",
]
license = "MIT"
repository = "https://github.com/jrmuizel/pdf-extract"
[lib]
name = "pdf_extract"
path = "src/lib.rs"
[dependencies.adobe-cmap-parser]
version = "0.4.1"
[dependencies.cff-parser]
version = "0.2.0"
[dependencies.encoding_rs]
version = "0.8.34"
[dependencies.euclid]
version = "0.20.5"
[dependencies.log]
version = "0.4.22"
[dependencies.lopdf]
version = "0.42"
features = ["wasm_js"]
default-features = false
[dependencies.postscript]
version = "0.14"
[dependencies.type1-encoding-parser]
version = "0.1.1"
[dependencies.unicode-normalization]
version = "0.1.19"
[lints.rust]
warnings = "allow"

24
vendor/pdf-extract/README.md vendored Normal file
View file

@ -0,0 +1,24 @@
## pdf-extract
[![Build Status](https://github.com/jrmuizel/pdf-extract/actions/workflows/rust.yml/badge.svg)](https://github.com/jrmuizel/pdf-extract/actions)
[![crates.io](https://img.shields.io/crates/v/pdf-extract.svg)](https://crates.io/crates/pdf-extract)
[![Documentation](https://docs.rs/pdf-extract/badge.svg)](https://docs.rs/pdf-extract)
A rust library to extract content from PDF files.
```rust
let bytes = std::fs::read("tests/docs/simple.pdf").unwrap();
let out = pdf_extract::extract_text_from_mem(&bytes).unwrap();
assert!(out.contains("This is a small demonstration"));
```
## See also
- https://github.com/elacin/PDFExtract/
- https://github.com/euske/pdfminer / https://github.com/pdfminer/pdfminer.six
- https://gitlab.com/crossref/pdfextract
- https://github.com/VikParuchuri/marker
- https://github.com/kermitt2/pdfalto used by [grobid](https://github.com/kermitt2/grobid/)
- https://github.com/opendatalab/MinerU (uses PyMuPDF and pdfminer.six)
### Not PDF specific
- https://github.com/Layout-Parser/layout-parser

18
vendor/pdf-extract/src/core_fonts.rs vendored Normal file
View file

File diff suppressed because one or more lines are too long

1810
vendor/pdf-extract/src/encodings.rs vendored Normal file
View file

File diff suppressed because it is too large Load diff

41
vendor/pdf-extract/src/glyphlist-export.py vendored Normal file
View file

@ -0,0 +1,41 @@
glyphlist = []
glyphs_seen = {}
def read_glyphs(name):
f = open(name)
lines = f.readlines()
import re
for l in lines:
if l[0] == '#' or l[0] == '\n':
continue
split = re.split('[; ,]+', l)
name = split[0]
val = int(split[1], 16)
if val > 0xffff:
val = int(split[-1], 16)
if val == 0xf766 and name != "Fsmall":
continue
if name in glyphs_seen:
continue
glyphs_seen[name] = True
glyphlist.append((name,val))
read_glyphs("glyphlist-extended.txt")
read_glyphs("texglyphlist.txt")
read_glyphs("additional.txt")
# there are some conflicts between these files
# e.g. tildewide=0x02dc, vs tildewide=0x0303
# for now we just ignore the subsequent ones
glyphlist.append(('mapsto', 0x21A6))
glyphlist = list(set(glyphlist))
glyphlist.sort()
print "/* Autogenerated from:"
print " https://github.com/michal-h21/htfgen/commits/master/glyphlist-extended.txt"
print " https://github.com/kohler/lcdf-typetools/blob/master/texglyphlist.txt"
print " https://github.com/apache/pdfbox/blob/trunk/pdfbox/src/main/resources/org/apache/pdfbox/resources/glyphlist/additional.txt"
print " */"
print "pub fn name_to_unicode(name: &str) -> Option<u16> {"
print " const names: [(&'static str, u16); %d] = [" % len(glyphlist)
print ",\n".join('(\"%s\", 0x%04x)' % (g[0], g[1]) for g in glyphlist)
print " ];"
print " let result = names.binary_search_by_key(&name, |&(name,_code)| &name);"
print " result.ok().map(|indx| names[indx].1)"
print "}"

4553
vendor/pdf-extract/src/glyphlist-extended.txt vendored Normal file
View file

File diff suppressed because it is too large Load diff

4711
vendor/pdf-extract/src/glyphnames.rs vendored Normal file
View file

File diff suppressed because it is too large Load diff

2446
vendor/pdf-extract/src/lib.rs vendored Normal file
View file

File diff suppressed because it is too large Load diff

210
vendor/pdf-extract/src/zapfglyphnames.rs vendored Normal file
View file

@ -0,0 +1,210 @@
pub fn zapfdigbats_names_to_unicode(name: &str) -> Option<u16> {
let names = [
("a1", 0x2701),
("a10", 0x2721),
("a100", 0x275e),
("a101", 0x2761),
("a102", 0x2762),
("a103", 0x2763),
("a104", 0x2764),
("a105", 0x2710),
("a106", 0x2765),
("a107", 0x2766),
("a108", 0x2767),
("a109", 0x2660),
("a11", 0x261b),
("a110", 0x2665),
("a111", 0x2666),
("a112", 0x2663),
("a117", 0x2709),
("a118", 0x2708),
("a119", 0x2707),
("a12", 0x261e),
("a120", 0x2460),
("a121", 0x2461),
("a122", 0x2462),
("a123", 0x2463),
("a124", 0x2464),
("a125", 0x2465),
("a126", 0x2466),
("a127", 0x2467),
("a128", 0x2468),
("a129", 0x2469),
("a13", 0x270c),
("a130", 0x2776),
("a131", 0x2777),
("a132", 0x2778),
("a133", 0x2779),
("a134", 0x277a),
("a135", 0x277b),
("a136", 0x277c),
("a137", 0x277d),
("a138", 0x277e),
("a139", 0x277f),
("a14", 0x270d),
("a140", 0x2780),
("a141", 0x2781),
("a142", 0x2782),
("a143", 0x2783),
("a144", 0x2784),
("a145", 0x2785),
("a146", 0x2786),
("a147", 0x2787),
("a148", 0x2788),
("a149", 0x2789),
("a15", 0x270e),
("a150", 0x278a),
("a151", 0x278b),
("a152", 0x278c),
("a153", 0x278d),
("a154", 0x278e),
("a155", 0x278f),
("a156", 0x2790),
("a157", 0x2791),
("a158", 0x2792),
("a159", 0x2793),
("a16", 0x270f),
("a160", 0x2794),
("a161", 0x2192),
("a162", 0x27a3),
("a163", 0x2194),
("a164", 0x2195),
("a165", 0x2799),
("a166", 0x279b),
("a167", 0x279c),
("a168", 0x279d),
("a169", 0x279e),
("a17", 0x2711),
("a170", 0x279f),
("a171", 0x27a0),
("a172", 0x27a1),
("a173", 0x27a2),
("a174", 0x27a4),
("a175", 0x27a5),
("a176", 0x27a6),
("a177", 0x27a7),
("a178", 0x27a8),
("a179", 0x27a9),
("a18", 0x2712),
("a180", 0x27ab),
("a181", 0x27ad),
("a182", 0x27af),
("a183", 0x27b2),
("a184", 0x27b3),
("a185", 0x27b5),
("a186", 0x27b8),
("a187", 0x27ba),
("a188", 0x27bb),
("a189", 0x27bc),
("a19", 0x2713),
("a190", 0x27bd),
("a191", 0x27be),
("a192", 0x279a),
("a193", 0x27aa),
("a194", 0x27b6),
("a195", 0x27b9),
("a196", 0x2798),
("a197", 0x27b4),
("a198", 0x27b7),
("a199", 0x27ac),
("a2", 0x2702),
("a20", 0x2714),
("a200", 0x27ae),
("a201", 0x27b1),
("a202", 0x2703),
("a203", 0x2750),
("a204", 0x2752),
("a205", 0x276e),
("a206", 0x2770),
("a21", 0x2715),
("a22", 0x2716),
("a23", 0x2717),
("a24", 0x2718),
("a25", 0x2719),
("a26", 0x271a),
("a27", 0x271b),
("a28", 0x271c),
("a29", 0x2722),
("a3", 0x2704),
("a30", 0x2723),
("a31", 0x2724),
("a32", 0x2725),
("a33", 0x2726),
("a34", 0x2727),
("a35", 0x2605),
("a36", 0x2729),
("a37", 0x272a),
("a38", 0x272b),
("a39", 0x272c),
("a4", 0x260e),
("a40", 0x272d),
("a41", 0x272e),
("a42", 0x272f),
("a43", 0x2730),
("a44", 0x2731),
("a45", 0x2732),
("a46", 0x2733),
("a47", 0x2734),
("a48", 0x2735),
("a49", 0x2736),
("a5", 0x2706),
("a50", 0x2737),
("a51", 0x2738),
("a52", 0x2739),
("a53", 0x273a),
("a54", 0x273b),
("a55", 0x273c),
("a56", 0x273d),
("a57", 0x273e),
("a58", 0x273f),
("a59", 0x2740),
("a6", 0x271d),
("a60", 0x2741),
("a61", 0x2742),
("a62", 0x2743),
("a63", 0x2744),
("a64", 0x2745),
("a65", 0x2746),
("a66", 0x2747),
("a67", 0x2748),
("a68", 0x2749),
("a69", 0x274a),
("a7", 0x271e),
("a70", 0x274b),
("a71", 0x25cf),
("a72", 0x274d),
("a73", 0x25a0),
("a74", 0x274f),
("a75", 0x2751),
("a76", 0x25b2),
("a77", 0x25bc),
("a78", 0x25c6),
("a79", 0x2756),
("a8", 0x271f),
("a81", 0x25d7),
("a82", 0x2758),
("a83", 0x2759),
("a84", 0x275a),
("a85", 0x276f),
("a86", 0x2771),
("a87", 0x2772),
("a88", 0x2773),
("a89", 0x2768),
("a9", 0x2720),
("a90", 0x2769),
("a91", 0x276c),
("a92", 0x276d),
("a93", 0x276a),
("a94", 0x276b),
("a95", 0x2774),
("a96", 0x2775),
("a97", 0x275b),
("a98", 0x275c),
("a99", 0x275d),
("space", 0x0020),
];
let result = names.binary_search_by_key(&name, |&(name,_code)| &name);
result.ok().map(|indx| names[indx].1)
}