Compare commits

..

No commits in common. "master" and "v1.1.0" have entirely different histories.

33 changed files with 54 additions and 14672 deletions

2
Cargo.lock generated
View file

@ -2920,6 +2920,8 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]] [[package]]
name = "pdf-extract" name = "pdf-extract"
version = "0.12.0" version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "417e8fdc940f1d5bc62c5f89864c3a2255f74f69aa353c98509213d67df61e73"
dependencies = [ dependencies = [
"adobe-cmap-parser", "adobe-cmap-parser",
"cff-parser", "cff-parser",

View file

@ -4,34 +4,9 @@ members = [
"crates/quicksearch-core", "crates/quicksearch-core",
"crates/quicksearch-gui", "crates/quicksearch-gui",
] ]
# `vendor/pdf-extract` is deliberately NOT a member: it is a third-party crate
# carried here for one patch, not part of this workspace's lints, tests or
# release profile. `[patch.crates-io]` below is what makes the dependency graph
# resolve to it.
exclude = ["vendor/pdf-extract"]
# pdf-extract 0.12.0, with the two unbounded recursions in it bounded.
#
# `get_inherited` follows `/Parent` and `process_stream`'s `Do` arm follows
# Form XObjects, neither with a depth counter or a visited set. A page whose
# `/Parent` is itself, or an XObject whose content stream draws itself, walks
# the stack until it hits the guard page — and a stack overflow is not a panic
# that `catch_unwind` can contain (`extract/pdf.rs` has one, for the parser's
# ordinary panics): Rust's handler calls `abort()`, so a ~600-byte file kills
# the process. It recurs on every run, because the row keeps
# `content_state = 0` and the feeder selects exactly those; and the live
# watcher re-extracts on-screen rows on the GUI thread, so such a file crashes
# the app when it merely appears in a result list.
#
# Vendored rather than forked-by-URL so the build stays offline, `--locked`
# keeps meaning what it means, and the cross-compile job needs no new host.
# The patch is marked LOCAL PATCH in the source and is upstreamable; the crate
# is MIT and the copy is recorded in `packaging/copyright`.
[patch.crates-io]
pdf-extract = { path = "vendor/pdf-extract" }
[workspace.package] [workspace.package]
version = "1.1.1" version = "1.1.0"
edition = "2021" edition = "2021"
license = "GPL-3.0-or-later" license = "GPL-3.0-or-later"
authors = ["Jeremy <jeremy@karsttech.com>"] authors = ["Jeremy <jeremy@karsttech.com>"]

View file

@ -549,31 +549,16 @@ impl Config {
/// Write back to the file this config was loaded from (or the default /// Write back to the file this config was loaded from (or the default
/// location), creating parent directories as needed. Raw values are /// location), creating parent directories as needed. Raw values are
/// written verbatim — relative paths in a portable config stay relative. /// written verbatim — relative paths in a portable config stay relative.
///
/// Atomic: see the comment on the rename below.
pub fn save(&self) -> Result<(), String> { pub fn save(&self) -> Result<(), String> {
let path = self.source.clone().unwrap_or_else(Self::config_path); let path = self.source.clone().unwrap_or_else(Self::config_path);
if let Some(dir) = path.parent() { if let Some(dir) = path.parent() {
crate::platform::create_dir_private(dir) fs::create_dir_all(dir)
.map_err(|e| format!("Failed to create config dir {}: {}", dir.display(), e))?; .map_err(|e| format!("Failed to create config dir {}: {}", dir.display(), e))?;
} }
let content = toml::to_string_pretty(self) let content = toml::to_string_pretty(self)
.map_err(|e| format!("Failed to serialize config: {}", e))?; .map_err(|e| format!("Failed to serialize config: {}", e))?;
// Written beside the target and renamed over it, rather than fs::write(&path, content)
// truncate-then-write. `[security].salt` exists *only* in this file: .map_err(|e| format!("Failed to write config file {}: {}", path.display(), e))?;
// it is not derivable from the index and not stored anywhere else, so
// a config truncated by a crash, a full disk or a power cut in the
// middle of `write` is an encrypted index that no password can ever
// open again. `rename` is atomic on both platforms, and the `sync_all`
// before it means the bytes are on the disk before the name points at
// them.
let tmp = path.with_extension("toml.tmp");
write_private(&tmp, content.as_bytes())
.map_err(|e| format!("Failed to write config file {}: {}", tmp.display(), e))?;
fs::rename(&tmp, &path).map_err(|e| {
let _ = fs::remove_file(&tmp);
format!("Failed to replace config file {}: {}", path.display(), e)
})?;
Ok(()) Ok(())
} }
@ -631,32 +616,6 @@ impl Config {
} }
} }
/// Write `bytes` to `path`, owner-readable only, and flush them to the disk
/// before returning.
///
/// `O_NOFOLLOW` on Unix: the config directory is not always somewhere only
/// this user can write — a portable install can sit in a shared or removable
/// directory — and a symlink left at the config's name would otherwise
/// redirect this write onto whatever it points at.
fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
use std::io::Write;
let mut opts = fs::OpenOptions::new();
opts.write(true).create(true).truncate(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.custom_flags(libc::O_NOFOLLOW);
opts.mode(0o600);
}
let mut f = opts.open(path)?;
f.write_all(bytes)?;
// The rename that follows is atomic with respect to the *directory*, not
// to the file's contents: without this, a crash can leave the new name
// pointing at a block of zeroes.
f.sync_all()
}
/// Reserved `content_extensions` entry standing for "files with no /// Reserved `content_extensions` entry standing for "files with no
/// extension". Matched case-insensitively, and it cannot collide with a real /// extension". Matched case-insensitively, and it cannot collide with a real
/// extension because the parentheses are not part of one. /// extension because the parentheses are not part of one.

View file

@ -36,7 +36,7 @@ use rusqlite::Connection;
use crate::config::{diff_actions, Config, IgnoreSet, IndexWork}; use crate::config::{diff_actions, Config, IgnoreSet, IndexWork};
use crate::db; use crate::db;
use crate::extract::Registry; use crate::extract::Registry;
use crate::incremental::{apply_fs_event, Applied, Budget}; use crate::incremental::apply_fs_event;
use crate::indexing::{ConfigChange, IndexingService, IndexingStatus, PrepStep, ReconcileProgress}; use crate::indexing::{ConfigChange, IndexingService, IndexingStatus, PrepStep, ReconcileProgress};
use crate::scope::WorkCursor; use crate::scope::WorkCursor;
use crate::watcher::{FsEvent, WatchError, WatchFilters, Watcher, WatcherConfig}; use crate::watcher::{FsEvent, WatchError, WatchFilters, Watcher, WatcherConfig};
@ -230,7 +230,6 @@ impl IndexCoordinator {
watcher_gen: 0, watcher_gen: 0,
pending: HashMap::new(), pending: HashMap::new(),
targeted: HashMap::new(), targeted: HashMap::new(),
resume_from: HashMap::new(),
last_event_at: None, last_event_at: None,
pending_since: None, pending_since: None,
needs_full_run: false, needs_full_run: false,
@ -379,35 +378,6 @@ impl Drop for IndexCoordinator {
} }
} }
/// The verb a path submitted through [`IndexCoordinator::update_paths`]
/// deserves, or `None` to leave the index alone.
///
/// The caller knows a file changed, not what it changed into. `is_file()` is
/// the fast answer and almost always the right one — one `stat`, and this runs
/// while results are on screen — but it folds every stat error into `false`,
/// and a `Remove` costs the row *and everything beneath it*. So the negative
/// answer, and only it, is confirmed with a second `stat` that can tell "gone"
/// from "cannot see it just now": a share that dropped, a drive pulled while
/// its rows were displayed, a parent another process chmod'd.
///
/// In doubt the index wins. A stale row is a wrong line on screen until the
/// next full run; a deleted live one is data no run brings back until the file
/// is walked again — and if the reason it could not be read was that its whole
/// tree went away, that walk will not reach it either.
fn verb_for(path: PathBuf) -> Option<FsEvent> {
if path.is_file() {
return Some(FsEvent::Modify(path));
}
// `metadata`, not `symlink_metadata`: it has to agree with `is_file()`
// above about following links, or the two can disagree about the verb.
match std::fs::metadata(&path) {
// There, but no longer something the walk would index.
Ok(_) => Some(FsEvent::Remove(path)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Some(FsEvent::Remove(path)),
Err(_) => None,
}
}
/// Fold `event` into the last-event-wins pending map. Renames split into /// Fold `event` into the last-event-wins pending map. Renames split into
/// their halves so downstream application never needs pair handling. /// their halves so downstream application never needs pair handling.
fn enqueue(pending: &mut HashMap<PathBuf, FsEvent>, event: FsEvent) { fn enqueue(pending: &mut HashMap<PathBuf, FsEvent>, event: FsEvent) {

View file

@ -28,11 +28,6 @@ pub(super) struct Inner {
/// in manual mode, because it exists to keep the rows a user is *reading* /// in manual mode, because it exists to keep the rows a user is *reading*
/// in step with the disk however the indexer is configured. /// in step with the disk however the indexer is configured.
pub(super) targeted: HashMap<PathBuf, FsEvent>, pub(super) targeted: HashMap<PathBuf, FsEvent>,
/// How far a directory event got before its turn's budget ran out, so the
/// next turn resumes rather than re-walking what it already applied. Keyed
/// by the same path as the queue the event went back into, and removed
/// when the event completes or is dropped.
pub(super) resume_from: HashMap<PathBuf, usize>,
/// When the most recent event arrived; the burst is over once this is /// When the most recent event arrived; the burst is over once this is
/// `pending_settle` old. /// `pending_settle` old.
pub(super) last_event_at: Option<Instant>, pub(super) last_event_at: Option<Instant>,
@ -186,10 +181,14 @@ impl Inner {
if !prefixes.iter().any(|lo| spelled.starts_with(lo.as_str())) { if !prefixes.iter().any(|lo| spelled.starts_with(lo.as_str())) {
continue; continue;
} }
// Existence decides the verb; `verb_for` also decides when // Existence decides the verb. The caller knows a file
// it cannot be decided at all, and says so with `None`. // changed, not what it changed into, and a `Modify` for a
let Some(event) = verb_for(path) else { // path that is gone would be silently skipped rather than
continue; // removing the row.
let event = if path.is_file() {
FsEvent::Modify(path)
} else {
FsEvent::Remove(path)
}; };
enqueue(&mut self.targeted, event); enqueue(&mut self.targeted, event);
} }
@ -422,11 +421,6 @@ impl Inner {
// `clear` keeps the map's capacity — up to 100k slots after a storm; // `clear` keeps the map's capacity — up to 100k slots after a storm;
// shrinking is the point. // shrinking is the point.
self.pending.shrink_to_fit(); self.pending.shrink_to_fit();
// Resume points describe events that no longer exist. Entries for
// `targeted` events survive, which is why this filters rather than
// clearing: that queue deliberately outlives this call.
self.resume_from
.retain(|p, _| self.targeted.contains_key(p));
self.last_event_at = None; self.last_event_at = None;
self.pending_since = None; self.pending_since = None;
} }
@ -496,35 +490,13 @@ impl Inner {
let Some(ev) = self.pending.remove(&path) else { let Some(ev) = self.pending.remove(&path) else {
continue; continue;
}; };
match apply_fs_event( if let Err(e) =
&mut conn, apply_fs_event(&mut conn, &ev, &self.config, &self.ignore, &self.registry)
&ev, {
&self.config, // As above: the event is out of `pending`, so only a full
&self.ignore, // run still picks the file up.
&self.registry, crate::log_warn!("coordinator: apply {:?}: {}; scheduling full run", ev, e);
&Budget { self.needs_full_run = true;
deadline,
cancel: &self.reconcile_stop.cancel,
resume_from: self.resume_from.remove(&path).unwrap_or(0),
},
) {
// A directory event can cover a whole moved-in tree. Put
// it back, with a note of how far it got, and let the next
// tick continue it — so one `mv` cannot hold this loop, or
// the shutdown queued behind it, for as long as the tree
// takes.
Ok(Applied::Unfinished { done }) => {
self.resume_from.insert(path.clone(), done);
self.pending.insert(path, ev);
break;
}
Ok(Applied::Done) => {}
Err(e) => {
// As above: the event is out of `pending`, so only a
// full run still picks the file up.
crate::log_warn!("coordinator: apply {:?}: {}; scheduling full run", ev, e);
self.needs_full_run = true;
}
} }
if Instant::now() >= deadline { if Instant::now() >= deadline {
break; break;
@ -592,27 +564,10 @@ impl Inner {
let Some(ev) = self.targeted.remove(&path) else { let Some(ev) = self.targeted.remove(&path) else {
continue; continue;
}; };
match apply_fs_event( if let Err(e) =
&mut conn, apply_fs_event(&mut conn, &ev, &self.config, &self.ignore, &self.registry)
&ev, {
&self.config, crate::log_warn!("coordinator: targeted apply {:?}: {}", ev, e);
&self.ignore,
&self.registry,
&Budget {
deadline,
cancel: &self.reconcile_stop.cancel,
resume_from: self.resume_from.remove(&path).unwrap_or(0),
},
) {
Ok(Applied::Unfinished { done }) => {
self.resume_from.insert(path.clone(), done);
self.targeted.insert(path, ev);
break;
}
Ok(Applied::Done) => {}
Err(e) => {
crate::log_warn!("coordinator: targeted apply {:?}: {}", ev, e);
}
} }
if Instant::now() >= deadline { if Instant::now() >= deadline {
break; break;

View file

@ -948,53 +948,6 @@ fn a_deletion_during_a_full_run_is_queued_then_applied() {
coord.shutdown(); coord.shutdown();
} }
/// A path that cannot be read is not a path that is gone.
///
/// `update_paths` is fed by the frontend from the rows it is displaying, and
/// its `Remove` verb takes the row *and its whole subtree*. `is_file()` cannot
/// tell "not a regular file" from "I could not look", so an unreadable file —
/// a network share that dropped, a removable drive unplugged with its results
/// on screen, a directory another process just chmod'd — used to read as a
/// deletion. The next full run cannot undo it: an unreachable root is recorded
/// unreadable rather than re-walked.
#[test]
fn an_unreadable_path_is_not_treated_as_deleted() {
let dir = crate::testutil::scratch_dir("verb");
let present = dir.join("here.txt");
std::fs::write(&present, b"x").unwrap();
assert!(
matches!(verb_for(present.clone()), Some(FsEvent::Modify(_))),
"a readable file is a Modify"
);
let gone = dir.join("never-existed.txt");
assert!(
matches!(verb_for(gone), Some(FsEvent::Remove(_))),
"a genuinely absent file is a Remove"
);
// A directory is not something the walk indexes, so it is still a Remove.
assert!(matches!(verb_for(dir.clone()), Some(FsEvent::Remove(_))));
// The case that matters: the file is there, and unreadable.
let locked = dir.join("locked");
std::fs::create_dir_all(&locked).unwrap();
let hidden = locked.join("file.txt");
std::fs::write(&hidden, b"x").unwrap();
if crate::platform::deny_read(&locked).is_ok() {
// Skipped when the test runs with rights that ignore the mode — CI
// drops CAP_DAC_OVERRIDE with capsh for exactly this reason.
if std::fs::metadata(&hidden).is_err() {
assert!(
verb_for(hidden).is_none(),
"an unreadable file must leave the index alone"
);
}
let _ = crate::platform::restore_read(&locked);
}
std::fs::remove_file(&present).ok();
}
#[test] #[test]
fn enqueue_last_wins_and_rename_splits() { fn enqueue_last_wins_and_rename_splits() {
let mut pending = HashMap::new(); let mut pending = HashMap::new();

View file

@ -42,18 +42,12 @@ pub(crate) fn open_or_recreate_keyed(
let path = Path::new(db_path).to_path_buf(); let path = Path::new(db_path).to_path_buf();
if let Some(dir) = path.parent() { if let Some(dir) = path.parent() {
if !dir.as_os_str().is_empty() { if !dir.as_os_str().is_empty() {
crate::platform::create_dir_private(dir) std::fs::create_dir_all(dir)
.map_err(|e| format!("Failed to create database dir {}: {}", dir.display(), e))?; .map_err(|e| format!("Failed to create database dir {}: {}", dir.display(), e))?;
} }
} }
let conn = Connection::open(db_path) let conn = Connection::open(db_path)
.map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?; .map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?;
// Before a single row is written. SQLite creates the file 0644 and hands
// that mode on to `-wal` and `-shm`, so on a default umask every other
// user on the machine could read the index — which holds the names and
// full text of everything under the configured roots, including files
// whose own permissions are 0600.
crate::platform::restrict_to_owner(&path);
key_and_probe(&conn, db_path, key)?; key_and_probe(&conn, db_path, key)?;
conn.execute_batch(PRAGMAS_FAST) conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas: {}", e))?; .map_err(|e| format!("Failed to apply pragmas: {}", e))?;
@ -351,9 +345,6 @@ fn wipe_and_reopen(
} }
let conn = Connection::open(path) let conn = Connection::open(path)
.map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?; .map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?;
// A rebuild creates the file afresh, so it needs narrowing again for the
// same reason the first open does.
crate::platform::restrict_to_owner(path);
key_and_probe(&conn, &path.to_string_lossy(), key)?; key_and_probe(&conn, &path.to_string_lossy(), key)?;
conn.execute_batch(PRAGMAS_FAST) conn.execute_batch(PRAGMAS_FAST)
.map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?; .map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?;

View file

@ -546,55 +546,3 @@ fn open_existing_rw_allows_delete() {
drop(conn); drop(conn);
std::fs::remove_file(&p).ok(); std::fs::remove_file(&p).ok();
} }
/// The index, and the WAL and SHM it hands its mode to, must not be readable
/// by other users on the machine.
///
/// SQLite creates its database file 0644 and copies that mode to `-wal` and
/// `-shm`; with the near-universal umask 022 that leaves the names and full
/// text of everything under the configured roots — including documents whose
/// own files are 0600 — readable by every account on a shared machine. The
/// README's carve-out is about other *processes of the same user*, not other
/// users, so nothing else covers this.
#[cfg(unix)]
#[test]
fn a_fresh_index_and_its_sidecars_are_owner_only() {
use std::os::unix::fs::PermissionsExt;
// A directory that does not exist yet, so the creation path is the one
// under test: an existing directory keeps whatever mode its owner chose.
let p = tmp_db_path()
.parent()
.unwrap()
.join("data")
.join("index.sqlite");
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
// A write, so the WAL and SHM exist to be checked.
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('a', '/perm-a', '/', 0, 0)",
[],
)
.unwrap();
let mode_of = |path: &std::path::Path| {
std::fs::metadata(path)
.unwrap_or_else(|e| panic!("stat {}: {e}", path.display()))
.permissions()
.mode()
& 0o777
};
assert_eq!(mode_of(&p), 0o600, "index at {}", p.display());
for suffix in ["-wal", "-shm"] {
let sidecar = std::path::PathBuf::from(format!("{}{}", p.display(), suffix));
if sidecar.exists() {
assert_eq!(mode_of(&sidecar), 0o600, "sidecar {}", sidecar.display());
}
}
// And the directory created for it, which would otherwise take the umask
// and let any account list what is indexed.
assert_eq!(mode_of(p.parent().unwrap()), 0o700);
drop(conn);
std::fs::remove_file(&p).ok();
}

View file

@ -524,22 +524,7 @@ pub fn pending_content_page(
) -> Result<Vec<PendingContentRow>, String> { ) -> Result<Vec<PendingContentRow>, String> {
let mut stmt = conn let mut stmt = conn
.prepare_cached( .prepare_cached(
// `INDEXED BY` rather than a hint, because the planner gets this "SELECT id, name, path, mime FROM files
// one wrong exactly when it costs most. Left to itself it takes
// `UNIQUE(path)` for the range and then sorts the survivors into a
// temp b-tree to satisfy `ORDER BY id` — which means every page
// walks the whole root's range and fetches each row's heap entry
// to test `content_state`. At `FEED_PAGE` rows per page that is
// quadratic over a run. The partial index below is already
// id-ordered, so it answers `id > ?` and the ORDER BY together and
// holds only pending rows. Measured on 500k rows with everything
// pending — the first index of a tree, i.e. the case that matters:
// 127 ms/page against 0.1 ms/page.
//
// The planner only prefers it once pending rows are a small
// minority, and never before ANALYZE has run at all, which is why
// this cannot be left to statistics.
"SELECT id, name, path, mime FROM files INDEXED BY idx_files_content_pending
WHERE content_state = 0 AND size <= ?1 AND id > ?2 WHERE content_state = 0 AND size <= ?1 AND id > ?2
AND path >= ?3 AND path < ?4 AND path >= ?3 AND path < ?4
ORDER BY id ORDER BY id

View file

@ -69,17 +69,6 @@ impl ExtractedContent {
/// should surface human-readable messages. /// should surface human-readable messages.
pub type ExtractError = String; pub type ExtractError = String;
/// Run `f`, turning a panic into an [`ExtractError`] naming the file.
///
/// The extractors drive third-party parsers — `pdf-extract`, `rtf-parser`,
/// `cfb`, `lofty`, `quick-xml` — over bytes chosen by whoever wrote the file,
/// and several of them are documented to panic on malformed input. See
/// [`Registry::extract`] for what each caller stands to lose.
fn contain_panic<T>(path: &Path, f: impl FnOnce() -> T) -> Result<T, ExtractError> {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(f))
.map_err(|_| format!("extractor panicked on {}", path.display()))
}
/// A pluggable content extractor. Stateless; implementors should not hold /// A pluggable content extractor. Stateless; implementors should not hold
/// file handles across calls. /// file handles across calls.
pub trait Extractor: Send + Sync { pub trait Extractor: Send + Sync {
@ -150,54 +139,26 @@ impl Registry {
/// Look up a handler for `mime` and run it against `path`. Returns /// Look up a handler for `mime` and run it against `path`. Returns
/// `Ok(None)` if no extractor claims the MIME — the caller should then /// `Ok(None)` if no extractor claims the MIME — the caller should then
/// decide whether the file is "not applicable" (text state NA). /// decide whether the file is "not applicable" (text state NA).
///
/// A panicking parser becomes an `Err`, here rather than at each call
/// site: this and [`Registry::extract_complete_head`] are the two places
/// third-party code is handed a file nobody vouched for, and every caller
/// has more than one file to lose. A content worker's panic silently
/// drops the row it claimed; a *walk* worker's costs the root its whole
/// content pass and disables stale cleanup run-wide; the live watcher's
/// costs every displayed row for the rest of the session. Containing it
/// at the boundary means a new caller cannot forget.
///
/// This cannot help with a stack overflow, which aborts rather than
/// unwinding — see `vendor/pdf-extract`, which bounds the recursion that
/// made that reachable.
pub fn extract( pub fn extract(
&self, &self,
path: &Path, path: &Path,
mime: &str, mime: &str,
) -> Result<Option<ExtractedContent>, ExtractError> { ) -> Result<Option<ExtractedContent>, ExtractError> {
let Some(extractor) = self.find(mime) else { self.find(mime).map(|e| e.extract(path)).transpose()
return Ok(None);
};
contain_panic(path, || extractor.extract(path))
.and_then(|r| r)
.map(Some)
} }
/// [`Registry::extract`] for a file whose complete contents the caller /// [`Registry::extract`] for a file whose complete contents the caller
/// already holds. `None` when no extractor claims the MIME or the one /// already holds. `None` when no extractor claims the MIME or the one
/// that does needs the file on disk — both mean "leave this to the /// that does needs the file on disk — both mean "leave this to the
/// content pass". /// content pass".
/// Contained the same way [`Registry::extract`] is, and this is the one
/// that runs on a walk worker.
pub fn extract_complete_head( pub fn extract_complete_head(
&self, &self,
path: &Path, path: &Path,
mime: &str, mime: &str,
head: &[u8], head: &[u8],
) -> Option<Result<ExtractedContent, ExtractError>> { ) -> Option<Result<ExtractedContent, ExtractError>> {
let extractor = self.find(mime)?; self.find(mime)
// `extract_from_head` returning `None` means "needs the file on .and_then(|e| e.extract_from_head(path, head))
// disk", which is not a failure and must stay distinguishable from
// one — so the guard wraps the whole `Option` and a panic becomes
// `Some(Err(..))`, i.e. a failure this file is charged with rather
// than a deferral to the content pass that would meet the same panic.
match contain_panic(path, || extractor.extract_from_head(path, head)) {
Ok(outcome) => outcome,
Err(e) => Some(Err(e)),
}
} }
/// The default set: RTF, plaintext, office docs, PDF, audio tags. /// The default set: RTF, plaintext, office docs, PDF, audio tags.

View file

@ -143,18 +143,6 @@ fn open_container(path: &Path) -> Result<Archive, Box<dyn Error>> {
/// a tiny archive can inflate without bound. /// a tiny archive can inflate without bound.
const MAX_XML_BYTES: usize = 64 * 1024 * 1024; const MAX_XML_BYTES: usize = 64 * 1024 * 1024;
/// Cap on the text taken from one *container*, mirroring [`ole::MAX_TEXT_BYTES`].
///
/// [`MAX_XML_BYTES`] bounds each member on its own, which is not the same
/// thing: a workbook or a deck holds one member per sheet or per slide, and
/// nothing stops a small archive from carrying dozens that each inflate to
/// that cap. The truncation to `maximum_text_size` happens only after the
/// whole string is built and handed back, so without a running total the peak
/// is members × 64 MiB — gigabytes from a file measured in megabytes, on every
/// extraction worker at once, and an allocation failure aborts rather than
/// unwinding.
const MAX_TEXT_BYTES: usize = 64 * 1024 * 1024;
/// One member's bytes as a string. An over-cap member keeps its prefix. /// One member's bytes as a string. An over-cap member keeps its prefix.
fn member_text<R: Read + Seek>( fn member_text<R: Read + Seek>(
archive: &mut ZipArchive<R>, archive: &mut ZipArchive<R>,
@ -213,12 +201,6 @@ fn extract_pptx(path: &Path) -> Result<String, Box<dyn Error>> {
let mut archive = open_container(path)?; let mut archive = open_container(path)?;
let mut out = String::new(); let mut out = String::new();
for name in xml_members_under(&mut archive, "ppt/slides/slide")? { for name in xml_members_under(&mut archive, "ppt/slides/slide")? {
// Per-container budget: see `MAX_TEXT_BYTES`. Whole slides are kept or
// dropped rather than cut mid-way, which is why the test is here
// rather than inside the collector.
if out.len() >= MAX_TEXT_BYTES {
break;
}
let xml = member_text(&mut archive, &name)?; let xml = member_text(&mut archive, &name)?;
collect_xml_text(&xml, &PPTX, &mut out)?; collect_xml_text(&xml, &PPTX, &mut out)?;
out.push_str("\n--- New Slide ---\n"); out.push_str("\n--- New Slide ---\n");
@ -269,20 +251,10 @@ fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(),
Ok(Event::Start(ref e)) if e.name().as_ref() == b"c" => { Ok(Event::Start(ref e)) if e.name().as_ref() == b"c" => {
in_cell = true; in_cell = true;
cell_type.clear(); cell_type.clear();
// `with_checks(false)`: the default duplicate-attribute-name for attr in e.attributes() {
// check compares each name against every name already seen on
// the tag, which is quadratic in the count and has no bound
// but the tag's own size (RUSTSEC-2026-0194). A member may be
// 64 MiB of inflated XML, so one crafted `<c>` can hold
// millions of attributes and hold this worker for hours —
// uncancellably, since the stop flag is only read between
// files. Rejecting duplicate names was never this extractor's
// job; it wants one attribute and stops at it.
for attr in e.attributes().with_checks(false) {
let attr = attr?; let attr = attr?;
if attr.key.as_ref() == b"t" { if attr.key.as_ref() == b"t" {
cell_type = String::from_utf8_lossy(&attr.value).to_string(); cell_type = String::from_utf8_lossy(&attr.value).to_string();
break;
} }
} }
} }
@ -322,10 +294,6 @@ fn extract_xlsx(path: &Path) -> Result<String, Box<dyn Error>> {
let strings = shared_strings(&mut archive); let strings = shared_strings(&mut archive);
let mut out = String::new(); let mut out = String::new();
for name in xml_members_under(&mut archive, "xl/worksheets/sheet")? { for name in xml_members_under(&mut archive, "xl/worksheets/sheet")? {
// Per-container budget: see `MAX_TEXT_BYTES`.
if out.len() >= MAX_TEXT_BYTES {
break;
}
let xml = member_text(&mut archive, &name)?; let xml = member_text(&mut archive, &name)?;
collect_sheet(&xml, &strings, &mut out)?; collect_sheet(&xml, &strings, &mut out)?;
} }

View file

@ -266,100 +266,4 @@ mod tests {
err err
); );
} }
/// A page whose `/Parent` is itself must be skipped, not fatal.
///
/// `get_inherited` walks `/Parent` looking for `Resources` and `MediaBox`,
/// and upstream did so without a depth bound — a cycle recursed until the
/// guard page. That is *not* a panic `catch_unwind` can hold: Rust's
/// handler aborts, so this test could not even be written before
/// `vendor/pdf-extract` bounded the walk; it would have taken the test
/// binary down with it. What arrives now is an ordinary contained failure.
#[test]
fn a_self_referential_page_parent_is_contained() {
let mut doc = Document::with_version("1.5");
let contents = doc.add_object(Stream::new(dictionary! {}, b"BT ET".to_vec()));
let page_id = doc.new_object_id();
// Neither `Resources` nor `MediaBox` here, so both lookups have to
// follow `/Parent` — which points back at this same dictionary.
doc.objects.insert(
page_id,
Object::Dictionary(dictionary! {
"Type" => "Page",
"Parent" => page_id,
"Contents" => contents,
}),
);
let pages_id = doc.add_object(dictionary! {
"Type" => "Pages",
"Kids" => vec![page_id.into()],
"Count" => 1,
});
let catalog = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => pages_id,
});
doc.trailer.set("Root", catalog);
let path = crate::testutil::scratch_dir("pdf-parent-cycle").join("cycle.pdf");
doc.save(&path).expect("write fixture pdf");
// The verdict that matters is that we reach this line at all.
let _ = PdfExtractor.extract(&path);
}
/// A Form XObject whose content stream draws itself must be skipped, not
/// fatal — the second unbounded recursion, in `process_stream`'s `Do` arm.
/// The same bound also caps the branching shape, where each level draws
/// the next twice and a shallow document costs 2^depth calls.
#[test]
fn a_self_drawing_form_xobject_is_contained() {
let mut doc = Document::with_version("1.5");
let form_id = doc.new_object_id();
// Its own resources name it, so `/X0 Do` inside it re-enters itself.
doc.objects.insert(
form_id,
Object::Stream(Stream::new(
dictionary! {
"Type" => "XObject",
"Subtype" => "Form",
"BBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
"Resources" => dictionary! {
"XObject" => dictionary! { "X0" => form_id },
},
},
b"/X0 Do".to_vec(),
)),
);
let resources = doc.add_object(dictionary! {
"XObject" => dictionary! { "X0" => form_id },
});
let contents = doc.add_object(Stream::new(dictionary! {}, b"/X0 Do".to_vec()));
let pages_id = doc.new_object_id();
let page = doc.add_object(dictionary! {
"Type" => "Page",
"Parent" => pages_id,
"Contents" => contents,
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
});
doc.objects.insert(
pages_id,
Object::Dictionary(dictionary! {
"Type" => "Pages",
"Kids" => vec![page.into()],
"Count" => 1,
"Resources" => resources,
}),
);
let catalog = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => pages_id,
});
doc.trailer.set("Root", catalog);
let path = crate::testutil::scratch_dir("pdf-xobject-cycle").join("cycle.pdf");
doc.save(&path).expect("write fixture pdf");
let _ = PdfExtractor.extract(&path);
}
} }

View file

@ -83,53 +83,6 @@ mod tests {
std::fs::remove_file(&p).ok(); std::fs::remove_file(&p).ok();
} }
/// A `\\u` escape naming a lone UTF-16 surrogate must fail the file, not
/// the thread.
///
/// `rtf-parser` reaches `String::from_utf16(..).unwrap()` with whatever
/// `\\uN` supplied, and screens nothing for the surrogate range. RTF is one
/// of the two extractors that also run at *walk* time, off
/// `extract_from_head`, where a panicking worker costs the root its entire
/// content pass and disables stale cleanup run-wide — so this is contained
/// in `decide_content` and `prepare_file_record` rather than left to the
/// parser. Both entry points are exercised here.
#[test]
fn a_lone_surrogate_escape_is_contained() {
let body = br"{\rtf1\u55296 }";
let p = tmp("surrogate", body);
// The on-disk path, as the content pass reaches it.
let outcome = crate::file_handling::decide_content(
p.to_str().unwrap(),
Some("application/rtf"),
&crate::extract::Registry::default_set(),
&crate::config::Config::default(),
);
assert!(
matches!(outcome, crate::file_handling::ContentOutcome::Failed(_)),
"a panicking parser must record a failure, not unwind: {:?}",
outcome
);
// And the head path, as a walk worker reaches it: through the
// registry, which is where the containment lives. The raw
// `RtfExtractor::extract_from_head` below it still panics — that is
// third-party code doing what it does, and the point is that no
// caller in this crate is exposed to it.
let head = crate::extract::Registry::default_set().extract_complete_head(
&p,
"application/rtf",
body,
);
assert!(
matches!(head, Some(Err(_))),
"a panicking parser must be charged to the file, not the worker: {:?}",
head.map(|r| r.map(|c| c.text))
);
std::fs::remove_file(&p).ok();
}
#[test] #[test]
fn malformed_input_errors_and_names_the_file() { fn malformed_input_errors_and_names_the_file() {
let p = tmp("broken", br"{\rtf1 truncated"); let p = tmp("broken", br"{\rtf1 truncated");

View file

@ -78,10 +78,7 @@ pub(super) fn get_file_hash(
path: &Path, path: &Path,
hash_length: usize, hash_length: usize,
) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> { ) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> {
// The caller's `is_file()` came from a `stat` taken before this open; a let mut f: File = File::open(path)?;
// FIFO renamed over the name in between would block this walk worker
// forever and park the pool behind it. See `platform::open_regular_file`.
let mut f: File = crate::platform::open_regular_file(path)?;
// Files shorter than the window hash whole; `min` keeps the cast sound // Files shorter than the window hash whole; `min` keeps the cast sound
// for large files on 32-bit targets. // for large files on 32-bit targets.
let mut head = vec![0u8; size.min(hash_length as u64) as usize]; let mut head = vec![0u8; size.min(hash_length as u64) as usize];
@ -236,9 +233,6 @@ pub fn prepare_file_record(
if size == 0 || size > config.processing.hash_length as u64 { if size == 0 || size > config.processing.hash_length as u64 {
return None; return None;
} }
// A panicking parser arrives here as `Some(Err(..))` — contained by
// the registry, which is what keeps a walk worker alive. See
// `Registry::extract`.
match registry.extract_complete_head(Path::new(path), m, &head) { match registry.extract_complete_head(Path::new(path), m, &head) {
Some(Ok(content)) => { Some(Ok(content)) => {
let mut text = content.text; let mut text = content.text;
@ -350,9 +344,7 @@ pub fn decide_content(
return ContentOutcome::NotApplicable; return ContentOutcome::NotApplicable;
} }
// `content_extractable` established that an extractor claims this MIME, // `content_extractable` established that an extractor claims this MIME,
// so the `Ok(None)` arm below is unreachable. A panicking parser is // so the `Ok(None)` arm below is unreachable.
// contained by the registry and arrives as `Err`, which becomes this
// row's recorded failure reason rather than a dead worker.
let result = match mime { let result = match mime {
Some(m) => registry.extract(p, m), Some(m) => registry.extract(p, m),
None => Ok(None), None => Ok(None),

View file

@ -11,8 +11,6 @@
//! containment check is repeated here. //! containment check is repeated here.
use std::path::Path; use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant;
use rusqlite::{Connection, OptionalExtension}; use rusqlite::{Connection, OptionalExtension};
@ -26,59 +24,22 @@ use crate::file_handling::{
use crate::platform::path_has_hidden_component_under; use crate::platform::path_has_hidden_component_under;
use crate::watcher::FsEvent; use crate::watcher::FsEvent;
/// How much of one event may be applied in this turn, and where the last turn
/// stopped.
///
/// One event is not always one file: a directory moved into a watched tree
/// arrives as a single `Create` covering everything beneath it. Shaped like
/// [`crate::scope::advance`]'s arguments and there for the same reason — this
/// runs on the coordinator's own thread, so an unbounded call is a command
/// loop that reads no commands, including the shutdown a closing window is
/// waiting on.
pub struct Budget<'a> {
pub deadline: Instant,
pub cancel: &'a AtomicBool,
/// Entries an earlier turn already applied for this event.
pub resume_from: usize,
}
impl Budget<'_> {
fn spent(&self) -> bool {
self.cancel.load(Ordering::Relaxed) || Instant::now() >= self.deadline
}
}
/// Whether an event was applied in full, or ran out of budget partway.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Applied {
/// Everything the event implied is in the index.
Done,
/// Budget spent. What was written is committed; the caller should re-queue
/// the same event with `resume_from` set to `done`.
Unfinished { done: usize },
}
/// Apply one filesystem event to the index. Missing files are treated as /// Apply one filesystem event to the index. Missing files are treated as
/// no-ops (a Create followed by a quick delete resolves via the Remove /// no-ops (a Create followed by a quick delete resolves via the Remove
/// event); unchanged mtimes short-circuit without touching the DB. /// event); unchanged mtimes short-circuit without touching the DB.
///
/// See [`Budget`] for the one event that is not small.
pub fn apply_fs_event( pub fn apply_fs_event(
conn: &mut Connection, conn: &mut Connection,
event: &FsEvent, event: &FsEvent,
config: &Config, config: &Config,
ignore: &IgnoreSet, ignore: &IgnoreSet,
registry: &Registry, registry: &Registry,
budget: &Budget<'_>, ) -> Result<(), String> {
) -> Result<Applied, String> {
match event { match event {
FsEvent::Create(p) | FsEvent::Modify(p) => { FsEvent::Create(p) | FsEvent::Modify(p) => upsert_path(conn, p, config, ignore, registry),
upsert_path(conn, p, config, ignore, registry, budget) FsEvent::Remove(p) => remove_path(conn, p),
}
FsEvent::Remove(p) => remove_path(conn, p).map(|()| Applied::Done),
FsEvent::Rename { from, to } => { FsEvent::Rename { from, to } => {
remove_path(conn, from)?; remove_path(conn, from)?;
upsert_path(conn, to, config, ignore, registry, budget) upsert_path(conn, to, config, ignore, registry)
} }
} }
} }
@ -89,10 +50,9 @@ fn upsert_path(
config: &Config, config: &Config,
ignore: &IgnoreSet, ignore: &IgnoreSet,
registry: &Registry, registry: &Registry,
budget: &Budget<'_>, ) -> Result<(), String> {
) -> Result<Applied, String> {
if ignore.matches_path(path) { if ignore.matches_path(path) {
return Ok(Applied::Done); return Ok(());
} }
// Measured from the innermost configured root: the walk never filters // Measured from the innermost configured root: the walk never filters
// the root it was handed, so a root that is itself hidden must not be // the root it was handed, so a root that is itself hidden must not be
@ -100,11 +60,11 @@ fn upsert_path(
if !config.indexing.include_hidden if !config.indexing.include_hidden
&& path_has_hidden_component_under(path, &config.resolved_indexing_paths()) && path_has_hidden_component_under(path, &config.resolved_indexing_paths())
{ {
return Ok(Applied::Done); return Ok(());
} }
let Ok(meta) = std::fs::metadata(path) else { let Ok(meta) = std::fs::metadata(path) else {
// Already gone again — the pending Remove event handles it. // Already gone again — the pending Remove event handles it.
return Ok(Applied::Done); return Ok(());
}; };
if meta.is_dir() { if meta.is_dir() {
// A moved-in tree surfaces as one directory event; walk it with // A moved-in tree surfaces as one directory event; walk it with
@ -116,37 +76,20 @@ fn upsert_path(
let Some(root) = path.to_str() else { let Some(root) = path.to_str() else {
return Err(format!("directory path is not valid UTF-8: {:?}", path)); return Err(format!("directory path is not valid UTF-8: {:?}", path));
}; };
// Streamed, not collected: `mv` of a large tree is one event, and let entries: Vec<_> = filtered_walk(
// materialising its entries first is a `DirEntry` per file resident
// before a single row is written. Each file is its own transaction,
// so stopping between two of them leaves the index consistent and the
// remainder for the next turn.
//
// `skip` rather than re-testing every entry: `upsert_file` on an
// unchanged file is cheap but not free, and paying it again for
// everything already done would make a large tree quadratic in the
// number of turns it takes. The walk order is deterministic for an
// unchanged tree; if the tree does change under us the count is only
// an optimisation, and the next full run is what makes it exact.
let mut done = budget.resume_from;
for entry in filtered_walk(
root, root,
config.indexing.follow_symlinks, config.indexing.follow_symlinks,
config.indexing.include_hidden, config.indexing.include_hidden,
ignore, ignore,
&UnreadableDirs::default(), &UnreadableDirs::default(),
) )
.skip(budget.resume_from) .collect();
{ for entry in entries {
if budget.spent() {
return Ok(Applied::Unfinished { done });
}
upsert_file(conn, entry.path(), config, registry)?; upsert_file(conn, entry.path(), config, registry)?;
done += 1;
} }
Ok(Applied::Done) Ok(())
} else { } else {
upsert_file(conn, path, config, registry).map(|()| Applied::Done) upsert_file(conn, path, config, registry)
} }
} }
@ -252,7 +195,6 @@ pub fn remove_paths(
mod tests { mod tests {
use super::*; use super::*;
use crate::db::open_or_recreate; use crate::db::open_or_recreate;
use std::time::Duration;
struct Fixture { struct Fixture {
conn: Connection, conn: Connection,
@ -282,32 +224,15 @@ mod tests {
} }
} }
/// Applies with an effectively unlimited budget: these tests are about
/// what lands in the index, not about the slicing. See
/// [`Fixture::apply_within`] for the budget itself.
fn apply(&mut self, event: &FsEvent) { fn apply(&mut self, event: &FsEvent) {
let done = self.apply_within(event, Duration::from_secs(3600));
assert_eq!(done, Applied::Done, "unexpectedly ran out of budget");
}
fn apply_within(&mut self, event: &FsEvent, budget: Duration) -> Applied {
self.apply_resuming(event, budget, 0)
}
fn apply_resuming(&mut self, event: &FsEvent, budget: Duration, from: usize) -> Applied {
apply_fs_event( apply_fs_event(
&mut self.conn, &mut self.conn,
event, event,
&self.config, &self.config,
&self.ignore, &self.ignore,
&self.registry, &self.registry,
&Budget {
deadline: Instant::now() + budget,
cancel: &AtomicBool::new(false),
resume_from: from,
},
) )
.unwrap() .unwrap();
} }
fn write(&self, name: &str, content: &str) -> std::path::PathBuf { fn write(&self, name: &str, content: &str) -> std::path::PathBuf {
@ -371,44 +296,6 @@ mod tests {
/// The collapse must not change what ends up deleted — only how much work /// The collapse must not change what ends up deleted — only how much work
/// it takes to get there. /// it takes to get there.
/// A directory event is applied in slices, and a slice resumes where the
/// last one stopped instead of re-walking what it already did.
///
/// `mv` of a large tree is one `Create`. Applying it in one go held the
/// coordinator's command loop — and the shutdown queued behind it — for as
/// long as the whole tree took; applying it in slices that each restarted
/// from the top would be quadratic instead. Asserted by resume point
/// rather than by clock, because a timing-based assertion says nothing
/// reliable on a loaded CI runner.
#[test]
fn a_directory_event_resumes_where_its_budget_ran_out() {
let mut f = Fixture::new();
for i in 0..4 {
f.write(&format!("sub/f{i}.txt"), "body");
}
let sub = f.dir.join("sub");
// Nothing may be spent, so nothing is applied and the resume point is
// where it started.
let outcome = f.apply_within(&FsEvent::Create(sub.clone()), Duration::ZERO);
assert_eq!(outcome, Applied::Unfinished { done: 0 });
assert_eq!(f.counts().0, 0, "a spent budget must write nothing");
// Resuming past the first two entries applies only what is left, which
// is what makes slicing linear rather than quadratic.
let outcome = f.apply_resuming(&FsEvent::Create(sub.clone()), Duration::from_secs(3600), 2);
assert_eq!(outcome, Applied::Done);
assert_eq!(
f.counts().0,
2,
"entries before the resume point must be skipped, not re-applied"
);
// And from the start, the rest arrive.
f.apply(&FsEvent::Create(sub));
assert_eq!(f.counts().0, 4);
}
#[test] #[test]
fn collapsed_removal_deletes_the_same_rows_as_the_full_set() { fn collapsed_removal_deletes_the_same_rows_as_the_full_set() {
let mut f = Fixture::new(); let mut f = Fixture::new();

View file

@ -643,10 +643,7 @@ impl Loop {
/// The first `limit` bytes of a file, for MIME sniffing. A short read is the /// The first `limit` bytes of a file, for MIME sniffing. A short read is the
/// whole file and is not an error; an unreadable file simply has no MIME. /// whole file and is not an error; an unreadable file simply has no MIME.
fn read_head(path: &Path, limit: usize) -> Option<Vec<u8>> { fn read_head(path: &Path, limit: usize) -> Option<Vec<u8>> {
// The row's own `stat` decided this was a file; by now it may be a FIFO, let file = std::fs::File::open(path).ok()?;
// and a blocking open on this thread costs every live row for the rest of
// the session — and hangs exit, since `LiveWatcher::stop` joins here.
let file = crate::platform::open_regular_file(path).ok()?;
let mut head = Vec::new(); let mut head = Vec::new();
file.take(limit as u64).read_to_end(&mut head).ok()?; file.take(limit as u64).read_to_end(&mut head).ok()?;
Some(head) Some(head)

View file

@ -490,89 +490,6 @@ pub fn release_free_heap() {
// spans to the kernel on free. // spans to the kernel on free.
} }
/// Create `dir` and its parents, readable only by their owner.
///
/// `create_dir_all` leaves the mode to the umask, which is 022 nearly
/// everywhere and so grants the world `+rx`. The two directories this is
/// called for hold the index and the config, i.e. the names and text of
/// everything under the configured roots, plus the salt.
///
/// Only directories *created here* are narrowed: an existing one keeps its
/// mode, because a user who chose `~/Documents` as their data directory did
/// not ask for it to be locked down.
pub fn create_dir_private(dir: &Path) -> std::io::Result<()> {
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(dir)
}
#[cfg(not(unix))]
{
// Windows: a file created under the user's profile inherits an ACL
// that already excludes other users.
std::fs::create_dir_all(dir)
}
}
/// Narrow `path` to owner-only access, best effort.
///
/// SQLite creates its database 0644 (`SQLITE_DEFAULT_FILE_PERMISSIONS`, which
/// the bundled build does not override) and then copies that mode to the
/// `-wal` and `-shm` it derives from it, so narrowing the main file before
/// anything else is opened covers all three. The index is a strictly larger
/// secret than any single file it was read from: it holds the full text of
/// documents whose own permissions may be far tighter.
///
/// Failure is ignored: on a filesystem with no Unix permissions (a FAT stick,
/// a network share) there is nothing to set and nothing to report.
pub fn restrict_to_owner(path: &Path) {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600));
}
#[cfg(not(unix))]
{
let _ = path;
}
}
/// Open `path` for reading, refusing anything that is not a regular file.
///
/// Every caller has already decided the path *was* a regular file, from a
/// `stat` taken earlier — during the walk, or when a result row was put on
/// screen. A rename can put a FIFO, a tty or a character device at that name
/// in between, and `open` on one of those blocks in the kernel until a writer
/// or a carrier appears: uninterruptibly, past any stop flag, and for as long
/// as the process lives. One such open strands a walk worker while it still
/// holds its job slot, which parks the whole pool behind it.
///
/// `O_NONBLOCK` makes the open itself return, and the handle is then asked
/// what it actually is — `fstat` on the descriptor, so nothing can swap the
/// name again underneath the answer. Both are free on a regular file, which
/// is the only case that matters for speed: the flag is ignored for ordinary
/// files and the `fstat` hits the inode already in hand.
pub fn open_regular_file(path: &Path) -> std::io::Result<std::fs::File> {
let mut opts = std::fs::OpenOptions::new();
opts.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.custom_flags(libc::O_NONBLOCK);
}
let file = opts.open(path)?;
if !file.metadata()?.file_type().is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"not a regular file",
));
}
Ok(file)
}
/// How long to keep retrying a delete that fails because something else holds /// How long to keep retrying a delete that fails because something else holds
/// the file open. /// the file open.
#[cfg(windows)] #[cfg(windows)]

View file

@ -67,18 +67,6 @@ impl<'a> Cx<'a> {
if scanned.is_multiple_of(cancel_every) && self.cancelled() { if scanned.is_multiple_of(cancel_every) && self.cancelled() {
return Ok(false); return Ok(false);
} }
// The display limit is already full, and holding a row proves at
// least one more match exists than will be shown — so `limited` is
// exactly true here, and everything below is work whose result
// `flush_pass` would throw away. That work is not small: the
// full-text passes decompress the document, fold a copy of it, and
// cut a snippet, per row. `cascade::run` makes the same test
// between passes; without this one a single pass over a common
// term runs to the end of the candidate set.
if self.remaining() == 0 {
self.limited = true;
break;
}
let file_id: i64 = col(row, 0)?; let file_id: i64 = col(row, 0)?;
// Borrowed from the statement rather than `col::<String>`: this // Borrowed from the statement rather than `col::<String>`: this
// runs for every *scanned* row — a full-table scan on the filename // runs for every *scanned* row — a full-table scan on the filename

View file

@ -109,12 +109,7 @@ pub fn verify_identical(paths: &[PathBuf], cancel: &AtomicBool, on: &mut dyn FnM
let mut reference: Option<(usize, File, u64)> = None; let mut reference: Option<(usize, File, u64)> = None;
let mut rest: Vec<(usize, File)> = Vec::new(); let mut rest: Vec<(usize, File)> = Vec::new();
for (i, path) in paths.iter().enumerate() { for (i, path) in paths.iter().enumerate() {
// These paths come from the index, which records what each file was let file = match File::open(path) {
// when it was walked. A member replaced by a FIFO since then needs no
// race at all to be sitting here — the walk keeps the old row when a
// regular file turns into something else — and a blocking open would
// strand this worker before it reported a single verdict.
let file = match crate::platform::open_regular_file(path) {
Ok(f) => f, Ok(f) => f,
Err(e) => { Err(e) => {
verdicts[i] = MemberVerdict::Unreadable(describe(path, &e)); verdicts[i] = MemberVerdict::Unreadable(describe(path, &e));

View file

@ -349,18 +349,9 @@ impl QuickSearchApp {
.resolved_database_path() .resolved_database_path()
.to_string_lossy() .to_string_lossy()
.into_owned(); .into_owned();
let previous_security = self.cfg.security.clone();
self.cfg.security = new_security; self.cfg.security = new_security;
if let Err(e) = self.cfg.save() { if let Err(e) = self.cfg.save() {
// Fail closed, exactly as `SecurityAction::SetKeychain` does: the
// salt this change depends on lives only in that file. Carrying on
// would install the new key and rebuild the index under it while
// the config on disk still describes the old state — an index
// encrypted with a salt that reached no disk, which no password
// can open afterwards. Put the config back and stop.
self.cfg.security = previous_security;
self.config_error = Some(e); self.config_error = Some(e);
return;
} }
match (&new_key, self.cfg.security.use_keychain) { match (&new_key, self.cfg.security.use_keychain) {
(Some(key), true) => { (Some(key), true) => {

View file

@ -211,20 +211,10 @@ impl UnlockScreen {
match result { match result {
Ok(key) => return self.unlocked(ctx, key), Ok(key) => return self.unlocked(ctx, key),
Err(e) => { Err(e) => {
// A tagged mismatch has three distinct causes and self.error = Some(if e.starts_with(db::KEY_MISMATCH_PREFIX) {
// `key_mismatch_message` already told them apart. Only one "Wrong password.".to_string()
// of them is a wrong password; the others are "the config } else {
// says protected but the index on disk is not" (a crash e
// between saving the config and rebuilding) and "this
// index wants a password at all". Collapsing them all into
// "Wrong password." tells a user with the right password
// that it is wrong, and the only button on this screen
// deletes their index and turns protection off. The detail
// names the database path and nothing secret.
self.error = Some(match e.strip_prefix(db::KEY_MISMATCH_PREFIX) {
Some(_) if e.contains("wrong password") => "Wrong password.".to_string(),
Some(detail) => detail.to_string(),
None => e,
}); });
// The field was cleared on submit; put the caret back // The field was cleared on submit; put the caret back
// for the retry. // for the retry.

View file

@ -78,11 +78,5 @@ Comment:
font data rather than as linked program code, which is the same basis on font data rather than as linked program code, which is the same basis on
which Debian ships these fonts and other egui-based applications. which Debian ships these fonts and other egui-based applications.
. .
One dependency is carried in-tree rather than fetched: vendor/pdf-extract
is pdf-extract 0.12.0 (MIT, Jeff Muizelaar,
https://github.com/jrmuizel/pdf-extract) with two unbounded recursions
given a depth limit, marked "LOCAL PATCH" in the source. Its licence is
unchanged and is the MIT stanza below.
.
Run `cargo metadata --all-features` against the source tree to reproduce Run `cargo metadata --all-features` against the source tree to reproduce
the per-crate licence list. the per-crate licence list.

View file

@ -1 +0,0 @@
{"v":1}

View file

@ -1,6 +0,0 @@
{
"git": {
"sha1": "b95bf9f6268772d5088f09b0034e488e64294835"
},
"path_in_vcs": ""
}

View file

@ -1,73 +0,0 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2018"
name = "pdf-extract"
version = "0.12.0"
authors = ["Jeff Muizelaar <jmuizelaar@mozilla.com>"]
build = false
include = [
"src/**/*",
"README.md",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "A library to extract content from pdfs"
documentation = "https://docs.rs/crate/pdf-extract/"
readme = "README.md"
keywords = [
"pdf2text",
"text",
"pdf",
"pdf2txt",
]
license = "MIT"
repository = "https://github.com/jrmuizel/pdf-extract"
[lib]
name = "pdf_extract"
path = "src/lib.rs"
[dependencies.adobe-cmap-parser]
version = "0.4.1"
[dependencies.cff-parser]
version = "0.2.0"
[dependencies.encoding_rs]
version = "0.8.34"
[dependencies.euclid]
version = "0.20.5"
[dependencies.log]
version = "0.4.22"
[dependencies.lopdf]
version = "0.42"
features = ["wasm_js"]
default-features = false
[dependencies.postscript]
version = "0.14"
[dependencies.type1-encoding-parser]
version = "0.1.1"
[dependencies.unicode-normalization]
version = "0.1.19"
[lints.rust]
warnings = "allow"

View file

@ -1,24 +0,0 @@
## pdf-extract
[![Build Status](https://github.com/jrmuizel/pdf-extract/actions/workflows/rust.yml/badge.svg)](https://github.com/jrmuizel/pdf-extract/actions)
[![crates.io](https://img.shields.io/crates/v/pdf-extract.svg)](https://crates.io/crates/pdf-extract)
[![Documentation](https://docs.rs/pdf-extract/badge.svg)](https://docs.rs/pdf-extract)
A rust library to extract content from PDF files.
```rust
let bytes = std::fs::read("tests/docs/simple.pdf").unwrap();
let out = pdf_extract::extract_text_from_mem(&bytes).unwrap();
assert!(out.contains("This is a small demonstration"));
```
## See also
- https://github.com/elacin/PDFExtract/
- https://github.com/euske/pdfminer / https://github.com/pdfminer/pdfminer.six
- https://gitlab.com/crossref/pdfextract
- https://github.com/VikParuchuri/marker
- https://github.com/kermitt2/pdfalto used by [grobid](https://github.com/kermitt2/grobid/)
- https://github.com/opendatalab/MinerU (uses PyMuPDF and pdfminer.six)
### Not PDF specific
- https://github.com/Layout-Parser/layout-parser

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load diff

View file

@ -1,41 +0,0 @@
glyphlist = []
glyphs_seen = {}
def read_glyphs(name):
f = open(name)
lines = f.readlines()
import re
for l in lines:
if l[0] == '#' or l[0] == '\n':
continue
split = re.split('[; ,]+', l)
name = split[0]
val = int(split[1], 16)
if val > 0xffff:
val = int(split[-1], 16)
if val == 0xf766 and name != "Fsmall":
continue
if name in glyphs_seen:
continue
glyphs_seen[name] = True
glyphlist.append((name,val))
read_glyphs("glyphlist-extended.txt")
read_glyphs("texglyphlist.txt")
read_glyphs("additional.txt")
# there are some conflicts between these files
# e.g. tildewide=0x02dc, vs tildewide=0x0303
# for now we just ignore the subsequent ones
glyphlist.append(('mapsto', 0x21A6))
glyphlist = list(set(glyphlist))
glyphlist.sort()
print "/* Autogenerated from:"
print " https://github.com/michal-h21/htfgen/commits/master/glyphlist-extended.txt"
print " https://github.com/kohler/lcdf-typetools/blob/master/texglyphlist.txt"
print " https://github.com/apache/pdfbox/blob/trunk/pdfbox/src/main/resources/org/apache/pdfbox/resources/glyphlist/additional.txt"
print " */"
print "pub fn name_to_unicode(name: &str) -> Option<u16> {"
print " const names: [(&'static str, u16); %d] = [" % len(glyphlist)
print ",\n".join('(\"%s\", 0x%04x)' % (g[0], g[1]) for g in glyphlist)
print " ];"
print " let result = names.binary_search_by_key(&name, |&(name,_code)| &name);"
print " result.ok().map(|indx| names[indx].1)"
print "}"

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -1,210 +0,0 @@
pub fn zapfdigbats_names_to_unicode(name: &str) -> Option<u16> {
let names = [
("a1", 0x2701),
("a10", 0x2721),
("a100", 0x275e),
("a101", 0x2761),
("a102", 0x2762),
("a103", 0x2763),
("a104", 0x2764),
("a105", 0x2710),
("a106", 0x2765),
("a107", 0x2766),
("a108", 0x2767),
("a109", 0x2660),
("a11", 0x261b),
("a110", 0x2665),
("a111", 0x2666),
("a112", 0x2663),
("a117", 0x2709),
("a118", 0x2708),
("a119", 0x2707),
("a12", 0x261e),
("a120", 0x2460),
("a121", 0x2461),
("a122", 0x2462),
("a123", 0x2463),
("a124", 0x2464),
("a125", 0x2465),
("a126", 0x2466),
("a127", 0x2467),
("a128", 0x2468),
("a129", 0x2469),
("a13", 0x270c),
("a130", 0x2776),
("a131", 0x2777),
("a132", 0x2778),
("a133", 0x2779),
("a134", 0x277a),
("a135", 0x277b),
("a136", 0x277c),
("a137", 0x277d),
("a138", 0x277e),
("a139", 0x277f),
("a14", 0x270d),
("a140", 0x2780),
("a141", 0x2781),
("a142", 0x2782),
("a143", 0x2783),
("a144", 0x2784),
("a145", 0x2785),
("a146", 0x2786),
("a147", 0x2787),
("a148", 0x2788),
("a149", 0x2789),
("a15", 0x270e),
("a150", 0x278a),
("a151", 0x278b),
("a152", 0x278c),
("a153", 0x278d),
("a154", 0x278e),
("a155", 0x278f),
("a156", 0x2790),
("a157", 0x2791),
("a158", 0x2792),
("a159", 0x2793),
("a16", 0x270f),
("a160", 0x2794),
("a161", 0x2192),
("a162", 0x27a3),
("a163", 0x2194),
("a164", 0x2195),
("a165", 0x2799),
("a166", 0x279b),
("a167", 0x279c),
("a168", 0x279d),
("a169", 0x279e),
("a17", 0x2711),
("a170", 0x279f),
("a171", 0x27a0),
("a172", 0x27a1),
("a173", 0x27a2),
("a174", 0x27a4),
("a175", 0x27a5),
("a176", 0x27a6),
("a177", 0x27a7),
("a178", 0x27a8),
("a179", 0x27a9),
("a18", 0x2712),
("a180", 0x27ab),
("a181", 0x27ad),
("a182", 0x27af),
("a183", 0x27b2),
("a184", 0x27b3),
("a185", 0x27b5),
("a186", 0x27b8),
("a187", 0x27ba),
("a188", 0x27bb),
("a189", 0x27bc),
("a19", 0x2713),
("a190", 0x27bd),
("a191", 0x27be),
("a192", 0x279a),
("a193", 0x27aa),
("a194", 0x27b6),
("a195", 0x27b9),
("a196", 0x2798),
("a197", 0x27b4),
("a198", 0x27b7),
("a199", 0x27ac),
("a2", 0x2702),
("a20", 0x2714),
("a200", 0x27ae),
("a201", 0x27b1),
("a202", 0x2703),
("a203", 0x2750),
("a204", 0x2752),
("a205", 0x276e),
("a206", 0x2770),
("a21", 0x2715),
("a22", 0x2716),
("a23", 0x2717),
("a24", 0x2718),
("a25", 0x2719),
("a26", 0x271a),
("a27", 0x271b),
("a28", 0x271c),
("a29", 0x2722),
("a3", 0x2704),
("a30", 0x2723),
("a31", 0x2724),
("a32", 0x2725),
("a33", 0x2726),
("a34", 0x2727),
("a35", 0x2605),
("a36", 0x2729),
("a37", 0x272a),
("a38", 0x272b),
("a39", 0x272c),
("a4", 0x260e),
("a40", 0x272d),
("a41", 0x272e),
("a42", 0x272f),
("a43", 0x2730),
("a44", 0x2731),
("a45", 0x2732),
("a46", 0x2733),
("a47", 0x2734),
("a48", 0x2735),
("a49", 0x2736),
("a5", 0x2706),
("a50", 0x2737),
("a51", 0x2738),
("a52", 0x2739),
("a53", 0x273a),
("a54", 0x273b),
("a55", 0x273c),
("a56", 0x273d),
("a57", 0x273e),
("a58", 0x273f),
("a59", 0x2740),
("a6", 0x271d),
("a60", 0x2741),
("a61", 0x2742),
("a62", 0x2743),
("a63", 0x2744),
("a64", 0x2745),
("a65", 0x2746),
("a66", 0x2747),
("a67", 0x2748),
("a68", 0x2749),
("a69", 0x274a),
("a7", 0x271e),
("a70", 0x274b),
("a71", 0x25cf),
("a72", 0x274d),
("a73", 0x25a0),
("a74", 0x274f),
("a75", 0x2751),
("a76", 0x25b2),
("a77", 0x25bc),
("a78", 0x25c6),
("a79", 0x2756),
("a8", 0x271f),
("a81", 0x25d7),
("a82", 0x2758),
("a83", 0x2759),
("a84", 0x275a),
("a85", 0x276f),
("a86", 0x2771),
("a87", 0x2772),
("a88", 0x2773),
("a89", 0x2768),
("a9", 0x2720),
("a90", 0x2769),
("a91", 0x276c),
("a92", 0x276d),
("a93", 0x276a),
("a94", 0x276b),
("a95", 0x2774),
("a96", 0x2775),
("a97", 0x275b),
("a98", 0x275c),
("a99", 0x275d),
("space", 0x0020),
];
let result = names.binary_search_by_key(&name, |&(name,_code)| &name);
result.ok().map(|indx| names[indx].1)
}