630 lines
22 KiB
Rust
630 lines
22 KiB
Rust
//! Parallel content extraction for one indexing root.
|
||
//!
|
||
//! The second half of a root's pipeline, and the sibling of [`crate::walk`]:
|
||
//! a pool of worker threads produces finished work over a bounded channel, and
|
||
//! the single writer drains it round-robin against every other root.
|
||
//!
|
||
//! That shape is the whole point. Extraction used to run *on* the writer
|
||
//! thread, a batch of files at a time, with the database connection held for
|
||
//! the duration — so one root reading a network share or a stack of large PDFs
|
||
//! stopped every other root dead for as long as its batch took, and blocked
|
||
//! the watcher's incremental writes with it. Roots are meant to be
|
||
//! independent; walking already was, and this makes the rest of the pipeline
|
||
//! match.
|
||
//!
|
||
//! The split within the pass mirrors the walk's too: **one feeder thread owns
|
||
//! the only database connection**, paging through the root's pending rows,
|
||
//! while N workers do nothing but filesystem work. A connection per worker
|
||
//! would multiply SQLite's page cache by the pool size, which is exactly what
|
||
//! [`crate::db::schema::PRAGMAS_WALK_READER`] exists to avoid.
|
||
//!
|
||
//! Deliberately std-only (`std::thread` + `std::sync::mpsc`), matching the
|
||
//! house style of [`crate::walk`] and [`crate::watcher`].
|
||
|
||
use std::sync::atomic::AtomicBool;
|
||
use std::sync::{mpsc, Arc, Condvar, Mutex};
|
||
use std::thread::JoinHandle;
|
||
|
||
use crate::config::Config;
|
||
use crate::extract::Registry;
|
||
use crate::file_handling::{decide_content, ContentOutcome, ExtractCursor};
|
||
use crate::indexing::should_abort;
|
||
use crate::walk::{try_recv_next, TryNext, WorkerStats};
|
||
|
||
/// Finished rows waiting for the writer.
|
||
///
|
||
/// Far shallower than the walk's 4096. An [`ExtractedRow`] carries up to
|
||
/// `maximum_text_size` of extracted text — 256 KiB by default, some 32× a
|
||
/// `WalkedFile` — so the walk's depth would put gigabytes in flight. At 32 the
|
||
/// ceiling is ~8 MiB per root, plus whatever the workers hold mid-document.
|
||
const READY_CAP: usize = 32;
|
||
|
||
/// Rows fetched but not yet claimed by a worker.
|
||
///
|
||
/// Small for the same reason [`crate::walk`] caps its prefetch: the feeder
|
||
/// must not run arbitrarily far ahead of a pool that is bound by file I/O.
|
||
/// These are only ids and paths, so the cost is the strings, not the text.
|
||
const QUEUE_AHEAD: usize = 256;
|
||
|
||
/// How many rows the feeder fetches per query. Large enough that a slow root
|
||
/// is not paying a round trip per file, small enough to stay inside
|
||
/// [`QUEUE_AHEAD`].
|
||
const FEED_PAGE: usize = 128;
|
||
|
||
/// One file's extracted content, ready to be written.
|
||
#[derive(Debug)]
|
||
pub struct ExtractedRow {
|
||
pub file_id: i64,
|
||
/// The `files.name` the FTS row is indexed under.
|
||
pub name: String,
|
||
pub outcome: ContentOutcome,
|
||
}
|
||
|
||
/// A row the feeder handed to the pool: everything a worker needs, and nothing
|
||
/// that would make it touch the database.
|
||
#[derive(Debug)]
|
||
struct Pending {
|
||
file_id: i64,
|
||
name: String,
|
||
path: String,
|
||
mime: Option<String>,
|
||
}
|
||
|
||
#[derive(Default)]
|
||
struct Queue {
|
||
rows: Vec<Pending>,
|
||
/// Set while the feeder is mid-query, holding rows that are in neither the
|
||
/// queue nor a worker. Without it a worker could see an empty queue
|
||
/// between two pages and declare the pass finished early.
|
||
feeding: bool,
|
||
/// True once the feeder has read the last page.
|
||
drained: bool,
|
||
done: bool,
|
||
}
|
||
|
||
struct Shared {
|
||
queue: Mutex<Queue>,
|
||
idle: Condvar,
|
||
}
|
||
|
||
impl Shared {
|
||
/// Claim a row, blocking while the feeder might still produce more.
|
||
///
|
||
/// `None` only when the queue is empty *and* the feeder is finished — at
|
||
/// that instant nobody is left who could add another row.
|
||
fn take(&self) -> Option<Pending> {
|
||
let mut q = self.queue.lock().unwrap();
|
||
loop {
|
||
if q.done {
|
||
return None;
|
||
}
|
||
if let Some(row) = q.rows.pop() {
|
||
// The feeder may be parked behind QUEUE_AHEAD.
|
||
self.idle.notify_all();
|
||
return Some(row);
|
||
}
|
||
if q.drained && !q.feeding {
|
||
q.done = true;
|
||
self.idle.notify_all();
|
||
return None;
|
||
}
|
||
q = self.idle.wait(q).unwrap();
|
||
}
|
||
}
|
||
|
||
/// Claim the right to fetch one page, or `None` once the pass is over.
|
||
/// Parks while the queue is already [`QUEUE_AHEAD`] deep.
|
||
fn take_feed_slot(&self) -> Option<()> {
|
||
let mut q = self.queue.lock().unwrap();
|
||
loop {
|
||
if q.done || q.drained {
|
||
return None;
|
||
}
|
||
if q.rows.len() < QUEUE_AHEAD {
|
||
q.feeding = true;
|
||
return Some(());
|
||
}
|
||
q = self.idle.wait(q).unwrap();
|
||
}
|
||
}
|
||
|
||
/// Publish a page and clear the in-flight flag together, under one lock —
|
||
/// the indivisibility [`Shared::take`]'s end-of-pass test relies on.
|
||
fn finish_feed(&self, rows: Vec<Pending>, last_page: bool) {
|
||
let mut q = self.queue.lock().unwrap();
|
||
// Reversed: `take` pops from the back, and rows should reach workers
|
||
// in id order so a partial run leaves a contiguous prefix done.
|
||
q.rows.extend(rows.into_iter().rev());
|
||
q.feeding = false;
|
||
if last_page {
|
||
q.drained = true;
|
||
}
|
||
self.idle.notify_all();
|
||
}
|
||
|
||
fn shutdown(&self) {
|
||
let mut q = self.queue.lock().unwrap();
|
||
q.done = true;
|
||
q.drained = true;
|
||
self.idle.notify_all();
|
||
}
|
||
}
|
||
|
||
/// A running content pass. Draining it yields finished rows; dropping it stops
|
||
/// the workers and joins them.
|
||
pub struct ContentPass {
|
||
rx: Option<mpsc::Receiver<ExtractedRow>>,
|
||
handles: Vec<JoinHandle<()>>,
|
||
feeder: Option<JoinHandle<()>>,
|
||
shared: Arc<Shared>,
|
||
stats: WorkerStats,
|
||
}
|
||
|
||
impl ContentPass {
|
||
/// Non-blocking pull, for the writer loop multiplexing several roots.
|
||
pub fn try_next(&mut self) -> TryNext<ExtractedRow> {
|
||
try_recv_next(self.rx.as_ref())
|
||
}
|
||
|
||
/// A cheap, cloneable handle for reading pool activity while the pass is
|
||
/// mutably borrowed by the writer loop. The sibling of
|
||
/// [`crate::walk::ParallelWalk::worker_stats`], and for the same reason:
|
||
/// once a root reaches extraction its walk pool is gone, so the progress
|
||
/// display has to read this one instead.
|
||
pub fn worker_stats(&self) -> WorkerStats {
|
||
self.stats.clone()
|
||
}
|
||
|
||
/// Join the workers and report whether every one finished cleanly. Needed
|
||
/// for the same reason as [`crate::walk::ParallelWalk::finish`], which
|
||
/// states it.
|
||
pub fn finish(&mut self) -> bool {
|
||
// Dropping the receiver first releases any worker parked in `send`.
|
||
self.rx = None;
|
||
self.shared.shutdown();
|
||
let mut clean = true;
|
||
for handle in self.handles.drain(..) {
|
||
if handle.join().is_err() {
|
||
clean = false;
|
||
}
|
||
}
|
||
if let Some(handle) = self.feeder.take() {
|
||
if handle.join().is_err() {
|
||
clean = false;
|
||
}
|
||
}
|
||
clean
|
||
}
|
||
}
|
||
|
||
impl Drop for ContentPass {
|
||
fn drop(&mut self) {
|
||
self.shared.shutdown();
|
||
// No-op if the caller already called `finish`.
|
||
self.finish();
|
||
}
|
||
}
|
||
|
||
/// Page the root's pending rows into the queue from one read-only connection.
|
||
///
|
||
/// A failed query ends the pass rather than retrying: the rows stay
|
||
/// `content_state = 0` and the next run picks them up, which is the same
|
||
/// outcome as being interrupted.
|
||
fn feeder(shared: &Shared, db_path: &str, mut cursor: ExtractCursor, max_size: i64) {
|
||
let conn = match crate::db::open::open_walk_reader(db_path) {
|
||
Ok(conn) => conn,
|
||
Err(e) => {
|
||
crate::log_warn!("content reader: {}", e);
|
||
shared.shutdown();
|
||
return;
|
||
}
|
||
};
|
||
|
||
while shared.take_feed_slot().is_some() {
|
||
let page =
|
||
match crate::db::repo::pending_content_page(&conn, &cursor, max_size, FEED_PAGE as i64)
|
||
{
|
||
Ok(page) => page,
|
||
Err(e) => {
|
||
crate::log_warn!("{}", e);
|
||
shared.shutdown();
|
||
return;
|
||
}
|
||
};
|
||
let last_page = page.len() < FEED_PAGE;
|
||
if let Some((id, _, _, _)) = page.last() {
|
||
cursor.last_id = *id;
|
||
}
|
||
let rows = page
|
||
.into_iter()
|
||
.map(|(file_id, name, path, mime)| Pending {
|
||
file_id,
|
||
name,
|
||
path,
|
||
mime,
|
||
})
|
||
.collect();
|
||
shared.finish_feed(rows, last_page);
|
||
if last_page {
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
fn worker(
|
||
shared: &Shared,
|
||
tx: &mpsc::SyncSender<ExtractedRow>,
|
||
registry: &Registry,
|
||
config: &Config,
|
||
stop_flag: &Arc<AtomicBool>,
|
||
suspend_flag: &Arc<AtomicBool>,
|
||
stats: &WorkerStats,
|
||
) {
|
||
while let Some(row) = shared.take() {
|
||
// Held for the whole of `decide_content` — reading and parsing a
|
||
// document is exactly the work the progress line is reporting.
|
||
let _busy = stats.enter();
|
||
if should_abort(stop_flag, suspend_flag) {
|
||
shared.shutdown();
|
||
return;
|
||
}
|
||
let outcome = decide_content(&row.path, row.mime.as_deref(), registry, config);
|
||
let sent = tx.send(ExtractedRow {
|
||
file_id: row.file_id,
|
||
name: row.name,
|
||
outcome,
|
||
});
|
||
if sent.is_err() {
|
||
// Receiver gone: the run was stopped or failed. Not an error.
|
||
shared.shutdown();
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Extract every pending row under `cursor`'s range, in parallel.
|
||
///
|
||
/// `workers` is the root's own count — the same value its walk uses, because
|
||
/// extraction over a share is round-trip bound for the same reason walking is.
|
||
/// Clamped to 1..=64.
|
||
#[allow(clippy::too_many_arguments)]
|
||
pub fn extract_content(
|
||
db_path: &str,
|
||
cursor: &ExtractCursor,
|
||
registry: Arc<Registry>,
|
||
config: Config,
|
||
stop_flag: Arc<AtomicBool>,
|
||
suspend_flag: Arc<AtomicBool>,
|
||
workers: usize,
|
||
) -> ContentPass {
|
||
let shared = Arc::new(Shared {
|
||
queue: Mutex::new(Queue::default()),
|
||
idle: Condvar::new(),
|
||
});
|
||
let max_size = i64::try_from(config.processing.maximum_text_file_size).unwrap_or(i64::MAX);
|
||
|
||
let (tx, rx) = mpsc::sync_channel(READY_CAP);
|
||
let stats = WorkerStats::new(workers.clamp(1, 64));
|
||
let handles = (0..stats.total())
|
||
.map(|_| {
|
||
let (shared, tx) = (shared.clone(), tx.clone());
|
||
let (registry, config) = (registry.clone(), config.clone());
|
||
let (stop_flag, suspend_flag) = (stop_flag.clone(), suspend_flag.clone());
|
||
let stats = stats.clone();
|
||
crate::platform::spawn_worker("qs-extract", move || {
|
||
crate::platform::set_background_priority();
|
||
worker(
|
||
&shared,
|
||
&tx,
|
||
®istry,
|
||
&config,
|
||
&stop_flag,
|
||
&suspend_flag,
|
||
&stats,
|
||
)
|
||
})
|
||
})
|
||
.collect();
|
||
// The workers must hold the only senders, or `try_recv` never reports the
|
||
// end of the pass. The feeder deliberately holds none: it produces rows to
|
||
// extract, not extracted rows.
|
||
drop(tx);
|
||
|
||
let feeder_handle = {
|
||
let (shared, db_path, cursor) = (shared.clone(), db_path.to_string(), cursor.clone());
|
||
crate::platform::spawn_worker("qs-feeder", move || {
|
||
crate::platform::set_background_priority();
|
||
feeder(&shared, &db_path, cursor, max_size)
|
||
})
|
||
};
|
||
|
||
ContentPass {
|
||
rx: Some(rx),
|
||
handles,
|
||
feeder: Some(feeder_handle),
|
||
shared,
|
||
stats,
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
// Only the tests sleep; the pass itself spawns through
|
||
// `platform::spawn_worker` and blocks on channels rather than polling.
|
||
use std::thread;
|
||
|
||
use crate::db::open_or_recreate;
|
||
use crate::db::repo::{self, insert_file, NewFile};
|
||
use crate::file_handling::{extract_scope_prepare, store_extracted};
|
||
use crate::mime::FileType;
|
||
use std::path::{Path, PathBuf};
|
||
/// A path that does not exist yet — the caller builds the tree under it.
|
||
fn tmp(tag: &str) -> PathBuf {
|
||
crate::testutil::scratch_dir(tag).join("tree")
|
||
}
|
||
|
||
/// A tree of `n` text files under `root/sub`, plus an index holding a
|
||
/// pending row for each.
|
||
fn seed(tag: &str, dirs: &[(&str, usize)]) -> (PathBuf, PathBuf) {
|
||
let tree = tmp(&format!("{}-tree", tag));
|
||
let db = tmp(&format!("{}-db", tag));
|
||
let mut conn = open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
|
||
let tx = conn.transaction().unwrap();
|
||
for (dir, n) in dirs {
|
||
let d = tree.join(dir);
|
||
std::fs::create_dir_all(&d).unwrap();
|
||
for i in 0..*n {
|
||
let f = d.join(format!("f{:04}.txt", i));
|
||
std::fs::write(&f, format!("sphinx of black quartz {} {}", dir, i)).unwrap();
|
||
insert_file(
|
||
&tx,
|
||
&NewFile {
|
||
name: f.file_name().unwrap().to_str().unwrap(),
|
||
path: f.to_str().unwrap(),
|
||
parent: d.to_str().unwrap(),
|
||
size: std::fs::metadata(&f).unwrap().len(),
|
||
mtime: 1,
|
||
inode: None,
|
||
device_id: None,
|
||
mime: Some("text/plain"),
|
||
ftype: FileType::TEXT,
|
||
hash: None,
|
||
needs_content: true,
|
||
},
|
||
)
|
||
.unwrap()
|
||
.expect("unique path");
|
||
}
|
||
}
|
||
tx.commit().unwrap();
|
||
drop(conn);
|
||
(tree, db)
|
||
}
|
||
|
||
fn pass_for(tree: &Path, db: &Path, sub: &str, workers: usize) -> ContentPass {
|
||
extract_content(
|
||
db.to_str().unwrap(),
|
||
&ExtractCursor::for_root(tree.join(sub).to_str().unwrap()),
|
||
Arc::new(Registry::default_set()),
|
||
Config::default(),
|
||
Arc::new(AtomicBool::new(false)),
|
||
Arc::new(AtomicBool::new(false)),
|
||
workers,
|
||
)
|
||
}
|
||
|
||
/// Drain a pass to exhaustion, blocking between polls the way the writer
|
||
/// loop's outer sleep does.
|
||
fn drain(pass: &mut ContentPass) -> Vec<ExtractedRow> {
|
||
let mut out = Vec::new();
|
||
loop {
|
||
match pass.try_next() {
|
||
TryNext::Item(row) => out.push(row),
|
||
TryNext::Empty => thread::sleep(std::time::Duration::from_millis(1)),
|
||
TryNext::Finished => return out,
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn every_pending_row_is_yielded_exactly_once() {
|
||
let (tree, db) = seed("once", &[("r1", 250)]);
|
||
let mut pass = pass_for(&tree, &db, "r1", 4);
|
||
let rows = drain(&mut pass);
|
||
assert!(pass.finish(), "no worker panicked");
|
||
|
||
assert_eq!(rows.len(), 250);
|
||
let ids: std::collections::HashSet<i64> = rows.iter().map(|r| r.file_id).collect();
|
||
assert_eq!(ids.len(), 250, "no id may be yielded twice");
|
||
assert!(
|
||
rows.iter()
|
||
.all(|r| matches!(r.outcome, ContentOutcome::Done { .. })),
|
||
"every plaintext file extracts"
|
||
);
|
||
|
||
std::fs::remove_dir_all(&tree).ok();
|
||
std::fs::remove_file(&db).ok();
|
||
}
|
||
|
||
/// The pass is scoped by the cursor's path range, so a sibling root's rows
|
||
/// are never touched. (Moved here with the extraction itself: the scoping
|
||
/// is now the feeder's query, not the writer's.)
|
||
#[test]
|
||
fn the_pass_is_scoped_to_its_root_range() {
|
||
let (tree, db) = seed("scope", &[("r1", 3), ("r2", 3)]);
|
||
let conn_mutex = Arc::new(Mutex::new(
|
||
open_or_recreate(db.to_str().unwrap(), "trigram").unwrap(),
|
||
));
|
||
let config = Config::default();
|
||
let cursor = ExtractCursor::for_root(tree.join("r1").to_str().unwrap());
|
||
|
||
let scope = extract_scope_prepare(&conn_mutex, &cursor, &config).unwrap();
|
||
assert_eq!(scope.pending, 3, "only r1's files are in range");
|
||
assert_eq!(scope.already_done, 0, "nothing extracted yet");
|
||
|
||
let mut pass = pass_for(&tree, &db, "r1", 2);
|
||
let rows = drain(&mut pass);
|
||
assert!(pass.finish());
|
||
assert_eq!(rows.len(), 3);
|
||
|
||
let stop = Arc::new(AtomicBool::new(false));
|
||
assert_eq!(
|
||
store_extracted(&conn_mutex, &rows, &stop, &config).unwrap(),
|
||
3
|
||
);
|
||
|
||
let state = |p: &Path| -> i64 {
|
||
conn_mutex
|
||
.lock()
|
||
.unwrap()
|
||
.query_row(
|
||
"SELECT content_state FROM files WHERE path = ?1",
|
||
rusqlite::params![p.to_str().unwrap()],
|
||
|r| r.get(0),
|
||
)
|
||
.unwrap()
|
||
};
|
||
assert_eq!(state(&tree.join("r1/f0000.txt")), repo::STATE_DONE);
|
||
assert_eq!(
|
||
state(&tree.join("r2/f0000.txt")),
|
||
repo::STATE_PENDING,
|
||
"out-of-range row untouched"
|
||
);
|
||
|
||
// A second run over the unchanged root reports it already extracted,
|
||
// so progress reads "3 of 3" rather than "0 of 0".
|
||
let scope2 = extract_scope_prepare(&conn_mutex, &cursor, &config).unwrap();
|
||
assert_eq!((scope2.pending, scope2.already_done), (0, 3));
|
||
|
||
let hits: i64 = conn_mutex
|
||
.lock()
|
||
.unwrap()
|
||
.query_row(
|
||
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH '\"sphinx\"'",
|
||
[],
|
||
|r| r.get(0),
|
||
)
|
||
.unwrap();
|
||
assert_eq!(hits, 3);
|
||
|
||
std::fs::remove_dir_all(&tree).ok();
|
||
std::fs::remove_file(&db).ok();
|
||
}
|
||
|
||
#[test]
|
||
fn an_empty_range_terminates_immediately() {
|
||
// The "nothing to do at t=0" corner: every worker must observe the pass
|
||
// as finished rather than waiting for rows that will never arrive.
|
||
let (tree, db) = seed("empty", &[("r1", 2)]);
|
||
let mut pass = pass_for(&tree, &db, "nonexistent", 4);
|
||
assert!(drain(&mut pass).is_empty());
|
||
assert!(pass.finish());
|
||
std::fs::remove_dir_all(&tree).ok();
|
||
std::fs::remove_file(&db).ok();
|
||
}
|
||
|
||
#[test]
|
||
fn an_already_stopped_pass_does_not_run_to_completion() {
|
||
let (tree, db) = seed("stop", &[("r1", 400)]);
|
||
let mut pass = extract_content(
|
||
db.to_str().unwrap(),
|
||
&ExtractCursor::for_root(tree.join("r1").to_str().unwrap()),
|
||
Arc::new(Registry::default_set()),
|
||
Config::default(),
|
||
Arc::new(AtomicBool::new(true)),
|
||
Arc::new(AtomicBool::new(false)),
|
||
4,
|
||
);
|
||
assert!(drain(&mut pass).len() < 400);
|
||
assert!(pass.finish());
|
||
std::fs::remove_dir_all(&tree).ok();
|
||
std::fs::remove_file(&db).ok();
|
||
}
|
||
|
||
#[test]
|
||
fn dropping_the_pass_early_does_not_hang() {
|
||
// Workers blocked in `send` must be released by the receiver going
|
||
// away, or `Drop` would join threads that never wake.
|
||
let (tree, db) = seed("early-drop", &[("r1", 500)]);
|
||
let mut pass = pass_for(&tree, &db, "r1", 4);
|
||
// Pull one, leave the rest queued and the channel full.
|
||
loop {
|
||
match pass.try_next() {
|
||
TryNext::Item(_) => break,
|
||
TryNext::Empty => thread::sleep(std::time::Duration::from_millis(1)),
|
||
TryNext::Finished => break,
|
||
}
|
||
}
|
||
drop(pass); // must return, not deadlock
|
||
std::fs::remove_dir_all(&tree).ok();
|
||
std::fs::remove_file(&db).ok();
|
||
}
|
||
|
||
#[test]
|
||
fn repeated_passes_agree_on_the_result_set() {
|
||
// The termination protocol is racy by nature; run it enough times under
|
||
// real contention that a premature exit would show up.
|
||
let (tree, db) = seed("repeat", &[("r1", 120)]);
|
||
for run in 0..20 {
|
||
let mut pass = pass_for(&tree, &db, "r1", 4);
|
||
let rows = drain(&mut pass);
|
||
assert!(pass.finish(), "run {}", run);
|
||
assert_eq!(rows.len(), 120, "run {}", run);
|
||
}
|
||
std::fs::remove_dir_all(&tree).ok();
|
||
std::fs::remove_file(&db).ok();
|
||
}
|
||
|
||
/// The pass counts its own busy threads, which is what the progress line
|
||
/// shows once a root leaves the walk behind.
|
||
#[test]
|
||
fn the_pool_reports_its_own_activity() {
|
||
let (tree, db) = seed("stats", &[("r1", 300)]);
|
||
let mut pass = pass_for(&tree, &db, "r1", 4);
|
||
let stats = pass.worker_stats();
|
||
assert_eq!(stats.total(), 4);
|
||
|
||
// Nothing is drained here, so the ready channel fills and every worker
|
||
// ends up parked mid-row inside `send` — busy by the definition the
|
||
// display uses, and there are more rows than the channel holds, so all
|
||
// four get there.
|
||
let mut peak = 0;
|
||
for _ in 0..500 {
|
||
peak = peak.max(stats.active());
|
||
if peak == 4 {
|
||
break;
|
||
}
|
||
thread::sleep(std::time::Duration::from_millis(2));
|
||
}
|
||
assert_eq!(peak, 4, "every worker busy while the channel is full");
|
||
|
||
assert_eq!(drain(&mut pass).len(), 300);
|
||
assert!(pass.finish());
|
||
assert_eq!(stats.active(), 0, "a finished pool is idle");
|
||
|
||
std::fs::remove_dir_all(&tree).ok();
|
||
std::fs::remove_file(&db).ok();
|
||
}
|
||
|
||
/// A file that vanished between the walk and extraction is a failure with
|
||
/// a reason, not a silent skip: the row records why so it is not retried
|
||
/// forever.
|
||
#[test]
|
||
fn a_missing_file_is_reported_as_failed() {
|
||
let (tree, db) = seed("missing", &[("r1", 2)]);
|
||
std::fs::remove_file(tree.join("r1/f0000.txt")).unwrap();
|
||
let mut pass = pass_for(&tree, &db, "r1", 2);
|
||
let rows = drain(&mut pass);
|
||
assert!(pass.finish());
|
||
assert_eq!(rows.len(), 2, "both rows are still reported");
|
||
assert_eq!(
|
||
rows.iter()
|
||
.filter(|r| matches!(r.outcome, ContentOutcome::Failed(_)))
|
||
.count(),
|
||
1
|
||
);
|
||
std::fs::remove_dir_all(&tree).ok();
|
||
std::fs::remove_file(&db).ok();
|
||
}
|
||
}
|