Jeremy Karst
be30ddc9af
- Duplicates tab: groups are confirmed by each file's last 8 KiB and keyed by head+tail; unchecked files are never shown. False-positive duplicates drop from 3.4% to 0.1%, and indexing is unchanged. - Encrypted search cache cap raised to 192 MiB at 150 B per file: a keystroke at 1M files takes 59 ms, down from 99 ms. - Case-insensitive filename matching uses a memchr2 scan (1.18x). - Benches run again on the compact index, seed with the indexer's 2 GiB WAL cap (peak scratch 6.2 GB -> 2.45 GB), and every recorded figure is re-measured. - A fresh index no longer logs a rebuild warning. - CI: the sandbox memory test's allocation survives release builds, and --toggle retries an interrupted read.
950 lines
33 KiB
Rust
950 lines
33 KiB
Rust
//! How `hash_length` trades bytes read per file against false duplicates.
|
||
//!
|
||
//! The index identifies a file as `sha256(size ‖ first hash_length bytes)`,
|
||
//! so one number decides both. Collisions are measured over `1..32 KiB`
|
||
//! against the 32 KiB baseline, which is right by definition; `--verify`
|
||
//! reads the baseline's groups byte for byte and reports how many of those
|
||
//! were wrong too. The `+tail` columns apply the Duplicates tab's tail check
|
||
//! (`search::split_by_tail`) at each sample size, the tail as long as the
|
||
//! head.
|
||
//!
|
||
//! ```text
|
||
//! cargo build -p quicksearch-core --example hashprobe --release
|
||
//! ./target/release/examples/hashprobe collide /media/shared --workers 8 --verify
|
||
//! ./target/release/examples/hashprobe collide /media/shared \
|
||
//! --ignore target --ignore build --ignore __pycache__
|
||
//! ```
|
||
//!
|
||
//! [`indexprobe`](indexprobe.rs) prices the read cost of a sample size. The
|
||
//! traversal walks the default [`IgnoreSet`]; `--ignore` adds to it.
|
||
|
||
// Timed on the allocator the shipped binaries install; see
|
||
// `platform::Allocator`.
|
||
#[global_allocator]
|
||
static GLOBAL: quicksearch_core::platform::Allocator = quicksearch_core::platform::Allocator;
|
||
|
||
mod common;
|
||
|
||
use std::collections::HashMap;
|
||
use std::fs::File;
|
||
use std::io::{Read, Seek, SeekFrom};
|
||
use std::path::{Path, PathBuf};
|
||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||
use std::sync::{Arc, Mutex};
|
||
use std::time::Instant;
|
||
|
||
use sha2::{Digest, Sha256};
|
||
|
||
use quicksearch_core::config::{Config, IgnoreSet};
|
||
use quicksearch_core::extract::Registry;
|
||
use quicksearch_core::file_handling::{content_extractable, filtered_walk, UnreadableDirs};
|
||
use quicksearch_core::mime::guess_mime_from_head;
|
||
use quicksearch_core::textenc::looks_like_text;
|
||
use quicksearch_core::verify::{verify_identical, VerifyUpdate};
|
||
|
||
use common::{mib, Io};
|
||
|
||
/// Sample sizes, smallest first; the last is the baseline. Widening the
|
||
/// window can only split a group, so the six partitions are successive
|
||
/// refinements: one traversal answers all of them, and the false-positive
|
||
/// counts fall monotonically ([`Sweep::check_invariants`]).
|
||
const SWEEP: [usize; 6] = [1 << 10, 2 << 10, 4 << 10, 8 << 10, 16 << 10, 32 << 10];
|
||
|
||
const BASE: usize = SWEEP.len() - 1;
|
||
|
||
/// Eight rather than the core count: the scan is round-trip bound, and on
|
||
/// virtiofs a thread spends almost all its time parked in `read`.
|
||
const DEFAULT_WORKERS: usize = 8;
|
||
|
||
/// Ceiling on bytes `--verify` reads; whatever it stops short of is
|
||
/// reported, not silently dropped.
|
||
const DEFAULT_VERIFY_BUDGET: u64 = 8 * 1024 * 1024 * 1024;
|
||
|
||
type Sha = [u8; 32];
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Scanning
|
||
// ---------------------------------------------------------------------------
|
||
|
||
struct Scan {
|
||
size: u64,
|
||
/// `sha256(size ‖ head[..n])` for each `n` in [`SWEEP`].
|
||
digests: [Sha; SWEEP.len()],
|
||
/// For each `n`, `sha256` of the bytes `search::split_by_tail` reads
|
||
/// with a tail of `n`: from `max(n, size - n)` to the end. Zero for a
|
||
/// file no larger than `n`, which the head hashed whole.
|
||
tails: [Sha; SWEEP.len()],
|
||
/// Bit `i`: the MIME guessed from `SWEEP[i]` bytes differs from the
|
||
/// baseline's, since a short sample also classifies on less evidence.
|
||
mime_differs: u8,
|
||
/// Bit `i`: [`looks_like_text`] disagrees with the baseline's answer.
|
||
text_differs: u8,
|
||
/// Whether the content pass would have had anything to extract.
|
||
extractable: bool,
|
||
}
|
||
|
||
/// Files the traversal saw but could not use, kept apart from the results so a
|
||
/// corpus that is half unreadable cannot masquerade as a clean measurement.
|
||
#[derive(Default, Debug)]
|
||
struct ScanErrors {
|
||
unreadable: u64,
|
||
/// Zero-length files, excluded as `search::duplicates` excludes them.
|
||
empty: u64,
|
||
irregular: u64,
|
||
}
|
||
|
||
/// Read one file's head (`min(size, 32 KiB)`) and the tail past it (at most
|
||
/// the last 32 KiB) once, and derive every sample size's answer from them, so
|
||
/// the six answers are consistent with each other as six separate runs could
|
||
/// not be.
|
||
fn scan_one(path: &Path, size: u64, config: &Config, registry: &Registry) -> Option<Scan> {
|
||
let base = SWEEP[BASE] as u64;
|
||
let mut head = vec![0u8; size.min(base) as usize];
|
||
let mut file = File::open(path).ok()?;
|
||
// `read_exact`, like the indexer: a file that shrank between the stat and
|
||
// this read is not a file we know the size of, so it is not scanned.
|
||
file.read_exact(&mut head).ok()?;
|
||
// Every smaller sample's tail lies inside this one or the head: below
|
||
// `2 × base` the two are the whole file, and above it every tail is a
|
||
// suffix of this one.
|
||
let rest_start = base.max(size.saturating_sub(base));
|
||
let mut rest = vec![0u8; size.saturating_sub(rest_start) as usize];
|
||
if !rest.is_empty() {
|
||
file.seek(SeekFrom::Start(rest_start)).ok()?;
|
||
file.read_exact(&mut rest).ok()?;
|
||
}
|
||
|
||
let mut digests = [[0u8; 32]; SWEEP.len()];
|
||
let mut tails = [[0u8; 32]; SWEEP.len()];
|
||
for ((slot, tail), n) in digests.iter_mut().zip(&mut tails).zip(SWEEP) {
|
||
let mut hasher = Sha256::new();
|
||
hasher.update(size.to_le_bytes());
|
||
hasher.update(&head[..head.len().min(n)]);
|
||
slot.copy_from_slice(&hasher.finalize());
|
||
|
||
let n = n as u64;
|
||
if size > n {
|
||
let start = n.max(size - n);
|
||
let mut hasher = Sha256::new();
|
||
hasher.update(&head[head.len().min(start as usize)..]);
|
||
hasher.update(&rest[start.saturating_sub(rest_start) as usize..]);
|
||
tail.copy_from_slice(&hasher.finalize());
|
||
}
|
||
}
|
||
|
||
let base_mime = guess_mime_from_head(path, &head);
|
||
let base_text = looks_like_text(&head);
|
||
let (mut mime_differs, mut text_differs) = (0u8, 0u8);
|
||
for (i, n) in SWEEP.iter().enumerate().take(BASE) {
|
||
let window = &head[..head.len().min(*n)];
|
||
if guess_mime_from_head(path, window) != base_mime {
|
||
mime_differs |= 1 << i;
|
||
}
|
||
if looks_like_text(window) != base_text {
|
||
text_differs |= 1 << i;
|
||
}
|
||
}
|
||
|
||
let extractable = size <= config.processing.maximum_text_file_size
|
||
&& content_extractable(path, base_mime, config, registry);
|
||
|
||
Some(Scan {
|
||
size,
|
||
digests,
|
||
tails,
|
||
mime_differs,
|
||
text_differs,
|
||
extractable,
|
||
})
|
||
}
|
||
|
||
/// Scan `roots` with the default config, `workers` at a time. A bounded
|
||
/// channel keeps a million pending paths from materialising at once, and each
|
||
/// worker's results merge once at the end.
|
||
fn scan(
|
||
roots: &[String],
|
||
workers: usize,
|
||
config: &Config,
|
||
) -> (Vec<PathBuf>, Vec<Scan>, ScanErrors) {
|
||
let (tx, rx) = std::sync::mpsc::sync_channel::<(PathBuf, u64)>(4096);
|
||
let rx = Arc::new(Mutex::new(rx));
|
||
let done = AtomicUsize::new(0);
|
||
|
||
let mut paths = Vec::new();
|
||
let mut scans = Vec::new();
|
||
let mut errors = ScanErrors::default();
|
||
|
||
std::thread::scope(|scope| {
|
||
let mut handles = Vec::with_capacity(workers);
|
||
for _ in 0..workers {
|
||
let rx = Arc::clone(&rx);
|
||
let done = &done;
|
||
handles.push(scope.spawn(move || {
|
||
// Building a registry per worker is cheaper than sharing one.
|
||
let registry = Registry::default_set();
|
||
let (mut paths, mut scans) = (Vec::new(), Vec::new());
|
||
let mut unreadable = 0u64;
|
||
loop {
|
||
// The lock is held for the handoff and never across a read.
|
||
let Ok((path, size)) = ({
|
||
let rx = rx.lock().expect("scan channel");
|
||
rx.recv()
|
||
}) else {
|
||
break;
|
||
};
|
||
match scan_one(&path, size, config, ®istry) {
|
||
Some(scan) => {
|
||
paths.push(path);
|
||
scans.push(scan);
|
||
}
|
||
None => unreadable += 1,
|
||
}
|
||
let n = done.fetch_add(1, Ordering::Relaxed) + 1;
|
||
if n.is_multiple_of(25_000) {
|
||
eprint!("\r scanned {} files...", n);
|
||
}
|
||
}
|
||
(paths, scans, unreadable)
|
||
}));
|
||
}
|
||
|
||
let ignore = IgnoreSet::compile(&config.indexing.ignore_patterns).expect("ignore patterns");
|
||
let failures = UnreadableDirs::default();
|
||
for root in roots {
|
||
for entry in filtered_walk(
|
||
root,
|
||
config.indexing.include_hidden,
|
||
&ignore,
|
||
&failures,
|
||
None,
|
||
false,
|
||
) {
|
||
if !entry.file_type().is_file() {
|
||
errors.irregular += 1;
|
||
continue;
|
||
}
|
||
let Ok(meta) = entry.metadata() else {
|
||
errors.unreadable += 1;
|
||
continue;
|
||
};
|
||
if meta.len() == 0 {
|
||
errors.empty += 1;
|
||
continue;
|
||
}
|
||
if tx.send((entry.into_path(), meta.len())).is_err() {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
drop(tx);
|
||
|
||
for handle in handles {
|
||
let (p, s, unreadable) = handle.join().expect("scan worker");
|
||
paths.extend(p);
|
||
scans.extend(s);
|
||
errors.unreadable += unreadable;
|
||
}
|
||
});
|
||
eprint!("\r");
|
||
|
||
(paths, scans, errors)
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Collision accounting
|
||
// ---------------------------------------------------------------------------
|
||
|
||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||
struct GroupStat {
|
||
members: u64,
|
||
/// Distinct baseline groups merged; `1` is honest, more is false positive.
|
||
baseline_groups: u64,
|
||
/// `C(members, 2)`.
|
||
reported_pairs: u64,
|
||
/// Of those, the pairs the baseline says are different files.
|
||
fp_pairs: u64,
|
||
/// The Duplicates tab's reclaimable-bytes figure minus what the baseline
|
||
/// subgroups hold: the number that could talk someone into deleting
|
||
/// something.
|
||
overstated_bytes: u64,
|
||
}
|
||
|
||
/// Measure one reported group against the baseline. Pairs are counted
|
||
/// arithmetically: a hardlink farm has `C(m, 2)` pairs, a number to compute
|
||
/// and not a loop to run.
|
||
fn group_stat(members: &[u32], base: &[u32], size: &[u64]) -> GroupStat {
|
||
#[derive(Default, Clone, Copy)]
|
||
struct Sub {
|
||
count: u64,
|
||
sum: u64,
|
||
max: u64,
|
||
}
|
||
|
||
let mut subs: HashMap<u32, Sub> = HashMap::new();
|
||
let (mut sum, mut max) = (0u64, 0u64);
|
||
for &i in members {
|
||
let (s, b) = (size[i as usize], base[i as usize]);
|
||
sum += s;
|
||
max = max.max(s);
|
||
let sub = subs.entry(b).or_default();
|
||
sub.count += 1;
|
||
sub.sum += s;
|
||
sub.max = sub.max.max(s);
|
||
}
|
||
|
||
let m = members.len() as u64;
|
||
let reported_pairs = m * m.saturating_sub(1) / 2;
|
||
let true_pairs: u64 = subs.values().map(|s| s.count * (s.count - 1) / 2).sum();
|
||
let truthful_bytes: u64 = subs.values().map(|s| s.sum - s.max).sum();
|
||
|
||
GroupStat {
|
||
members: m,
|
||
baseline_groups: subs.len() as u64,
|
||
reported_pairs,
|
||
fp_pairs: reported_pairs - true_pairs,
|
||
overstated_bytes: (sum - max) - truthful_bytes,
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||
struct Collisions {
|
||
groups: u64,
|
||
files: u64,
|
||
reported_pairs: u64,
|
||
fp_groups: u64,
|
||
fp_files: u64,
|
||
fp_pairs: u64,
|
||
overstated_bytes: u64,
|
||
}
|
||
|
||
/// Sum over a partition, skipping groups of one as the `HAVING cnt > 1` in
|
||
/// `search::duplicates` does.
|
||
fn tally(partition: &[Vec<u32>], base: &[u32], size: &[u64]) -> Collisions {
|
||
let mut total = Collisions::default();
|
||
for members in partition {
|
||
if members.len() < 2 {
|
||
continue;
|
||
}
|
||
let stat = group_stat(members, base, size);
|
||
total.groups += 1;
|
||
total.files += stat.members;
|
||
total.reported_pairs += stat.reported_pairs;
|
||
if stat.baseline_groups > 1 {
|
||
total.fp_groups += 1;
|
||
total.fp_files += stat.members;
|
||
total.fp_pairs += stat.fp_pairs;
|
||
total.overstated_bytes += stat.overstated_bytes;
|
||
}
|
||
}
|
||
total
|
||
}
|
||
|
||
fn partition(scans: &[Scan], which: usize) -> Vec<Vec<u32>> {
|
||
let mut by_digest: HashMap<Sha, Vec<u32>> = HashMap::new();
|
||
for (i, scan) in scans.iter().enumerate() {
|
||
by_digest
|
||
.entry(scan.digests[which])
|
||
.or_default()
|
||
.push(i as u32);
|
||
}
|
||
by_digest.into_values().collect()
|
||
}
|
||
|
||
/// [`partition`], each group then split by its members' tails as
|
||
/// `search::split_by_tail` splits it with `tail_len = SWEEP[which]`. The
|
||
/// size is already in the head digest.
|
||
fn partition_with_tail(scans: &[Scan], which: usize) -> Vec<Vec<u32>> {
|
||
let mut by_digest: HashMap<(Sha, Sha), Vec<u32>> = HashMap::new();
|
||
for (i, scan) in scans.iter().enumerate() {
|
||
by_digest
|
||
.entry((scan.digests[which], scan.tails[which]))
|
||
.or_default()
|
||
.push(i as u32);
|
||
}
|
||
by_digest.into_values().collect()
|
||
}
|
||
|
||
fn baseline_ids(scans: &[Scan]) -> Vec<u32> {
|
||
let mut ids = vec![0u32; scans.len()];
|
||
let mut next: HashMap<Sha, u32> = HashMap::new();
|
||
for (i, scan) in scans.iter().enumerate() {
|
||
let n = next.len() as u32;
|
||
ids[i] = *next.entry(scan.digests[BASE]).or_insert(n);
|
||
}
|
||
ids
|
||
}
|
||
|
||
struct Sweep {
|
||
rows: Vec<Collisions>,
|
||
/// The same, after the tail check.
|
||
tailed: Vec<Collisions>,
|
||
}
|
||
|
||
impl Sweep {
|
||
/// Assert the invariants that hold by construction: the baseline's
|
||
/// false-positive columns are zero, and every "wrong" column is
|
||
/// non-increasing as the sample widens. The tail check only splits, so
|
||
/// its rows obey the same and never report more than the head alone
|
||
/// (head and tail at `2n` cover what they cover at `n`). A probe that
|
||
/// reports a plausible wrong number is worse than none.
|
||
fn check_invariants(&self) {
|
||
Sweep::check_rows(&self.rows);
|
||
Sweep::check_rows(&self.tailed);
|
||
for (head, tailed) in self.rows.iter().zip(&self.tailed) {
|
||
assert!(tailed.files <= head.files, "the tail check only splits");
|
||
assert!(
|
||
tailed.fp_files <= head.fp_files,
|
||
"the tail check only splits"
|
||
);
|
||
assert!(
|
||
tailed.fp_pairs <= head.fp_pairs,
|
||
"the tail check only splits"
|
||
);
|
||
}
|
||
}
|
||
|
||
fn check_rows(rows: &[Collisions]) {
|
||
let base = &rows[BASE];
|
||
assert_eq!(base.fp_groups, 0, "the baseline defines truth");
|
||
assert_eq!(base.fp_files, 0, "the baseline defines truth");
|
||
assert_eq!(base.fp_pairs, 0, "the baseline defines truth");
|
||
assert_eq!(base.overstated_bytes, 0, "the baseline defines truth");
|
||
|
||
for pair in rows.windows(2) {
|
||
let (small, large) = (&pair[0], &pair[1]);
|
||
assert!(
|
||
large.files <= small.files,
|
||
"a wider window cannot merge more files"
|
||
);
|
||
assert!(
|
||
large.reported_pairs <= small.reported_pairs,
|
||
"a wider window cannot report more pairs"
|
||
);
|
||
assert!(
|
||
large.fp_files <= small.fp_files,
|
||
"false positives must shrink"
|
||
);
|
||
assert!(
|
||
large.fp_pairs <= small.fp_pairs,
|
||
"false positives must shrink"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// collide
|
||
// ---------------------------------------------------------------------------
|
||
|
||
fn pct(part: u64, whole: u64) -> String {
|
||
if whole == 0 {
|
||
return "-".to_string();
|
||
}
|
||
let p = part as f64 * 100.0 / whole as f64;
|
||
if p > 0.0 && p < 0.01 {
|
||
return "<0.01%".to_string();
|
||
}
|
||
format!("{:.2}%", p)
|
||
}
|
||
|
||
fn sample_name(n: usize) -> String {
|
||
if n.is_multiple_of(1024) {
|
||
format!("{} KiB", n / 1024)
|
||
} else {
|
||
format!("{} B", n)
|
||
}
|
||
}
|
||
|
||
fn collide(roots: &[String], workers: usize, ignore: &[String], verify_budget: Option<u64>) {
|
||
let mut config = Config::default();
|
||
// Build output is where a disk keeps its near-identical files, so a
|
||
// corpus with and without it answers different questions; `--ignore`
|
||
// asks the second.
|
||
config
|
||
.indexing
|
||
.ignore_patterns
|
||
.extend(ignore.iter().cloned());
|
||
eprintln!(
|
||
"scanning {} root(s) with {} workers, {} per file, ignoring {:?}",
|
||
roots.len(),
|
||
workers,
|
||
sample_name(SWEEP[BASE]),
|
||
config.indexing.ignore_patterns,
|
||
);
|
||
|
||
let io_start = Io::read();
|
||
let start = Instant::now();
|
||
let (paths, scans, errors) = scan(roots, workers, &config);
|
||
let elapsed = start.elapsed();
|
||
let io = Io::read().since(&io_start);
|
||
|
||
let sizes: Vec<u64> = scans.iter().map(|s| s.size).collect();
|
||
let base = baseline_ids(&scans);
|
||
|
||
eprintln!(
|
||
"\nscanned {} files in {:.1?} ({:.0} files/sec), read {} in {} calls",
|
||
scans.len(),
|
||
elapsed,
|
||
scans.len() as f64 / elapsed.as_secs_f64(),
|
||
mib(io.rchar),
|
||
io.syscr,
|
||
);
|
||
eprintln!(
|
||
"skipped {} empty, {} irregular (FIFO/device/socket), {} unreadable",
|
||
errors.empty, errors.irregular, errors.unreadable,
|
||
);
|
||
eprintln!(
|
||
"total bytes indexed-as-content: {} across {} distinct baseline groups\n",
|
||
mib(sizes.iter().sum::<u64>()),
|
||
base.iter().max().map(|m| m + 1).unwrap_or(0),
|
||
);
|
||
|
||
let sweep = Sweep {
|
||
rows: (0..SWEEP.len())
|
||
.map(|i| tally(&partition(&scans, i), &base, &sizes))
|
||
.collect(),
|
||
tailed: (0..SWEEP.len())
|
||
.map(|i| tally(&partition_with_tail(&scans, i), &base, &sizes))
|
||
.collect(),
|
||
};
|
||
sweep.check_invariants();
|
||
|
||
let files = scans.len() as u64;
|
||
println!(
|
||
"### Duplicate grouping vs. the {} baseline\n",
|
||
sample_name(SWEEP[BASE])
|
||
);
|
||
println!(
|
||
"| sample | dup groups | dup files | reported pairs | FP groups | FP files | FP pairs | FP rate (pairs) | FP rate (files) | overstated | FP files +tail | FP rate (files) +tail | overstated +tail |"
|
||
);
|
||
println!("|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|");
|
||
for (i, (row, tailed)) in sweep.rows.iter().zip(&sweep.tailed).enumerate() {
|
||
println!(
|
||
"| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |",
|
||
sample_name(SWEEP[i]),
|
||
row.groups,
|
||
row.files,
|
||
row.reported_pairs,
|
||
row.fp_groups,
|
||
row.fp_files,
|
||
row.fp_pairs,
|
||
pct(row.fp_pairs, row.reported_pairs),
|
||
pct(row.fp_files, files),
|
||
mib(row.overstated_bytes),
|
||
tailed.fp_files,
|
||
pct(tailed.fp_files, files),
|
||
mib(tailed.overstated_bytes),
|
||
);
|
||
}
|
||
|
||
println!("\n### Where the false positives go when the window grows\n");
|
||
println!("| step | FP pairs resolved | FP files resolved | overstated bytes resolved |");
|
||
println!("|---|---:|---:|---:|");
|
||
for i in 0..BASE {
|
||
let (small, large) = (&sweep.rows[i], &sweep.rows[i + 1]);
|
||
println!(
|
||
"| {} -> {} | {} | {} | {} |",
|
||
sample_name(SWEEP[i]),
|
||
sample_name(SWEEP[i + 1]),
|
||
small.fp_pairs - large.fp_pairs,
|
||
small.fp_files - large.fp_files,
|
||
mib(small.overstated_bytes - large.overstated_bytes),
|
||
);
|
||
}
|
||
|
||
// The head's other two jobs: how much evidence the MIME and text sniffs
|
||
// get, and how many files the walk can finish inline.
|
||
println!("\n### What else the window decides\n");
|
||
println!("| sample | inlinable files | share | MIME differs | text sniff differs |");
|
||
println!("|---:|---:|---:|---:|---:|");
|
||
for (i, n) in SWEEP.iter().enumerate() {
|
||
let inlinable = scans
|
||
.iter()
|
||
.filter(|s| s.extractable && s.size <= *n as u64)
|
||
.count() as u64;
|
||
let (mime, text) = if i == BASE {
|
||
(0, 0)
|
||
} else {
|
||
(
|
||
scans
|
||
.iter()
|
||
.filter(|s| s.mime_differs & (1 << i) != 0)
|
||
.count() as u64,
|
||
scans
|
||
.iter()
|
||
.filter(|s| s.text_differs & (1 << i) != 0)
|
||
.count() as u64,
|
||
)
|
||
};
|
||
println!(
|
||
"| {} | {} | {} | {} | {} |",
|
||
sample_name(*n),
|
||
inlinable,
|
||
pct(inlinable, files),
|
||
mime,
|
||
text,
|
||
);
|
||
}
|
||
|
||
top_offenders(&scans, &paths, &base, &sizes);
|
||
|
||
if let Some(budget) = verify_budget {
|
||
verify_baseline(&scans, &paths, &sizes, budget);
|
||
}
|
||
}
|
||
|
||
/// Print the five worst false-positive groups at the default `hash_length`:
|
||
/// what collides.
|
||
fn top_offenders(scans: &[Scan], paths: &[PathBuf], base: &[u32], sizes: &[u64]) {
|
||
let default_index = SWEEP
|
||
.iter()
|
||
.position(|n| *n == Config::default().processing.hash_length)
|
||
.unwrap_or(BASE);
|
||
|
||
let mut worst: Vec<(GroupStat, Vec<u32>)> = partition(scans, default_index)
|
||
.into_iter()
|
||
.filter(|members| members.len() >= 2)
|
||
.map(|members| (group_stat(&members, base, sizes), members))
|
||
.filter(|(stat, _)| stat.baseline_groups > 1)
|
||
.collect();
|
||
worst.sort_by_key(|(stat, _)| std::cmp::Reverse((stat.overstated_bytes, stat.members)));
|
||
|
||
println!(
|
||
"\n### Worst false-positive groups at the {} default\n",
|
||
sample_name(SWEEP[default_index])
|
||
);
|
||
if worst.is_empty() {
|
||
println!("None: every group the default reports is real.");
|
||
return;
|
||
}
|
||
for (stat, members) in worst.iter().take(5) {
|
||
println!(
|
||
"- {} members from {} distinct files, {} each, {} overstated",
|
||
stat.members,
|
||
stat.baseline_groups,
|
||
mib(sizes[members[0] as usize]),
|
||
mib(stat.overstated_bytes),
|
||
);
|
||
// Two members from different baseline groups: a pair claimed wrongly,
|
||
// for a reader to `cmp`.
|
||
let first = members[0];
|
||
if let Some(other) = members
|
||
.iter()
|
||
.find(|m| base[**m as usize] != base[first as usize])
|
||
{
|
||
println!(" - {}", paths[first as usize].display());
|
||
println!(" - {}", paths[*other as usize].display());
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Read the baseline's duplicate groups byte for byte and report how many
|
||
/// were wrong; only groups holding a file larger than the sample can be.
|
||
/// Opt-in and budgeted, because this is the only part of the probe that reads
|
||
/// whole files.
|
||
fn verify_baseline(scans: &[Scan], paths: &[PathBuf], sizes: &[u64], budget: u64) {
|
||
let baseline = partition(scans, BASE);
|
||
let baseline_dup_groups = baseline.iter().filter(|m| m.len() >= 2).count();
|
||
let mut groups: Vec<Vec<u32>> = baseline
|
||
.into_iter()
|
||
.filter(|m| m.len() >= 2 && sizes[m[0] as usize] > SWEEP[BASE] as u64)
|
||
.collect();
|
||
// Digest order: uniformly random and stable between runs.
|
||
// Largest-claim-first would spend the budget on static libraries and
|
||
// disk images — long identical headers — and inflate the rate severalfold.
|
||
groups.sort_by_key(|m| scans[m[0] as usize].digests[BASE]);
|
||
|
||
let cancel = AtomicBool::new(false);
|
||
let (mut checked, mut wrong, mut differing, mut bytes) = (0u64, 0u64, 0u64, 0u64);
|
||
let (mut skipped, mut skipped_bytes) = (0u64, 0u64);
|
||
let mut examples: Vec<String> = Vec::new();
|
||
|
||
let mut exhausted = false;
|
||
for members in &groups {
|
||
let cost: u64 = members.iter().map(|m| sizes[*m as usize]).sum();
|
||
// Stop at the budget: carrying on would admit whatever still fits, the
|
||
// small groups, and weight the sample toward files the baseline is
|
||
// least likely to get wrong.
|
||
exhausted |= bytes + cost > budget;
|
||
if exhausted {
|
||
skipped += 1;
|
||
skipped_bytes += cost;
|
||
continue;
|
||
}
|
||
let group: Vec<PathBuf> = members.iter().map(|m| paths[*m as usize].clone()).collect();
|
||
let mut report = None;
|
||
verify_identical(&group, &cancel, &mut |update| {
|
||
if let VerifyUpdate::Done(r) = update {
|
||
report = Some(r);
|
||
}
|
||
});
|
||
let Some(report) = report else { continue };
|
||
checked += 1;
|
||
bytes += report.bytes_read;
|
||
if !report.all_identical() {
|
||
wrong += 1;
|
||
differing += report.differing() as u64;
|
||
if examples.len() < 5 {
|
||
examples.push(format!(
|
||
"{} ({} members, {} differ)",
|
||
group[0].display(),
|
||
group.len(),
|
||
report.differing()
|
||
));
|
||
}
|
||
}
|
||
if checked % 100 == 0 {
|
||
eprint!("\r verified {} groups, {}...", checked, mib(bytes));
|
||
}
|
||
}
|
||
eprint!("\r");
|
||
|
||
println!("\n### Is the baseline itself right?\n");
|
||
println!(
|
||
"{} of {} baseline groups hold a file larger than {}, so only those can be wrong.",
|
||
groups.len(),
|
||
baseline_dup_groups,
|
||
sample_name(SWEEP[BASE]),
|
||
);
|
||
println!(
|
||
"Compared {} of them byte for byte, sampled in digest order ({} read): \
|
||
**{} were not identical** ({} of the sample), {} members differing.",
|
||
checked,
|
||
mib(bytes),
|
||
wrong,
|
||
pct(wrong, checked),
|
||
differing,
|
||
);
|
||
if skipped > 0 {
|
||
println!(
|
||
"{} groups ({}) went unchecked against the {} budget — not counted either way.",
|
||
skipped,
|
||
mib(skipped_bytes),
|
||
mib(budget),
|
||
);
|
||
}
|
||
for example in &examples {
|
||
println!("- {}", example);
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
|
||
fn main() {
|
||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||
let mode = args.first().map(String::as_str).unwrap_or_default();
|
||
|
||
let mut positional = Vec::new();
|
||
let mut workers = DEFAULT_WORKERS;
|
||
let mut verify_budget = None;
|
||
let mut ignore = Vec::new();
|
||
let mut rest = args.iter().skip(1);
|
||
while let Some(arg) = rest.next() {
|
||
match arg.as_str() {
|
||
"--workers" => workers = rest.next().and_then(|v| v.parse().ok()).unwrap_or(workers),
|
||
"--ignore" => ignore.extend(rest.next().cloned()),
|
||
"--verify" => verify_budget = Some(verify_budget.unwrap_or(DEFAULT_VERIFY_BUDGET)),
|
||
"--verify-max-bytes" => {
|
||
verify_budget = rest.next().and_then(|v| v.parse().ok()).or(verify_budget)
|
||
}
|
||
other => positional.push(other.to_string()),
|
||
}
|
||
}
|
||
|
||
match mode {
|
||
"collide" if !positional.is_empty() => {
|
||
collide(&positional, workers.max(1), &ignore, verify_budget);
|
||
}
|
||
_ => {
|
||
eprintln!(
|
||
"usage:\n \
|
||
hashprobe collide <root>... [--workers N] [--ignore PATTERN]... \
|
||
[--verify] [--verify-max-bytes B]"
|
||
);
|
||
std::process::exit(2);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Brute-force the pair counts [`group_stat`] shortcuts, so the shortcut
|
||
/// is checked against the definition rather than itself.
|
||
fn brute_force_pairs(members: &[u32], base: &[u32]) -> (u64, u64) {
|
||
let (mut reported, mut fp) = (0u64, 0u64);
|
||
for (i, a) in members.iter().enumerate() {
|
||
for b in &members[i + 1..] {
|
||
reported += 1;
|
||
if base[*a as usize] != base[*b as usize] {
|
||
fp += 1;
|
||
}
|
||
}
|
||
}
|
||
(reported, fp)
|
||
}
|
||
|
||
/// [`group_stat`] across the group shapes that matter, cross-checked
|
||
/// against [`brute_force_pairs`].
|
||
#[test]
|
||
fn group_stat_matches_enumeration_across_group_shapes() {
|
||
struct Case {
|
||
label: &'static str,
|
||
members: Vec<u32>,
|
||
base: Vec<u32>,
|
||
sizes: Vec<u64>,
|
||
want: GroupStat,
|
||
}
|
||
let cases = [
|
||
Case {
|
||
label: "honest",
|
||
members: vec![0, 1, 2],
|
||
base: vec![7, 7, 7],
|
||
sizes: vec![100, 100, 100],
|
||
want: GroupStat {
|
||
members: 3,
|
||
baseline_groups: 1,
|
||
reported_pairs: 3,
|
||
fp_pairs: 0,
|
||
overstated_bytes: 0,
|
||
},
|
||
},
|
||
Case {
|
||
label: "mixed",
|
||
members: vec![0, 1, 2, 3],
|
||
base: vec![1, 1, 2, 2],
|
||
sizes: vec![100; 4],
|
||
want: GroupStat {
|
||
members: 4,
|
||
baseline_groups: 2,
|
||
reported_pairs: 6,
|
||
fp_pairs: 4,
|
||
overstated_bytes: 100,
|
||
},
|
||
},
|
||
Case {
|
||
label: "all singletons",
|
||
members: vec![0, 1, 2, 3],
|
||
base: vec![1, 2, 3, 4],
|
||
sizes: vec![500; 4],
|
||
want: GroupStat {
|
||
members: 4,
|
||
baseline_groups: 4,
|
||
reported_pairs: 6,
|
||
fp_pairs: 6,
|
||
overstated_bytes: 1500,
|
||
},
|
||
},
|
||
Case {
|
||
label: "lumpy",
|
||
members: (0..10).collect(),
|
||
base: vec![0, 0, 0, 1, 1, 2, 3, 3, 3, 3],
|
||
sizes: vec![64; 10],
|
||
want: GroupStat {
|
||
members: 10,
|
||
baseline_groups: 4,
|
||
reported_pairs: 45,
|
||
fp_pairs: 45 - ((3 + 1) + 6),
|
||
overstated_bytes: 3 * 64,
|
||
},
|
||
},
|
||
Case {
|
||
label: "empty",
|
||
members: vec![],
|
||
base: vec![],
|
||
sizes: vec![],
|
||
want: GroupStat::default(),
|
||
},
|
||
Case {
|
||
label: "lone",
|
||
members: vec![0],
|
||
base: vec![5],
|
||
sizes: vec![900],
|
||
want: GroupStat {
|
||
members: 1,
|
||
baseline_groups: 1,
|
||
reported_pairs: 0,
|
||
fp_pairs: 0,
|
||
overstated_bytes: 0,
|
||
},
|
||
},
|
||
];
|
||
for c in &cases {
|
||
let stat = group_stat(&c.members, &c.base, &c.sizes);
|
||
assert_eq!(stat, c.want, "{}", c.label);
|
||
let (reported, fp) = brute_force_pairs(&c.members, &c.base);
|
||
assert_eq!(
|
||
(stat.reported_pairs, stat.fp_pairs),
|
||
(reported, fp),
|
||
"{} vs enumeration",
|
||
c.label
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn tally_skips_singletons_and_sums_the_rest() {
|
||
let partition = vec![vec![0, 1], vec![2], vec![3, 4, 5]];
|
||
let base = [1, 1, 9, 2, 3, 3];
|
||
let sizes = [10u64, 10, 10, 20, 20, 20];
|
||
let total = tally(&partition, &base, &sizes);
|
||
assert_eq!(total.groups, 2, "the singleton is not a duplicate group");
|
||
assert_eq!(total.files, 5);
|
||
assert_eq!(total.reported_pairs, 1 + 3);
|
||
assert_eq!(total.fp_groups, 1);
|
||
assert_eq!(
|
||
total.fp_files, 3,
|
||
"every member of a mixed group is misinformed"
|
||
);
|
||
assert_eq!(total.fp_pairs, 2);
|
||
assert_eq!(total.overstated_bytes, 20);
|
||
}
|
||
|
||
/// Each sample's tail digest covers the bytes `split_by_tail` reads,
|
||
/// wherever the size falls against the head and the 32 KiB read past it.
|
||
#[test]
|
||
fn tail_digests_cover_what_the_tail_check_reads() {
|
||
let dir = quicksearch_core::testutil::scratch_dir("hashprobe-tails");
|
||
let (config, registry) = (Config::default(), Registry::default_set());
|
||
let base = SWEEP[BASE];
|
||
for size in [
|
||
1,
|
||
1024,
|
||
1025,
|
||
3000,
|
||
base,
|
||
base + 1,
|
||
base + 5000,
|
||
2 * base,
|
||
2 * base + 1,
|
||
5 * base + 7,
|
||
] {
|
||
let body: Vec<u8> = (0..size).map(|i| (i * 31 % 251) as u8).collect();
|
||
let path = dir.join(format!("{size}.bin"));
|
||
std::fs::write(&path, &body).unwrap();
|
||
let scan = scan_one(&path, size as u64, &config, ®istry).expect("scanned");
|
||
for (tail, n) in scan.tails.iter().zip(SWEEP) {
|
||
let want: Sha = if size > n {
|
||
Sha256::digest(&body[n.max(size - n)..]).into()
|
||
} else {
|
||
[0; 32]
|
||
};
|
||
assert_eq!(*tail, want, "size {size}, tail {n}");
|
||
}
|
||
}
|
||
std::fs::remove_dir_all(&dir).ok();
|
||
}
|
||
|
||
/// The hardlink-farm case: the pair count is quadratic, so it must be
|
||
/// computed, not enumerated.
|
||
#[test]
|
||
fn a_group_of_thousands_does_not_need_enumerating() {
|
||
let members: Vec<u32> = (0..10_000).collect();
|
||
let base = vec![0u32; 10_000];
|
||
let sizes = vec![1u64; 10_000];
|
||
let stat = group_stat(&members, &base, &sizes);
|
||
assert_eq!(stat.reported_pairs, 10_000 * 9_999 / 2);
|
||
assert_eq!(stat.fp_pairs, 0);
|
||
}
|
||
}
|