From d2d326fb06f5c7ecf865a3769672a18eff29a3d7 Mon Sep 17 00:00:00 2001 From: Jeremy Karst Date: Sat, 12 Sep 2026 20:27:28 -0400 Subject: [PATCH] Added to findings, generated report --- .gitignore | 5 +- ergodic_sampling_test.py | 26 +++-- findings.md | 104 ++++++++++++++++-- make_report.py | 222 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 333 insertions(+), 24 deletions(-) create mode 100644 make_report.py diff --git a/.gitignore b/.gitignore index aab52d9..599e8b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -*.png \ No newline at end of file +*.png +report.docx +report.pdf +Geophysical Prospecting - 2023 - Zhang - Ergodic sampling Acquisition design to maximize information from limited samples.pdf diff --git a/ergodic_sampling_test.py b/ergodic_sampling_test.py index f7fff5f..d960edd 100644 --- a/ergodic_sampling_test.py +++ b/ergodic_sampling_test.py @@ -357,11 +357,11 @@ C_ENS, C_ONE = '#1f77b4', '#ff7f0e' # blue = ensemble comparison, orange = sin COLS = ['Ground truth', 'Sparse + spline interpolation\n(naive control)', 'Sparse + compressive sensing\n(Fourier)', - 'Sparse + compressive sensing\n(symlet wavelet, as in the paper)', + 'Sparse + compressive sensing\n(symlet wavelet, from paper)', 'Sparse + kriging\n(Gaussian process)', 'Ergodic + spline interpolation', 'Ergodic + compressive sensing\n(Fourier)', - 'Ergodic + compressive sensing\n(symlet wavelet, as in the paper)', + 'Ergodic + compressive sensing\n(symlet wavelet, from paper)', 'Ergodic + kriging\n(Gaussian process)'] COL_MASKS = [np.ones_like(mask_reg), mask_reg, mask_reg, mask_reg, mask_reg, mask_erg, mask_erg, mask_erg, mask_erg] @@ -381,7 +381,7 @@ for fname, gt, imgs, note in [ for c, (a, x, t) in enumerate(zip(axes[0], imgs, COLS)): im = a.imshow(x.T, origin='lower', vmin=-v, vmax=v, cmap='RdBu_r') sub = '' if c == 0 else (f'\nleakage {leakage(x, gt):+.2f} · ' - f'root-mean-square error {rms(x, gt):.3f}') + f'RMS error {rms(x, gt):.3f}') a.set_title(t + sub, fontsize=9) axes[0, 0].set_ylabel('Reconstruction\n\ngrid y (cells)') fig.colorbar(im, ax=axes[0], shrink=0.9, @@ -439,10 +439,10 @@ def sigma_study(fname, desc, make_truth, truth, x_rec, sig_raw, rec_fn, rec_name axes[0, 0].set_title('Ground truth', fontsize=10) axes[0, 1].imshow(x_rec.T, origin='lower', vmin=-v, vmax=v, cmap='RdBu_r') axes[0, 1].set_title(f'Reconstruction: ergodic + {rec_name}\n' - f'leakage {leakage(x_rec, truth):+.2f} · root-mean-square error ' + f'leakage {leakage(x_rec, truth):+.2f} · RMS error ' f'{rms(x_rec, truth):.3f}', fontsize=10) axes[0, 2].imshow(err.T, origin='lower', vmin=-v, vmax=v, cmap='RdBu_r') - axes[0, 2].set_title('Signed error:\nreconstruction − truth', fontsize=10) + axes[0, 2].set_title('Signed error:\nreconstruction - truth', fontsize=10) fig.colorbar(im0, ax=axes[0], shrink=0.9, label='field value or error\n(units of the truth standard deviation)') for a, x, t in zip(axes[1], @@ -450,7 +450,7 @@ def sigma_study(fname, desc, make_truth, truth, x_rec, sig_raw, rec_fn, rec_name ['Predicted standard deviation:\nGaussian process posterior (truth-free)', f'Empirical standard deviation\n({ENS_K} independent truths, ' 'same sampling pattern)', - 'Actual absolute error, this realization:\n|reconstruction − truth|']): + 'Actual absolute error, this realization:\n|reconstruction - truth|']): im1 = a.imshow(x.T, origin='lower', vmin=0, vmax=v, cmap='turbo') a.set_title(t, fontsize=10) a.set_xlabel('grid x (cells)') @@ -493,21 +493,19 @@ def sigma_study(fname, desc, make_truth, truth, x_rec, sig_raw, rec_fn, rec_name a.plot([0, 1], [0, 1], 'k--', lw=.8, label='perfect calibration') a.set_xlim(0, 1.0); a.set_ylim(0, 1.0); a.set_aspect(1) a.set_xlabel('mean predicted standard deviation within decile') - a.set_ylabel('root-mean-square error or\nempirical standard deviation within decile') + a.set_ylabel('RMS error or\nempirical standard deviation within decile') a.ticklabel_format(style='sci', scilimits=(-2, 3)) a.set_title('Calibration of the Gaussian process prediction\n(cells binned into deciles of ' 'the prediction)', fontsize=10) a.legend(loc='upper left', fontsize=8) - fig.suptitle('Truth-free predicted standard deviation versus actual reconstruction error\n' + fig.suptitle('Statistical analysis of reconstruction error\n' f'({desc}; ergodic sampling; {rec_name} reconstruction)\n' - 'Uniform ergodic coverage intentionally flattens the prediction; its narrow ' - 'range caps single-realization correlation;\nthe ensemble comparison is the ' - 'decisive test.', fontsize=11) + ,fontsize=11) plt.savefig(f'./{fname}.png', dpi=120) DESC_SP = f'Fourier-sparse truth ({N_MODES} modes)' DESC_PL = f'potential-field truth (spectral exponent {BETA}, source depth {Z_SRC} cells)' -REC_GP, REC_SYM = 'kriging (Gaussian process)', 'compressive sensing (symlet wavelet, as in the paper)' +REC_GP, REC_SYM = 'kriging (Gaussian process)', 'compressive sensing (symlet wavelet, from paper)' KRIG_FAST_SP = gp_predictor(mask_erg, *GP_SP) KRIG_FAST_PL = gp_predictor(mask_erg, BETA_HAT, Z_HAT) sigma_study('plt2a1', DESC_SP, lambda: fourier_sparse_field(N_MODES), truth_sp, x_sp_gp, sig_sp_raw, @@ -587,13 +585,13 @@ for a, (cname, _) in zip(axes, CLASSES): if cname == CLASSES[0][0]: t += f'\n(dashed line: best possible = predict zero everywhere, {np.sqrt(1 - DELTA):.2f})' a.set_title(t, fontsize=10) - a.set_xlabel('root-mean-square error\n(units of the truth standard deviation)') + a.set_xlabel('RMS error\n(units of the truth standard deviation)') a.set_xlim(0, xmax*1.32) a.grid(axis='x', lw=0.4, alpha=0.4) a.set_axisbelow(True) axes[0].set_yticks(ypos, [DISPLAY[m] for m in methods]) axes[0].axvline(np.sqrt(1 - DELTA), ls='--', lw=1, color='k') -fig.suptitle('Monte-Carlo root-mean-square reconstruction error: ' +fig.suptitle('Monte-Carlo RMS reconstruction error: ' f'mean ± standard deviation over {N_TRIALS} trials per truth class ' f'({DELTA:.1%} of samples, error bars = ± one standard deviation)') plt.savefig('./plt3.png', dpi=120) diff --git a/findings.md b/findings.md index 127d001..23eec7f 100644 --- a/findings.md +++ b/findings.md @@ -1,4 +1,4 @@ -# Findings: sampling patterns, reconstruction methods, and what the ground truth is made of +# Sampling patterns, reconstruction methods, and how they relate to ground-truth structure Written 2026-09-12. Refer to [ergodic_sampling_test.py](ergodic_sampling_test.py) for analysis and figures. @@ -8,7 +8,7 @@ The analysis uses a 64 by 64 grid and keeps only 11.8% of the cells (484 samples sparse grid (every third cell) or in the "ergodic" irregular pattern from the Zhang and Li paper (found by minimizing equation 4). From those samples we rebuild the full grid with several methods and measure how much the reconstruction differs from the ground truth. The value of merit chosen is -root-mean-squared error: the typical size of the difference between the reconstruction and the true +RMS error: the typical size of the difference between the reconstruction and the true field at a cell, normalized such that 1.0 is 1 standard deviation (lower is better). Each RMS error value shown is the average over 25 iterations with a freshly generated ground truth. @@ -39,7 +39,7 @@ The three kinds of ground truth: ## The numbers -Root-mean-square error, mean plus or minus spread over 25 trials (best per column in bold): +RMS error, mean plus or minus spread over 25 trials (best per column in bold): | method | white noise (control) | Fourier-sparse | potential-field | |---|---|---|---| @@ -51,6 +51,13 @@ Root-mean-square error, mean plus or minus spread over 25 trials (best per colum | ergodic + compressive sensing (symlet) | 1.304 ± 0.034 | 0.737 ± 0.107 | 0.145 ± 0.020 | | ergodic + kriging | **1.002 ± 0.007** | 0.558 ± 0.074 | 0.094 ± 0.016 | +![Figure 1](plt3.png) + +*Figure 1. Reconstruction error broken down by sampling pattern + reconstruction formulation. Bar +color encodes the sampling pattern: blue for sparse grid, orange for ergodic. Bars are the mean +over 25 trials per truth class. The dashed line in the left panel marks the best score attainable +on structureless / random data, which is predicting zero at every unsampled cell (0.94).* + ## What was found **1. On truly random ground truth, ergodic + kriging came out best of everything tested.** @@ -63,17 +70,37 @@ that do not exist (1.22 on the sparse grid, 1.45 on the ergodic pattern) and the paints in wavelet texture (1.30). That gap, up to roughly 50% worse than guessing zero, is the cost of hallucinated structure made concrete. -**2. On the potential-field ground truth, the plain sparse grid is the best pattern.** This +**2. On the potential-field ground truth, the plain sparse grid is the best pattern (figure 3).** This field is smooth enough that every third cell is dense enough sampling, the situation classical sampling theory covers. There the sparse grid wins simply on coverage: its farthest cell from any sample is 1.41 cells away, while the ergodic pattern, being irregular, leaves gaps up to 3 cells. The ergodic pattern costs about 1.5 times the error here. That cost is the premium for insurance that pays out on the Fourier-sparse truth, where half the signal varies too fast for the sparse grid: there the sparse grid garbles the fast waves into false slow ones -(this is aliasing, and no processing can undo it). THIS CASE IS MOSTLY THEORETICAL: If we knew -that there were no small-scale variation to capture, the sparse grid would be the right choice -outright; in practice that is rarely known before sampling, which is exactly what motivates the -signal-agnostic ergodic pattern. +(this is aliasing, and is destructive; compare the sparse-grid and ergodic columns of +figure 2). THIS CASE IS MOSTLY THEORETICAL: If we knew that there was no small-scale variation +to capture, the sparse grid would be the right choice outright; in practice that is rarely known +before sampling, which is exactly what motivates the signal-agnostic ergodic pattern. See +the "Potential Future Work" section for some related thoughts. + +![Figure 2](plt1a.png) + +*Figure 2. One realization of the Fourier-sparse ground truth with all eight pattern and method +combinations. Top row: the ground truth on the left followed by each reconstruction, labelled with +its leakage and RMS error. Middle row: the signed error of each reconstruction, on a single shared color +scale. Bottom row: the sampling locations that produced the column above. Ergodic sampling with Fourier +compressive sensing on a fourier sparse ground truth recovers the field exactly, leaving a blank error +panel. Every sparse sampling method captures only the slow part of the field and leaves the unobserved +high frequency error.* + +![Figure 3](plt1b.png) + +*Figure 3. The same layout for one realization of the potential-field ground truth. Because this field is +smooth on the scale of the sample spacing, every method except Fourier compressive sensing reproduces it +closely; Fourier compressive sensing insists on a handful of dominant waves hallucinates high frequency +noise, which shows up as the striped error panel in its column. What error remains for the other methods +gathers in the widest gaps between samples, which is why the more even coverage of the sparse grid wins +here: its farthest cell from a sample is 1.41 cells away, against 3 cells for the ergodic pattern.* **3. Matching the reconstruction method to how the ground truth was generated wins, as expected.** Each column of the table is won by the method whose built-in assumption mirrors the generator: @@ -84,6 +111,54 @@ fail hard: Fourier compressive sensing assumes a few dominant waves and scores 0 smooth field, and the symlet version, which assumes the wrong kind of building block for a sum-of-waves signal, reaches only 0.737 where the Fourier version is exact. +## Predicting the error without the ground truth + +Everything above scores a reconstruction against a truth we happen to know. In a real survey the truth +is exactly what is missing, so the practical question is whether the reconstruction can say, by itself, +where it is likely to be wrong. Kriging answers that for free: alongside an estimate at every cell it +reports the standard deviation of that estimate, computed from the sample locations and from a +similarity model fitted to the sample values alone. No ground truth enters at any point. + +Figures 4 to 7 test that predicted error map for both ground truth classes and for both the kriging and +the symlet compressive sensing reconstruction. Each figure has the same layout. The top row shows the +truth, the reconstruction, and their signed difference. The middle row puts the truth-free predicted +standard deviation beside two things it should resemble: the error actually measured over 64 independent +truths drawn from the same generator and sampled with the same pattern, and the absolute error in the +single realization above. The bottom row shows the sampling pattern, a cell-by-cell scatter of predicted +against actual error, and a calibration curve in which cells are binned into deciles of the prediction. + +Two cautions matter when reading these. First, one realization of an error is one draw of a random +variable, so even a perfect prediction correlates with it only up to a ceiling, which is printed on each +scatter panel; the comparison against the 64-truth ensemble is the decisive test, and there the +prediction tracks the actual error pattern with correlations of 0.91 to 0.99. Second, uniform ergodic +coverage intentionally flattens the predicted map, since the whole point of the pattern is to leave no +cell poorly covered, and that narrow range is what pushes the single-realization ceiling down. The +calibration curves run close to the one-to-one line in every case, slightly below it, meaning the +predicted magnitudes are roughly correct and not merely correctly ranked. + +![Figure 4](plt2a1.png) + +*Figure 4. Fourier-sparse ground truth, ergodic sampling, kriging reconstruction. Kriging is the wrong +prior for this field class, so the errors are large, but the predicted map still identifies where errors +are most likely.* + +![Figure 5](plt2a2.png) + +*Figure 5. Fourier-sparse ground truth, ergodic sampling, symlet compressive sensing reconstruction: the +same predicted map tested against a reconstruction built on symlet wavelets. Provided as a direct +comparison to the reconstruction methods used in the paper.* + +![Figure 6](plt2b1.png) + +*Figure 6. Potential-field ground truth, ergodic sampling, kriging reconstruction. This is the matched +case, where the fitted similarity model recovers the generator's parameters, and it is where the +prediction is most accurate.* + +![Figure 7](plt2b2.png) + +*Figure 7. Potential-field ground truth, ergodic sampling, symlet compressive sensing reconstruction. +Provided as a direct comparison to the reconstruction methods used in the paper.* + ## The bigger picture Choosing how to reconstruct in essence is a choice about how much you trust what you know about @@ -93,11 +168,22 @@ assumptions: a method that expects structure will manufacture it out of nothing, structureless control every such method lost to simply predicting zero, with kriging only slightly worse than predicting zero. The paper's ergodic pattern is best understood in this light: it is deliberately designed without any knowledge of the signal, as insurance that keeps every option open. -Alongside it, our error-prediction experiment (the plt2 figures) shows that a map of the expected +Alongside it, our error-prediction experiment (figures 4 to 7) shows that a map of the expected error at every cell can be computed from nothing but the sample values and their locations, and it tracks the actual error pattern closely (correlation 0.91 to 0.99 against the error measured over 64 independent trials, with correctly calibrated magnitudes). +## Does kriging reconstruction strictly dominate symlet wavelet reconstruction? + +Largely yes. Kriging beat symlet compressive sensing in every Monte-Carlo column above, and a +follow-up test showed it wins even on truths that are exactly sparse in the symlet basis: the +decimated wavelet transform and point sampling are both spatially localized, a coherent pair that +violates compressive sensing's incoherence requirement. Two caveats: for signals of a few isolated +wavelet-like anomalies (the paper's Figure 15 regime) symlet keeps a relative edge, though at +11.8% sampling both methods there do worse than predicting zero; and for sparse-spectrum signals +Fourier compressive sensing, not kriging, is the right choice (exact recovery kriging cannot +match). Symlet was never the best method in any regime tested. + ## Potential future work Since we have shown that for cases where no small scale variation exists to capture, sparse sampling diff --git a/make_report.py b/make_report.py new file mode 100644 index 0000000..058681f --- /dev/null +++ b/make_report.py @@ -0,0 +1,222 @@ +"""Render findings.md (text plus the generated plots) into a report: report.docx and report.pdf. + +Handles the markdown subset findings.md uses: headings, paragraphs with bold and links, bullet +lists, pipe tables, and image lines followed by an italic "*Figure n. ...*" caption line. + +Wide figures (aspect ratio above 2) get their own landscape page so the panel labels stay legible; +everything else flows inline on portrait pages. The PDF is produced by converting the .docx with +LibreOffice, which must be on the PATH. +""" +import re +import shutil +import subprocess +import sys +from pathlib import Path + +from PIL import Image +from docx import Document +from docx.enum.section import WD_ORIENT, WD_SECTION +from docx.enum.text import WD_ALIGN_PARAGRAPH +from docx.shared import Emu, Inches, Pt, RGBColor + +SRC = Path(__file__).parent / 'findings.md' +OUT_DOCX = Path(__file__).parent / 'report.docx' + +PORTRAIT = (Inches(8.5), Inches(11), Inches(1.0)) # width, height, side margin +LANDSCAPE = (Inches(11), Inches(8.5), Inches(0.5)) +MARGIN_TB = Inches(0.8) +WIDE_ASPECT = 2.0 # aspect above this gets a dedicated landscape page +MAX_INLINE_HEIGHT = 7.4 # inches, leaves room for the caption on a portrait page + +IMG_RE = re.compile(r'^!\[(?P[^\]]*)\]\((?P[^)]+)\)\s*$') +CAPTION_RE = re.compile(r'^\*(?PFigure .*)\*\s*$') +# bold, inline code, or link, in one pass so the pieces stay in document order +INLINE_RE = re.compile(r'\*\*(?P[^*]+)\*\*|`(?P[^`]+)`|\[(?P[^\]]+)\]\([^)]+\)') + + +def set_page(section, geometry): + width, height, side = geometry + section.orientation = WD_ORIENT.LANDSCAPE if width > height else WD_ORIENT.PORTRAIT + section.page_width, section.page_height = width, height + section.left_margin = section.right_margin = side + section.top_margin = section.bottom_margin = MARGIN_TB + + +def usable_width(geometry): + width, _, side = geometry + return Emu(width - 2*side).inches # Length arithmetic returns plain integer emu + + +class Report: + """Builds the document, tracking which page orientation is currently active.""" + + def __init__(self): + self.doc = Document() + self.geometry = PORTRAIT + set_page(self.doc.sections[0], PORTRAIT) + normal = self.doc.styles['Normal'] + normal.font.name = 'Calibri' + normal.font.size = Pt(10.5) + normal.paragraph_format.space_after = Pt(8) + + def use(self, geometry): + """Switch orientation lazily, at the moment content is added, so no empty page is left + behind when two figures of the same orientation follow each other.""" + if geometry is self.geometry: + return + set_page(self.doc.add_section(WD_SECTION.NEW_PAGE), geometry) + self.geometry = geometry + + def runs(self, paragraph, text): + """Emit `text` into `paragraph`, honouring bold, inline code and link markup.""" + pos = 0 + for m in INLINE_RE.finditer(text): + if m.start() > pos: + paragraph.add_run(text[pos:m.start()]) + if m.group('bold') is not None: + paragraph.add_run(m.group('bold')).bold = True + elif m.group('code') is not None: + run = paragraph.add_run(m.group('code')) + run.font.name = 'Consolas' + run.font.size = Pt(9.5) + else: + run = paragraph.add_run(m.group('text')) + run.font.color.rgb = RGBColor(0x1F, 0x4E, 0x79) + pos = m.end() + paragraph.add_run(text[pos:]) + + def heading(self, text, level): + self.use(PORTRAIT) + p = self.doc.add_heading('', level=level) + self.runs(p, text) + return p + + def paragraph(self, text, style=None): + self.use(PORTRAIT) + p = self.doc.add_paragraph(style=style) + self.runs(p, text) + return p + + def figure(self, path, caption): + img = Image.open(path) + aspect = img.width / img.height + wide = aspect >= WIDE_ASPECT + self.use(LANDSCAPE if wide else PORTRAIT) + width = usable_width(self.geometry) + height = width/aspect + if not wide and height > MAX_INLINE_HEIGHT: # tall figure: fit the page height instead + height, width = MAX_INLINE_HEIGHT, MAX_INLINE_HEIGHT*aspect + p = self.doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + p.paragraph_format.space_after = Pt(4) + p.add_run().add_picture(str(path), width=Inches(width)) + cap = self.doc.add_paragraph() + cap.alignment = WD_ALIGN_PARAGRAPH.CENTER + for run in [cap.add_run(caption)]: + run.italic = True + run.font.size = Pt(9) + + def table(self, rows): + self.use(PORTRAIT) + t = self.doc.add_table(rows=len(rows), cols=len(rows[0])) + t.style = 'Table Grid' + t.autofit = True + for r, row in enumerate(rows): + for c, cell in enumerate(row): + para = t.cell(r, c).paragraphs[0] + self.runs(para, cell) + para.paragraph_format.space_after = Pt(2) + for run in para.runs: + run.font.size = Pt(9) + run.bold = run.bold or r == 0 + self.doc.add_paragraph() + + +def split_row(line): + return [c.strip() for c in line.strip().strip('|').split('|')] + + +def is_separator(line): + return bool(re.fullmatch(r'\|[\s:|-]+\|', line.strip())) + + +def render(md_lines, report, base): + i, n = 0, len(md_lines) + while i < n: + line = md_lines[i].rstrip() + if not line.strip(): + i += 1 + continue + + m = IMG_RE.match(line) + if m: + j = i + 1 + while j < n and not md_lines[j].strip(): + j += 1 + caption_lines = [] + while j < n and md_lines[j].strip(): # captions wrap over several lines + caption_lines.append(md_lines[j].strip()) + j += 1 + caption = ' '.join(caption_lines) + cm = CAPTION_RE.match(caption) + report.figure(base/m.group('path'), + cm.group('text') if cm else m.group('alt')) + i = j if cm else i + 1 + continue + + if line.startswith('#'): + level = len(line) - len(line.lstrip('#')) + report.heading(line[level:].strip(), level - 1 if level > 1 else 0) + i += 1 + continue + + if line.lstrip().startswith('- '): + while i < n and md_lines[i].lstrip().startswith('- '): + indent = len(md_lines[i]) - len(md_lines[i].lstrip()) + item = [md_lines[i].lstrip()[2:].strip()] + i += 1 + while (i < n and md_lines[i].strip() # wrapped continuation + and not md_lines[i].lstrip().startswith('- ') + and len(md_lines[i]) - len(md_lines[i].lstrip()) > indent + 1): + item.append(md_lines[i].strip()) + i += 1 + style = 'List Bullet' if indent < 2 else 'List Bullet 2' + report.paragraph(' '.join(item), style=style) + continue + + if line.startswith('|'): + rows = [] + while i < n and md_lines[i].strip().startswith('|'): + if not is_separator(md_lines[i]): + rows.append(split_row(md_lines[i])) + i += 1 + report.table(rows) + continue + + block = [line.strip()] + i += 1 + while i < n and md_lines[i].strip() and not md_lines[i].lstrip().startswith(('#', '|', '- ', '![')): + block.append(md_lines[i].strip()) + i += 1 + report.paragraph(' '.join(block)) + + +def to_pdf(docx_path): + soffice = shutil.which('soffice') or shutil.which('libreoffice') + if soffice is None: + print('LibreOffice not found; wrote the .docx only', file=sys.stderr) + return None + subprocess.run([soffice, '--headless', '--convert-to', 'pdf', + '--outdir', str(docx_path.parent), str(docx_path)], + check=True, stdout=subprocess.DEVNULL) + return docx_path.with_suffix('.pdf') + + +if __name__ == '__main__': + report = Report() + render(SRC.read_text().splitlines(), report, SRC.parent) + report.doc.save(OUT_DOCX) + print(f'wrote {OUT_DOCX}') + pdf = to_pdf(OUT_DOCX) + if pdf: + print(f'wrote {pdf}')