90 lines
3.5 KiB
Python
90 lines
3.5 KiB
Python
|
"""Gap-length binning for the fill comparison table.
|
||
|
|
|
||
|
|
The bins exist because exact gap lengths produced columns resting on as few as
|
||
|
|
five independent targets, where a tenth of a dB is indistinguishable from noise.
|
||
|
|
Getting the boundaries or the trial count wrong would reintroduce exactly that.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
import bench
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("length, expected", [
|
||
|
|
(1, "1"), (2, "2"), (3, "3-4"), (4, "3-4"), (5, "5-8"), (8, "5-8"),
|
||
|
|
(9, "9-16"), (16, "9-16"), (17, "17-32"), (32, "17-32"),
|
||
|
|
(33, "33-64"), (64, "33-64"), (65, "65-128"), (128, "65-128"),
|
||
|
|
(129, "129-256"), (256, "129-256"), (257, "257-512"), (512, "257-512"),
|
||
|
|
(513, "513+"), (5000, "513+"),
|
||
|
|
])
|
||
|
|
def test_every_boundary_lands_in_the_intended_bin(length, expected):
|
||
|
|
assert bench.bin_label(bench.gap_bin(length)) == expected
|
||
|
|
|
||
|
|
|
||
|
|
def test_the_bins_are_contiguous_and_cover_every_positive_length():
|
||
|
|
"""A hole would silently drop results out of the table."""
|
||
|
|
previous = None
|
||
|
|
for low, high in bench.GAP_BINS:
|
||
|
|
if previous is not None:
|
||
|
|
assert low == previous + 1, f"gap between bins at {low}"
|
||
|
|
previous = high
|
||
|
|
assert bench.GAP_BINS[0][0] == 1
|
||
|
|
assert bench.GAP_BINS[-1][1] is None # open-ended at the top
|
||
|
|
for length in range(1, 600):
|
||
|
|
assert bench.gap_bin(length) is not None
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_gap_below_one_has_no_bin():
|
||
|
|
"""gap_length 0 means 'not injected'; it must not be counted as a 1-slot gap."""
|
||
|
|
assert bench.gap_bin(0) is None
|
||
|
|
assert bench.gap_bin(-3) is None
|
||
|
|
|
||
|
|
|
||
|
|
def test_bin_label_is_stable_for_single_width_and_open_bins():
|
||
|
|
assert bench.bin_label((1, 1)) == "1"
|
||
|
|
assert bench.bin_label((5, 8)) == "5-8"
|
||
|
|
assert bench.bin_label((513, None)) == "513+"
|
||
|
|
|
||
|
|
|
||
|
|
# ------------------------------------------------- learned versus hand-written
|
||
|
|
|
||
|
|
|
||
|
|
A, B, C = (1, 1), (2, 2), (3, 4) # three gap bins
|
||
|
|
|
||
|
|
|
||
|
|
def test_classical_fillers_are_the_registry_minus_the_model():
|
||
|
|
assert set(bench.CLASSICAL) == {"hold_last", "linear_blend", "optical_flow",
|
||
|
|
"crosssat", "solar_rotation"}
|
||
|
|
|
||
|
|
|
||
|
|
def test_the_bar_is_the_best_hand_written_filler_per_bin():
|
||
|
|
table = {"optical_flow": {A: [40.0, 42.0], B: [30.0]},
|
||
|
|
"solar_rotation": {A: [39.0], B: [33.0]},
|
||
|
|
"v8": {A: [50.0], B: [50.0]}}
|
||
|
|
best = bench.best_classical(table, [A, B, C])
|
||
|
|
assert best == {A: ("optical_flow", 41.0), B: ("solar_rotation", 33.0)}
|
||
|
|
|
||
|
|
|
||
|
|
def test_named_checkpoints_are_never_the_bar_they_are_measured_against():
|
||
|
|
"""Scored under --learned-name, a model is 'v8', not 'learned'; it used to
|
||
|
|
win the 'best non-learned' slot and leave no learned row to compare."""
|
||
|
|
table = {"optical_flow": {A: [40.0]}, "v8": {A: [45.0]}, "v7_s128": {A: [44.0]},
|
||
|
|
"prior": {A: [42.0]}, "learned": {A: [43.0]}}
|
||
|
|
best = bench.best_classical(table, [A])
|
||
|
|
assert best == {A: ("optical_flow", 40.0)}
|
||
|
|
margins = bench.learned_margins(table, best)
|
||
|
|
assert margins == {"v8": {A: 5.0}, "v7_s128": {A: 4.0}, "prior": {A: 2.0},
|
||
|
|
"learned": {A: 3.0}}
|
||
|
|
|
||
|
|
|
||
|
|
def test_margins_cover_only_bins_with_both_a_bar_and_a_score():
|
||
|
|
table = {"optical_flow": {A: [40.0], B: [30.0]}, "v8": {B: [31.5], C: [20.0]}}
|
||
|
|
margins = bench.learned_margins(table, bench.best_classical(table, [A, B, C]))
|
||
|
|
assert margins == {"v8": {B: 1.5}}
|
||
|
|
|
||
|
|
|
||
|
|
def test_without_hand_written_fillers_there_is_nothing_to_compare_against():
|
||
|
|
table = {"v8": {A: [45.0]}, "v7": {A: [44.0]}}
|
||
|
|
best = bench.best_classical(table, [A])
|
||
|
|
assert best == {} and bench.learned_margins(table, best) == {}
|