tests: close F-034; bound the derived trio at 8 ULP of the oracle record
The oracle records six significant figures, so the last digit of an area near 200 mm2 is worth 0.001 mm2. Every measured disagreement between port and reference is one, two or three units in that place. SECTION_AREA_MM2, VOLUME_MM3 and MASS_G are now bounded at 8 ULP of the expected value. Everything that positions material keeps the declared 1e-4 mm and remains exact in all 123 cases. Option 1 scope kept, derivation rejected on measurement. Max |dA|/P over the accepted set is 1.434e-05 mm, one seven-hundredth of the 0.01 mm criterion, so a perimeter x 0.01 bound would have run 1800 to 4500 times the worst real discrepancy and caught nothing. Perimeter also anti-correlates with the error. Suite 468 passed, 0 failed. The 30 expected failures are resolved, not suppressed. Mutation tested before landing: worst case uses 37.5 percent of its bound, 12 ULP offsets and 1e-4 relative scalings are caught in all 113 cases, 1e-6 and 1e-5 correctly are not. Adds docs/ACCEPTANCE.md as the specification. Adds F-036, the venv interpreter error, same class as F-035. Corrects the F-034 per-profile distribution to Three-Fin 10, Y 7, A Frame 6, Rectangle 5, T 1, Four-Fin 1, which sums to the stated 30.
This commit is contained in:
+140
-5
@@ -10,6 +10,21 @@ skips with a clear reason. Those integrity checks always run -- an oracle that
|
||||
has been edited is worse than no oracle, and that should fail loudly on any
|
||||
machine, at any time, with no dependencies.
|
||||
|
||||
TOLERANCE
|
||||
Two regimes, and `docs/ACCEPTANCE.md` holds the reasoning.
|
||||
|
||||
A declared absolute -- the oracle's own tolerance block. Everything that
|
||||
positions material. Exact in all 123 cases; never loosen it.
|
||||
|
||||
B six-significant-figure last place -- SECTION_AREA_MM2, VOLUME_MM3 and
|
||||
MASS_G only. The oracle records what OpenSCAD's echo printed, which is
|
||||
%g at six significant figures, so the last recorded digit of an area
|
||||
near 200 mm2 is worth 0.001 mm2. Measured disagreement across the whole
|
||||
set is one, two or three units in that place, never more.
|
||||
|
||||
The change belongs here and never in the oracle JSON: the tolerance block is
|
||||
inside the hashed document and `test_integrity_hash` covers it.
|
||||
|
||||
pytest -n auto # all of it
|
||||
pytest -m oracle # acceptance only
|
||||
pytest -k integrity # oracle checks alone, always runnable
|
||||
@@ -17,11 +32,61 @@ machine, at any time, with no dependencies.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.oracle
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tolerance policy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The three keys limited by arc-segment discretisation rather than by the port.
|
||||
# One underlying quantity reported three times: volume is area times the model
|
||||
# length, mass is volume times density over 1000. See F-034.
|
||||
SIXFIG_KEYS = ("SECTION_AREA_MM2", "VOLUME_MM3", "MASS_G")
|
||||
|
||||
# How many units in that last place are allowed. Worst measured is 3 for area
|
||||
# and volume, 5 for mass; 8 leaves margin without reaching 1e-4 relative, which
|
||||
# `test_tolerance_policy_is_not_vacuous` enforces.
|
||||
SIXFIG_ULPS = 8
|
||||
|
||||
# Counts, compared exactly.
|
||||
EXACT_KEYS = ("SECTION_PARTS", "STRAP_CHANNELS", "BUNDLE_COUNT")
|
||||
|
||||
# The ceiling every derived bound must stay under, as a fraction of the value.
|
||||
RELATIVE_CEILING = 1e-4
|
||||
|
||||
|
||||
def last_place(value: float) -> float:
|
||||
"""
|
||||
The value of the last digit the reference actually recorded.
|
||||
|
||||
The oracle holds `echo` output, six significant figures. For 213.950 that
|
||||
is 0.001; for 21395.0 it is 0.1. Returns 0.0 for a zero value, which has no
|
||||
meaningful last place -- callers fall back to the declared tolerance.
|
||||
"""
|
||||
if value == 0.0 or not math.isfinite(value):
|
||||
return 0.0
|
||||
return 10.0 ** (math.floor(math.log10(abs(value))) - 5)
|
||||
|
||||
|
||||
def bound_for(key: str, want: float, tolerance: dict) -> float:
|
||||
"""The tolerance in force for one key at one magnitude."""
|
||||
if key in SIXFIG_KEYS:
|
||||
ulp = last_place(want)
|
||||
if ulp:
|
||||
return SIXFIG_ULPS * ulp
|
||||
return tolerance["areas_mm2"] if key.endswith("_MM2") else tolerance["lengths_mm"]
|
||||
|
||||
|
||||
def comparable(key: str, want: object) -> bool:
|
||||
"""Float-valued report keys that are not counts."""
|
||||
return isinstance(want, float) and key not in EXACT_KEYS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integrity - no dependency on the port
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -74,6 +139,75 @@ def test_integrity_invariants_hold_in_the_oracle(accepted_cases):
|
||||
f"{where}: wall below its declared minimum"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The tolerance policy itself - no dependency on the port
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_tolerance_policy_is_not_vacuous(accepted_cases, tolerance):
|
||||
"""
|
||||
No derived bound ever exceeds 1e-4 relative.
|
||||
|
||||
This is the guard against the failure mode that matters here. A comparison
|
||||
quietly loosened until it no longer catches a real regression passes every
|
||||
other test in this file while proving nothing, and it does so silently --
|
||||
which is worse than the 30 known failures this policy replaced, because
|
||||
those were visible. ACCEPTANCE.md E-6.
|
||||
|
||||
Scoped to regime B, which is the part of the policy that lives in this
|
||||
unhashed file and could therefore be widened by an edit. Regime A is the
|
||||
oracle's own declared tolerance and is protected by `test_integrity_hash`;
|
||||
it is also legitimately absolute rather than relative, since it covers
|
||||
quantities well below 1 mm such as CLEARANCE_MM and STRAP_THICK_MM.
|
||||
"""
|
||||
for case in accepted_cases:
|
||||
where = f"{case['profile']}/{case['label']}"
|
||||
for key in SIXFIG_KEYS:
|
||||
want = case["report"][key]
|
||||
if want == 0.0:
|
||||
continue
|
||||
bound = bound_for(key, want, tolerance)
|
||||
assert bound <= abs(want) * RELATIVE_CEILING, (
|
||||
f"{where}: the bound for {key} is {bound} against a value of "
|
||||
f"{want}, which is looser than {RELATIVE_CEILING} relative. "
|
||||
f"The comparison has stopped being able to catch a regression."
|
||||
)
|
||||
|
||||
|
||||
def test_sixfig_keys_are_the_derived_trio(accepted_cases):
|
||||
"""
|
||||
Volume is area times the model length; mass is volume times density.
|
||||
|
||||
One bound governs all three keys only because they are one quantity
|
||||
reported three times. If that stops being true, E-5's justification is
|
||||
gone and the three need separate treatment. ACCEPTANCE.md E-5.
|
||||
"""
|
||||
ratios = set()
|
||||
|
||||
for case in accepted_cases:
|
||||
r = case["report"]
|
||||
where = f"{case['profile']}/{case['label']}"
|
||||
|
||||
area = r["SECTION_AREA_MM2"]
|
||||
volume = r["VOLUME_MM3"]
|
||||
ratio = volume / area
|
||||
ratios.add(round(ratio, 6))
|
||||
|
||||
if "LENGTH_MM" in r:
|
||||
assert ratio == pytest.approx(r["LENGTH_MM"], rel=1e-6), (
|
||||
f"{where}: VOLUME_MM3 / SECTION_AREA_MM2 is {ratio}, but the "
|
||||
f"report says LENGTH_MM is {r['LENGTH_MM']}"
|
||||
)
|
||||
|
||||
assert r["MASS_G"] == pytest.approx(volume * 1.24 / 1000, rel=5e-5), (
|
||||
f"{where}: MASS_G is not VOLUME_MM3 times density over 1000"
|
||||
)
|
||||
|
||||
assert len(ratios) == 1, (
|
||||
f"the extrusion length is not constant across the oracle: {sorted(ratios)}. "
|
||||
f"One bound can no longer serve all three keys."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Acceptance - requires the port
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -96,17 +230,18 @@ def test_accepted_case_matches_oracle(port, accepted_case, tolerance):
|
||||
got = result.report
|
||||
|
||||
# Counts are exact.
|
||||
for key in ("SECTION_PARTS", "STRAP_CHANNELS", "BUNDLE_COUNT"):
|
||||
for key in EXACT_KEYS:
|
||||
assert got[key] == expected[key], f"{where}: {key}"
|
||||
|
||||
# Lengths and areas carry the tolerance the oracle declares.
|
||||
# Everything else carries the bound its regime declares.
|
||||
for key, want in expected.items():
|
||||
if not isinstance(want, float) or key in ("SECTION_PARTS", "STRAP_CHANNELS"):
|
||||
if not comparable(key, want):
|
||||
continue
|
||||
tol = tolerance["areas_mm2"] if key.endswith("_MM2") else tolerance["lengths_mm"]
|
||||
tol = bound_for(key, want, tolerance)
|
||||
assert key in got, f"{where}: port did not report {key}"
|
||||
assert abs(got[key] - want) <= tol, (
|
||||
f"{where}: {key} is {got[key]}, oracle says {want}"
|
||||
f"{where}: {key} is {got[key]}, oracle says {want} "
|
||||
f"(difference {got[key] - want:.6g}, bound {tol:.6g})"
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user