mechanical-compiler/tests/test_oracle.py

"""
Acceptance of any reimplementation against the frozen rev-8.0.0 oracle.

This file is the specification of the port's public API. It was written before
the port existed, deliberately: the shape of the interface should be decided by
what has to be verified, not by what happens to be convenient to implement.

Until `mechcomp.profiles.build` exists, everything except the integrity checks
skips with a clear reason. Those integrity checks always run -- an oracle that
has been edited is worse than no oracle, and that should fail loudly on any
machine, at any time, with no dependencies.

TOLERANCE
    Two regimes, and `docs/ACCEPTANCE.md` holds the reasoning.

    A  declared absolute -- the oracle's own tolerance block. Everything that
       positions material. Exact in all 123 cases; never loosen it.

    B  six-significant-figure last place -- SECTION_AREA_MM2, VOLUME_MM3 and
       MASS_G only. The oracle records what OpenSCAD's echo printed, which is
       %g at six significant figures, so the last recorded digit of an area
       near 200 mm2 is worth 0.001 mm2. Measured disagreement across the whole
       set is one, two or three units in that place, never more.

    The change belongs here and never in the oracle JSON: the tolerance block is
    inside the hashed document and `test_integrity_hash` covers it.

    pytest -n auto              # all of it
    pytest -m oracle            # acceptance only
    pytest -k integrity         # oracle checks alone, always runnable
"""

from __future__ import annotations

import math

import pytest

pytestmark = pytest.mark.oracle


# ---------------------------------------------------------------------------
# Tolerance policy
# ---------------------------------------------------------------------------

# The three keys limited by arc-segment discretisation rather than by the port.
# One underlying quantity reported three times: volume is area times the model
# length, mass is volume times density over 1000. See F-034.
SIXFIG_KEYS = ("SECTION_AREA_MM2", "VOLUME_MM3", "MASS_G")

# How many units in that last place are allowed. Worst measured is 3 for area
# and volume, 5 for mass; 8 leaves margin without reaching 1e-4 relative, which
# `test_tolerance_policy_is_not_vacuous` enforces.
SIXFIG_ULPS = 8

# Counts, compared exactly.
EXACT_KEYS = ("SECTION_PARTS", "STRAP_CHANNELS", "BUNDLE_COUNT")

# The ceiling every derived bound must stay under, as a fraction of the value.
RELATIVE_CEILING = 1e-4


def last_place(value: float) -> float:
    """
    The value of the last digit the reference actually recorded.

    The oracle holds `echo` output, six significant figures. For 213.950 that
    is 0.001; for 21395.0 it is 0.1. Returns 0.0 for a zero value, which has no
    meaningful last place -- callers fall back to the declared tolerance.
    """
    if value == 0.0 or not math.isfinite(value):
        return 0.0
    return 10.0 ** (math.floor(math.log10(abs(value))) - 5)


def bound_for(key: str, want: float, tolerance: dict) -> float:
    """The tolerance in force for one key at one magnitude."""
    if key in SIXFIG_KEYS:
        ulp = last_place(want)
        if ulp:
            return SIXFIG_ULPS * ulp
    return tolerance["areas_mm2"] if key.endswith("_MM2") else tolerance["lengths_mm"]


def comparable(key: str, want: object) -> bool:
    """Float-valued report keys that are not counts."""
    return isinstance(want, float) and key not in EXACT_KEYS


# ---------------------------------------------------------------------------
# Integrity - no dependency on the port
# ---------------------------------------------------------------------------

def test_integrity_hash(oracle_raw):
    """The committed oracle has not been modified."""
    import hashlib
    import json

    doc = dict(oracle_raw)
    recorded = doc.pop("fixtures_sha256")
    actual = hashlib.sha256(json.dumps(doc, indent=2, sort_keys=True).encode()).hexdigest()
    assert actual == recorded, (
        "The oracle has been edited. Its hash covers the document without the "
        "hash field, serialised with indent=2 and sort_keys=True. If this was "
        "intentional, regenerate it inside the reference toolchain image and "
        "record why in FAILURES.md."
    )


def test_integrity_shape(oracle_raw):
    """The case matrix is the one the documents describe."""
    assert oracle_raw["generator_revision"] == "8.0.0"
    assert oracle_raw["toolchain"]["openscad"] == "2021.01"
    assert oracle_raw["toolchain"]["bosl2_commit"].startswith("92d697c2")

    summary = oracle_raw["summary"]
    assert summary["cases"] == 123
    assert summary["accepted"] == 113
    assert summary["rejected"] == 10
    assert len(oracle_raw["cases"]) == summary["cases"]


def test_integrity_invariants_hold_in_the_oracle(accepted_cases):
    """
    Every accepted case in the oracle satisfies the invariants.

    This validates the oracle itself rather than the port. If it ever fails,
    the fixture set is describing geometry that should never have been
    accepted, and no port should be measured against it.
    """
    for case in accepted_cases:
        r = case["report"]
        where = f"{case['profile']}/{case['label']}"
        expected_members = 3 if "3x" in case["generator"] else 4

        assert r["SECTION_PARTS"] == 1, f"{where}: not one connected solid"
        assert r["STRAP_CHANNELS"] == expected_members, f"{where}: channels merged"
        assert r["MIN_WALL_ACTUAL_MM"] >= r["MIN_WALL_SPEC_MM"] - 1e-4, \
            f"{where}: wall below its declared minimum"


# ---------------------------------------------------------------------------
# The tolerance policy itself - no dependency on the port
# ---------------------------------------------------------------------------

def test_tolerance_policy_is_not_vacuous(accepted_cases, tolerance):
    """
    No derived bound ever exceeds 1e-4 relative.

    This is the guard against the failure mode that matters here. A comparison
    quietly loosened until it no longer catches a real regression passes every
    other test in this file while proving nothing, and it does so silently --
    which is worse than the 30 known failures this policy replaced, because
    those were visible. ACCEPTANCE.md E-6.

    Scoped to regime B, which is the part of the policy that lives in this
    unhashed file and could therefore be widened by an edit. Regime A is the
    oracle's own declared tolerance and is protected by `test_integrity_hash`;
    it is also legitimately absolute rather than relative, since it covers
    quantities well below 1 mm such as CLEARANCE_MM and STRAP_THICK_MM.
    """
    for case in accepted_cases:
        where = f"{case['profile']}/{case['label']}"
        for key in SIXFIG_KEYS:
            want = case["report"][key]
            if want == 0.0:
                continue
            bound = bound_for(key, want, tolerance)
            assert bound <= abs(want) * RELATIVE_CEILING, (
                f"{where}: the bound for {key} is {bound} against a value of "
                f"{want}, which is looser than {RELATIVE_CEILING} relative. "
                f"The comparison has stopped being able to catch a regression."
            )


def test_sixfig_keys_are_the_derived_trio(accepted_cases):
    """
    Volume is area times the model length; mass is volume times density.

    One bound governs all three keys only because they are one quantity
    reported three times. If that stops being true, E-5's justification is
    gone and the three need separate treatment. ACCEPTANCE.md E-5.
    """
    ratios = set()

    for case in accepted_cases:
        r = case["report"]
        where = f"{case['profile']}/{case['label']}"

        area = r["SECTION_AREA_MM2"]
        volume = r["VOLUME_MM3"]
        ratio = volume / area
        ratios.add(round(ratio, 6))

        if "LENGTH_MM" in r:
            assert ratio == pytest.approx(r["LENGTH_MM"], rel=1e-6), (
                f"{where}: VOLUME_MM3 / SECTION_AREA_MM2 is {ratio}, but the "
                f"report says LENGTH_MM is {r['LENGTH_MM']}"
            )

        assert r["MASS_G"] == pytest.approx(volume * 1.24 / 1000, rel=5e-5), (
            f"{where}: MASS_G is not VOLUME_MM3 times density over 1000"
        )

    assert len(ratios) == 1, (
        f"the extrusion length is not constant across the oracle: {sorted(ratios)}. "
        f"One bound can no longer serve all three keys."
    )


# ---------------------------------------------------------------------------
# Acceptance - requires the port
# ---------------------------------------------------------------------------

def test_accepted_case_matches_oracle(port, accepted_case, tolerance):
    """
    A case the reference accepted must be accepted, with matching geometry.

    Both halves matter. Reproducing the measured values while accepting a case
    the reference rejected is not a passing port.
    """
    expected = accepted_case["report"]
    where = f"{accepted_case['profile']}/{accepted_case['label']}"

    result = port.build(
        family=accepted_case["family"],
        profile=accepted_case["profile"],
        params=accepted_case["params"],
    )
    got = result.report

    # Counts are exact.
    for key in EXACT_KEYS:
        assert got[key] == expected[key], f"{where}: {key}"

    # Everything else carries the bound its regime declares.
    for key, want in expected.items():
        if not comparable(key, want):
            continue
        tol = bound_for(key, want, tolerance)
        assert key in got, f"{where}: port did not report {key}"
        assert abs(got[key] - want) <= tol, (
            f"{where}: {key} is {got[key]}, oracle says {want} "
            f"(difference {got[key] - want:.6g}, bound {tol:.6g})"
        )


def test_rejected_case_is_rejected(port, rejected_case):
    """
    A case the reference rejected must be rejected.

    These ten are the part of the contract a naive reimplementation loses: it
    is easy to reproduce the geometry and quietly drop the constraint that made
    it trustworthy. Accepting any of them is a failure, however good the
    numbers look elsewhere.
    """
    where = f"{rejected_case['profile']}/{rejected_case['label']}"
    with pytest.raises(port.ProfileRejected) as excinfo:
        port.build(
            family=rejected_case["family"],
            profile=rejected_case["profile"],
            params=rejected_case["params"],
        )
    assert str(excinfo.value).strip(), (
        f"{where}: rejected without a message. A rejection must name the "
        f"parameter and the limit, as the reference does:\n"
        f"  {rejected_case['rejection']}"
    )


def test_no_cad_dependency_on_the_2d_path(port, accepted_case):
    """
    Building a cross-section must not import the 3D kernel.

    ENVIRONMENT.md section 1.1: a slow OCCT import must never land in the
    request path for a page that only draws a cross-section. This test also
    runs in the CI job where requirements-cad.txt is absent, where an accidental
    import fails outright rather than merely being slow.
    """
    import sys

    for module in ("cadquery", "OCP", "build123d"):
        sys.modules.pop(module, None)

    port.build(
        family=accepted_case["family"],
        profile=accepted_case["profile"],
        params=accepted_case["params"],
    )

    leaked = [m for m in ("cadquery", "OCP", "build123d") if m in sys.modules]
    assert not leaked, f"the 2D path imported {leaked}"