eng-lab/labs/el01_hover/tests/test_tinker.py

"""EL01 lab test for step 9: three knob experiments, each predicted before it was run and explained after."""
import csv
import unittest

from _impl import REPO, artifact

PATH = artifact("tinker.csv", REPO / "logs" / "tinker.csv")


class Step9Tinker(unittest.TestCase):
    def test_three_explained_experiments(self):
        if not PATH.exists():
            self.fail(f"no tinker log yet ({PATH}); run `python -m hwe tinker KEY=VALUE --predict \"...\"`")
        with open(PATH, newline="", encoding="utf-8") as f:
            rows = list(csv.DictReader(f))
        complete = [r for r in rows if r.get("prediction", "").strip() and r.get("explanation", "").strip()]
        self.assertGreaterEqual(len(complete), 3, "need three entries with both a prediction and an explanation")
        for r in complete:
            self.assertTrue(r.get("result", "").strip(), f"entry {r.get('id')}: the result column is empty; "
                                                          "log entries come from `python -m hwe tinker`, not by hand")
        knobs = {r["change"].split("=")[0] for r in complete}
        self.assertGreaterEqual(len(knobs), 3, "turn three different knobs")


if __name__ == "__main__":
    unittest.main()