scorer:v0
1
2
3
4
5
6
7
8
import weave
@weave.op()
def scorer(expected_result: str, model_output: dict) -> dict:
if model_output is None or model_output["test_report"].status is None:
return {"solution_passed": False}
return {"solution_passed": expected_result == model_output["test_report"].status} # check if the test_report status == passed