OpenAIJudgeModel.score:v0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
import weave
from pydantic.main import BaseModel
from pydantic.fields import Field
class JudgeMent(BaseModel):
think_out_loud: str = Field(
description="Think out loud about your eventual judgement"
)
score: float = Field(description="A score between 0 and 1")
judgement: str = Field(description="Output either 'correct' or 'incorrect'")
@weave.op()
def score(self, base_prompt: str, model_output: dict) -> dict:
judgement: JudgeMent = self.predict(
base_prompt=base_prompt, generated_image=model_output["image"]
)
return {
"score": judgement.score,
"is_image_correct": judgement.judgement == "correct",
}