llm_response_scorer:v0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import weave
import json
from typing import List
from typing import Dict
from pydantic.main import BaseModel
from pydantic.functional_validators import field_validator
from typing import Any
@weave.op
async def parse_and_validate_response(response_text: str) -> Dict[str, Any]:
"""
Parse and validate the response text to ensure it meets the expected format.
Args:
response_text (str): The response text to be parsed and validated.
Returns:
Dict[str, Any]: The validated response as a dictionary.
Raises:
ValueError: If the 'final_score' is not in [0, 1, 2] or if the 'decision' is not 'correct' or 'incorrect'.
"""
class CorrectnessScore(BaseModel):
reason: str
final_score: int
decision: str