llm_retrieval_scorer:v0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
import weave
from typing import List
from typing import Dict
from typing import Any
import json
import litellm
from pydantic.main import BaseModel
from pydantic.functional_validators import field_validator
@weave.op
async def parse_and_validate_response(
response_text: str, num_contexts: int
) -> Dict[str, Any]:
"""
Parse and validate the response text.
Args:
response_text (str): The response text to be parsed and validated.
num_contexts (int): The expected number of contexts.
Returns:
Dict[str, Any]: The validated response as a dictionary.
Raises:
ValueError: If the relevance score is not 0, 1, or 2.
ValueError: If the IDs are not unique.
ValueError: If the number of scores does not match the number of contexts.