context_precision_score:v0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import weave
import google.generativeai as genai
import json
@weave.op()
async def context_precision_score(question, model_output):
model = genai.GenerativeModel("gemini-1.5-pro-latest")
prompt = f"""Given question, answer and context verify if the context was useful in arriving at the given answer. Give verdict as 1 if useful and 0 if not with json output.
Output in only valid JSON format.
Use this JSON schema:
Result = {{"verdict": int}}
Return: Result
question: {question}
context: {model_output['context']}
answer: {model_output['answer']}
verdict: """
result = model.generate_content(prompt)
result_text = result.candidates[0].content.parts[0].text.strip('`').lstrip('json\n')
response_json = json.loads(result_text)
return {
"verdict": int(response_json["verdict"]) == 1,
}