eval_retrieval:v0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
import weave
@weave.op()
def eval_retrieval(model_output: dict, main_source: str) -> dict:
"""Evaluate the retrieval accuracy of the predictions: check whether top source document returned by the
RetrievalQA chain equals the original source document.
Args:
- model_output: the dict that will be provided by the model that is evaluated
- main_source: the target source - as defined in the dataset"""
# post-process prediction results from RetrievalQA in the weave setup.RagNodel
nr1_retrieval = model_output["source_documents"][0]["url"]
return {"first retrieval correct": nr1_retrieval == main_source}