AI scorers: Evaluating AI-generated text with ROUGE

pip install litellm
pip install -qq rouge
git clone -b xtra-scorers https://github.com/wandb/weave.git && cd weave && pip install -qq -e .

import weave
from weave.scorers import RougeScorer

# Initialize Weave
weave.init("rouge-scorer-demo")

# Define the RougeScorer
scorer = RougeScorer(column_map={"output": "prediction", "ground_truth": "ground_truth"})

# Prepare your data
data = [
    {"prediction": "The cat sat on the mat.", "ground_truth": "A cat was sitting on the mat."},
    {"prediction": "The quick brown fox jumps over the lazy dog.", "ground_truth": "A fast brown fox jumped over a sleeping dog."},
    {"prediction": "The sun rises in the east.", "ground_truth": "Sunlight emerges from the eastern horizon."},
]

# Iterate through the data and compute ROUGE scores
for item in data:
    scores = scorer.score(item["prediction"], item["ground_truth"])
    print(f"Prediction: {item['prediction']}")
    print(f"Ground Truth: {item['ground_truth']}")
    print(f"ROUGE Scores: {scores}")
    print("-" * 50)


import weave
import time 
from litellm import acompletion
import asyncio
import nest_asyncio
from weave.scorers import RougeScorer
from weave import Evaluation


weave_client = weave.init("rouge-scorer")

dataset = weave.ref(
    "weave:///c-metrics/rouge-scorer/object/longbench_gov_report_subset:qGNjItwJSEw1NF6xMXX2a0syHJfXVMjeYqwqVwWsdbs"
).get()



class GPT4oMini(weave.Model):
    model_name: str = "gpt-4o-mini"
    temp: float = 0.0
    max_tokens: int = 2048
    top_p: float = 1.0

    @weave.op()
    async def predict(self, query: str) -> str:
        response = await acompletion(
            model=self.model_name,
            api_key="your api key",
            messages=[
                {
                    "role": "system",
                    "content": "You are provided with government reports. Summarize the report in a few sentences but make sure to include all the important information."
                },
                {
                    "role": "user",
                    "content": query
                }
            ],
            temperature=self.temp,
            max_tokens=self.max_tokens,
            top_p=self.top_p
        )
        return response.choices[0].message.content

class GPT4o(weave.Model):
    model_name: str = "gpt-4o-2024-08-06"
    temp: float = 0.0
    max_tokens: int = 2048
    top_p: float = 1.0

    @weave.op()
    async def predict(self, query: str) -> str:
        time.sleep(2)
        response = await acompletion(
            model=self.model_name,
            api_key="your api key",
            messages=[
                {
                    "role": "system",
                    "content": "You are provided with government reports. Summarize the report in a few sentences but make sure to include all the important information."
                },
                {
                    "role": "user",
                    "content": query
                }
            ],
            temperature=self.temp,
            max_tokens=self.max_tokens,
            top_p=self.top_p
        )
        return response.choices[0].message.content

gpt4o = GPT4o()
gpt4omini = GPT4oMini()


nest_asyncio.apply()


scorer = RougeScorer(column_map={"output": "query", "ground_truth": "ground_truth"})

evaluation = Evaluation(
    dataset = dataset.rows[:30],
    scorers=[scorer],
)

asyncio.run(evaluation.evaluate(gpt4o))
asyncio.run(evaluation.evaluate(gpt4omini))