AI scorers: Evaluating AI-generated text with BLEU

git clone https://github.com/wandb/weave.git && cd weave && git fetch origin pull/3006/head:xtra-scorers && git checkout xtra-scorers && pip install -qq -e .

export OPENAI_API_KEY='your api key' 

import weave; weave.init('bleu-scorer-demo')
from weave.scorers import BLEUScorer

# Initialize Weave BLEU scorer
weave_scorer = BLEUScorer()

# Define sample hypotheses and references
examples = [
    {"hypothesis": "The cat sat on the mat.", "references": ["A cat was sitting on the mat."]},
    {"hypothesis": "The quick brown fox jumps over the lazy dog.", "references": ["A fast brown fox jumped over a sleeping dog."]},
    {"hypothesis": "The sun rises in the east.", "references": ["Sunlight emerges from the eastern horizon."]},
]

# Prepare input for Weave's summarize function
weave_inputs = []

print("Sentence-Level BLEU:")
for example in examples:
    # Compute sentence-level BLEU for each example
    sentence_bleu_result = weave_scorer.score(
        ground_truths=example["references"],
        output=example["hypothesis"]
    )
    weave_inputs.append({
        "output_pred": example["hypothesis"],
        "output_refs": example["references"],
        "sentence_bleu": sentence_bleu_result["sentence_bleu"],  # Use the sentence-level score from Weave
    })

    # Print sentence-level BLEU results
    print(f"Hypothesis: {sentence_bleu_result['output_pred']}")
    print(f"References: {sentence_bleu_result['output_refs']}")
    print(f"BLEU Score: {sentence_bleu_result['sentence_bleu']}")
    print(f"Brevity Penalty: {sentence_bleu_result['sentence_bp']}")
    print("-" * 50)

# Calculate corpus-level BLEU using Weave
weave_corpus_score = weave_scorer.summarize(weave_inputs)

# Print corpus-level BLEU results
print("\nWeave Corpus-Level BLEU:")
print(f"BLEU Score: {weave_corpus_score['corpus_level']['bleu']}")
print(f"Brevity Penalty: {weave_corpus_score['corpus_level']['brevity_penalty']}")


import weave
import time
from litellm import acompletion
import asyncio
import nest_asyncio
from weave.scorers import BLEUScorer
from weave import Evaluation

# Initialize Weave with BLEU Scorer
weave_client = weave.init("bleu-scorer-demo")

dataset = weave.ref(
    "weave:///c-metrics/rouge-scorer/object/longbench_gov_report_subset:qGNjItwJSEw1NF6xMXX2a0syHJfXVMjeYqwqVwWsdbs"
).get()



class GPT4oMini(weave.Model):
    model_name: str = "gpt-4o-mini"
    temp: float = 0.0
    max_tokens: int = 2048
    top_p: float = 1.0

    @weave.op()
    async def predict(self, query: str) -> str:
        response = await acompletion(
            model=self.model_name,
            messages=[
                {
                    "role": "system",
                    "content": "You are provided with government reports. Summarize the report in a few sentences but make sure to include all the important information."
                },
                {
                    "role": "user",
                    "content": query
                }
            ],
            temperature=self.temp,
            max_tokens=self.max_tokens,
            top_p=self.top_p
        )
        return response.choices[0].message.content

class GPT4o(weave.Model):
    model_name: str = "gpt-4o-2024-08-06"
    temp: float = 0.0
    max_tokens: int = 2048
    top_p: float = 1.0

    @weave.op()
    async def predict(self, query: str) -> str:
        time.sleep(2)
        response = await acompletion(
            model=self.model_name,
            messages=[
                {
                    "role": "system",
                    "content": "You are provided with government reports. Summarize the report in a few sentences but make sure to include all the important information."
                },
                {
                    "role": "user",
                    "content": query
                }
            ],
            temperature=self.temp,
            max_tokens=self.max_tokens,
            top_p=self.top_p
        )
        return response.choices[0].message.content

gpt4o = GPT4o()
gpt4omini = GPT4oMini()

nest_asyncio.apply()

# Initialize BLEU Scorer with column map
scorer = BLEUScorer(column_map={"output": "query", "ground_truths": "ground_truth"})

# Create the evaluation
evaluation = Evaluation(
    dataset=dataset.rows[:30],
    scorers=[scorer],
)

# Evaluate the models
asyncio.run(evaluation.evaluate(gpt4o))
asyncio.run(evaluation.evaluate(gpt4omini))