Amazon Bedrock에서 LLM 평가하기

aws configure

pip install boto3 botocore weave wandb

import os
import arxiv
import fitz  # PyMuPDF
import json
import boto3
from botocore.exceptions import ClientError
import re
import random
import time
from time import sleep

# Directory to save downloaded papers
download_dir = "arxiv_papers"
os.makedirs(download_dir, exist_ok=True)

# Set up Amazon Bedrock client
bedrock_client = boto3.client("bedrock-runtime", region_name="us-east-1")
MODEL_ID = "anthropic.claude-3-sonnet-20240229-v1:0"

# Fixed questions for paper analysis
FIXED_QUESTIONS = """
What is the primary objective of this research?
What methodologies or algorithms are proposed or evaluated?
What datasets or experimental setups are used in this study?
What are the key findings and contributions of this research?
What are the implications of these findings for the broader field of AI?
What limitations or challenges are acknowledged by the authors?
What are the proposed future directions or next steps in this research?
"""

# Define AI-specific search queries
search_queries = [
    "Large Language Models for vision tasks AND cat:cs.AI",
    "Multimodal AI techniques AND cat:cs.CV",
    "Applications of Transformers in healthcare AI AND cat:cs.LG",
    "Few-shot learning in AI and ML AND cat:cs.LG",
    "Vision and language models integration AND cat:cs.CV",
    "Domain-specific fine-tuning for ML models AND cat:cs.LG",
    "Foundational models in AI and CV applications AND cat:cs.AI",
    "NLP in robotics and vision systems AND cat:cs.AI",
    "Bias and fairness in AI for CV AND cat:cs.CV",
    "Evaluation metrics for multimodal AI AND cat:cs.LG"
]

def download_papers(max_pages=15, max_attempts_per_query=20):
    """Download one suitable paper for each query, retrying if papers exceed page limit."""
    papers = []
    downloaded_titles = set()
    client = arxiv.Client()

    for query in search_queries:
        paper_found = False
        attempt = 0
        
        while not paper_found and attempt < max_attempts_per_query:
            search = arxiv.Search(
                query=query,
                max_results=100,
                sort_by=arxiv.SortCriterion.SubmittedDate
            )
            
            try:
                results = list(client.results(search))
                start_idx = attempt * 5
                end_idx = start_idx + 5
                current_batch = results[start_idx:end_idx]
                
                for result in current_batch:
                    if result.title not in downloaded_titles:
                        print(f"Downloading: {result.title}")
                        paper_id = result.entry_id.split('/')[-1]
                        pdf_filename = f"{paper_id}.pdf"
                        pdf_path = os.path.join(download_dir, pdf_filename)
                        
                        result.download_pdf(dirpath=download_dir, filename=pdf_filename)
                        
                        try:
                            with fitz.open(pdf_path) as pdf:
                                if pdf.page_count <= max_pages:
                                    papers.append({
                                        "title": result.title,
                                        "file_path": pdf_path,
                                        "arxiv_id": paper_id
                                    })
                                    downloaded_titles.add(result.title)
                                    print(f"Accepted: {result.title}")
                                    paper_found = True
                                    break
                                else:
                                    os.remove(pdf_path)
                                    print(f"Skipped (too many pages: {pdf.page_count}): {result.title}")
                        except Exception as e:
                            print(f"Error checking PDF {pdf_path}: {e}")
                            if os.path.exists(pdf_path):
                                os.remove(pdf_path)
            
                attempt += 1
                if not paper_found:
                    print(f"Attempt {attempt}/{max_attempts_per_query} for query: {query}")
                    sleep(3)
                    
            except Exception as e:
                print(f"Error during download: {e}")
                sleep(3)
                attempt += 1
                continue
        
        if not paper_found:
            print(f"Failed to find suitable paper for query after {max_attempts_per_query} attempts: {query}")

    print(f"\nSuccessfully downloaded {len(papers)} papers")
    return papers

def extract_text(pdf_path):
    """Extract text from the entire PDF."""
    with fitz.open(pdf_path) as pdf:
        text = ""
        for page in pdf:
            text += page.get_text()
        return text

def generate_summary_with_claude(text, title):
    """Generate a 300-word summary using Claude via Amazon Bedrock with exponential backoff."""
    prompt = (
        f"Please analyze the following research paper titled '{title}' and provide a comprehensive 300-word summary. "
        f"Consider these key aspects when analyzing the paper:\n\n{FIXED_QUESTIONS}\n\n"
        f"Based on these questions, synthesize a coherent summary that captures the essential elements "
        f"of the research while maintaining a natural flow. Ensure the summary is 300 words.\n\n"
        f"Paper content:\n\n{text}"
    )
    
    request = {
        "anthropic_version": "bedrock-2023-05-31",
        "max_tokens": 4096,
        "temperature": 0.0,
        "messages": [
            {
                "role": "user",
                "content": [{"type": "text", "text": prompt}]
            }
        ]
    }
    
    max_retries = 15
    backoff_time = 10  # Start with a 10-second delay
    
    for attempt in range(max_retries):
        try:
            response = bedrock_client.invoke_model(
                modelId=MODEL_ID,
                body=json.dumps(request)
            )
            response_body = json.loads(response["body"].read())
            summary = response_body["content"][0]["text"]
            return {"summary": summary}
        except ClientError as e:
            if e.response['Error']['Code'] == 'ThrottlingException':
                print(f"ThrottlingException encountered. Retrying in {backoff_time} seconds...")
                time.sleep(backoff_time + random.uniform(0, 1))  # Add jitter
                backoff_time *= 2  # Exponential backoff
            else:
                print(f"Error generating summary for {title}: {e}")
                break
    print(f"Failed to generate summary for {title} after {max_retries} retries.")
    return {"summary": ""}

def count_words(text):
    """Count words excluding punctuation and special characters."""
    cleaned_text = re.sub(r'[^\w\s]', ' ', text.lower())
    words = [word for word in cleaned_text.split() if word.strip()]
    return len(words)

def main():
    # Download papers
    papers = download_papers()
    print(f"\nDownloaded {len(papers)} papers. Generating summaries...\n")
    
    # Process papers and generate summaries
    paper_data = []
    for paper in papers:
        title = paper["title"]
        pdf_path = paper["file_path"]
        
        print(f"Processing: {title}")
        paper_text = extract_text(pdf_path)
        summary_json = generate_summary_with_claude(paper_text, title)
        summary_text = summary_json.get('summary', '')
        word_count = count_words(summary_text)
        
        paper_data.append({
            "title": title,
            "file_path": pdf_path,
            "summary": summary_text,
            "word_count": word_count,
            "arxiv_id": paper["arxiv_id"]
        })
        
        sleep(5)  # Wait before processing the next paper to avoid throttling

    # Save to JSONL file
    output_file = "paper_summaries.jsonl"
    with open(output_file, "w") as f:
        for entry in paper_data:
            json.dump(entry, f)
            f.write("\n")

    print(f"\nProcessed {len(paper_data)} papers. Results saved to {output_file}")

if __name__ == "__main__":
    main()


import weave
from weave import Model
import json
import boto3
from botocore.exceptions import ClientError
from time import sleep
import asyncio
from rouge_score.rouge_scorer import RougeScorer
from typing import Dict, Any
import bert_score
import fitz
import os
from weave.trace.box import unbox
import time

# Initialize Weave
weave.init('bedrock_abstract_eval')
client = boto3.client("bedrock-runtime", region_name="us-east-1")

def extract_paper_text(pdf_path: str) -> str:
    """Extract text from PDF paper."""
    if isinstance(pdf_path, weave.trace.box.BoxedStr):
        pdf_path = unbox(pdf_path)

    text = ""
    try:
        with fitz.open(unbox(pdf_path)) as pdf:
            for page in pdf:
                text += page.get_text()
    except Exception as e:
        print(f"Error extracting text from PDF {pdf_path}: {e}")
    return text

def format_prompt(text: str, title: str) -> str:
    """Format prompt for model."""
    return f"""
    <|begin_of_text|><|start_header_id|>user<|end_header_id|>
    Please analyze this research paper and provide a comprehensive 300-word summary that covers:
    - Primary research objective
    - Methodology and approach
    - Key findings and results
    - Main contributions to the field

    Title: {title}

    Content: {text}

    Please analyze this research paper and provide a comprehensive 300-word summary that covers:
    - Primary research objective
    - Methodology and approach
    - Key findings and results
    - Main contributions to the field

    Generate a clear, coherent summary that captures the essence of the research.
    <|eot_id|>
    <|start_header_id|>assistant<|end_header_id|>
    """

def model_forward(model_id: str, title: str, pdf_path: str) -> str:
    """Core prediction logic to be called by predict methods."""
    max_retries = 15
    backoff_time = 10  # Start with 10 seconds delay

    for attempt in range(max_retries):
        try:
            # Extract text from paper
            paper_text = extract_paper_text(pdf_path)

            # Prepare request
            request = {
                "prompt": format_prompt(paper_text, title),
                "max_gen_len": 4096,
                "temperature": 0.0,
            }
            print(f"Invoking model (Attempt {attempt + 1}/{max_retries})...")
            
            # Make prediction
            response = client.invoke_model(
                modelId=model_id,
                body=json.dumps(request)
            )
            print("Done invoking")

            # Extract and clean prediction
            response_body = json.loads(response["body"].read())
            prediction = response_body["generation"].strip()
            
            return prediction

        except ClientError as e:
            if e.response['Error']['Code'] == 'ThrottlingException':
                print(f"ThrottlingException encountered. Retrying in {backoff_time} seconds...")
                time.sleep(backoff_time)
                backoff_time *= 2  # Exponential backoff
            else:
                print(f"Error generating prediction with {model_id}: {e}")
                break
        except Exception as e:
            print(f"Unexpected error: {e}")
            break

    print(f"Failed to generate prediction after {max_retries} retries.")
    return ""

class Llama8B(Model):
    @weave.op
    def predict(self, title: str, pdf_path: str) -> dict:
        prediction = model_forward("us.meta.llama3-1-8b-instruct-v1:0", title, pdf_path)
        return {"model_output": prediction}

class Llama11B(Model):
    """Llama 11B model."""
    @weave.op
    def predict(self, title: str, pdf_path: str) -> dict:
        """Generate prediction using Llama 11B."""
        prediction = model_forward("us.meta.llama3-2-11b-instruct-v1:0", title, pdf_path)
        return {"model_output": prediction}

class Llama1B(Model):
    """Llama 1B model."""    
    @weave.op
    def predict(self, title: str, pdf_path: str) -> dict:
        """Generate prediction using Llama 1B."""
        prediction = model_forward("us.meta.llama3-2-1b-instruct-v1:0", title, pdf_path)
        return {"model_output": prediction}

@weave.op
def bert_scorer(gt_abstract: str, model_output: dict) -> Dict[str, float]:
    """Calculate BERTScore for the abstract."""
    if not model_output or 'model_output' not in model_output:
        return {'bert_score': 0.0}
    
    try:
        P, R, F1 = bert_score.score(
            [model_output['model_output']],
            [gt_abstract],
            lang='en',
            model_type='microsoft/deberta-xlarge-mnli'
        )
        return {'bert_score': float(F1.mean())}
    except Exception as e:
        print(f"Error calculating BERTScore: {e}")
        return {'bert_score': 0.0}

@weave.op
def claude_scorer(gt_abstract: str, model_output: dict) -> dict:
    """Evaluate abstract using Claude."""
    if not model_output or 'model_output' not in model_output:
        return {'claude_score': 0.0}
    print("claude evaluating")
    # client = boto3.client("bedrock-runtime", region_name="us-east-1")
    
    prompt = json.dumps({
        "anthropic_version": "bedrock-2023-05-31",
        "max_tokens": 1024,
        "temperature": 0.0,
        "messages": [{
            "role": "user",
            "content": [
                {"type": "text", "text": f'''Rate how well this generated abstract captures the key information from the ground truth abstract on a scale from 1-5, where:
1: Poor - Missing most key information or seriously misrepresenting the research
2: Fair - Captures some information but misses crucial elements
3: Good - Captures most key points but has some gaps or inaccuracies
4: Very Good - Accurately captures nearly all key information with minor omissions
5: Excellent - Perfectly captures all key information and maintains accuracy

Ground Truth Abstract:
{gt_abstract}

Generated Abstract:
{model_output["model_output"]}

Provide your rating as a JSON object with this schema:
{{"score": <integer 1-5>}}'''}
            ]
        }]
    })
    
    try:
        response = client.invoke_model(
            modelId="anthropic.claude-3-sonnet-20240229-v1:0",
            body=prompt
        )
        
        result = json.loads(response["body"].read())
        score = json.loads(result["content"][0]["text"])["score"]
        print(score)
        sleep(2)  # Rate limiting
        return {'claude_score': float(score)}
        
    except Exception as e:
        print(f"Error in Claude evaluation: {e}")
        return {'claude_score': 0.0}

@weave.op
def rouge_scorer(gt_abstract: str, model_output: dict) -> Dict[str, float]:
    """Calculate ROUGE scores for the abstract."""
    if not model_output or 'model_output' not in model_output:
        return {
            'rouge1_f': 0.0,
            'rouge2_f': 0.0,
            'rougeL_f': 0.0
        }
    
    try:
        scorer = RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)
        scores = scorer.score(gt_abstract, model_output['model_output'])
        
        return {
            'rouge1_f': float(scores['rouge1'].fmeasure),
            'rouge2_f': float(scores['rouge2'].fmeasure),
            'rougeL_f': float(scores['rougeL'].fmeasure)
        }
    except Exception as e:
        print(f"Error calculating ROUGE scores: {e}")
        return {
            'rouge1_f': 0.0,
            'rouge2_f': 0.0,
            'rougeL_f': 0.0
        }

@weave.op
def compression_scorer(gt_abstract: str, model_output: dict) -> Dict[str, float]:
    """Calculate compression ratio of the abstract."""
    if not model_output or 'model_output' not in model_output:
        return {'compression_ratio': 0.0}
    
    try:
        gt_words = len(gt_abstract.split())
        generated_words = len(model_output['model_output'].split())
        
        compression_ratio = min(gt_words, generated_words) / max(gt_words, generated_words)
        
        return {'compression_ratio': float(compression_ratio)}
    except Exception as e:
        print(f"Error calculating compression ratio: {e}")
        return {'compression_ratio': 0.0}

@weave.op
def coverage_scorer(gt_abstract: str, model_output: dict) -> Dict[str, float]:
    """Calculate content coverage using word overlap."""
    if not model_output or 'model_output' not in model_output:
        return {'coverage_score': 0.0}
    
    try:
        gt_words = set(gt_abstract.lower().split())
        generated_words = set(model_output['model_output'].lower().split())
        
        intersection = len(gt_words.intersection(generated_words))
        union = len(gt_words.union(generated_words))
        
        coverage_score = intersection / union if union > 0 else 0.0
        
        return {'coverage_score': float(coverage_score)}
    except Exception as e:
        print(f"Error calculating coverage score: {e}")
        return {'coverage_score': 0.0}

def create_evaluation_dataset(gt_file: str):
    """Create dataset from ground truth file."""
    dataset = []
    with open(gt_file, 'r') as f:
        for line in f:
            entry = json.loads(line)
            dataset.append({
                "title": entry["title"],
                "gt_abstract": entry["summary"],
                "pdf_path": entry["file_path"]
            })
    return dataset

async def run_evaluations(gt_file: str):
    """Run evaluations for each model."""
    eval_dataset = create_evaluation_dataset(gt_file)
    
    # Initialize models
    models = {
        "llama_8b": Llama8B(),
        "llama_11b": Llama11B(),
        "llama_1b": Llama1B()
    }

    # Setup scorers
    scorers = [
        claude_scorer,
        rouge_scorer,
        compression_scorer,
        coverage_scorer,
        bert_scorer
    ]
    
    # Run evaluations
    results = {}
    for model_name, model in models.items():
        print(f"\nEvaluating {model_name}...")
        evaluation = weave.Evaluation(
            dataset=eval_dataset,
            scorers=scorers,
            name=model_name + " Eval"
        )
        results[model_name] = await evaluation.evaluate(model)
    
    # Print results
    print("\nEvaluation Results:")
    for model_name, result in results.items():
        print(f"\n{model_name} Results:")
        print(json.dumps(result, indent=2))
    
    # Save results to file
    output_file = "llama_evaluation_results.json"
    with open(output_file, 'w') as f:
        json.dump(results, f, indent=2)
    print(f"\nResults saved to {output_file}")
    
    return results

if __name__ == "__main__":
    gt_file = "paper_summaries.jsonl"
    asyncio.run(run_evaluations(gt_file))

import weave
from weave import Model
import json
import boto3
from botocore.exceptions import ClientError
from time import sleep
import asyncio
from rouge_score.rouge_scorer import RougeScorer
from typing import Dict, Any
import bert_score
import fitz
import os
from weave.trace.box import unbox
import time
import logging
logging.basicConfig(level=logging.DEBUG)
# Initialize Weave
weave.init('bedrock_abstract_eval')

client = boto3.client(service_name="bedrock-runtime", region_name="us-east-1")

def extract_paper_text(pdf_path: str) -> str:
    """Extract text from PDF paper."""
    if isinstance(pdf_path, weave.trace.box.BoxedStr):
        pdf_path = unbox(pdf_path)

    text = ""
    try:
        with fitz.open(unbox(pdf_path)) as pdf:
            for page in pdf:
                text += page.get_text()
    except Exception as e:
        print(f"Error extracting text from PDF {pdf_path}: {e}")
    return text

def format_prompt(text: str, title: str) -> str:
    """Format prompt for model."""
    return f"""
    <|begin_of_text|><|start_header_id|>user<|end_header_id|>
    Please analyze this research paper and provide a comprehensive 300-word summary that covers:
    - Primary research objective
    - Methodology and approach
    - Key findings and results
    - Main contributions to the field

    Title: {title}

    Content: {text}

    Please analyze this research paper and provide a comprehensive 300-word summary that covers:
    - Primary research objective
    - Methodology and approach
    - Key findings and results
    - Main contributions to the field

    Generate a clear, coherent summary that captures the essence of the research.
    <|eot_id|>
    <|start_header_id|>assistant<|end_header_id|>
    """


def model_forward_nova(title: str, pdf_path: str) -> str:
    """Core prediction logic for Amazon Nova Pro."""
    max_retries = 15
    backoff_time = 10  # Start with 10 seconds delay

    for attempt in range(max_retries):
        try:
            paper_text = extract_paper_text(pdf_path)
            messages = [
                {"role": "user", "content": [{"text": format_prompt(paper_text, title)}]},
            ]

            # Make prediction
            print(f"Invoking Nova Pro model (Attempt {attempt + 1}/{max_retries})...")
            response = client.converse(
                modelId="us.amazon.nova-pro-v1:0",
                messages=messages
            )
            prediction = response["output"]["message"]["content"][0]["text"].strip()
            print("Done invoking")
            return prediction

        except Exception as e:
            print(f"Error generating prediction with Nova Pro (Attempt {attempt + 1}/{max_retries}): {e}")
            if attempt < max_retries - 1:  # Avoid sleeping on the last attempt
                time.sleep(backoff_time)
                backoff_time *= 2  # Exponential backoff

    print(f"Failed to generate prediction after {max_retries} retries.")
    return ""

class NovaPro(Model):
    """Amazon Nova Pro model."""
    @weave.op
    def predict(self, title: str, pdf_path: str) -> dict:
        """Generate prediction using Amazon Nova Pro."""
        prediction = model_forward_nova(title, pdf_path)
        return {"model_output": prediction}

@weave.op
def bert_scorer(gt_abstract: str, model_output: dict) -> Dict[str, float]:
    """Calculate BERTScore for the abstract."""
    if not model_output or 'model_output' not in model_output:
        return {'bert_score': 0.0}
    
    try:
        P, R, F1 = bert_score.score(
            [model_output['model_output']],
            [gt_abstract],
            lang='en',
            model_type='microsoft/deberta-xlarge-mnli'
        )
        return {'bert_score': float(F1.mean())}
    except Exception as e:
        print(f"Error calculating BERTScore: {e}")
        return {'bert_score': 0.0}

@weave.op
def rouge_scorer(gt_abstract: str, model_output: dict) -> Dict[str, float]:
    """Calculate ROUGE scores for the abstract."""
    if not model_output or 'model_output' not in model_output:
        return {
            'rouge1_f': 0.0,
            'rouge2_f': 0.0,
            'rougeL_f': 0.0
        }
    
    try:
        scorer = RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)
        scores = scorer.score(gt_abstract, model_output['model_output'])
        
        return {
            'rouge1_f': float(scores['rouge1'].fmeasure),
            'rouge2_f': float(scores['rouge2'].fmeasure),
            'rougeL_f': float(scores['rougeL'].fmeasure)
        }
    except Exception as e:
        print(f"Error calculating ROUGE scores: {e}")
        return {
            'rouge1_f': 0.0,
            'rouge2_f': 0.0,
            'rougeL_f': 0.0
        }



@weave.op
def compression_scorer(gt_abstract: str, model_output: dict) -> Dict[str, float]:
    """Calculate compression ratio of the abstract."""
    if not model_output or 'model_output' not in model_output:
        return {'compression_ratio': 0.0}
    
    try:
        gt_words = len(gt_abstract.split())
        generated_words = len(model_output['model_output'].split())
        
        compression_ratio = min(gt_words, generated_words) / max(gt_words, generated_words)
        
        return {'compression_ratio': float(compression_ratio)}
    except Exception as e:
        print(f"Error calculating compression ratio: {e}")
        return {'compression_ratio': 0.0}

@weave.op
def coverage_scorer(gt_abstract: str, model_output: dict) -> Dict[str, float]:
    """Calculate content coverage using word overlap."""
    if not model_output or 'model_output' not in model_output:
        return {'coverage_score': 0.0}
    
    try:
        gt_words = set(gt_abstract.lower().split())
        generated_words = set(model_output['model_output'].lower().split())
        
        intersection = len(gt_words.intersection(generated_words))
        union = len(gt_words.union(generated_words))
        
        coverage_score = intersection / union if union > 0 else 0.0
        
        return {'coverage_score': float(coverage_score)}
    except Exception as e:
        print(f"Error calculating coverage score: {e}")
        return {'coverage_score': 0.0}


@weave.op
def claude_scorer(gt_abstract: str, model_output: dict) -> dict:
    """Evaluate abstract using Claude."""
    if not model_output or 'model_output' not in model_output:
        return {'claude_score': 0.0}
    print("claude evaluating")
    # client = boto3.client("bedrock-runtime", region_name="us-east-1")
    
    prompt = json.dumps({
        "anthropic_version": "bedrock-2023-05-31",
        "max_tokens": 1024,
        "temperature": 0.0,
        "messages": [{
            "role": "user",
            "content": [
                {"type": "text", "text": f'''Rate how well this generated abstract captures the key information from the ground truth abstract on a scale from 1-5, where:
1: Poor - Missing most key information or seriously misrepresenting the research
2: Fair - Captures some information but misses crucial elements
3: Good - Captures most key points but has some gaps or inaccuracies
4: Very Good - Accurately captures nearly all key information with minor omissions
5: Excellent - Perfectly captures all key information and maintains accuracy

Ground Truth Abstract:
{gt_abstract}

Generated Abstract:
{model_output["model_output"]}

Provide your rating as a JSON object with this schema:
{{"score": <integer 1-5>}}'''}
            ]
        }]
    })
    
    try:
        response = client.invoke_model(
            modelId="anthropic.claude-3-sonnet-20240229-v1:0",
            body=prompt
        )
        
        result = json.loads(response["body"].read())
        score = json.loads(result["content"][0]["text"])["score"]
        print(score)
        sleep(2)  # Rate limiting
        return {'claude_score': float(score)}
        
    except Exception as e:
        print(f"Error in Claude evaluation: {e}")
        return {'claude_score': 0.0}


def create_evaluation_dataset(gt_file: str):
    """Create dataset from ground truth file."""
    dataset = []
    with open(gt_file, 'r') as f:
        for line in f:
            entry = json.loads(line)
            dataset.append({
                "title": entry["title"],
                "gt_abstract": entry["summary"],
                "pdf_path": entry["file_path"]
            })
    return dataset



async def run_evaluations(gt_file: str):
    """Run evaluations for each model."""
    eval_dataset = create_evaluation_dataset(gt_file)
    
    # Initialize models
    models = {
        "nova_pro": NovaPro(),
    }

    # Setup scorers
    scorers = [
        claude_scorer,
        rouge_scorer,
        compression_scorer,
        coverage_scorer,
        bert_scorer
    ]
    # Run evaluations
    results = {}
    for model_name, model in models.items():
        print(f"\nEvaluating {model_name}...")
        evaluation = weave.Evaluation(
            dataset=eval_dataset,
            scorers=scorers,
            name=model_name + " Eval"
        )
        results[model_name] = await evaluation.evaluate(model)
    
    # Print results
    print("\nEvaluation Results:")
    for model_name, result in results.items():
        print(f"\n{model_name} Results:")
        print(json.dumps(result, indent=2))
    
    # Save results to file
    output_file = "llama_vs_nova_evaluation_results.json"
    with open(output_file, 'w') as f:
        json.dump(results, f, indent=2)
    print(f"\nResults saved to {output_file}")
    
    return results

if __name__ == "__main__":
    gt_file = "paper_summaries.jsonl"
    asyncio.run(run_evaluations(gt_file))