Chroma, OpenAI, Weights & Biases로 질문 응답 시스템 강화

!pip install openai pandas wandb

import json
import pandas as pd
Import requests
import openai
import wandb
from sklearn.model_selection import train_test_split

data_path = "/content/COVID-QA.json"
with open(data_path, "r") as f:
    data = json.load(f)

questions = []
answers = []
contexts = []

for entry in data['data']:
    for paragraph in entry['paragraphs']:
        context = paragraph['context']
        for qa in paragraph['qas']:
            questions.append(qa['question'])
            answers.append(qa['answers'][0]['text'])
            contexts.append(context)

df = pd.DataFrame({
    'question': questions,
    'answer': answers,
    'context': contexts
})

import chromadb
from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction

openai.api_key = 'your_openai_api_key'

embedding_function = OpenAIEmbeddingFunction(api_key=openai.api_key)

chroma_client = chromadb.Client() # Ephemeral by default
scifact_corpus_collection = chroma_client.create_collection(name='scifact_corpus', embedding_function=embedding_function)

batch_size = 100

for i in range(0, len(df), batch_size):
    batch_df = df[i:i+batch_size]
    scifact_corpus_collection.add(
        ids=batch_df['doc_id'].apply(lambda x: str(x)).tolist(), # Chroma takes string IDs.
        documents=(batch_df['title'] + '. ' + batch_df['abstract'].apply(lambda x: ' '.join(x))).to_list(), # We concatenate the title and abstract.
        metadatas=[{"structured": structured} for structured in batch_df['structured'].to_list()] # We also store the metadata, though we don't use it in this example.
    )

from openai import OpenAI

def get_improved_answer(question, context, api_key):
    client = OpenAI(api_key=api_key)      
    response = client.completions.create(
        model="gpt-3.5-turbo-instruct",
        prompt=f"Question: {question}\nContext: {context}\nAnswer:",
        max_tokens=150
    )
    return response.choices[0].text.strip()

df['generated_answer'] = df.apply(lambda row: get_answer(row['augmented_question'], row['augmented_context']), axis=1)

import wandb
wandb.init(project='enhanced_qa_with_openai_chroma')

old_answers = df['answer'].head(5).tolist()
old_questions = df['question'].head(5).tolist()

old_answers_table = wandb.Table(data=[old_questions, old_answers], columns=["Question", "Old Answer"])
wandb.log({"Old Answers Table": old_answers_table})

improved_answers = []

for index, row in df.head(5).iterrows():
    improved_answer = get_improved_answer(row['question'], row['context'])
    improved_answers.append(improved_answer)

improved_answers_table = wandb.Table(data=[old_questions, improved_answers], columns=["Question", "Improved Answer"])
wandb.log({"Improved Answers Table": improved_answers_table})

improvement_description = "The table above compares the answers generated by the model before fine-tuning and after fine-tuning. After fine-tuning, the model provides more accurate and relevant answers, showcasing the effectiveness of the fine-tuning process in enhancing the model's performance."
wandb.log({"Improvement Description": improvement_description})