GPT-3 미세 조정으로 챗봇 대화를 한 단계 끌어올리기

!pip install wandb
!pip install openai 

import json


def convert_dataset_to_jsonl(input_json_path, output_jsonl_path):
    with open(input_json_path, 'r', encoding='utf-8') as f:
        data = json.load(f)


    with open(output_jsonl_path, 'w', encoding='utf-8') as outfile:
        for article in data['data']:
            for paragraph in article['paragraphs']:
                context = paragraph['context']
                for qa in paragraph['qas']:
                    question = qa['question']
                    is_impossible = qa.get('is_impossible', False)
                    prompt = f"Context: {context}\nQuestion: {question}\nAnswer:"
                
                    if is_impossible:
                        completion = " Impossible"
                    else:
                        
                        answer = qa['answers'][0]['text'] if qa['answers'] else "Unknown"
                        completion = f" {answer}"
               
                    jsonl_entry = json.dumps({"prompt": prompt, "completion": completion})
                    outfile.write(jsonl_entry + '\n')


convert_dataset_to_jsonl('/train-v2.0.json', '/train.jsonl')
convert_dataset_to_jsonl('/dev-v2.0.json', '/dev.jsonl')

import openai
import wandb
import os


# Set OpenAI API Key
openai.api_key = 'your api key'
client = OpenAI(api_key)

wandb.login()
wandb.init(project='project_name', entity='entity_name')

def upload_file_to_openai(file_path, purpose='fine-tune'):
    response = openai.File.create(file=open(file_path), purpose=purpose)
    return response.id


train_file_id = upload_file_to_openai("/train.jsonl")
dev_file_id = upload_file_to_openai("/dev.jsonl")
print(train_file_id)

# Define hyperparameters
hyperparameters = {
    "n_epochs": 2,  # Number of training epochs
    "batch_size": 4,  # Batch size for training
    "learning_rate_multiplier": 0.1,  # Learning rate adjustment factor
}


# Log hyperparameters to wandb
wandb.config.update(hyperparameters)

openai.FineTuningJob.retrieve(fine_tune_id)
fine_tune_response = openai.FineTuningJob.create(
    training_file=train_file_id,
    validation_file=dev_file_id,
    model="babbage-002",
    hyperparameters=hyperparameters
)


print(f"Fine-tuning started with ID: {fine_tune_response['id']}")
wandb.log({"fine_tune_id": fine_tune_response["id"]})


fine_tune_id= fine_tune_response['id']


fine_tune_status = openai.FineTuningJob.retrieve(fine_tune_id)


print(f"Fine-tuning job status: {fine_tune_status['status']}")

import signal
import datetime


fine_tune_id= fine_tune_response['id']
def signal_handler(sig, frame):
    status = openai.FineTuningJob.retrieve(fine_tune_id)['status']  # Access status correctly
    print(f"Stream interrupted. Job is still {status}.")
    return
print(f"Streaming events for the fine-tuning job: {fine_tune_id}")
signal.signal(signal.SIGINT, signal_handler)
try:
    events_response = openai.FineTuningJob.list_events(id=fine_tune_id)
    events = events_response['data']  # Access the list of events
   
    for event in events:
        event_time = datetime.datetime.fromtimestamp(event['created_at']).strftime('%Y-%m-%d %H:%M:%S')
        print(f"{event_time} {event['message']}")
except Exception as e:
    print(f"Stream interrupted (client disconnected). Error: {str(e)}")

wandb.login()
wandb.init(project='project_name', entity='entity_name')

# Test dataset
test_data = [
    {
        "context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.",
        "qas": [
            {
                "question": "In what country is Normandy located?",
                "answer": "France"
            },
            {
                "question": "When were the Normans in Normandy?",
                "answer": "10th and 11th centuries"
            },
            {
                "question": "From which countries did the Norse originate?",
                "answer": "Denmark, Iceland and Norway"
            },
            {
                "question": "Who was the Norse leader?",
                "answer": "Rollo"
            },
            {
                "question": "What century did the Normans first gain their separate identity?",
                "answer": "10th century"
            }
        ]
    },
    {
        "context": "The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.",
        "qas": [
            {
                "question": "Who was the duke in the battle of Hastings?",
                "answer": "William the Conqueror"
            },
            {
                "question": "Who ruled the duchy of Normandy",
                "answer": "Richard I"
            }
        ]
    }
]

# Function to normalize answers (removing punctuation, lowercase, etc.)
def normalize_answer(s):
    import re
    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)
    def white_space_fix(text):
        return ' '.join(text.split())
    def remove_punct(text):
        return re.sub(r'[\W]', ' ', text)
    def lower(text):
        return text.lower()
    return white_space_fix(remove_articles(remove_punct(lower(s))))

# Calculate F1 score
def f1_score(prediction, truth):
    prediction_tokens = normalize_answer(prediction).split()
    truth_tokens = normalize_answer(truth).split()
    common_tokens = Counter(prediction_tokens) & Counter(truth_tokens)
    num_same = sum(common_tokens.values())
    if num_same == 0: return 0
    precision = 1.0 * num_same / len(prediction_tokens)
    recall = 1.0 * num_same / len(truth_tokens)
    f1 = (2 * precision * recall) / (precision + recall)
    return f1

# Calculate Exact Match score
def exact_match_score(prediction, truth):
    return int(normalize_answer(prediction) == normalize_answer(truth))

# Function to query the model and get the answer
def query_model(question, context, model):
    openai.api_key = '<key-here>'
    response = openai.Completion.create(
        model=model,
        prompt=f"Question: {question}\nContext: {context}\nAnswer:",
        temperature=0,
        max_tokens=50,
        top_p=1.0,
        frequency_penalty=0.0,
        presence_penalty=0.0,
        stop=["\n"]
    )
    return response.choices[0].text.strip()

# Adjusted part of the script to compile results into a DataFrame
detailed_results = []  # List to store detailed results


for item in test_data:
    context = item['context']
    for qa in item['qas']:
        question = qa['question']
        true_answer = qa['answer']
        model_answer = query_model(question, context, model="modelid")
       
        # Append detailed results for each question
        detailed_results.append({
            "question": question,
            "model_answer": model_answer,
            "true_answer": true_answer        })


# Convert detailed results list to DataFrame
df_results = pd.DataFrame(detailed_results)




# Log the entire DataFrame as a table to W&B
wandb.log({"results_table": wandb.Table(dataframe=df_results)})


# Optional: Save the DataFrame to CSV for local use
df_results.to_csv('evaluation_results.csv', index=False)