Fine-Tuning ChatGPT for Sentiment Analysis With W&B

{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]}

{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]}

{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]}

!pip install openai
!pip install wandb
!pip install git+https://github.com/wandb/wandb.git@openai-finetuning

import os
import openai
import wandb
import pandas as pd
import json
from openai import OpenAI
from wandb.integration.openai import WandbLogger

openai.api_key = "Insert your OpenAI API key here”

filename = "Insert the path to your data set here"

# Read the CSV
df = pd.read_csv(filename)

# Drop rows with NaN values in 'clean_comment' and 'category'
df.dropna(subset=['clean_comment', 'category'], inplace=True)

# Sample 100 rows from the dataset
df = df.sample(100)

WandbLogger.sync(project="chatgpt3.5-fine-tuning-for-Sentiment-Analysis")

df = df.sample(100) 

def convert_response_to_sentiment(response):
    response = response.lower()
    if 'positive' in response:
        return 1
    elif 'negative' in response:
        return -1
    elif 'neutral' in response:
        return 0
    else:
        return -1  # Unknown sentiment
    
def convert_numeric_to_string_sentiment(value):
    if value == 1:
        return "positive"
    elif value == -1:
        return "negative"
    elif value == 0:
        return "neutral"
    else:
        return "unknown"

client = openai.Client()

correct_predictions = 0
loop_count = 0  # Counter for loop iterations

results = []

# Iterate over each row in the DataFrame
for index, row in df.iterrows():
    loop_count += 1  # Increment the loop count
    text = row['clean_comment']  # Adjusted column name
    
    try:
        completion = client.chat.completions.create(
            model="gpt-3.5-turbo",
            messages=[
                {"role": "system", "content": "What is the sentiment of the following text? Please respond with 'positive', 'negative', or 'neutral'."},
                {"role": "user", "content": text},
            ]
        )
        response = completion.choices[0].message.content
        predicted_sentiment = convert_response_to_sentiment(response)
        
        results.append({
        "sentiment": text,  
        "labeled_prediction": convert_numeric_to_string_sentiment(row['category']),
        "old_model_prediction": response
    })
        
        # Check if the predicted sentiment matches the actual sentiment
        if predicted_sentiment == row['category']:  # Adjusted column name
            correct_predictions += 1
        
        # Print the current progress using loop_count
        total_rows = len(df)
        print(f"Processed {loop_count}/{total_rows} rows.")
        
    except Exception as e:
        print(f"Error on index {index}: {e}")
        continue

accuracy = (correct_predictions / total_rows) * 100

wandb.log({"Old Accuracy": accuracy})
print(f'Model Accuracy before: {accuracy:.2f}%')

output_filename = "insert the directory in which to save the newly processed data set "

# Convert DataFrame to the desired JSONL format
with open(output_filename, "w") as file:
    for _, row in df.iterrows():
        # Map the target to its corresponding string label
        target_label = {
            0: 'neutral',
            1: 'positive',  # Corrected the spelling here
            -1: 'negative'
        }.get(row['category'], 'unknown')
        
        data = {
            "messages": [
                {
                    "role": "system",
                    "content": "What is the sentiment of the following text? Please respond with 'positive', 'negative', or 'neutral'."
                },
                {
                    "role": "user",
                    "content": row['clean_comment']
                },
                {
                    "role": "assistant",
                    "content": target_label
                }
            ]
        }
        
        # Write each data point as a separate line in the JSONL file
        file.write(json.dumps(data) + "\n")

training_file = client.files.create(
  file=open(output_filename, "rb"),
  purpose='fine-tune'
)

training_file_id = training_file.id

fine_tuning_job = client.fine_tuning.jobs.create(
  training_file=training_file_id,
  model="gpt-3.5-turbo"
)

job_id = fine_tuning_job.id

model_id = "Insert here the new model’s ID"
correct_predictions = 0
loop_count = 0  # Counter for loop iterations
loop_index = 0  # Initialize loop_index


# Iterate over each row in the DataFrame for the new model
for index, row in df.iterrows():
    loop_count += 1  # Increment the loop count
    text = row['clean_comment']  # Adjusted column name
    
    try:
        completion = client.chat.completions.create(
            model=model_id,
            messages=[
                {"role": "system", "content": "What is the sentiment of the following text? Please respond with 'positive', 'negative', or 'neutral'."},
                {"role": "user", "content": text},
            ]
        )
        response = completion.choices[0].message.content
        predicted_sentiment = convert_response_to_sentiment(response)
        
        results[loop_index].update({"new_model_prediction": response})
        loop_index += 1  # Increment the loop index
        
        # Check if the predicted sentiment matches the actual sentiment
        if predicted_sentiment == row['category']:  # Adjusted column name
            correct_predictions += 1
        
        # Print the current progress using loop_count
        print(f"Processed {loop_count}/{total_rows} rows.")
        
    except Exception as e:
        print(f"Error on index {index}: {e}")
        continue

accuracy = (correct_predictions / total_rows) * 100

wandb.log({"New Accuracy": accuracy})
print(f'Model Accuracy after: {accuracy:.2f}%')

# Convert results list to DataFrame
df_results = pd.DataFrame(results)

# Log the entire DataFrame as a table to W&B
wandb.log({"results_table": wandb.Table(dataframe=df_results)})

wandb.finish()