Crafting Superior Summaries: The ChatGPT Fine-Tuning Guide

{"messages": [{"role": "system", "content": "GPT is a great and to-the-point dialogue summarization tool."}, {"role": "user", "content": "#Person1#: hey, you look great! how's everything?\n#Person2#: yeah, you know what? I've been going to the club regularly. The training really pays off. Now I am in a good shape and I know more about how to keep fit.\n#Person1#: really? tell me about it. I haven't gone to the club for a long time. I am too busy with work.\n#Person2#: it's important to do proper exercises.\n#Person1#: you're right. Too much or too little won't do any good.\n#Person2#: the trainer tells me, besides regular sports activities, I should also have a healthy and balanced diet.\n#Person1#: sounds reasonable.\n#Person2#: we should eat more vegetables instead of junk food to stay energetic.\n#Person1#: and fruits!\n#Person2#: surely it is. Getting enough sleep is also crucial for fitness.\n#Person1#: I've heard that. Does your trainer tell you anything about keeping fit?\n#Person2#: yeah, he advises me to stay in a good mood. That can help one to keep sound physical health.\n#Person1#: I think if you follow your trainer's advice, you'll be on the right track.\n#Person2#: you bet it!"}, {"role": "assistant", "content": "#Person2# looks great because #Person2#'s been to the training club regularly. #Person2# tells #Person1# that having a healthy and balanced diet, getting enough sleep, and staying in a good mood help keep physical health."}]}

!pip install openai
!pip install wandb
!pip install git+https://github.com/wandb/wandb.git@openai-finetuning

import os
import openai
import wandb
import pandas as pd
import json
from openai import OpenAI
from wandb.integration.openai import WandbLogger

openai.api_key = "Insert your own personal OpenAI key here"

client = openai.Client()

WandbLogger.sync(project="chatgpt3.5-fine-tuning-for-Summarization")

# Read the CSV data
df = pd.read_csv('Insert path to summarization dataset”)

# Sample 100 rows
df = df.sample(100)

output_filename = "Insert the new path to save the newly modified JSONL data set"

# Create a new JSONL file
with open(output_filename, 'w') as jsonl_file:
    for _, row in df.iterrows():
        # Create the desired format for each row in the CSV
        data = {
            "messages": [
                {
                    "role": "system",
                    "content": "GPT is a great and to-the-point dialogue summarization tool."
                },
                {
                    "role": "user",
                    "content": row['dialogue']
                },
                {
                    "role": "assistant",
                    "content": row['summary']
                }
            ]
        }
        # Write the data to the JSONL file
        jsonl_file.write(json.dumps(data) + '\n')

training_file = client.files.create(
  file=open(output_filename, "rb"),
  purpose='fine-tune'
)

training_file_id = training_file.id

fine_tuning_job = client.fine_tuning.jobs.create(
  training_file=training_file_id, 
  model="gpt-3.5-turbo"
)

job_id = fine_tuning_job.id

def evaluate_model(model_name, model_id):
    # Initialize Weights & Biases
    run = wandb.init(project="text-summarization-with-openai", reinit=True)

    correct_predictions = 0
    loop_count = 0
    results = []

    # Iterate over each row in the DataFrame
    for index, row in df.iterrows():
        loop_count += 1
        dialogue_text = row['dialogue']

        try:
            completion = client.chat.completions.create(
                model=model_id,
                messages=[
                    {"role": "system", "content": "GPT is a great and to-the-point dialogue summarization tool."},
                    {"role": "user", "content": dialogue_text},
                ]
            )
            response = completion.choices[0].message.content
            
            results.append({
                "dialogue": dialogue_text,
                "actual_summary": row['summary'],
                "predicted_summary": response
            })

            if compare_summaries(response, row['summary']):
                correct_predictions += 1

            print(f"[{model_name}] Processed {loop_count}/{len(df)} rows.")

        except Exception as e:
            print(f"Error on index {index} with {model_name}: {e}")
            continue

    accuracy = (correct_predictions / len(df)) * 100
    wandb.log({f"{model_name} Accuracy": accuracy})

    # Convert results list to DataFrame
    df_results = pd.DataFrame(results)

    # Log the entire DataFrame as a table to W&B
    wandb.log({"results_table": wandb.Table(dataframe=df_results)})

    print(f'{model_name} Summarization Accuracy: {accuracy:.2f}%')
    
    # Finish the Weights & Biases run for the current model
    run.finish()

def compare_summaries(predicted, actual):
    return predicted.strip() == actual.strip()

filename = "/kaggle/input/dialogsum/CSV/test.csv"
df = pd.read_csv(filename)

# Remove any NaN values in 'dialogue' and 'summary'
df.dropna(subset=['dialogue', 'summary'], inplace=True)
df = df.sample(100)  # Use a subset of the data for quick testing

evaluate_model("base", "gpt-3.5-turbo")

evaluate_model("fine_tuned", "Insert fine_tuned model ID")