Azure, OpenAI, 그리고 Weights & Biases가 함께 작동하는 방식

!pip install wandb
!pip install openai azureml-core
!pip install openai

import openai
import wandb
from azureml.core import Workspace
import os


# Set OpenAI API Key
openai.api_key = 'your api key'
client = OpenAI(api_key)

wandb.login()
wandb.init(project='project_name', entity='entity_name')

import json

def convert_dataset_to_jsonl(input_json_path, output_jsonl_path):
    with open(input_json_path, 'r', encoding='utf-8') as f:
        data = json.load(f)


    with open(output_jsonl_path, 'w', encoding='utf-8') as outfile:
        for article in data['data']:
            for paragraph in article['paragraphs']:
                context = paragraph['context']
                for qa in paragraph['qas']:
                    question = qa['question']
                    is_impossible = qa.get('is_impossible', False)
                    prompt = f"Context: {context}\nQuestion: {question}\nAnswer:"
                    # Handle both possible and impossible questions
                    if is_impossible:
                        completion = " Impossible"
                    else:
                        # Using the first answer for simplicity
                        answer = qa['answers'][0]['text'] if qa['answers'] else "Unknown"
                        completion = f" {answer}"
                    # Write the JSONL entry
                    jsonl_entry = json.dumps({"prompt": prompt, "completion": completion})
                    outfile.write(jsonl_entry + '\n')

# Convert your dataset
convert_dataset_to_jsonl('/train-v2.0.json', '/train.jsonl')
convert_dataset_to_jsonl('/dev-v2.0.json', '/dev.jsonl')

def upload_file_to_openai(file_path, purpose='fine-tune'):
    response = openai.File.create(file=open(file_path), purpose=purpose)
    return response.id


train_file_id = upload_file_to_openai("/train.jsonl")
dev_file_id = upload_file_to_openai("/dev.jsonl")
print(train_file_id)

# Define hyperparameters
hyperparameters = {
    "n_epochs": 2,  # Number of training epochs
    "batch_size": 4,  # Batch size for training
    "learning_rate_multiplier": 0.1,  # Learning rate adjustment factor
}


# Log hyperparameters to wandb
wandb.config.update(hyperparameters)

fine_tune_response = openai.FineTuningJob.create(
    training_file=train_file_id,
    validation_file=dev_file_id,
    model="babbage-002",
    hyperparameters=hyperparameters
)


print(f"Fine-tuning started with ID: {fine_tune_response['id']}")
wandb.log({"fine_tune_id": fine_tune_response["id"]})


fine_tune_id= fine_tune_response['id']


fine_tune_status = openai.FineTuningJob.retrieve(fine_tune_id)


print(f"Fine-tuning job status: {fine_tune_status['status']}")

import signal
import datetime

fine_tune_id= fine_tune_response['id']
def signal_handler(sig, frame):
    status = openai.FineTuningJob.retrieve(fine_tune_id)['status']  # Access status correctly
    print(f"Stream interrupted. Job is still {status}.")
    return
print(f"Streaming events for the fine-tuning job: {fine_tune_id}")
signal.signal(signal.SIGINT, signal_handler)
try:
    events_response = openai.FineTuningJob.list_events(id=fine_tune_id)
    events = events_response['data']  # Access the list of events
   
    for event in events:
        event_time = datetime.datetime.fromtimestamp(event['created_at']).strftime('%Y-%m-%d %H:%M:%S')
        print(f"{event_time} {event['message']}")
except Exception as e:
    print(f"Stream interrupted (client disconnected). Error: {str(e)}")

import re
import wandb
# Assuming the messages are stored in a list variable named `messages`
# messages = [
#     "2024-02-21 17:33:45 Step 5901/5937: training loss=1.78, validation loss=0.69"
#     "2024-02-21 17:33:26 Step 5801/5937: training loss=1.29, validation loss=0.62"]
for message in messages:
    # Extract step, training loss, and validation loss using regex
    match = re.search(r"Step (\d+/\d+): training loss=([\d.]+), validation loss=([\d.]+)", message)
    if match:
        step, training_loss, validation_loss = match.groups()
        step = int(step.split('/')[0])  # Extract the current step number
        training_loss = float(training_loss)
        validation_loss = float(validation_loss) 
        # Log the metrics to wandb
        wandb.log({"Step": step, "Training Loss": training_loss, "Validation Loss": validation_loss})
 
# Finish the wandb run
wandb.finish()

# Function to normalize answers (removing punctuation, lowercase, etc.)
def normalize_answer(s):
    import re
    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)
    def white_space_fix(text):
        return ' '.join(text.split())
    def remove_punct(text):
        return re.sub(r'[\W]', ' ', text)
    def lower(text):
        return text.lower()
    return white_space_fix(remove_articles(remove_punct(lower(s))))

# Calculate F1 score
def f1_score(prediction, truth):
    prediction_tokens = normalize_answer(prediction).split()
    truth_tokens = normalize_answer(truth).split()
    common_tokens = Counter(prediction_tokens) & Counter(truth_tokens)
    num_same = sum(common_tokens.values())
    if num_same == 0: return 0
    precision = 1.0 * num_same / len(prediction_tokens)
    recall = 1.0 * num_same / len(truth_tokens)
    f1 = (2 * precision * recall) / (precision + recall)
    return f1

# Calculate Exact Match score
def exact_match_score(prediction, truth):
    return int(normalize_answer(prediction) == normalize_answer(truth))

# Function to query the model and get the answer
def query_model(question, context, model):
    openai.api_key = '<key-here>'
    response = openai.Completion.create(
        model=model,
        prompt=f"Question: {question}\nContext: {context}\nAnswer:",
        temperature=0,
        max_tokens=50,
        top_p=1.0,
        frequency_penalty=0.0,
        presence_penalty=0.0,
        stop=["\n"]
    )
    return response.choices[0].text.strip()

# Test dataset
test_data = [
    {
        "context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.",
        "qas": [
            {
                "question": "In what country is Normandy located?",
                "answer": "France"
            },
            {
                "question": "When were the Normans in Normandy?",
                "answer": "10th and 11th centuries"
            },
            {
                "question": "From which countries did the Norse originate?",
                "answer": "Denmark, Iceland and Norway"
            },
            {
                "question": "Who was the Norse leader?",
                "answer": "Rollo"
            },
            {
                "question": "What century did the Normans first gain their separate identity?",
                "answer": "10th century"
            }
        ]
    },
    {
        "context": "The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.",
        "qas": [
            {
                "question": "Who was the duke in the battle of Hastings?",
                "answer": "William the Conqueror"
            },
            {
                "question": "Who ruled the duchy of Normandy",
                "answer": "Richard I"
            }
        ]
    }
]

# Adjusted part of the script to compile results into a DataFrame
detailed_results = []  # List to store detailed results

for item in test_data:
    context = item['context']
    for qa in item['qas']:
        question = qa['question']
        true_answer = qa['answer']
        model_answer = query_model(question, context, model="modelid")
       
        # Append detailed result for each question
        detailed_results.append({
            "question": question,
            "model_answer": model_answer,
            "true_answer": true_answer        })

# Convert detailed results list to DataFrame
df_results = pd.DataFrame(detailed_results)



# Log the entire DataFrame as a table to W&B
wandb.log({"results_table": wandb.Table(dataframe=df_results)})

# Optional: Save the DataFrame to CSV for local use
df_results.to_csv('evaluation_results.csv', index=False)

from azureml.core import Workspace

# Provide your Azure subscription ID, resource group name, and workspace name
subscription_id = 'your-subscription-id'
resource_group = 'your-resource-group'
workspace_name = 'your-workspace-name'

# Access the workspace
ws = Workspace(subscription_id=subscription_id, resource_group=resource_group, workspace_name=workspace_name)

print("Loaded workspace:", ws.name)

import openai
import json

def init():
    global openai_model
    openai.api_key = 'your_openai_api_key'
    openai_model = 'your_model_id'  # Replace with your fine-tuned model ID (find it here openai.FineTuningJob.list())

def run(raw_data):
    data = json.loads(raw_data)
    response = openai.Completion.create(
        model=openai_model,
        prompt=data['prompt'],
        temperature=0.7,
        max_tokens=150
    )
    return response.choices[0].text

%%writefile score.py

import openai
import json

def init():
    global openai_model
    openai.api_key = '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'
    openai_model = 'ft:babbage-002:personal::8ug5NMO9'  # Replace with your fine-tuned model ID

def run(raw_data):
    data = json.loads(raw_data)
    response = openai.Completion.create(
        model=openai_model,
        prompt=data['prompt'],
        temperature=0.7,
        max_tokens=150
    )
    return response.choices[0].text

from azureml.core import Workspace, Model
from azureml.core import Environment
from azureml.core.model import InferenceConfig
from azureml.core.webservice import AciWebservice

# Create an environment from the yml file
env = Environment.from_conda_specification(name="myenv", file_path="myenv.yml")

# Create an inference configuration
inference_config = InferenceConfig(entry_script="score.py", environment=env)

# Set the deployment configuration
deployment_config = AciWebservice.deploy_configuration(cpu_cores=1, memory_gb=1)

# Deploy the model as a web service
service = Model.deploy(workspace=ws,
                       name="openai-model-service",
                       models=[],  # No models are registered in Azure ML since we're using an external OpenAI model
                       inference_config=inference_config,
                       deployment_config=deployment_config,
                       overwrite=True)

service.wait_for_deployment(show_output=True)
print(f"Service state: {service.state}")
print(f"Scoring URI: {service.scoring_uri}")