Bulk creating high-quality titles and descriptions with GPT-4o

import os
import re
import requests
from bs4 import BeautifulSoup
import openai
from google.oauth2 import service_account
from googleapiclient.discovery import build
from urllib.parse import urlparse
import itertools
import wandb

# Set up your OpenAI API key
openai.api_key = 'sk-YOUR_API_KEY' 

# Set up Google Sheets API credentials
SERVICE_ACCOUNT_FILE = 'C:/Users/USERNAME/YOUR_FILE.json'  # Replace with your credentials path and file.
SCOPES = ['https://www.googleapis.com/auth/spreadsheets']

credentials = service_account.Credentials.from_service_account_file(
    SERVICE_ACCOUNT_FILE, scopes=SCOPES)
service = build('sheets', 'v4', credentials=credentials)

# Initialize Weights & Biases
wandb_api_key = 'YOUR_WANDB_KEY'
wandb.login(key=wandb_api_key)

# Replace with your spreadsheet ID and sheet name
SPREADSHEET_ID = 'YOUR_SPREADSHEET_ID'
SHEET_NAME = 'TAB_NAME'  # Replace with the actual name of your tab

# Initialize W&B run
wandb.init(project='new_title_descriptions', name='Title and Description Generation')

# Create a W&B Table
table = wandb.Table(columns=['Row Index', 'URL', 'Current Title', 'New Title', 'Current Description', 'New Description', 'Reason'])

# Read data from the Google Sheet
sheet_range = f'{SHEET_NAME}!A:F'
result = service.spreadsheets().values().get(
    spreadsheetId=SPREADSHEET_ID,
    range=sheet_range
).execute()
values = result.get('values', [])

if not values:
    print('No data found.')
    data_rows = []
else:
    headers = values[0]
    data_rows = values[1:]  # Skip the header row

# Define batch size
batch_size = 10

# Helper function to split data into batches
def batch_data(data, batch_size):
    """Yield successive batch_size chunks from data."""
    for i in range(0, len(data), batch_size):
        yield data[i:i + batch_size]

# Group data_rows into batches
batches = list(batch_data(data_rows, batch_size))

for batch_num, batch in enumerate(batches, start=1):
    print(f"Processing batch {batch_num}/{len(batches)}")
    batch_updates = []
    batch_wandb_data = []
    
    for idx_in_batch, row in enumerate(batch):
        row_idx = batch_size * (batch_num - 1) + idx_in_batch + 2  # +2 to account for header and zero indexing
        url = row[0] if len(row) > 0 else ''
        if not url:
            continue  # Skip if URL is empty

        parsed_url = urlparse(url)
        current_title = row[1] if len(row) > 1 else ''  # Column B
        current_description = row[4] if len(row) > 4 else ''  # Column E

        print(f"Processing URL at row {row_idx}: {url}")

        # Scrape the content
        try:
            response = requests.get(url)
            soup = BeautifulSoup(response.content, 'html.parser')
            page_text = soup.get_text(separator=' ', strip=True)
        except Exception as e:
            print(f"Error fetching URL at row {row_idx}: {e}")
            continue

        # Summarize the content using GPT-4-mini
        summary_prompt = f"Please summarize the following content in 400 words:\n\n{page_text}"
        try:
            summary_response = openai.chat.completions.create(
                model='gpt-4o-mini',
                messages=[
                    {"role": "system", "content": "You are a helpful assistant summarizing web page content for further processing."},
                    {"role": "user", "content": summary_prompt}
                ]
            )
            summary = summary_response.choices[0].message.content
        except Exception as e:
            print(f"Error summarizing content at row {row_idx}: {e}")
            continue

        # Generate new title and description using GPT-4o
        title_description_prompt = f"""
Based on the following current title and description, generate a new title and description for the content.
Also, explain why the suggested title and description are better than the current ones.

Current Title: {current_title}
Current Description: {current_description}

Summary:
{summary}

URL: {url}

Obey the following rules:
- You write in sentence case.
- Use the URL itself to understand the content that may exist on a page.
- While we want sentence case, if a title or description has a colon, the word after should start capitalized..
- do not use the terms: deep dive, delve
- When referencing people in title tags, include their company and position whenever possible.

Provide the output in the following format:

Title: [Your New Title - maximum 65 characters in sentence case]
Description: [Your New Description - max 165 characters in sentence case]
Reason: [Explain why the suggested title and description are better than the current ones, focusing on improvements in clarity, relevance, or engagement.]
"""
        try:
            td_response = openai.chat.completions.create(
                model='gpt-4o',
                messages=[
                    {"role": "system", "content": "You are a helpful assistant for generating web page titles and descriptions. You write everything in sentence case."},
                    {"role": "user", "content": title_description_prompt}
                ]
            )
            td_content = td_response.choices[0].message.content
        except Exception as e:
            print(f"Error generating title and description at row {row_idx}: {e}")
            continue

        # Extract the title, description, and reason
        title_match = re.search(r'Title:\s*(.*)', td_content)
        description_match = re.search(r'Description:\s*(.*)', td_content)
        reason_match = re.search(r'Reason:\s*(.*)', td_content)

        new_title = title_match.group(1).strip() if title_match else ''
        new_description = description_match.group(1).strip() if description_match else ''
        reason = reason_match.group(1).strip() if reason_match else ''

        # Append updates for Google Sheets
        batch_updates.extend([
            {
                'range': f'{SHEET_NAME}!C{row_idx}',
                'values': [[new_title]]
            },
            {
                'range': f'{SHEET_NAME}!E{row_idx}',
                'values': [[new_description]]
            },
            {
                'range': f'{SHEET_NAME}!F{row_idx}',
                'values': [[reason]]
            }
        ])

        # Append data for W&B Table
        batch_wandb_data.append([
            row_idx,
            url,
            current_title,
            new_title,
            current_description,
            new_description,
            reason
        ])

    # Update Google Sheets with batch updates
    if batch_updates:
        body = {
            'valueInputOption': 'RAW',
            'data': batch_updates
        }

        try:
            service.spreadsheets().values().batchUpdate(
                spreadsheetId=SPREADSHEET_ID,
                body=body
            ).execute()
            print(f"Updated batch {batch_num}")
        except Exception as e:
            print(f"Error updating sheet at batch {batch_num}: {e}")
            continue
    else:
        print(f"No updates to perform for batch {batch_num}")

    # Add data to W&B Table
    for data in batch_wandb_data:
        table.add_data(*data)

# Log the table to W&B
wandb.log({'Processed Data': table})

# Finish the W&B run
wandb.finish()