Generating content outlines with prompt engineering, entities and GPT-4o

# Define what you're trying to do
query = input ("What do you want to rank for :") 
print(query)

!pip install google-api-python-client
import os
from getpass import getpass
!pip install requests
import requests
!pip install beautifulsoup4
from bs4 import BeautifulSoup
!pip install google-cloud-language
from googleapiclient.discovery import build
from google.cloud import language_v1
!pip install google-auth
from google.oauth2 import service_account
import re
!pip install numpy
import numpy as np
!pip install openai
from openai import OpenAI
!pip install weave --upgrade
import weave
!pip install wandb
import wandb
wandb.login()

#Instructions on getting keys, etc. at http://wandb.me/smx_advanced

# Google API Key and credentials
google_api = 'YOUR_GOOGLE_API_KEY'
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'C:/Users/dave_/Desktop/daves-awesome-articles-effc6fff54dc.json'

#Google Custom Search API
google_search_id = 'CUSTOM_SEARCH_ENGINE_ID'

# Set up OpenAI API Key
%env OPENAI_API_KEY=sk-OPENAI_API

# Setup Google Search API
def google_search(search_term, api_key, cse_id, **kwargs):
    service = build("customsearch", "v1", developerKey=api_key)
    res = service.cse().list(q=search_term, cx=cse_id, **kwargs).execute()
    return res['items']

def analyze_entities(text_content):
    client = language_v1.LanguageServiceClient()
    type_ = language_v1.Document.Type.PLAIN_TEXT
    document = {"content": text_content, "type_": type_}
    encoding_type = language_v1.EncodingType.UTF8
    response = client.analyze_entities(request = {'document': document, 'encoding_type': encoding_type})
    return response

client = OpenAI()
def extract_questions(text):
    # Prompt GPT-4 to generate questions from the text
    prompt = "Extract questions from the following text:\n" + text + "Related to the query: " + query + "\n Questions:"
    response = client.chat.completions.create(
        messages = [{"role": "system", "content": "You are a highly skilled writer, who wants to maximize the impact of their work by answering common questions on a topic in articles you write."},
                  {"role": "user", "content": prompt}],
        model="gpt-4o",
        max_tokens=1000,
        temperature=0.1,
        n=1  # You can adjust this parameter based on the desired number of questions generated
    )
    # Extract questions from GPT-4's response
    questions = [choice.message.content.strip() for choice in response.choices]
    print(questions)
    return questions

def top_questions(text):
    # Prompt GPT-4 to generate the top questions from the text
    prompt = "Across all of these questions, what 5 questions from this list or generated new, would answer the questions you think are the most important to the topic and would be most useful for the user who entered the keyword."
    response = client.chat.completions.create(
        messages = [{"role": "system", "content": "You're an SEO and content strategies and you're analyzing all the questions answered in the top 10 search pages for the keyword:" + query + ". The questions you come up with are: /n" + str(all_questions)},
                  {"role": "user", "content": prompt}],
        model="gpt-4o",
        max_tokens=1000,
        temperature=0.1,
        n=1  # You can adjust this parameter based on the desired number of questions generated
    )
    # Extract questions from GPT-4's response
    questions = [choice.message.content.strip() for choice in response.choices]
    print(questions)
    return questions

# Initialize Weights & Biases
wandb.init(project="smx-article-outline-demo")

# Create W&B Tables to store questions and entities
questions_table = wandb.Table(columns=["URL", "Question"])
entities_table = wandb.Table(columns=["URL", "Entity", "Salience"])

# Search and scrape top 10 pages
search_results = google_search(query, google_api, google_search_id, num=10)
entity_sums = {}
all_questions = []

for result in search_results:
    page = requests.get(result['link'])
    if page.status_code == 403:
        print(f"Access to {result['link']} was denied with a 403 Forbidden error.")
        continue
    soup = BeautifulSoup(page.content, 'html.parser')
    page_text = soup.get_text()
    entities_response = analyze_entities(page_text)
    
    # Processing entities and their salience scores
    for entity in entities_response.entities:
        entity_name = entity.name
        salience = entity.salience
        if entity_name == '403 Forbidden':
            continue  # Skip this entity
        if entity_name in entity_sums:
            entity_sums[entity_name] += salience
        else:
            entity_sums[entity_name] = salience
        entities_table.add_data(result['link'], entity_name, salience)
            
    # Extract and process questions from the current page
    questions = extract_questions(page_text)
    all_questions.extend(questions)    

    # Log questions to W&B Tables
    for question in questions:
        questions_table.add_data(result['link'], question)

# Getting the top 10 entities based on entity_sum
top_entities = sorted(entity_sums.items(), key=lambda item: item[1], reverse=True)[:10]

# Getting the top 5 questions
top_questions_list = top_questions(all_questions)

# Log top questions and entities to W&B
wandb.log({
    "questions_table": questions_table,
    "entities_table": entities_table,
})

# Generate an article outline
@weave.op()
def generate_outline(top_entities, top_questions, query):
    entities_str = ', '.join([entity[0] for entity in top_entities])
    questions_str = ', '.join(top_questions)
    response = client.chat.completions.create(
        messages = [{"role": "assistant", "content": "You are a highly skilled writer, and you want to produce an outline for an article that will appeal to users and rank well."},
                  {"role": "user", "content": "The following entities appear to be relevant to ranking in the top 10 and should be worked into the page: /n" + entities_str + ". Try to ensure the outline will make it easy to work these into the article prominently and explain how this might be done in comments. Additionally, the following questions appear to be important to answer in the article: /n" + questions_str + "/n.Try to ensure that it will be easy to answer these questions in the article, and again, explain how you would recommend to do this in a way that will seem useful to the user. The article outline should provide begin by explaining /n - all of the core concepts required to understand the topic, and/n - also include a tutorial to accomplish the task defined by the keyword " + query + " if possible./nAfter you have provided the outline, explain clearly how this article outline could be used to create an article that will rank well using best-practice SEO strategies as well as be helpful to users. You will be judged based on how well the article ranks, as well as how engaging the article is to readers and provide the metrics you would suggest be used to judge whether you are successful."}],
        model="gpt-4o",
        max_tokens=4000,
        n=1, #How many results to produce per prompt
        stop=None,
        temperature=0.2 #A number between 0 and 1, where higher numbers add randomness
)
    return response.choices[0].message.content.strip()
    
weave.init('smx-article-outline-demo')

# Generate the outline
outline = generate_outline(top_entities, top_questions_list, query)
print(outline)

# Log the outline to Weights & Biases
wandb.log({"article_outline": outline})

# Define what you're trying to do
userquestion = input ("Based on the output above, do you have any questions?")
def generate_outline(userquestion,outline):
    response = client.chat.completions.create(
        messages = [{"role": "assistant", "content": "You are a helpful agent, responding to a question about the outline you created above: /n" + outline + "/n Be brief and clearly answer the question."},
                  {"role": "user", "content": userquestion}],
        model="gpt-4o",
        max_tokens=500,
        n=1, #How many results to produce per prompt
        stop=None,
        temperature=0.2 #A number between 0 and 1, where higher numbers add randomness
)
    return response.choices[0].message.content.strip()

userreply = generate_outline(userquestion,outline)
print(userreply)