Autonomous AI Agents: Capabilities, challenges, and future trends

import requests
import os
import imaplib
import email as em
import re
from crewai_tools import ScrapeWebsiteTool
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from litellm import completion
from datetime import datetime
import weave; weave.init("hackernews_agent")

# Email configuration
EMAIL_ADDRESS = 'your_email@gmail.com'
EMAIL_PASSWORD = 'your app password'
RECIPIENT_EMAIL = 'your_email@gmail.com'

# File to track sent stories
SENT_FILE = "sent.txt"

# Function to fetch top HN stories
def fetch_top_stories(count=50):
    url = 'https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty'
    response = requests.get(url)
    if response.status_code == 200:
        return response.json()[:count]
    else:
        print(f"Error fetching top stories: {response.status_code}")
        return []

# Function to fetch story details
def fetch_story_details(story_id):
    url = f'https://hacker-news.firebaseio.com/v0/item/{story_id}.json?print=pretty'
    response = requests.get(url)
    if response.status_code == 200:
        return response.json()
    else:
        print(f"Error fetching story details for ID {story_id}: {response.status_code}")
        return None

# Load previously sent stories - just get the titles
def load_sent_stories():
    sent_stories = []
    if os.path.exists(SENT_FILE):
        with open(SENT_FILE, "r") as f:
            for line in f:
                title = line.strip()
                if title:
                    sent_stories.append(title)
    return sent_stories

# Add a story to the sent list - just store the title
def add_to_sent_stories(title):
    with open(SENT_FILE, "a") as f:
        f.write(f"{title}\n")

# Get the most recently sent story
def get_last_sent_story():
    if not os.path.exists(SENT_FILE) or os.path.getsize(SENT_FILE) == 0:
        return None
    
    with open(SENT_FILE, "r") as f:
        lines = f.readlines()
        if not lines:
            return None
        return lines[-1].strip()  # Just get the title

# Load preferred and unwanted keywords
def load_preferences():
    preferred = []
    unwanted = []
    
    try:
        if os.path.exists("preferred.txt"):
            with open("preferred.txt", "r") as f:
                preferred = [line.strip() for line in f.readlines() if line.strip()]
    except Exception as e:
        print(f"Error loading preferred.txt: {e}")
    
    try:
        if os.path.exists("unwanted.txt"):
            with open("unwanted.txt", "r") as f:
                unwanted = [line.strip() for line in f.readlines() if line.strip()]
    except Exception as e:
        print(f"Error loading unwanted.txt: {e}")
    
    return preferred, unwanted

# Check for feedback emails and update preferences
def check_feedback():
    try:
        # Connect to email
        mail = imaplib.IMAP4_SSL("imap.gmail.com")
        mail.login(EMAIL_ADDRESS, EMAIL_PASSWORD)
        mail.select("inbox")
        
        # Get the most recently sent story title
        last_sent_title = get_last_sent_story()
        if not last_sent_title:
            print("No previously sent stories found.")
            return
        
        print(f"Checking feedback for most recent story: {last_sent_title}")
        
        # Search for unread emails from the recipient
        status, messages = mail.search(None, f'(UNSEEN FROM "{EMAIL_ADDRESS}")')
        
        if not messages[0]:
            print("No new feedback emails found.")
            return
        
        # Process the most recent email
        message_ids = messages[0].split()
        latest_message_id = message_ids[-1]  # Get the most recent unread email
        
        status, msg_data = mail.fetch(latest_message_id, '(RFC822)')
        raw_email = msg_data[0][1]
        email_message = em.message_from_bytes(raw_email)
        
        # Get the email body
        body = ""
        if email_message.is_multipart():
            for part in email_message.walk():
                if part.get_content_type() == "text/plain":
                    body = part.get_payload(decode=True).decode()
                    break
        else:
            body = email_message.get_payload(decode=True).decode()
        
        # Process the feedback
        body = body.lower().strip()
        
        if "good" in body:
            print(f"Received positive feedback for: {last_sent_title}")
            # Add to preferred.txt if not already there
            with open("preferred.txt", "a+") as f:
                f.seek(0)
                content = f.read()
                if last_sent_title not in content:
                    f.write(f"{last_sent_title}\n")
                    print(f"Added '{last_sent_title}' to preferred.txt")
        
        elif "bad" in body:
            print(f"Received negative feedback for: {last_sent_title}")
            # Add to unwanted.txt if not already there
            with open("unwanted.txt", "a+") as f:
                f.seek(0)
                content = f.read()
                if last_sent_title not in content:
                    f.write(f"{last_sent_title}\n")
                    print(f"Added '{last_sent_title}' to unwanted.txt")
        
        # Mark as read
        mail.store(latest_message_id, '+FLAGS', '\\Seen')
        mail.close()
        mail.logout()
        
    except Exception as e:
        print(f"Error checking feedback: {e}")

# Select the best story based on preferences
@weave.op
def select_best_story(stories, preferred, unwanted, sent_stories, model="gpt-4o-mini", api_base=None):
    # Filter out already sent stories
    filtered_stories = []
    for story in stories:
        if 'title' in story and story['title'] not in sent_stories:
            filtered_stories.append(story)
    
    if not filtered_stories:
        print("All stories have already been sent!")
        return None
    
    print(f"After filtering out sent stories, {len(filtered_stories)} stories remain.")
    
    # Create story summaries for comparison
    story_texts = []
    for i, story in enumerate(filtered_stories):
        if 'title' in story and 'url' in story:
            story_texts.append(f"Story {i+1}: {story['title']} - {story['url']}")
    
    # Build the prompt
    system_message = "You are a helpful assistant that selects the most relevant news stories based on user preferences."
    user_message = f"""Select the most interesting Hacker News story based on the user's preferences.

PREFERRED TOPICS/KEYWORDS:
{', '.join(preferred)}

UNWANTED TOPICS/KEYWORDS:
{', '.join(unwanted)}

Here are the available stories:
{chr(10).join(story_texts)}

Select the story number that best matches the preferred topics and avoids the unwanted ones.
Respond with ONLY the story number, for example: 3
"""
    
    # Get the model's choice using litellm
    try:
        response = completion(
            model=model,
            messages=[
                {"role": "system", "content": system_message},
                {"role": "user", "content": user_message}
            ],
            api_base=api_base if api_base else None
        )
        
        choice_text = response['choices'][0]['message']['content'].strip()
        # Extract just the number from the response
        choice = int(''.join(filter(str.isdigit, choice_text))) - 1
        
        if 0 <= choice < len(filtered_stories):
            return filtered_stories[choice]
        
    except Exception as e:
        print(f"Error in story selection: {e}")
    
    # Fallback to the first story if there's an issue
    return filtered_stories[0] if filtered_stories else None

# Scrape and summarize the content
@weave.op
def scrape_and_summarize(url, title, model="gpt-4o-mini", api_base=None):
    # Scrape the page content
    scrape_tool = ScrapeWebsiteTool(website_url=url)
    content = scrape_tool.run()
    
    # Limit content to avoid token limits
    limited_content = content[:10000]
    
    # Summarize the content using litellm
    system_message = "You are a helpful assistant that summarizes news articles clearly and concisely."
    user_message = f"""Summarize the following article from Hacker News titled "{title}":

{limited_content}

Provide a concise summary in 3-4 paragraphs that captures the main points, key insights, and any interesting details.
"""
    
    try:
        response = completion(
            model=model,
            messages=[
                {"role": "system", "content": system_message},
                {"role": "user", "content": user_message}
            ],
            api_base=api_base if api_base else None
        )
        
        return response['choices'][0]['message']['content']
    except Exception as e:
        print(f"Error summarizing content: {e}")
        return f"Error summarizing the article: {str(e)}"

# Send email
def send_email(title, url, summary):
    msg = MIMEMultipart()
    msg['From'] = EMAIL_ADDRESS
    msg['To'] = RECIPIENT_EMAIL
    msg['Subject'] = f"HN Summary: {title}"
    
    body = f"""
    <h2>{title}</h2>
    <p><a href="{url}">Original Article</a></p>
    <hr>
    <h3>Summary:</h3>
    {summary}
    <hr>
    <p>Reply with only the word "good" if you liked this story selection, or only the word "bad" if you didn't.</p>
    """
    
    msg.attach(MIMEText(body, 'html'))
    
    try:
        server = smtplib.SMTP('smtp.gmail.com', 587)
        server.starttls()
        server.login(EMAIL_ADDRESS, EMAIL_PASSWORD)
        server.send_message(msg)
        server.quit()
        print(f"Email sent: {title}")
        
        # Add to sent stories file - just the title
        add_to_sent_stories(title)
        
    except Exception as e:
        print(f"Error sending email: {e}")

# Main function
def main(model="gpt-4o-mini", api_base=None):
    print("Checking for feedback emails...")
    check_feedback()
    
    print("Loading preferences...")
    preferred, unwanted = load_preferences()
    
    print("Loading list of sent stories...")
    sent_stories = load_sent_stories()
    print(f"Found {len(sent_stories)} previously sent stories.")
    
    print(f"Fetching top Hacker News stories...")
    top_story_ids = fetch_top_stories(50)
    
    # Get details for each story
    stories = []
    for story_id in top_story_ids:
        story = fetch_story_details(story_id)
        if story and 'title' in story and 'url' in story:
            stories.append(story)
    
    print(f"Selecting the best story from {len(stories)} candidates...")
    best_story = select_best_story(stories, preferred, unwanted, sent_stories, model=model, api_base=api_base)
    
    if best_story:
        title = best_story['title']
        url = best_story['url']
        
        # Double-check that the story hasn't already been sent
        if title in sent_stories:
            print(f"Story '{title}' has already been sent! Selecting another story...")
            # Remove this story from the list and try again
            stories = [s for s in stories if s['title'] != title]
            best_story = select_best_story(stories, preferred, unwanted, sent_stories, model=model, api_base=api_base)
            
            if not best_story:
                print("No alternative story found after filtering.")
                return
            
            title = best_story['title']
            url = best_story['url']
        
        print(f"Selected story: {title}")
        print("Scraping and summarizing content...")
        summary = scrape_and_summarize(url, title, model=model, api_base=api_base)
        
        print("Sending email...")
        send_email(title, url, summary)
    else:
        print("No suitable story found.")

if __name__ == "__main__":
    # Example of how to specify a different model and API base
    # main(model="ollama/llama2", api_base="http://localhost:11434")
    
    # Default usage with gpt-4o-mini
    main()

def fetch_top_stories(count=50):
    url = 'https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty'
    response = requests.get(url)
    return response.json()[:count] if response.status_code == 200 else []

def fetch_story_details(story_id):
    url = f'https://hacker-news.firebaseio.com/v0/item/{story_id}.json?print=pretty'
    response = requests.get(url)
    return response.json() if response.status_code == 200 else None

@weave.op
def select_best_story(stories, preferred, unwanted, sent_stories, model="gpt-4o-mini"):
    filtered_stories = [s for s in stories if 'title' in s and s['title'] not in sent_stories]
    if not filtered_stories:
        return None

    story_texts = [f"Story {i+1}: {s['title']} - {s['url']}" for i, s in enumerate(filtered_stories)]
    user_message = f"Select the most relevant Hacker News story based on user preferences:\nPREFERRED: {', '.join(preferred)}\nUNWANTED: {', '.join(unwanted)}\nAvailable stories:\n{chr(10).join(story_texts)}\nRespond with only the story number."

    response = completion(model=model, messages=[{"role": "user", "content": user_message}])
    choice = int(''.join(filter(str.isdigit, response['choices'][0]['message']['content']))) - 1
    return filtered_stories[choice] if 0 <= choice < len(filtered_stories) else filtered_stories[0]

@weave.op
def scrape_and_summarize(url, title, model="gpt-4o-mini"):
    scrape_tool = ScrapeWebsiteTool(website_url=url)
    content = scrape_tool.run()
    limited_content = content[:10000]

    system_message = "You are an assistant that summarizes news articles concisely."
    user_message = f"Summarize the following article titled '{title}':\n{limited_content}\nProvide a concise summary in 3-4 paragraphs."
    
    response = completion(model=model, messages=[{"role": "user", "content": user_message}])
    return response['choices'][0]['message']['content']

def send_email(title, url, summary):
    msg = MIMEMultipart()
    msg['From'] = EMAIL_ADDRESS
    msg['To'] = RECIPIENT_EMAIL
    msg['Subject'] = f"HN Summary: {title}"

    body = f"""
    <h2>{title}</h2>
    <p><a href="{url}">Original Article</a></p>
    <hr>
    <h3>Summary:</h3>
    {summary}
    <hr>
    <p>Reply with "good" if you liked this selection, or "bad" if you didn’t.</p>
    """
    
    msg.attach(MIMEText(body, 'html'))

    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.starttls()
    server.login(EMAIL_ADDRESS, EMAIL_PASSWORD)
    server.send_message(msg)
    server.quit()

def check_feedback():
    mail = imaplib.IMAP4_SSL("imap.gmail.com")
    mail.login(EMAIL_ADDRESS, EMAIL_PASSWORD)
    mail.select("inbox")

    last_sent_title = get_last_sent_story()
    if not last_sent_title:
        return

    status, messages = mail.search(None, f'(UNSEEN FROM "{EMAIL_ADDRESS}")')
    if not messages[0]:
        return

    latest_message_id = messages[0].split()[-1]
    status, msg_data = mail.fetch(latest_message_id, '(RFC822)')
    email_message = em.message_from_bytes(msg_data[0][1])

    body = ""
    if email_message.is_multipart():
        for part in email_message.walk():
            if part.get_content_type() == "text/plain":
                body = part.get_payload(decode=True).decode()
                break
    else:
        body = email_message.get_payload(decode=True).decode()

    body = body.lower().strip()

    if "good" in body:
        with open("preferred.txt", "a+") as f:
            f.seek(0)
            if last_sent_title not in f.read():
                f.write(f"{last_sent_title}\n")

    elif "bad" in body:
        with open("unwanted.txt", "a+") as f:
            f.seek(0)
            if last_sent_title not in f.read():
                f.write(f"{last_sent_title}\n")

    mail.store(latest_message_id, '+FLAGS', '\\Seen')
    mail.close()
    mail.logout()