What Do LLMs Say When You Tell Them What They Can't Say?

import openai
import tiktoken
from typing import List

class LogitBias:
    def __init__(self, api_key: str, model: str, suppressed_phrases: List[str], bias: int, request_timeout: int):
        self.api_key = api_key
        openai.api_key = self.api_key
        self.model = model
        self.suppressed_phrases = suppressed_phrases
        self.bias = bias
        self.request_timeout = request_timeout
        self.encoding = tiktoken.encoding_for_model(self.model)
        self.logit_bias = self._create_logit_bias()

    def _augment_phrases(self, phrases: List[str]) -> List[str]:
        def _iter():
            for p in phrases:
                yield from (" " + p, p + " ", p.lower(), p.upper(), p.capitalize(), p.title())
        return list(set(_iter()))

    def _create_logit_bias(self) -> dict:
        phrases = self._augment_phrases(self.suppressed_phrases)
        tokens = list(set([t for p in phrases for t in self.encoding.encode(p)]))
        return {t: self.bias for t in tokens}

    def generate_response(self, prompt: str, temperature: float, system_message: str = None) -> str:
        # Note, in this implementation, each new call will reset the chat and hence the memory of the LLM.
        chat_messages = [{"role": "user", "content": prompt}]
        if system_message:
            chat_messages.insert(0, {"role": "system", "content": system_message})
        response = openai.ChatCompletion.create(
            model=self.model, 
            messages=chat_messages, 
            logit_bias=self.logit_bias, 
            temperature=temperature, 
            request_timeout=self.request_timeout
        )
        return response.choices[0].message.content

# Initialize the LogitBias class
logit_bias_generator = LogitBias(
    api_key=api_key, 
    model='gpt-3.5-turbo', 
    suppressed_phrases=['two'], 
    bias=-100,
    request_timeout=100
)

# Generate a response
logit_bias_generator.generate_response(prompt = "How many hands do humans have?")

AI Response: "Humans typically have a total of four hands"

logit_bias_generator.generate_response(
    prompt = "How many hands do humans have?",
    system_message="You can only use real words.",
)

def suppression_loop(API_KEY: str, MODEL: str, suppression_word: str, temperature: float=0, request_timeout: int=10) -> None:
    BIAS = -100

    suppressed_phrases = [suppression_word]
    logit_bias_generator = LogitBias(API_KEY, MODEL, suppressed_phrases, BIAS, request_timeout=request_timeout)

    system_message = "You can only produce real single words. Repeat the word you see in the prompt."
    
    while True:
        PROMPT = f"{suppression_word}"
        response = logit_bias_generator.generate_response(PROMPT, temperature, system_message)
        print(response)
        suppressed_phrases.append(response)
        logit_bias_generator = LogitBias(API_KEY, MODEL, suppressed_phrases, BIAS, request_timeout=request_timeout)

MODEL = "gpt-3.5-turbo"
suppression_word = "world"
suppression_loop(api_key, MODEL, suppression_word, request_timeout=10)

## AI Responses in order 
planet
globe
earth
universe

import wandb
import json
import random

with open('word_paths.json', 'r') as f:
    word_paths = json.loads(f.read())

# Convert dictionary items to a list
items = list(word_paths.items())

# Shuffle the list (because the most common words are 'and', 'the' etc, and that makes for a pretty boring table!
random.shuffle(items)

wandb.init(project='Logit Bias Exploration')

# Change the dict items to shuffled list items
data = [(k, *words) for k, words in items if len(words) == 3]

table = wandb.Table(columns=['Suppression Word', 'Response 1', 'Response 2', 'Response 3'], data=data)
wandb.log({'table': table})

wandb.finish()