How to Evaluate an LLM, Part 1: Building an Evaluation Dataset for our LLM System

import pandas as pd

# read the csv file as Pandas DataFrame
df = pd.read_csv("path/to/file/you_want_to_log.csv")

# Log the DataFrame to W&B - it will automatically be converted to W&B Tables.
run = wandb.init(project="your_project_name")
run.log({"my_dataframe": df}

drop_ids = [3, 42, 194, 268]

for drop_id in drop_ids:
    public_df = public_df.drop(drop_id)

public_df = public_df.reset_index(drop=True)

import tiktoken
enc = tiktoken.encoding_for_model("gpt-4")
enc.encode("hello world")

drop_ids = [
    705, 336, 635, 769, 267, 284, 772, 77, 420, 562, 125, 207, 285, 374,
    514, 518, 576, 661, 223, 337, 429, 499, 577, 621, 4, 298, 327, 434,
    553, 638, 759, 415
]

from nomic import AtlasProject

map = AtlasProject(name='Public Questions Wandbot')
atlas_df = map.maps[0].data.df

dropped_ids = []

for dropped_id in dropped_index:
    id_ = atlas_df.loc[dropped_id].id_
    dropped_ids.append(id_)

with map.wait_for_project_lock():
    map.delete_data(ids=droped_ids)
    map.rebuild_maps()

import wandb
import umap # pip install map-learn
from sklearn.preprocessing import StandardScaler
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import KMeans

# List of embeddings for each query
embeddings = [...] # 1536 dimensional
# Initialize UMAP model
umap_model = umap.UMAP(n_components=50)
# Fit and transform the data
embeddings = umap_model.fit_transform(embeddings)
# Standardize
scaler = StandardScaler()
embeddings = scaler.fit_transform(embeddings)
# Get cosine similarity
cosine_sim = cosine_similarity(embeddings, embeddings)

# The `public_df` is a DataFrame where the column 
# "processed_query" contain the questions.
df = public_df[["processed_query"]]
# Perform clustering - replace KMeans with a different algorithm
kmeans = KMeans(n_clusters=80, random_state=42, max_iter=500, init="k-means++")
df['k_mean_cluster'] = kmeans.fit_predict(cosine_sim)

# Log the clusters as a W&B Table
run = wandb.init(project="wandbot-eval", job_type="clustering")
wandb.log({"public_questions_clusters": df})

from llama_index import VectorStoreIndex
from llama_index import Document
import networkx as nx

query_list = [...] # List of 671 questions
query_docs = [Document(text=t) for t in query_list] # To make it LlamaIndex compatible

# Get the top 2 semantic search retriever
vector_index = VectorStoreIndex.from_documents(query_docs)
vector_retriever = vector_index.as_retriever(similarity_top_k=2)

# Create a similarity graph
G = nx.Graph()

for i, text in enumerate(query_list):
    similar_texts = vector_retriever.retrieve(text)
    for similar_text in similar_texts:
        G.add_edge(text, similar_text.text)

from community import community_louvain # pip install python-louvain

# Apply Louvain Community Detection
partition = community_louvain.best_partition(G)

prompt_template = """
Wandbot is a question answering bot for Weights & Biases documentation. I have provided a list of questions
that are semantically similar. As a smart assistant, your job is to find one question from the list of questions
that will be best to evaluate wandbot on. Find one hard question from the list of question.

# QUESTIONS

{questions}

Make sure to return just one question from the list of question in the given format:

```
['question']
```

Everything between ``` should be a valid python list.

ANSWER: 
"""