LlamaIndex와 Claude 3를 활용한 RAG 파이프라인 구축

!pip install langchain
!pip install chromadb
!pip install sentence-transformers
!pip install openai
!pip install pypdf
!pip install anthropic
!pip install weave

import openai
import chromadb
from langchain.text_splitter import RecursiveCharacterTextSplitter, SentenceTransformersTokenTextSplitter
from langchain_community.vectorstores import Chroma
from langchain.embeddings import OpenAIEmbeddings
from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction
import pandas as pd
from pypdf import PdfReader
import anthropic
import weave

client = anthropic.Anthropic(
    api_key="YOUR API KEY HERE",
)

weave.init('RAG_Claude_Research')

reader = PdfReader("/content/19_CVPR_Semantic Segmentation of Crop Type in Africa.pdf")
pdf_texts = [p.extract_text().strip() for p in reader.pages]
combined_text = ' '.join(pdf_texts)

character_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200
)
character_split_texts = character_splitter.split_text((combined_text))


token_splitter = SentenceTransformersTokenTextSplitter(chunk_overlap=100, tokens_per_chunk=256)


token_split_texts = []
for combined_text in character_split_texts:
    token_split_texts += token_splitter.split_text(combined_text)

embedding_function = SentenceTransformerEmbeddingFunction()


chroma_client = chromadb.Client()
chroma_collection = client.create_collection(name = "research_paper", embedding_function=embedding_function)


ids = [str(i) for i in range(len(token_split_texts))]
chroma_collection.add(ids=ids, documents=token_split_texts)

def llm_response(query, model):
  results = chroma_collection.query(query_texts=[query], n_results=10)
  retrieved_documents = results['documents'][0]
  relevant_passage="".join(retrieved_documents)
  return make_rag_prompt(model, relevant_passage, query)

model = 'claude-3-opus-20240229'

@weave.op()
def make_rag_prompt(model, context, query):
    response = client.messages.create(
        system =  "You are a helpful reserach assistant. You will be shown data from a research paper and you have to answer questions about the paper",
        messages=[
                    {"role": "user", "content":  "Context: " + context + "\n\n Query: " + query},
                ],
        model= model,  # Choose the model you wish to use
        temperature=0,
        max_tokens=160
    )
    return response.content[0].text

rag_results_claude=[]
query =  "Which crops are mapped in the paper?"
rag_results_claude.append({
    "Query": query,
    "Response": llm_response(query, model)
})
print(rag_results_claude)