지능형 시스템에서 인컨텍스트 검색의 역할

pip install elasticsearch nltk wandb

import sklearn.datasets
# Load the 20 Newsgroups dataset
newsgroups_data = sklearn.datasets.fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))

# Create a list of dictionaries for indexing
documents = [{'title': f'Document {i}', 'content': newsgroups_data.data[i]} for i in range(len(newsgroups_data.data))]

!pip install Elasticsearch -q
# download elasticsearch
!wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.1-linux-x86_64.tar.gz -q
!tar -xzf elasticsearch-7.9.1-linux-x86_64.tar.gz
!chown -R daemon:daemon elasticsearch-7.9.1

# start server
import os
from subprocess import Popen, PIPE, STDOUT
es_server = Popen(['elasticsearch-7.9.1/bin/elasticsearch'],
                 stdout=PIPE, stderr=STDOUT,
                 preexec_fn=lambda: os.setuid(1)
                )
# wait a bit then test
!curl -X GET "localhost:9200/"

# client-side
!pip install elasticsearch==7.9.1
import elasticsearch
from elasticsearch import Elasticsearch


# Connect to Elasticsearch
from datetime import datetime
es = Elasticsearch([{'host': 'localhost' , 'port': 9200, "scheme": "http"}])
es.ping()

# Define the index mapping
index_name = "newsgroups_index"
mapping = {
   "properties": {
       "title": {"type": "text"},
       "content": {"type": "text"}
   }
}

# Create the index
es.indices.create(index=index_name, ignore=400)
es.indices.put_mapping(index=index_name, body=mapping)
# Index the documents
for doc in documents:
   es.index(index=index_name, body=doc)

nltk.download('punkt')
nltk.download('stopwords')
import nltk
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer    

# Initialize stopwords and stemmer
stop_words = set(stopwords.words('english'))
stemmer = PorterStemmer()

# Preprocess the data
preprocessed_documents = []
for document in newsgroups_data.data:
   # Tokenize the document
   tokens = nltk.word_tokenize(document.lower())

# Remove stopwords and stem the tokens
   filtered_tokens = [stemmer.stem(token) for token in tokens if token not in stop_words]

# Join the tokens back into a single string
   preprocessed_text = ' '.join(filtered_tokens)
   preprocessed_documents.append(preprocessed_text)

# Create a list of dictionaries for indexing
documents = [{'title': f'Document {i}', 'content': preprocessed_documents[i]} for i in range(len(preprocessed_documents))]

from elasticsearch import Elasticsearch


# Connect to Elasticsearch
es = Elasticsearch()
index_name = "newsgroups_index"


# Basic query
query = {
   "query": {
       "match": {
           "content": "comput"
       }
   }
}

results = es.search(index=index_name, body=query)
print(f"Number of results: {len(results['hits']['hits'])}")


# More complex query using Elasticsearch's Query DSL
query = {
   "query": {
       "bool": {
           "must": [
               {"match": {"title": "Document"}},
               {"match": {"content": "computer science"}}
           ]
       }
   }
}
results = es.search(index=index_name, body=query)
print(f"Number of results: {len(results['hits']['hits'])}")

!pip install wandb
import wandb


# Initialize Weights & Biases
wandb.init(project="newsgroups-retrieval-system")


# Log system metrics
for i in range(100):
   query = {
       "query": {
           "match": {
               "content": "comput"
           }
       }
   }
      metrics = {
       "query_time": results["took"],
       "num_results": len(results["hits"]["hits"])
   }
   wandb.log(metrics)


# Visualize system performance
wandb.finish()