Python으로 추천 엔진 구축하기

import requests
import zipfile
import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix
from scipy.sparse.linalg import svds
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from math import sqrt
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
!pip install wandb
!wandb login
import wandb

# Download the dataset
url = 'http://files.grouplens.org/datasets/movielens/ml-100k.zip'
r = requests.get(url, allow_redirects=True)
open('ml-100k.zip', 'wb').write(r.content)


# Extract the dataset
with zipfile.ZipFile('ml-100k.zip', 'r') as zip_ref:
    zip_ref.extractall('ml-100k')


# Load the datasets
df_ratings = pd.read_csv('ml-100k/ml-100k/u.data', sep='\t', names=['user id', 'movie id', 'rating', 'timestamp'], header=None)

# Initialize a new wandb run
wandb.init(project='movie_recommendation_system', name='Content_Based_Filtering')

# Read the dataset, specify column names for movie attributes and use '|' as the delimiter to address the dataset's format and character encoding (Latin-1). The usecols parameter limits the imported columns to just movie IDs and titles.


df_movies = pd.read_csv('ml-100k/ml-100k/u.item', sep='|', encoding='latin-1', header=None,
                        names=['movie id', 'movie_title', 'release_date', 'video_release_date',
                               'IMDb_URL', 'unknown', 'Action', 'Adventure', 'Animation',
                               'Children', 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy',
                               'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi',
                               'Thriller', 'War', 'Western'], usecols=[0, 1]

# Split the data into training and testing sets
train_data, test_data = train_test_split(df_ratings, test_size=0.2, random_state=42)

# Reshaping the training and test sets
test_matrix = test_data.pivot_table(index='user id', columns='movie id', values='rating', fill_value=0)
train_matrix = train_data.pivot_table(index='user id', columns='movie id', values='rating', fill_value=0)

# Convert the matrices to sparse matrices
train_matrix_sparse = csr_matrix(train_matrix, dtype=np.float32)
test_matrix_sparse = csr_matrix(test_matrix, dtype=np.flo

number_of_factors = 50  # Placeholder for the number of latent factors
regularization_strength = 0.1  # Placeholder for regularization strength


# Perform SVD on the training matrix
U, sigma, Vt = svds(train_matrix_sparse, k=number_of_factors)
sigma = np.diag(sigma)


# Log SVD model parameters and hyperparameters
wandb.config.update({
    "number_of_factors": number_of_factors,
    "regularization_strength": regularization_strength
})


# Add user mean back to get actual rating prediction for training set
user_ratings_mean_train = np.array(train_matrix.mean(axis=1))
predicted_ratings_train = np.dot(np.dot(U, sigma), Vt) + user_ratings_mean_train.reshape(-1, 1)

#Function to calculate the rmse between prediction and ground_truth. 
def rmse(prediction, ground_truth): 
     #Filters and reshape the prediction array by selecting only those predictions where the corresponding ground_truth rating exists (is non-zero). flatten() then converts the array into a one-dimensional array.
    prediction = prediction[ground_truth.nonzero()].flatten()


    #Similar to the prediction array, this line filters the ground_truth array for non-zero values.
    ground_truth = ground_truth[ground_truth.nonzero()].flatten()


    #Compute the mean squared error between the filtered prediction and ground_truth arrays. The sqrt (square root) function is then applied to this value to obtain the RMSE.
    return sqrt(mean_squared_error(prediction, ground_truth))


# Calculate RMSE for train data
train_rmse = rmse(predicted_ratings_train, train_matrix.to_numpy())
# Calculate RMSE for test data
test_rmse = rmse(predicted_ratings_train, test_matrix.to_numpy())
#log the RMSE
wandb.log({"RMSE Train": train_rmse,
           "RMSE Test": test_rmse})
print('User-based Train CF RMSE: ' + str(train_rmse))
print('User-based Test CF RMSE: ' + str(test_rmse))

# Function to recommend movies for the test dataset
def recommend_movies_test(user_id, num_recommendations=3):
    user_row_number = user_id - 1


#Sorting predicted ratings for the user
    sorted_user_predictions = pd.Series(predicted_ratings_train[user_row_number]).sort_values(ascending=False)


#Filtering previously user's rated movies
    user_data = test_data[test_data['user id'] == user_id]
    user_full = (user_data.merge(df_movies, how='left', left_on='movie id', right_on='movie id').
                 sort_values(['rating'], ascending=False)
                )
    #Generating recommendations
    recommendations = (df_movies[~df_movies['movie id'].isin(user_full['movie id'])].


#Selecting top recommendations:  merge(pd.DataFrame(sorted_user_predictions).reset_index(), how='left',
                             left_on='movie id',
                             right_on='index').
                       rename(columns={0: 'Predictions'}).
                       sort_values('Predictions', ascending=False).
                       iloc[:num_recommendations, :-1]
                      )


    return user_full, recommendations

# Get recommendations for a specific user
user_id = 5  # Change the user_id to the desired user
actual_movies, recommended_movies = recommend_movies_test(user_id, num_recommendations=3)
print("\nMovies watched by user:")
print(actual_movies)
print("\nRecommended movies:")
print(recommended_movies)


#log the user and movie metics 
wandb.log({
    "user_id": user_id,
    "watched_movies": actual_movies['movie_title'].tolist(),
    "recommended_movies": recommended_movies['movie_title'].tolist()
})
# Finish the run
wandb.finish()

# Function to get the title from the movie id
def get_title_from_index(index):
    return df_movies[df_movies.index == index]["movie_title"].values[0]


# Function to get index from the movie id
def get_index_from_movie_id(movie_id):
    matches = df_movies[df_movies['movie id'] == movie_id].index.values
    return matches[0] if len(matches) > 0 else None

def recommend_movies_based_on_one_movie(user_id, num_recommendations=3):
    # Find the highest-rated movie by the user
    user_movies = df_ratings[df_ratings['user id'] == user_id]
    highest_rated_movie_row = user_movies.sort_values(by='rating', ascending=False).iloc[0]
    highest_rated_movie_id = highest_rated_movie_row['movie id']


    # Find the index and title of this movie in df_movies
    movie_index = get_index_from_movie_id(highest_rated_movie_id)
    if movie_index is None:
        return "No movies found for the given user.", None


    movie_title_based_on = get_title_from_index(movie_index)
	
    # Calculate the cosine similarities with other movies
    cosine_similarities = cosine_similarity(tfidf_matrix[movie_index], tfidf_matrix).flatten()
    similar_movies = list(enumerate(cosine_similarities))
    sorted_similar_movies = sorted(similar_movies, key=lambda x: x[1], reverse=True)[1:]


    # Get the recommended movies
    recommended_movies = []
    for i in range(num_recommendations):
        index = sorted_similar_movies[i][0]
        movie_title = get_title_from_index(index)
        recommended_movies.append(movie_title)


    return movie_title_based_on, recommended_movies

#  Read the dataset, specify column names for movie attributes and use '|' as the delimiter to address the dataset's format and character encoding (Latin-1).
df_movies = pd.read_csv('ml-100k/ml-100k/u.item', sep='|', encoding='latin-1', header=None,
                        names=['movie id', 'movie_title', 'release_date', 'video_release_date',
                               'IMDb_URL', 'unknown', 'Action', 'Adventure', 'Animation',
                               'Children', 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy',
                               'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi',
                               'Thriller', 'War', 'Western'])


# Combine movie genres into a single string for each movie
genre_columns = ['Action', 'Adventure', 'Animation', 'Children', 'Comedy', 'Crime',
                 'Documentary', 'Drama', 'Fantasy', 'Film-Noir', 'Horror', 'Musical',
                 'Mystery', 'Romance', 'Sci-Fi', 'Thriller', 'War', 'Western']
df_movies['combined_features'] = df_movies[genre_columns].apply(lambda x: ' '.join(x.index[x == 1]), axis=1)

# Creating a TF-IDF Vectorizer to convert genres to a matrix of TF-IDF features
tfidf_vectorizer = TfidfVectorizer()
tfidf_matrix = tfidf_vectorizer.fit_transform(df_movies['combined_features'])

# Get recommendations based on one movie for a specific user
user_id = 20  # Change the user_id to the desired user
based_on_movie, recommended_movies = recommend_movies_based_on_one_movie(user_id)5)5

# Log sample recommendations
wandb.log({
    "user_id": user_id,
    "based_on_movie": based_on_movie,
    "recommended_movies": recommended_movies
})
# Finish the run
wandb.finish()