Recommender System with Collaborative Filtering

import numpy as np
import pandas as pd
import sys

movies = pd.read_csv("./ml-20m/movies.csv")
tags = pd.read_csv("./ml-20m/tags.csv")
ratings = pd.read_csv("./ml-20m/ratings.csv", nrows=16000000) 

movies.head()

tags.head()

ratings.head()

tags.drop(['timestamp'], axis=1, inplace=True)
ratings.drop(['timestamp'], axis=1, inplace=True)

len(ratings.movieId.unique())

movies['genres'] = movies['genres'].str.replace('|', ' ')

# Restrict to users that have rated atleast 60 movies
ratings_df = ratings.groupby('userId').filter(lambda x: len(x) >= 60)

ratings_df.shape

ratings.shape

len(ratings.userId.unique())

len(ratings_df.userId.unique())

# whihc all  movies are there in ratings_df, keep only those in movies 
ratings_movie_list = ratings_df['movieId'].unique().tolist()
movies = movies[movies['movieId'].isin(ratings_movie_list)]
movies.head()

merged_df = pd.merge(movies, tags, on='movieId', how='left')
merged_df.head()

merged_df.fillna("", inplace=True)
merged_df = pd.DataFrame(merged_df.groupby('movieId')['tag'].apply(' '.join))

merged_df.head()

final_df = pd.merge(movies, merged_df, on='movieId', how='left')

final_df.head()

final_df['metadata'] = final_df[['tag', 'genres']].apply(' '.join, axis=1)

final_df.head()

final_df.shape

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(stop_words='english')
tfidf_mat = tfidf.fit_transform(final_df['metadata'])

tfidf_df = pd.DataFrame(tfidf_mat.toarray(), index=final_df.index.tolist()) 
tfidf_df.head() 

tfidf_df.shape  #each row is a movie

from sklearn.decomposition import TruncatedSVD

svd = TruncatedSVD(n_components=200)
latent_matrix = svd.fit_transform(tfidf_df)

latent_matrix_df1 = pd.DataFrame(latent_matrix[:,0:200], index = final_df['title'].tolist())

import matplotlib.pyplot as plt
%matplotlib inline

explained_var = svd.explained_variance_ratio_.cumsum()
plt.plot(explained_var, '.-')
plt.xlabel("SVD components")
plt.ylabel("cumulative var explained(%)")
plt.show()

latent_matrix.shape

ratings_df.head()

ratings_df1 = pd.merge(movies[["movieId"]], ratings_df, on="movieId", how = "right")

ratings_df1.head()

ratings_df2 = ratings_df1.pivot(index='movieId', columns='userId', values = 'rating').fillna(0)
ratings_df2.head()

ratings_df2.shape

svd = TruncatedSVD(n_components=200)
latent_matrix = svd.fit_transform(ratings_df2)
latent_matrix_df2 = pd.DataFrame(latent_matrix, index = final_df['title'].tolist())

latent_matrix_df2.head()

explained_var = svd.explained_variance_ratio_.cumsum()
plt.plot(explained_var, '.-')
plt.xlabel("SVD components")
plt.ylabel("cumulative var explained(%)")
plt.show()

from sklearn.metrics.pairwise import cosine_similarity

latent_matrix_df1.head()

latent_matrix_df2.head()

# Check similaruty of movie with content and collaboratice matricess
movie_content_vector = np.array(latent_matrix_df1.loc['Toy Story (1995)']).reshape(1,-1)
movie_collab_vector = np.array(latent_matrix_df2.loc['Toy Story (1995)']).reshape(1,-1)

score_1 = cosine_similarity(latent_matrix_df1, movie_content_vector).reshape(-1)
score_2 = cosine_similarity(latent_matrix_df2, movie_collab_vector).reshape(-1)

#average score 
av_score = (score_1 + score_2)/2.0

movie_sim = {'content':score_1, 'collab': score_2, 'hybrid':av_score}
simil_df = pd.DataFrame(movie_sim, index=latent_matrix_df1.index)

simil_df.head()

simil_df.sort_values('content', ascending=False) # based on movie content toystory is similar to toy story2

simil_df.sort_values('collab', ascending=False) # based on user who likes Toystory would like Toystory2 

simil_df.sort_values('hybrid', ascending=False)