个性化推荐系统——1. 数据集探索

- 日理万妓 2023-07-03 04:35 69阅读 0赞

使用MovieLens数据集

# 下载数据集 #

下面这段代码写得很完备，很高大上

import pandas as pd
    from sklearn.model_selection import train_test_split
    import numpy as np
    import seaborn as sns
    from collections import Counter
    import tensorflow as tf
    
    import os
    import pickle
    import re
    from tensorflow.python.ops import math_ops

from urllib.request import urlretrieve
    from os.path import isfile, isdir
    from tqdm import tqdm
    import zipfile
    import hashlib
    
    def _unzip(save_path, _, database_name, data_path):
        """ Unzip wrapper with the same interface as _ungzip :param save_path: The path of the gzip files :param database_name: Name of database :param data_path: Path to extract to :param _: HACK - Used to have to same interface as _ungzip """
        print('Extracting {}...'.format(database_name))
        with zipfile.ZipFile(save_path) as zf:
            zf.extractall(data_path)
    
    def download_extract(database_name, data_path):
        """ Download and extract database :param database_name: Database name """
        DATASET_ML1M = 'ml-1m'
    
        if database_name == DATASET_ML1M:
            url = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip'
            hash_code = 'c4d9eecfca2ab87c1945afe126590906'
            extract_path = os.path.join(data_path, 'ml-1m')
            save_path = os.path.join(data_path, 'ml-1m.zip')
            extract_fn = _unzip
    
        if os.path.exists(extract_path):
            print('Found {} Data'.format(database_name))
            return
    
        if not os.path.exists(data_path):
            os.makedirs(data_path)
    
        if not os.path.exists(save_path):
            with DLProgress(unit='B', unit_scale=True, miniters=1, desc='Downloading {}'.format(database_name)) as pbar:
                urlretrieve(
                    url,
                    save_path,
                    pbar.hook)
    
        assert hashlib.md5(open(save_path, 'rb').read()).hexdigest() == hash_code, \
            '{} file is corrupted. Remove the file and try again.'.format(save_path)
    
        os.makedirs(extract_path)
        try:
            extract_fn(save_path, extract_path, database_name, data_path)
        except Exception as err:
            shutil.rmtree(extract_path)  # Remove extraction folder if there is an error
            raise err
    
        print('Done.')
        # Remove compressed data
    # os.remove(save_path)
    
    class DLProgress(tqdm):
        """ Handle Progress Bar while Downloading """
        last_block = 0
    
        def hook(self, block_num=1, block_size=1, total_size=None):
            """ A hook function that will be called once on establishment of the network connection and once after each block read thereafter. :param block_num: A count of blocks transferred so far :param block_size: Block size in bytes :param total_size: The total size of the file. This may be -1 on older FTP servers which do not return a file size in response to a retrieval request. """
            self.total = total_size
            self.update((block_num - self.last_block) * block_size)
            self.last_block = block_num

data_dir = './'
    download_extract('ml-1m', data_dir)

Found ml-1m Data

# 数据探索 #

import pandas as pd
    import matplotlib.pyplot as plt

## rating ##

def getRatings(file_path):
        rates = pd.read_table(
            file_path,
            header=None,
            sep="::",
            names=["userID", "movieID", "rate", "timestamp"],
        )
        print("userID的范围为: <{},{}>"
              .format(min(rates["userID"]), max(rates["userID"])))
        print("movieID的范围为: <{},{}>"
              .format(min(rates["movieID"]), max(rates["movieID"])))
        print("评分值的范围为: <{},{}>"
              .format(min(rates["rate"]), max(rates["rate"])))
        print("数据总条数为:\n{}".format(rates.count()))
        print("数据前5条记录为:\n{}".format(rates.head(5)))
        df = rates["userID"].groupby(rates["userID"])
        print("用户评分记录最少条数为：{}".format(df.count().min()))
    
        scores = rates["rate"].groupby(rates["rate"]).count()
        # 图上添加数字
        for x, y in zip(scores.keys(), scores.values):
            plt.text(x, y + 2, "%.0f" % y, ha="center", va="bottom", fontsize=12)
        plt.bar(scores.keys(), scores.values, fc="r", tick_label=scores.keys())
        plt.xlabel("Rating Score")
        plt.ylabel("People Number")
        plt.title("Rating Scores And related People Number")
        plt.show()

getRatings("./ml-1m/ratings.dat")

/home/gz/anaconda3/envs/tf2/lib/python3.7/site-packages/ipykernel_launcher.py:6: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.
    
    
    
    userID的范围为: <1,6040>
    movieID的范围为: <1,3952>
    评分值的范围为: <1,5>
    数据总条数为:
    userID       1000209
    movieID      1000209
    rate         1000209
    timestamp    1000209
    dtype: int64
    数据前5条记录为:
       userID  movieID  rate  timestamp
    0       1     1193     5  978300760
    1       1      661     3  978302109
    2       1      914     3  978301968
    3       1     3408     4  978300275
    4       1     2355     5  978824291
    用户评分记录最少条数为：20

![\[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-YxD37vHb-1580564733297)(output\_9\_2.png)\]][img-YxD37vHb-1580564733297_output_9_2.png]

## Movies ##

def getMovies(file_path):
        movies = pd.read_table(
            file_path,
            header=None,
            sep="::",
            names=["movieID", "title", "genres"]
        )
    
        print("movieID的范围为: <{},{}>"
              .format(min(movies["movieID"]), max(movies["movieID"])))
        print("数据总条数为:\n{}".format(movies.count()))
        # 电影分类统计
        moviesDict = dict()
        for line in movies["genres"].values:
            for one in line.split("|"):
                moviesDict.setdefault(one, 0)
                moviesDict[one] += 1
    
        print("电影类型总数为:{}".format(len(moviesDict)))
        print("电影类型分别为:{}".format(moviesDict.keys()))
        print(moviesDict)
    
        newMD = sorted(moviesDict.items(), key=lambda x: x[1], reverse=True)
        # 设置标签
        labels = [newMD[i][0] for i in range(len(newMD))]
        values = [newMD[i][1] for i in range(len(newMD))]
        # 与labels对应，数值越大离中心区越远
        explode = [x * 0.01 for x in range(len(newMD))]
        # 设置X轴 Y轴比例
        plt.axes(aspect=1)
        # labeldistance表示标签离中心距离，pctdistance表示百分百数据离中心区距离
        # autopct表示百分比的格式，shadow表示阴影
        plt.pie(
            x=values,
            labels=labels,
            explode=explode,
            autopct="%3.1f %%",
            shadow=False,
            labeldistance=1.1,
            startangle=0,
            pctdistance=0.8,
            center=(-1, 0),
        )
        # 控制位置：在bbox_to_anchor数组中，前者控制左右移动，后者控制上下
        # ncol控制图例所列的列数，默认为1
        plt.legend(loc=7, bbox_to_anchor=(1.3, 1.0), ncol=3, fancybox=True, shadow=True, fontsize=6)
        plt.show()

getMovies("./ml-1m/movies.dat")

/home/gz/anaconda3/envs/tf2/lib/python3.7/site-packages/ipykernel_launcher.py:6: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.
    
    
    
    movieID的范围为: <1,3952>
    数据总条数为:
    movieID    3883
    title      3883
    genres     3883
    dtype: int64
    电影类型总数为:18
    电影类型分别为:dict_keys(['Animation', "Children's", 'Comedy', 'Adventure', 'Fantasy', 'Romance', 'Drama', 'Action', 'Crime', 'Thriller', 'Horror', 'Sci-Fi', 'Documentary', 'War', 'Musical', 'Mystery', 'Film-Noir', 'Western'])
    {'Animation': 105, "Children's": 251, 'Comedy': 1200, 'Adventure': 283, 'Fantasy': 68, 'Romance': 471, 'Drama': 1603, 'Action': 503, 'Crime': 211, 'Thriller': 492, 'Horror': 343, 'Sci-Fi': 276, 'Documentary': 127, 'War': 143, 'Musical': 114, 'Mystery': 106, 'Film-Noir': 44, 'Western': 68}

![\[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-XlU4xku0-1580564733298)(output\_12\_2.png)\]][img-XlU4xku0-1580564733298_output_12_2.png]

## User ##

def getUsers(file_path):
        users = pd.read_table(
            file_path,
            header=None,
            sep="::",
            names=["userID", "gender", "age", "Occupation", "zip-code"],
        )
        print("userID的范围为: <{},{}>".format(min(users["userID"]), max(users["userID"])))
        print("数据总条数为:\n{}".format(users.count()))
    
        usersGender = users["gender"].groupby(users["gender"]).count()
        print(usersGender)
    
        plt.axes(aspect=1)
        plt.pie(x=usersGender.values, labels=usersGender.keys(), autopct="%3.1f %%")
        plt.legend(bbox_to_anchor=(1.0, 1.0))
        plt.show()
    
        usersAge = users["age"].groupby(users["age"]).count()
        print(usersAge)
    
        plt.plot(
            usersAge.keys(),
            usersAge.values,
            label="用户年龄信息展示",
            linewidth=3,
            color="r",
            marker="o",
            markerfacecolor="blue",
            markersize=12,
        )
        # 图上添加数字
        for x, y in zip(usersAge.keys(), usersAge.values):
            plt.text(x, y+10, "%.0f" % y, ha="center", va="bottom", fontsize=12)
        plt.xlabel("Age")
        plt.ylabel("Users Number")
        plt.title("Age of users")
        plt.show()

getUsers("./ml-1m/users.dat")

![\[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-SscEGWdg-1580564733299)(output\_15\_2.png)\]][img-SscEGWdg-1580564733299_output_15_2.png]

age
    1      222
    18    1103
    25    2096
    35    1193
    45     550
    50     496
    56     380
    Name: age, dtype: int64

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoZXJwYWh1_size_16_color_FFFFFF_t_70]

[img-YxD37vHb-1580564733297_output_9_2.png]: https://img-blog.csdnimg.cn/2020020121463922.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoZXJwYWh1,size_16,color_FFFFFF,t_70
[img-XlU4xku0-1580564733298_output_12_2.png]: https://img-blog.csdnimg.cn/20200201214622220.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoZXJwYWh1,size_16,color_FFFFFF,t_70
[img-SscEGWdg-1580564733299_output_15_2.png]: https://img-blog.csdnimg.cn/20200201214555551.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoZXJwYWh1_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200201214550815.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoZXJwYWh1,size_16,color_FFFFFF,t_70