基于词嵌入的词语分析代码说明

1）在PyCharm中导入所需的gensim库（点击代码可复制）：

        pip install -i  https://mirrors.aliyun.com/pypi/simple/ gensim

2）将数据文件转换为二进制格式，完整代码为（点击代码可复制）：

        from gensim.models import KeyedVectors

        file = 'tencent-ailab-embedding-zh-d200-v0.2.0-s.txt'
        model = KeyedVectors.load_word2vec_format(file, binary=False)
        model.save('Tencent_AILab_ChineseEmbedding.bin')

3）查看当前所有词语的词嵌入表示，完整代码为（点击代码可复制）：

        from gensim.models import KeyedVectors

        model = KeyedVectors.load('Tencent_AILab_ChineseEmbedding.bin')
        print(model['紫金山'])

4）查看词语的同义词，完整代码为（点击代码可复制）：

        from gensim.models import KeyedVectors

        model = KeyedVectors.load('Tencent_AILab_ChineseEmbedding.bin')
        print(model.most_similar('紫金山', 10))

5）对任意词语相关性的测度，完整代码为（点击代码可复制）：

        from gensim.models import KeyedVectors

        model = KeyedVectors.load('Tencent_AILab_ChineseEmbedding.bin')
        print(model.similarity('南京', '江苏'))
        print(model.similarity('南京', '南京路'))

6）找出不同意义的词语，完整代码为（点击代码可复制）：

        from gensim.models import KeyedVectors

        model = KeyedVectors.load('Tencent_AILab_ChineseEmbedding.bin')
        print(model.doesnt_match("上海路 宝马车 广州塔 北京电视台".split()))

7）可视化词语聚类，完整代码为（点击代码可复制）：

        import jieba
        from sklearn.manifold import TSNE
        import numpy as np
        import random
        import matplotlib.pyplot as plt
        from gensim.models import KeyedVectors

        w2v_model = KeyedVectors.load('Tencent_AILab_ChineseEmbedding.bin')
        terms = ['狮子', '老虎', '熊猫', '大象', '长颈鹿', '猴子', '狐狸', '兔子',
                    '猫', '狗', '孔子', '秦始皇', '武则天', '李白', '诸葛亮', '岳飞',
                    '成吉思汗', '朱元璋', '郑成功', '康熙', '丰田', '本田', '日产',
                    '宝马', '奔驰', '奥迪', '特斯拉', '保时捷', '路虎', '捷豹', '北京',
                    '上海', '广州', '深圳', '杭州', '西安', '成都', '重庆', '南京',
                    '武汉', '牡丹', '玫瑰', '菊花', '莲花', '梅花', '郁金香', '茉莉花',
                    '百合', '康乃馨', '山茶花']

        def reduce_dimensions(vectors, labels, num_dimensions=2):
            vectors = np.asarray(vectors)
            tsne = TSNE(n_components=num_dimensions, random_state=0)
            vectors = tsne.fit_transform(vectors)
            x_vals = np.asarray([v[0] for v in vectors])
            y_vals = np.asarray([v[1] for v in vectors])
            return x_vals, y_vals, labels

        def plot_word2vec(x_vals, y_vals, words):
            plt.rcParams["font.sans-serif"] = ["SimHei"]
            plt.rcParams["axes.unicode_minus"] = False
            random.seed(0)
            plt.figure(figsize=(40, 40), dpi=500)
            plt.scatter(x_vals, y_vals, c='y')
            for i in list(range(len(words))):
                plt.text(x_vals[i], y_vals[i], words[i],
                         fontdict={
                             'fontsize': 3,
                             'color': "black",
                         }
                         )
            plt.show()

        terms_vector = {}
        for place in terms:
            if place not in w2v_model:
                words = jieba.lcut(place)
                vector = np.zeros((200,), dtype='float32')
                for word in words:
                    vector += w2v_model[word]
                terms_vector[place] = vector / len(words)
            else:
                terms_vector[place] = w2v_model[place]
        labels = []
        vectors = []
        for l, v in terms_vector.items():
            labels.append(l)
            vectors.append(v)
        x, y, words = reduce_dimensions(vectors, labels, num_dimensions=2)
        plot_word2vec(x, y, words)