介绍

​ 今日从AINLP的公众号上面看到一个推送，关于短文本聚类的工具，刚好公司项目有用到短文本聚类，就进去看看。项目不复杂，使用了jaccard相似度量方法，当然也可以修改度量办法。其特点是内存友好，对于大批量的数据聚类有帮助。

代码解析

cluster.py

cluster.py文件的main方法是工具的主入口

def main():
    args = _get_parser()

    # preliminary work
    check_file(args.infile)
    ensure_dir(args.output)
    # ==检查并确保输入文件和输出文件夹的存在
    if args.name_len_update:
        line_cnt = line_counter(args.infile)
        args.name_len = len(str(line_cnt)) + 1
    # ==删除输出文件夹中与聚类输出文件名模式相匹配的文件
    clean_dir(args.output, args.name_len)
    # end preliminary work

    p_bucket = defaultdict(list)
    save_idx = 0
    id_name = '{0:0' + str(args.name_len) + 'd}'
    # load stop words
    # ==载入停用词
    stop_words = get_stop_words(args.stop_words) if os.path.exists(args.stop_words) else list()
    # load tokenizer
    # ==建立分词类，分词有两种，支持中英文
    seg = Segmentor(args)

    print('Splitting sentence into different clusters ...')
    infile = open(args.infile, 'r', encoding="utf-8")
    for line in tqdm(infile):
    # 遍历输入文件的每一行
        line = line.rstrip()
        is_match = False
        # 分词并去除停用词
        seg_list = list(seg.cut(line))
        if stop_words:
            seg_list = list(filter(lambda x: x not in stop_words, seg_list))
        for wd in seg_list:
        # ==p_bucket是一个根据词汇来查找已有聚类簇的列表，记录的列表名
            w_bucket = p_bucket[wd]
            # is_match = False
            for bucket in w_bucket:
            # == 如果某个聚类簇包含该词汇，那么就去该列表下面寻找已经有的样本行，不足五行，就全选，反之采样五行。
                bucket_path = os.path.join(args.output, bucket)
                check_file(bucket_path)
                selected = sample_file(bucket_path, args.sample_number)
                #  == 将选取的样本进行分词并去除通用词
                selected = list(map(lambda x: list(seg.cut(x)), selected))
                if stop_words:
                    filt_selected = list()
                    for sen in selected:
                        sen = list(filter(lambda x: x not in stop_words, sen))
                        filt_selected.append(sen)
                    selected = filt_selected
                # ==如果待分析样本与采样选取的每个样本都满足jaccard系数大于阈值，就归入该聚类簇
                if all(jaccard(seg_list, cmp_list) > args.threshold for cmp_list in selected):
                    is_match = True
                    with open(bucket_path, 'a', encoding='utf-8') as outfile:
                        outfile.write(line+'\n')
                    break
            # == 这里应该是一个不足之处，因为如果在这里就进行判断的话，无法对不同顺序单词组成的句子进行聚类
            
            if not is_match:
                bucket_name = ('tmp' + id_name).format(save_idx)
                w_bucket.append(bucket_name)
                bucket_path = os.path.join(args.output, bucket_name)
                with open(bucket_path, 'a', encoding='utf-8') as outfile:
                    outfile.write(line+'\n')
                save_idx += 1
                break

    infile.close()

    # sort and rename file
    file_list = os.listdir(args.output)
    file_list = list(filter(lambda x: x.startswith('tmp'), file_list))
    cnt = dict()
    for file in file_list:
        file_path = os.path.join(args.output, file)
        cnt[file] = line_counter(file_path)

    sorted_cnt = sorted(cnt.items(), key=lambda kv: kv[1], reverse=True)
    for idx, (file_name, times) in enumerate(sorted_cnt):
        origin_path = os.path.join(args.output, file_name)
        new_path = os.path.join(args.output, id_name.format(idx))
        os.rename(origin_path, new_path)

    print('All is well')

评价

优点：内存友好

缺点：项目有个bug，无法将不同顺序单词组成的句子归到同一类中，建议加入原有的聚类簇之后，不要break，只是不要再进行聚类类别判定或者新建，而对该样本的每个单词的p_bucket列表都加入该类，同时需要调整新建类别的逻辑，如果第一个单词匹配不通过，建立一个tmp词汇，将该单词放入，然后对下一个单词进行判定，直到所有的都不通过，才建立新的类别，并将tmp词汇里的p_bucket都加上这个新的类别。

项目地址

原始项目地址：（已经提了个issue，可能会修改这个问题

https://github.com/RandyPen/TextCluster

有bug的项目fork了一份：

https://github.com/612yese/TextCluster

追加一

def main():
    args = _get_parser()

    # preliminary work
    check_file(args.infile)
    ensure_dir(args.output)

    if args.name_len_update:
        line_cnt = line_counter(args.infile)
        args.name_len = len(str(line_cnt)) + 1

    clean_dir(args.output, args.name_len)
    # end preliminary work

    p_bucket = defaultdict(list)
    save_idx = 0
    id_name = '{0:0' + str(args.name_len) + 'd}'
    # load stop words
    stop_words = get_stop_words(args.stop_words) if os.path.exists(args.stop_words) else list()
    # load tokenizer
    seg = Segmentor(args)

    print('Splitting sentence into different clusters ...')
    infile = open(args.infile, 'r', encoding="utf-8")
    for line in tqdm(infile):
        line = line.rstrip()
        is_match = False
        seg_list = list(seg.cut(line))
        if stop_words:
            seg_list = list(filter(lambda x: x not in stop_words, seg_list))
        for wd in seg_list:
        ################# 追加之处 begin ####################
            if is_match:
                break
        ################# 追加之处 end ####################
            w_bucket = p_bucket[wd]
            for bucket in w_bucket:
                bucket_path = os.path.join(args.output, bucket)
                check_file(bucket_path)
                selected = sample_file(bucket_path, args.sample_number)
                selected = list(map(lambda x: list(seg.cut(x)), selected))
                # remove stop words
                if stop_words:
                    filt_selected = list()
                    for sen in selected:
                        sen = list(filter(lambda x: x not in stop_words, sen))
                        filt_selected.append(sen)
                    selected = filt_selected
                # calculate similarity with each bucket
                if all(jaccard(seg_list, cmp_list) > args.threshold for cmp_list in selected):
                    is_match = True
                    with open(bucket_path, 'a', encoding='utf-8') as outfile:
                        outfile.write(line+'\n')
################# 追加之处 begin ####################
                    for w in seg_list:
                        if bucket not in p_bucket[w]:
                            p_bucket[w].append(bucket)
################# 追加之处 end ####################
                    break
################追加改变，缩进改变##################
        if not is_match:
            bucket_name = ('tmp' + id_name).format(save_idx)
            bucket_path = os.path.join(args.output, bucket_name)
            with open(bucket_path, 'a', encoding='utf-8') as outfile:
                outfile.write(line+'\n')
################# 追加之处 begin ####################
            for w in seg_list:
                p_bucket[w].append(bucket_name)
################# 追加之处 end ####################
            save_idx += 1

    infile.close()

    # sort and rename file
    file_list = os.listdir(args.output)
    file_list = list(filter(lambda x: x.startswith('tmp'), file_list))
    cnt = dict()
    for file in file_list:
        file_path = os.path.join(args.output, file)
        cnt[file] = line_counter(file_path)

    sorted_cnt = sorted(cnt.items(), key=lambda kv: kv[1], reverse=True)
    for idx, (file_name, times) in enumerate(sorted_cnt):
        origin_path = os.path.join(args.output, file_name)
        new_path = os.path.join(args.output, id_name.format(idx))
        os.rename(origin_path, new_path)

    print('All is well')

作者的追加显然是可以解决该问题的，将新增的聚类逻辑放大了单词for循环之外，对于已有聚类簇，也将该类加入到所有单词的bucket中。