第二周第三课时

#断点续传功能，来自周作业的main函数
from multiprocessing import Pool
from Gchannel_extract import All_channnel_links
from Gpage_parsing import get_detailinfo, getdetail_links, detail_info, detail_urls


def get_all_links(channel):
    for i in range(1, 100):
        getdetail_links(channel, i)
if __name__ == '__main__':
    pool = Pool()
    pool.map(get_all_links, All_channnel_links.split())
    #断点续传功能
    db_urls = [item['url'] for item in detail_urls.find()]
    index_urls = [item['url'] for item in detail_info.find()]
    x = set(db_urls)
    y = set(index_urls)
    rest_urls = x - y
    pool.map(get_detailinfo, rest_urls)