美文网首页
async_generator

async_generator

作者: Sharp_Yi | 来源:发表于2019-05-05 13:19 被阅读0次
@async_generator
async def fetch(url, row, loop):
    async with ClientSession() as ses:
        if choice_:
            pro = await get_proxy(loop)
        else:
            pro = None
        async with ses.get(url, headers=HEADERS, proxy=pro) as res:
            with closing(res) as resp:
                text = await resp.text()
                response = Selector(text)
                tr_list = response.xpath(row[xquery.c.table_tr_query])
                for tr in tr_list:
                    ip = tr.xpath(
                        row[xquery.c.extract_ip_query]).extract_first()
                    port = tr.xpath(
                        row[xquery.c.extract_port_query]).extract_first()
                    http_scheme = tr.xpath(
                        row[xquery.c.extract_type_query]).extract_first()
                    timeout = tr.xpath(
                        row[xquery.c.extract_timeout_query]).extract_first()
                    validate_time = tr.xpath(
                        row[xquery.c.extract_validate_query]).extract_first()
                    if ip:
                        await yield_({'ip': ip, 'port': port, 'http_scheme': http_scheme, 'timeout': timeout.replace('秒', ''),
                                      'validate_time': validate_time})
                    HEADERS['Referer'] = url
            if res.status > 400:
                logging.error(f'{url}:{res.status}')
                await yield_from_(fetch(url, row, loop))

相关文章

网友评论

      本文标题:async_generator

      本文链接:https://www.haomeiwen.com/subject/sgicoqtx.html