2020-03-12

const Base = require('./base.js');
const puppeteer = require('puppeteer');

module.exports = class extends Base {

    async haiwaiListAction() {
        const browser = await puppeteer.launch({
            // 不显示浏览器界面
            headless: true,
            // 浏览器界面路径，安装时自动下载了可以不用设置
            executablePath: 'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe',
        });

        let page = await browser.newPage();
        page.setDefaultNavigationTimeout(30000);

        // 打开页面，直到dom加载完毕
        await page.goto('https://usa.haiwai.anjuke.com/pg1/', {waitUntil: 'domcontentloaded'});
        const modelHouseHaiwaiList = think.getModel('house_anjuke_haiwai_list');
        let nextPageLink = null;

        do {
            // 以下为每个列表项中涉及的字段，均使用css选择器写法，css、xpath规则参考文档链接
            // 选择当前页所有列表项
            const haiwaiList = await page.$$('div#content div.item-mod.const-sd', divs => divs);
            for (const item of haiwaiList) {
                // 涉及具体的API使用，参考中文文档
                // https://zhaoqize.github.io/puppeteer-api-zh_CN/#?product=Puppeteer&version=v2.1.1&show=api-pageclickselector-options

                const detail_url = await item.$eval('a', a => a.href);

                const image_url = await item.$eval('div.pic-container img.pic', img => img.src);

                const title = await item.$eval('.item-title h3', h3 => h3.innerText);

                // 有的有这两个字段，有的没有，可以通过page.$先定位选择器，如果选择器不存在，则不处理
                const hui = await item.$eval('.item-title i', i => i.innerText).catch(e => 0);
                const layout = await item.$eval('.item-info p.type', p => p.innerText).catch(e => "");

                const city = await item.$eval('.item-info p.loc', p => p.innerText);

                const new_or_old = await item.$eval('.item-info p.loc+p', p => p.innerText);

                const property_type = await item.$eval('.item-info p.serv', p => p.innerText);

                const price_dollar = await item.$eval('.item-price span:first-child', span => span.innerText);

                const price_rmb = await item.$eval('.item-price span:last-child', span => span.innerText);

                // 简单去重
                const exist = await modelHouseHaiwaiList.where({detail_url}).count();
                if (!think.isEmpty(exist)) {
                    continue;
                }

                const id = await modelHouseHaiwaiList.add({
                    detail_url,
                    image_url,
                    title,
                    city,
                    new_or_old,
                    property_type,
                    layout,
                    price_dollar,
                    price_rmb
                });
            }

            // <span class="next-page stat-disable">»</span>
            // <a href="//usa.haiwai.anjuke.com/pg4/" class="next-page next-link">»</a>
            nextPageLink = await page.$eval('.pagination .next-page', a => a ? a.href : null);

            // https://zhaoqize.github.io/puppeteer-api-zh_CN/#?product=Puppeteer&version=v2.1.1&show=api-pageclickselector-options
            // await Promise.all([
            //     await page.waitForNavigation(),
            //     await page.click('.pagination .next-page')
            // ]);

            if (nextPageLink) {
                await page
                    .goto(nextPageLink, {waitUntil: 'domcontentloaded', timeout: 30000})
                    .catch(async (e) => {
                        think.logger.debug(e);
                        console.log('...... error, nextPageLink:' + nextPageLink);
                    });
            }

        } while (nextPageLink)

        await browser.close();
    }

}