网页抓取保存到本地和解析

网页抓取保存到本地

savedata_Chrome_byurl.py

---

from selenium import webdriver

import time

import io

import csv

import pymysql

import os

import re

from lxml import etree

import codecs

def savepage(browser, filepath, pagename):

    try:

      if not os.path.exists(filepath):

            os.mkdir(filepath)

        textContent = browser.find_element_by_xpath('//html').get_attribute('outerHTML')

        str_utf8 = textContent.encode("UTF-8")

        textContent = str_utf8.decode('UTF-8', 'strict')

        pagepath = filepath +'//'+ pagename + '.html'

        fp = open(pagepath, "w", encoding='UTF-8');

        fp.write(textContent);

        fp.close()

    except Exception as excpt:

        print(excpt)

def getDbConn(db):

    isonserver = True

    osname = os.name

    if osname == 'nt':

        isonserver = False

        print('windows')

    else:

        isonserver = True

        print(os.name)

    isonserver = False

    if isonserver:

        host = 'localhost'

        user = 'root'

        passwd = '123456'

    else:

        host = ''

        user = ''

        passwd = ''

    # db = 'couponcategory'

    port = 3306

    conn = pymysql.connect(host=host, port=port, user=user, password=passwd, db=db)

    return conn

def parse_data_page_step1(browser, url, urlname):

    print('doing.......')

    creditcard__items = browser.find_elements_by_xpath('//div[@class="creditcard__item"]')

    for creditcard__item in creditcard__items:

        try:

            # title = creditcard__item.find_element_by_xpath('.//h2[@class="creditcard__heading"]').get_attribute('textContent')

            article = creditcard__item.find_element_by_xpath('./article');

            href = article.find_element_by_xpath('./div[@class="compare"]').find_element_by_xpath('./div[last()]/a').get_attribute('href')

            # .get_attribute('href')

            item = {}

            item['url'] = url

            item['url2'] = href

            item['info0'] = urlname

            # item['info1'] = title

            print(urlname)

            print(url)

            print(href)

            stu1 = [url, href, urlname, '']

            out = open('fix10004.csv', 'a', newline='')

            # out = open('d:/data_source10004_v1.csv', 'a', newline='')

            # 设定写入模式

            csv_write = csv.writer(out, dialect='excel')

            # 写入具体内容

            csv_write.writerow(stu1)

            out.close()

        except Exception as aas:

            print(aas)

        # print('write item.............................................')

        # print(item)

        # dbname = 'brcardsdata'

        # dbtablename = 'data_source10004_url_v2'

        # updateToDatabase(dbname, dbtablename, item)

        # print('write item..............................................')

def get_key_url_map(dbname, tablename):

    conn = getDbConn(dbname)

    cursor = conn.cursor()

    print("mysql connect success")

    sql = "select url,pagecode from " + tablename

    cursor.execute(sql)

    dataresult = cursor.fetchall()

    conn.close()

    return dataresult

def scrapyStart1(browser, url, pagecode):

    # 返回一个

    # get_attribute('textContent')

    # get_attribute('innerHTML')

    # get_attribute('outerHTML')

    print('4')

    time.sleep(1)

    print('6')

    browser.get(url)

    print('7')

    time.sleep(5)

    print('8')

    try:

        savepage(browser, '10004', pagecode)

    except Exception as errr:

        print('........currpage....error......................')

        print(errr)

    try:

        targetElem = browser.find_element_by_xpath('//div[@class="pagehero__button"]')

        browser.execute_script("arguments[0].focus();", targetElem)

        time.sleep(0.5)

        targetElem.click()

        time.sleep(1.8)

        print('8')

        pagecode2 = pagecode + '_nextpage'

        savepage(browser, '10004', pagecode2)

    except Exception as eerr:

        print('........nextpage....error......................')

        print(eerr)

# re.sub(r'\?.*','',url)

browser = webdriver.Chrome()

time.sleep(0.5)

browser.maximize_window()

time.sleep(1)

key_url_map = get_key_url_map('pagedata', 'data_source10004_url')

# key_url_map = [['https://www.foregon.com/solicitar/cartaodecredito/agillitas/fgn/cartao-pre-pago-agillitas-mundo-livre-visa/1028','1']]

for key_url in key_url_map:

    url = key_url[0]

    pagecode = key_url[1]

    pagecode = str(pagecode)

    print(url)

    scrapyStart1(browser, url, pagecode)

time.sleep(100)

browser.close()

parsepagedata.py

---

from selenium import webdriver

import time

import io

import csv

import pymysql

import os

import re

from lxml import etree

from bs4 import BeautifulSoup

import numpy as np

import codecs

def etreeWebElemToOuterHtml(webitem):

    outerHtml = etree.tostring(webitem)

    outerHtml = outerHtml.decode('utf-8')

    return outerHtml

def trimDataHtmlProAndImg(divstr):

    divstr = re.sub(r' href=".*?"', "", divstr)

    divstr = re.sub(r' class=".*?"', "", divstr)

    divstr = re.sub(r' target=".*?"', "", divstr)

    divstr = re.sub(r' align=".*?"', "", divstr)

    divstr = re.sub(r' rel=".*?"', "", divstr)

    divstr = re.sub(r'<img.*?>', "", divstr)

    divstr = re.sub(r' data-cfemail=".*?"', "", divstr)

    divstr = re.sub(r' id=".*?"', "", divstr)

    divstr = re.sub(r' name=".*?"', "", divstr)

    divstr = re.sub(r' style=".*?"', "", divstr)

    divstr = re.sub(r' src=".*?"', "", divstr)

    divstr = re.sub(r' dir=".*?"', "", divstr)

    divstr = re.sub(r'<div .*?>', "<p>", divstr)

    divstr = re.sub(r'<strong .*?>', "<p>", divstr)

    divstr = re.sub(r'<a .*?</a>', "", divstr)

    divstr = re.sub(r'<p .*?>', "<p>", divstr)

    divstr = re.sub(r'<button .*?</button>', "", divstr)

    divstr = divstr.replace('<div>', '<p>')

    divstr = divstr.replace('<strong>', '<p>')

    divstr = divstr.replace('</div>', '</p>')

    divstr = divstr.replace('</strong>', '</p>')

    return divstr

def loadpage(filepath, pagename):

    try:

        pagepath = filepath + '//' + pagename + '.html'

        htmlf = open(pagepath,'r',encoding="utf-8")

        htmlContent = htmlf.read()

        return htmlContent

    except Exception as excpt:

        print(excpt)

    return ''

def parseWithBeautifulSoup(htmlContent):

    soup = BeautifulSoup(htmlContent, 'lxml')

    mululist = soup.find_all(class_='mulu')

def parseWithXpath(htmlContent):

    html = etree.HTML(htmlContent)

    mululist = html.xpath('.//*[@class="mulu"]')

def getDbConn(db):

    isonserver = True

    osname = os.name

    if osname == 'nt':

        isonserver = False

        print('windows')

    else:

        isonserver = True

        print(os.name)

    isonserver = False

    if isonserver:

        host = 'localhost'

        user = 'root'

        passwd = '123456'

    else:

        host = ''

        user = ''

        passwd = ''

    port = 3306

    conn = pymysql.connect(host=host, port=port, user=user, password=passwd, db=db)

    return conn

def updateToDatabase(dbname, tablename, item):

    url2 = item['url2']

    updatevalue = {'url2': url2}

    setsqllist = []

    collist = ['info0', 'info1', 'info2', 'info3', 'info4', 'info5', 'info6', 'info7', 'info8', 'info9', 'info10', 'url']

    for idx in range(len(collist)):

        colname = collist[idx]

        if colname in item:

            if item[colname]:

                updatevalue[colname] = item[colname]

                setsqllist.append(colname + '=%(' + colname + ')s')

    setsqllistlen = len(setsqllist)

    if setsqllistlen > 0:

        updatesql = 'update ' + tablename + ' set '

        setsqlliststr = ','.join(setsqllist)

        wherestr = ' where url2=%(url2)s'

        updatesql = updatesql + setsqlliststr + wherestr

        print(updatesql)

        # print(updatevalue)

        conn = getDbConn(dbname)

        cursor = conn.cursor()

        try:

            cursor.execute(updatesql, updatevalue)

        except Exception as e:

            print('Insert Error1', e)

            conn.rollback()

        else:

            conn.commit()

        conn.close()

def parse_data_page_step1(htmlContent, pageid):

    print('doing.......')

    html = etree.HTML(htmlContent)

    divcon = html.xpath('//div[@class="pagehero__content"]')[0]

    str1 = divcon.xpath('./div[@class="pagehero__wrapper"]/h1[@class="pagehero__heading"]')[0].text

    str2 = divcon.xpath('./div[@class="pagehero__wrapper"]/strong[@class="pagehero__description"]')[0].text

    item = {}

    item['url2'] = url

    item['info1'] = str1

    item['info8'] = str2

    print('write item.............................................')

    print(item)

    # dbname = 'brcardsdata'

    # dbtablename = 'data_source10004_url'

    # updateToDatabase(dbname, dbtablename, item)

    print('write item..............................................')

def parse_data_page_step2(htmlContent, pageid):

    print('doing.......')

    html = etree.HTML(htmlContent)

    itemlist= html.xpath('//div[@class="box--list"]/div[@class="box--list-item"]')

    info5 = ''

    info6 = ''

    info7 = ''

    info10 = ''

    for item in  itemlist:

        itemcon = item.xpath('./div[@class="box--container"]')[0]

        str1 = itemcon.xpath('./div[@class="box--header"]/h3')[0].text

        print(str1)

        itemconbody = itemcon.xpath('./div[@class="box--body"]')[0]

        str1 = str1.lower()

        str1 = str1.strip()

        # print(str1)

        if str1 == 'online':

            str2item = itemconbody.xpath('./div[contains(@class,"notsignedin")]')[0]

            str2 = etreeWebElemToOuterHtml(str2item)

            # print(str2)

            str2 = trimDataHtmlProAndImg(str2)

            str2 = str2.replace('<a></a>', '')

            info5 = '<p>' + str2 + '</p>'

            print('info5')

            print(info5)

        if str1 == 'no local':

            str2item = itemconbody

            str2 = etreeWebElemToOuterHtml(str2item)

            str2 = trimDataHtmlProAndImg(str2)

            info6 = '<p>' + str2 + '</p>'

        if str1 == 'por telefone':

            str2item = itemconbody

            str2 = etreeWebElemToOuterHtml(str2item)

            str2 = trimDataHtmlProAndImg(str2 )

            info7 = '<p>' + str2 + '</p>'

        if str1 == 'online':

            try:

                info10 = itemconbody.xpath('./div[contains(@class,"notsignedin")]').getAttributeValue('data-redirect') #申请链接

            except Exception as exx:

                print('....................errr1.......................')

                print(exx)

                try:

                    info10 = itemconbody.find_element_by_xpath('./div[contains(@class,"notsignedin")]/button').getAttributeValue('data-redirect')  # 申请链接

                except Exception as exx:

                    print('....................errr2.......................')

                    print(exx)

            info10 = 'https://www.foregon.com' + info10

    item = {}

    item['url2'] = url

    item['info5'] = info5

    item['info6'] = info6

    item['info7'] = info7

    item['info10'] = info10

    print('write item.............................................')

    print(item)

    # dbname = 'brcardsdata'

    # dbtablename = 'data_source10004_url'

    # updateToDatabase(dbname, dbtablename, item)

    print('write item.................................................')

def get_key_url_map(dbname, tablename):

    conn = getDbConn(dbname)

    cursor = conn.cursor()

    print("mysql connect success")

    sql = "select url,pagecode from " + tablename

    cursor.execute(sql)

    dataresult = cursor.fetchall()

    conn.close()

    return dataresult

def scrapyStart1(url, pagecode):

    htmlContent = loadpage('10004', pagecode)

    parse_data_page_step1(htmlContent, pagecode)

    pagecode2 = pagecode + '_nextpage'

    htmlContent = loadpage('10004', pagecode2)

    parse_data_page_step2(htmlContent, pagecode2)

# key_url_map = get_key_url_map('pagedata', 'data_source10004_url')

key_url_map = [['https://www.foregon.com/solicitar/cartaodecredito/agillitas/fgn/cartao-pre-pago-agillitas-mundo-livre-visa/1028','1']]

for key_url in key_url_map:

    url = key_url[0]

    pagecode = key_url[1]

    pagecode = str(pagecode)

    print(url)

    scrapyStart1(url, pagecode)