selenium实战外贸（中国制造）剪刀

1，先看结果

image.png

2.思路
1.反爬虫机制:user-agent设置
2.自动翻页机制
3.一键建表入库
4.phantomjs无界面抓取

3.上代码

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import re
import time
import pymysql

'''
    描述:
        1.网址:http://cn.made-in-china.com    中国制造,外贸类型
        2.使用seleium+bs4
    成果:
        1.实现自动开启浏览器爬取
        2.任意商品可随意输入
        3.一键导入mysql建表
    问题:
        1.不能点击下一页
            使用滚动条下拉至底部
            browser.execute_script('windows.scrollBy(0,1000)')
            解决


'''



conn = pymysql.connect(host='lx',user='x',password='x',port=x,database='x',charset='x')
cursor=conn.cursor()

sql = "CREATE TABLE IF NOT EXISTS %s(ID INT(10) NOT NULL PRIMARY KEY AUTO_INCREMENT," \
      "A VARCHAR(255)," \
      "B VARCHAR(255)," \
      "C VARCHAR(255)," \
      "D VARCHAR(255)," \
      "E VARCHAR(255)," \
      "F VARCHAR(255)," \
      "G VARCHAR(255)," \
      "H VARCHAR(255)," \
      "Z TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)ENGINE=INNODB DEFAULT CHARSET=utf8"
names = input('数据库名:')
cursor.execute(sql%names)
print('创建成功')

url = 'http://cn.made-in-china.com/hot-search/%E5%89%AA%E5%88%80-2.html'

#url = 'http://cn.made-in-china.com/'
borwser = webdriver.Phantomjs()

timeout = WebDriverWait(borwser,10)
borwser.get(url)
def get_url():

    input_a = timeout.until(EC.presence_of_element_located((By.ID, 'inputWord')))
    input_a.clear()
    input_a.send_keys('剪刀')
    time.sleep(2)

    click_a = timeout.until(EC.element_to_be_clickable((By.ID, 'searchBtn')))
    click_a.submit()
    try:

        while True:
            html = borwser.page_source
            #print(borwser.page_source)
            soup = BeautifulSoup(html,'lxml')
            item = soup.find_all('li',class_='pro-item')
            for a in item:
                name = a.find('h3').get_text().strip() #商品名称，
                commodity_url= a.find('a')['href']       #商品页面链接，前面加上，可以获得商品主页面http: // cn.made - in -china.com /

                namber=[a.get_text().strip() for a in a.find_all('p',class_='pro-des')] #现货数量,商品描述
                numbers = ','.join(list(namber))

                company = a.find('div',attrs={'class':'co-name company-name-js'}).get_text().replace('\n','').strip().replace('\n\n','').replace(' ','').strip()

                price = a.find('div',class_='pro-price').get_text().strip()
                detatime = a.find('span',class_='pro-date').get_text()
                city = a.find('div',class_='gray-light').get_text()

                sql = "INSERT INTO %s(A,B,C,D)VALUES('公司:%s','商品:%s,价钱:%s','城市:%s,商品URL:%s','数量:%s,时间:%s')"
                values=(names,company,name,price,city,commodity_url,numbers,detatime)
                cursor.execute(sql % values)

                print('公司:{}\n商品:{}\n价钱:{}\n城市:{}\n商品URL:{}\n数量:{}\n时间:{}\n'.format(company,name,price,city,commodity_url,numbers,detatime))
                print('导入成功！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！')

            conn.commit()
            #下拉滚动条到底部
            time.sleep(2)
            borwser.execute_script('window.scrollBy(0,10000)')

            #下拉滚动条到底部
            # borwser.execute_script('var q=document.documentElement.scrollTop=100000')
            # time.sleep(3)


            #点击下一页

            click_a = timeout.until(EC.element_to_be_clickable((By.CLASS_NAME,'page-next')))
            click_a.click()
            time.sleep(2)

            #click_a = timeout.until(EC.element_to_be_clickable((By.ID,'')))

    except Exception as e:
        print(e)
    finally:

        cursor.close()
        conn.close()

if __name__ == "__main__":
    get_url()