1.解析xml

import xml.etree.ElementTree as ET
import pprint

tree = ET.parse('exampleResearchArticle.xml') 
root = tree.getroot()

print "Children of root:"
for child in root:
    print child.tag                #使用tag属性打印出每个子元素的标签名
    
title = root.find('./fm/bibl/title')                    #xpath表达式
title_text=''
for p in title:   
    title_text += p.text
    print "\nTitle:\n",title_text
    
print "\nAuthor email addresses:"
for a in root.findall('./fm/bibl/aug/au'):
    email = a.find('email')
    if email is not None:
        print email.text

2.提取数据（xml）

从 xml 中提取关于文章作者的数据，并将其添加到列表中，一个作者对应一个条目。
如下是我们期望的格式。名字、姓氏和电子邮箱标签应该直接对应字典关键字

olution = [{'fnm': 'Omer', 'snm': 'Mei-Dan', 'email': 'omer@extremegate.com'}, {'fnm': 'Mike', 'snm': 'Carmont', 'email': 'mcarmont@hotmail.com'}, {'fnm': 'Lior', 'snm': 'Laver', 'email': 'laver17@gmail.com'}, {'fnm': 'Meir', 'snm': 'Nyska', 'email': 'nyska@internet-zahav.net'}, {'fnm': 'Hagay', 'snm': 'Kammar', 'email': 'kammarh@gmail.com'}, {'fnm': 'Gideon', 'snm': 'Mann', 'email': 'gideon.mann.md@gmail.com'}, {'fnm': 'Barnaby', 'snm': 'Clarck', 'email': 'barns.nz@gmail.com'}, {'fnm': 'Eugene', 'snm': 'Kots', 'email': 'eukots@gmail.com'}]

import xml.etree.ElementTree as ET
article_file = "exampleResearchArticle.xml"
def get_root(fname):
    tree = ET.parse(fname)
    return tree.getroot()
root = get_root(article_file)
def get_authors(root):
    authors = []
    for author in root.findall('./fm/bibl/aug/au'):
        data = {
                "fnm": author.find('./fnm').text,
                "snm": author.find('./snm').text,
                "email": author.find('./email').text
        }

        authors.append(data)

    return authors
get_authors(root)

3.处理属性（xml）

从"insr"标签中提取属性iid的值，并将其添加到字典关键字“insr”列表中
<insr iid="I2"/>

import xml.etree.ElementTree as ET

article_file = "exampleResearchArticle.xml"


def get_root(fname):
    tree = ET.parse(fname)
    return tree.getroot()

root = get_root(article_file)

def get_authors(root):
    authors = []
    for author in root.findall('./fm/bibl/aug/au'):
        data = {
                "fnm": author.find('./fnm').text,
                "snm": author.find('./snm').text,
                "email": author.find('./email').text,
                "insr": []
        }
        
        insr = author.findall('./insr')
        for i in insr:
            data['insr'].append(i.attrib['iid'])


        authors.append(data)

    return authors

4.提取实体(html)

from bs4 import BeautifulSoup

def options(soup,id):
    option_values = []
    carrier_list = soup.find(id = id)
    for option in carrier_list.find_all('option'):
        option_values.append(option['value'])
    return option_values

def print_list(label,codes):
    print "\n%s:" % label
    for c in codes:
        print c
        
def main():
    soup = BeautifulSoup(open('virgin_and_logan_airport.html'))
    codes = options(soup,'CarrierList')
    print_list('Carriers',codes)
    
    codes = options(soup,'AirportList')
    print_list('Airports',codes)

5.使用BeautifulSoup(html)

使用 BeautifulSoup 处理 HTML，提取出"__EVENTVALIDATION”和“__VIEWSTATE”的隐藏表格字段值，并在数据字典中设置相应的值。

from bs4 import BeautifulSoup
import requests
import json

html_page = "page_source.html"

def extract_data(page):
    data = {"eventvalidation": "",
            "viewstate": ""}
    with open(page, "r") as html:
        soup = BeautifulSoup(html,'lxml')
        ev = soup.find(id = '__EVENTVALIDATION')
        data['eventvalidation'] = ev['value']
        
        vs = soup.find(id="__VIEWSTATE")
        data['viewstate'] = vs['value']
        # do something here to find the necessary values


    return data

def make_request(data):
    eventvalidation = data["eventvalidation"]
    viewstate = data["viewstate"]

    r = requests.post("http://www.transtats.bts.gov/Data_Elements.aspx?Data=2",
                    data={'AirportList': "BOS",
                          'CarrierList': "VX",
                          'Submit': 'Submit',
                          "__EVENTTARGET": "",
                          "__EVENTARGUMENT": "",
                          "__EVENTVALIDATION": eventvalidation,
                          "__VIEWSTATE": viewstate
                    })

    return r.text

6.抓取解法

from bs4 import BeautifulSoup
import requests
import json

s = requests.Session()

r=s.get('https://www.transtats.bts.gov/Data_Elements.aspx?Data=2')
soup = BeautifulSoup(r.text)

viewstate_element = soup.find(id='__VIEWSTATE')
viewstate = viewstate_element['value']

eventvalidation_element = soup.find(id='__EVENTVALIDATION')
eventvalidation= eventvalidation_element['value']

r=s.post('https://www.transtats.bts.gov/Data_Elements.aspx?Data=2',
         data={'AirportList': "BOS",
                          'CarrierList': "VX",
                          'Submit': 'Submit',
                          "__EVENTTARGET": "",
                          "__EVENTARGUMENT": "",
                          "__EVENTVALIDATION": eventvalidation,
                          "__VIEWSTATE": viewstate
                    })

f = open('virgin_and_logan_airport.html','w')
f.write(r.text)

7.习题集

7.1运营商列表(html)

获取一个包含所有航空公司的列表。在你所返回的数据中要去掉所有类似 “All U.S. Carriers” 的组合。最终你应该返回一个含有运营商编码的列表。

from bs4 import BeautifulSoup
html_page = "options.html"


def extract_carriers(page):
    data = []

    with open(page, "r") as html:
        # do something here to find the necessary values
        soup = BeautifulSoup(html, "lxml")
        carrier_list = soup.find(id="CarrierList")
        carriers = carrier_list.find_all('option')
        
        for carrier in carriers:
            if len(carrier['value'])==2:
                data.append(carrier['value'])
        
    return data


def make_request(data):
    eventvalidation = data["eventvalidation"]
    viewstate = data["viewstate"]
    airport = data["airport"]
    carrier = data["carrier"]

    r = s.post("https://www.transtats.bts.gov/Data_Elements.aspx?Data=2",
               data = (("__EVENTTARGET", ""),
                       ("__EVENTARGUMENT", ""),
                       ("__VIEWSTATE", viewstate),
                       ("__VIEWSTATEGENERATOR",viewstategenerator),
                       ("__EVENTVALIDATION", eventvalidation),
                       ("CarrierList", carrier),
                       ("AirportList", airport),
                       ("Submit", "Submit")))

    return r.text

7.2机场列表(html)

其返回机场代码列表，并删除任何组合内容，例如“All”。

from bs4 import BeautifulSoup
html_page = "options.html"
def extract_airports(page):
    data = []
    with open(page, "r") as html:
        # do something here to find the necessary values
        soup = BeautifulSoup(html, "lxml")
        airport_list = soup.find(id='AirportList')
        airports = airport_list.find_all('option')
        
        for airport in airports:
            if 'All' not in airport['value']:
                data.append(airport['value'])

    return data

7.3处理所有数据(html)

Let's assume that you combined the code from the previous 2 exercises with code
from the lesson on how to build requests, and downloaded all the data locally.
The files are in a directory "data", named after the carrier and airport:
"{}-{}.html".format(carrier, airport), for example "FL-ATL.html".

The table with flight info has a table class="dataTDRight". Your task is to
use 'process_file()' to extract the flight data from that table as a list of
dictionaries, each dictionary containing relevant data from the file and table
row. This is an example of the data structure you should return:

data = [{"courier": "FL",
"airport": "ATL",
"year": 2012,
"month": 12,
"flights": {"domestic": 100,
"international": 100}
},
{"courier": "..."}
]
Note - year, month, and the flight data should be integers.
You should skip the rows that contain the TOTAL data for a year.

from bs4 import BeautifulSoup
from zipfile import ZipFile
import os

datadir = "data"


def open_zip(datadir):
    with ZipFile('{0}.zip'.format(datadir), 'r') as myzip:
        myzip.extractall()


def process_all(datadir):
    files = os.listdir(datadir)
    return files


def process_file(f):
    """
    This function extracts data from the file given as the function argument in
    a list of dictionaries. This is example of the data structure you should
    return:

    data = [{"courier": "FL",
             "airport": "ATL",
             "year": 2012,
             "month": 12,
             "flights": {"domestic": 100,
                         "international": 100}
            },
            {"courier": "..."}
    ]

    Note: create a new dictionary for each entry in the output data list.
    If you use the info dictionary defined here each element in the list 
    will be a reference to the same info dictionary.
    """
  
    with open("{}/{}".format(datadir, f), "r") as html:
        data = []
        info = {"courier": "",
                "airport": "",
                "year": "",
                "month": "",
                "flights": {"domestic": "",
                         "international": ""}
            }
        info["courier"], info["airport"] = f[:6].split("-")

        soup = BeautifulSoup(html,'lxml')
        table_data = soup.find('table',{'class':'dataTDRight'})
        for tr in table_data.find_all('tr'):
            td = tr.find_all('td')
            
            if td[1].string =='Month' or td[1].string =='TOTAL':
                continue
            else:
                info['year'] = int(td[0].string)
                info['month'] = int(td[1].string)
                info['flights']['domestic']= int(td[2].string.replace(',',''))
                info['flights']['international']= int(td[3].string.replace(',',''))
                
            data.append(info)
        

    return data

7.4
xml文件不是有效的 XML，因为它有几个根元素和 XML 声明,是多个相连的 XML 文档构成的。
本题需要将文件拆分为多个文档，并将这些文档处理为有效的 XML 文档。

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# So, the problem is that the gigantic file is actually not a valid XML, because
# it has several root elements, and XML declarations.
# It is, a matter of fact, a collection of a lot of concatenated XML documents.
# So, one solution would be to split the file into separate documents,
# so that you can process the resulting files as valid XML documents.

import xml.etree.ElementTree as ET
PATENTS = 'patent.data'

def get_root(fname):
    tree = ET.parse(fname)
    return tree.getroot()


def split_file(filename):
    with open(filename) as infile:
        n = -1 
        outfile = open('{}-{}'.format(filename, n), 'w') 
        for line in infile:
            if line.startswith('<?xml'):
                outfile.close() 
                n += 1       
                outfile = open('{}-{}'.format(filename, n),'w')

            outfile.write(line)
            
        outfile.close() 
    """
    Split the input file into separate files, each containing a single patent.
    As a hint - each patent declaration starts with the same line that was
    causing the error found in the previous exercises.
    
    The new files should be saved with filename in the following format:
    "{}-{}".format(filename, n) where n is a counter, starting from 0.
    """

def test():
    split_file(PATENTS)
    for n in range(4):
        try:
            fname = "{}-{}".format(PATENTS, n)
            f = open(fname, "r")
            if not f.readline().startswith("<?xml"):
                print "You have not split the file {} in the correct boundary!".format(fname)
            f.close()
        except:
            print "Could not find file {}. Check if the filename is correct!".format(fname)


test()