爬虫实现抓取RIR分配的ip地址号并进行前缀处理

需求：python 2.7 从FTP中读取已经分配的IP前缀，计算出对应的IP前缀所包含的IP地址范围，用IP_start和IP_end来标记对应前缀所标识的范围，采用inet_pton函数对其进行编码转换，保存转码后的地址范围

#!/usr/bin/python
#coding=utf-8
import urllib2
import MySQLdb
import win_inet_pton
import socket
from netaddr import IPNetwork
import time
import math

def viewPartInfo(fs):
#文件解析函数
    V4_list = []
    V6_list = []
    rtype4 = socket.AF_INET
    rtype6 = socket.AF_INET6
    for line in fs:
        if line[0] == '#':
            continue
            #注释部分
        list = line.split('|')
        if list[1] == '*':
            continue
            #统计部分
        if list[1] == '':
            continue
        if list[2] == 'asn':
                continue
        if list[2] == 'ipv6':
            var_list = [list[3], list[4]]
            a = '/'
            ip_prefix = a.join(var_list)
            network = IPNetwork(ip_prefix.strip("\n "))
            V6_list.append((
                str(network),
                socket.inet_pton(rtype6, str(network.network)),
                socket.inet_pton(rtype6, str(network.broadcast)),
                time.strftime('%Y-%m-%d', time.localtime(time.mktime(time.strptime(list[5], '%Y%m%d')))),
                ####对于list[5]可能是空的的情况需要处理
                list[1]
            ))
        elif list[2] == 'ipv4':
            m=32 - int(math.log(int(list[4]), 2))
            b=str(m)
            a = '/'
            var_list = [list[3], b]
            ip_prefix = a.join(var_list)
            network = IPNetwork(ip_prefix.strip("\n "))
            V4_list.append((
                str(network),
                socket.inet_pton(rtype4, str(network.network)),
                socket.inet_pton(rtype4, str(network.broadcast)),
                time.strftime('%Y-%m-%d', time.localtime(time.mktime(time.strptime(list[5],'%Y%m%d')))),
                list[1]
            ))
    fs.close()
    try:
        conn = MySQLdb.connect('###地址','database',"user_name","password")
        cur = conn.cursor()
        print len(V4_list)
        print len(V6_list)
        cur.executemany(
            """INSERT INTO table_name(ip_prefix, ip_start, ip_end,date,country) VALUES (%s,%s,%s,%s,%s)""",
            V4_list)
        conn.commit()
        cur.executemany(
            """INSERT INTO table_name(ip_prefix, ip_start, ip_end,date,country) VALUES (%s,%s,%s,%s,%s)""",
            V6_list)
        conn.commit()
    except MySQLdb.Error,e:
        print "Mysql Error %d: %s" % (e.args[0], e.args[1])
        conn.rollback()
    finally :
        cur.close()
        conn.close()
    return

print 'Downloading files, please wait...'
try:
    req = urllib2.Request('ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest')
    response = urllib2.urlopen(req)
    the_page = response.read()
    fs = open('a.txt', 'w')
    fs.write(the_page)
    print 'writed'
    fs.close()
except:
    print '[-] Request data error!'
conn = MySQLdb.connect('118.229.239.28','bgpuser',"niclab433","bgpdata")
cur = conn.cursor()
sql = "truncate table bgp_ip"
cur.execute(sql)
fs = open('a.txt', 'r')
viewPartInfo(fs)

print 'Downloading files, please wait...'
try:
    req = urllib2.Request('ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest')
    response = urllib2.urlopen(req)
    the_page = response.read()
    fs = open('a.txt', 'w')
    fs.write(the_page)
    print 'writed'
    fs.close()
except:
    print '[-] Request data error!'

fs = open('a.txt', 'r')
viewPartInfo(fs)

print 'Downloading files, please wait...'
try:
    req = urllib2.Request('ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest')
    response = urllib2.urlopen(req)
    the_page = response.read()
    fs = open('a.txt', 'w')
    fs.write(the_page)
    print 'writed'
    fs.close()
except:
    print '[-] Request data error!'

fs = open('a.txt', 'r')
viewPartInfo(fs)

print 'Downloading files, please wait...'
try:
    req = urllib2.Request('ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest')
    response = urllib2.urlopen(req)
    the_page = response.read()
    fs = open('a.txt', 'w')
    fs.write(the_page)
    print 'writed'
    fs.close()
except:
    print '[-] Request data error!'

fs = open('a.txt', 'r')
viewPartInfo(fs)

print 'Downloading files, please wait...'
try:
    req = urllib2.Request('ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest')
    response = urllib2.urlopen(req)
    the_page = response.read()
    fs = open('a.txt', 'w')
    fs.write(the_page)
    print 'writed'
    fs.close()
except:
    print '[-] Request data error!'

fs = open('a.txt', 'r')
viewPartInfo(fs)