记录

pool = Pool(processes=10)

########################################################################
class path(threading.Thread):
    """"""

    #----------------------------------------------------------------------
    def __init__(self, name, q):
        """Constructor"""
        threading.Thread.__init__(self, name = name)
        self.data = q
    #----------------------------------------------------------------------
    def run(self):
        """"""
        with open('dict/yujian.txt') as f:
            path_list = f.readlines()
            for i in path_list:
                i = i.strip('\r').strip('\n')
                self.data.put(i)
                print self.data.get()

########################################################################
class conn(threading.Thread):
    """"""

    #----------------------------------------------------------------------
    def __init__(self, name, q):
        """Constructor"""
        threading.Thread.__init__(self, name  = name)
        self.data = q
    #----------------------------------------------------------------------
    def run(self):
        """"""
        path = self.data.get()
        #print self.data.qsize()
        self.data.task_done()
        print path
        #try:
            #req = get_request(name + path)
            #if str(req.status_code).startswith('4'):
                #pass
            #else:
                #print i
                #print '[+]  '  + str(req.status_code) +'  ' + name + path + '----Success!'
                #print path_list.index(i)
        #except Exception,e:
            #print str(e)
            #pass        
        
#----------------------------------------------------------------------
def get_path():
    """"""
    q = queue.Queue()
    pro = path('1', q)
    con = conn('http://www.jianshu.com', q)
    
    pro.start()
    con.start()
    
    pro.join()
    con.join()


t2 = datetime.datetime.now()  
#get_path()

print t2 - t1
    
#----------------------------------------------------------------------
def path_brute(url):
    """"""
    with open('dict/yujian.txt') as f:
        path_list = f.readlines()
        for i in path_list:
            i = i.strip('\r').strip('\n')
            if  i.startswith('/'):           
                try:
                    print i
                    req = get_request(url + i)
                    if str(req.status_code).startswith('4'):
                        pass
                    else:
                        #print i
                        print '[+]  '  + str(req.status_code) +'  ' + url + i + '----Success!'
                        #print path_list.index(i)
                except Exception,e:
                    print str(e)
                    pass


#req = get_request('http://www.jianshu.com' + '/robots.txt')
#print req.status_code
#print type(req.status_code)

import queue
q = queue.Queue()
#----------------------------------------------------------------------
def test():
    """"""

    for i in xrange(50):
        
        t = threading.Thread(target=path_brute('http://jianshu.com'))
        t.setDaemon(True)
        thread_list.append(t)    
    for k in thread_list:
        t.start()
    for k in thread_list:
        t.join()
#test()
           
path_brute('http://www.jianshu.com')
t2 = datetime.datetime.now()  
print t1
print t2
print t2 - t1