数学建模社区-数学中国

标题: 爬虫抓取信息 [打印本页]

作者: 2744557306 时间: 2024-3-31 17:18
标题: 爬虫抓取信息
#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
Python爬虫,抓取一卡通相关企业信息
Anthor: yangyongzhen
Version: 0.0.2
Date: 2014-12-14
Language: Python2.7.5
Editor: Sublime Text2
"""

import urllib2, re, string
import threading, Queue, time
import sys
import os
from bs4 import BeautifulSoup
#from pprint import pprint

reload(sys)
sys.setdefaultencoding('utf8')
_DATA = []
FILE_LOCK = threading.Lock()
SHARE_Q = Queue.Queue()  #构造一个不限制大小的的队列
_WORKER_THREAD_NUM = 3  #设置线程的个数

_Num = 0 #总条数
class MyThread(threading.Thread) :

def __init__(self, func,num) :
      super(MyThread, self).__init__()  #调用父类的构造函数
      self.func = func  #传入线程函数逻辑
      self.thread_num = num
def run(self) :
      self.func()
      #print u'线程ID:',self.thread_num

def worker() :
global SHARE_Q
while not SHARE_Q.empty():
      url = SHARE_Q.get() #获得任务
      my_page = get_page(url)
      find_data(my_page)  #获得当前页面的数据
      #write_into_file(temp_data)
      time.sleep(1)
      SHARE_Q.task_done()

def get_page(url) :
"""
根据所给的url爬取网页HTML
Args:
      url: 表示当前要爬取页面的url
Returns:
      返回抓取到整个页面的HTML(unicode编码)
Raises:
      URLError:url引发的异常
"""
try :
html = urllib2.urlopen(url).read()
my_page = html.decode("gbk",'ignore')
#my_page = unicode(html,'utf-8','ignore').encode('utf-8','ignore')
      #my_page = urllib2.urlopen(url).read().decode("utf8")
except urllib2.URLError, e :
      if hasattr(e, "code"):
         print "The server couldn't fulfill the request."
         print "Error code: %s" % e.code
      elif hasattr(e, "reason"):
         print "We failed to reach a server. Please check your url and read the Reason"
         print "Reason: %s" % e.reason
return my_page

def find_data(my_page) :
"""
通过返回的整个网页HTML, 正则匹配名称

Args:
      my_page: 传入页面的HTML文本用于正则匹配
"""
global _Num
temp_data = []
items = BeautifulSoup(my_page).find_all("div", style="width:96%;margin:10px;border-bottom:1px #CCC dashed;padding-bottom:10px;")
for index, item in enumerate(items) :
#print item
#print item.h1
      #print h.group()
#temp_data.append(item)
#print item.find(re.compile("^a"))
href = item.find(re.compile("^a"))
#soup = BeautifulSoup(item)
#公司名称
if item.a:
data = item.a.string.encode("gbk","ignore")
print data
temp_data.append(data)

goods = item.find_all("div", style="font-size:12px;")

#经营产品与联系方式
for i in goods:
data = i.get_text().encode("gbk","ignore")
temp_data.append(data)
print data
#b = item.find_all("b")
#print b
#链接地址
pat = re.compile(r'href="([^"]*)"')
h = pat.search(str(item))
if h:
#print h.group(0)
href = h.group(1)
print href
temp_data.append(h.group(1))

_Num += 1
#b = item.find_all(text=re.compile("Dormouse"))
#pprint(goods)
#print href
#pat = re.compile(r'title="([^"]*)"')
#h = pat.search(str(href))
#if h:
#print h.group(1)
#temp_data.append(h.group(1))
_DATA.append(temp_data)

#headers = {'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}##浏览器请求头（大部分网站没有这个请求头会报错、请务必加上哦）
#all_url = 'http://www.mzitu.com/all'  ##开始的URL地址
#start_html = requests.get(all_url,  headers=headers)  ##使用requests中的get方法来获取all_url(就是：http://www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释
#print(start_html.text) ##打印出start_html (请注意，concent是二进制的数据，一般用于下载图片、视频、音频、等多媒体内容是才使用concent, 对于打印网页内容请使用text)

def main() :
global SHARE_Q
threads = []
start = time.clock()
douban_url = "http://company.yktworld.com/comapny_search.asp?page={page}"
#向队列中放入任务, 真正使用时, 应该设置为可持续的放入任务
for index in xrange(20) :
      SHARE_Q.put(douban_url.format(page = index * 1))
for i in xrange(_WORKER_THREAD_NUM) :
      thread = MyThread(worker,i)
      thread.start()  #线程开始处理任务

      threads.append(thread)
for thread in threads :
      thread.join()
SHARE_Q.join()
i = 0
with open("down.txt", "w+") as my_file :
      for page in _DATA :
        i += 1
        for name in page:
        my_file.write(name + "\n")

print "Spider Successful!!!"
end = time.clock()
print u'抓取完成！'
print u'总页数:',i
print u'总条数:',_Num
print u'一共用时：',end-start,u'秒'

if __name__ == '__main__':
main()

欢迎光临数学建模社区-数学中国 (http://www.madio.net/)