数学建模社区-数学中国
标题:
爬虫抓取信息
[打印本页]
作者:
2744557306
时间:
2024-3-31 17:18
标题:
爬虫抓取信息
#!/usr/bin/env python
4 U7 q5 P' F$ N. g
# -*- coding:utf-8 -*-
: ?$ b6 S8 G% U; \3 x
"""
/ j' v: |9 Z* ? }, V5 S
Python爬虫,抓取一卡通相关企业信息
2 |9 Z8 _5 d Q7 e P: U
Anthor: yangyongzhen
! L5 M7 F8 [9 k. A9 l# J
Version: 0.0.2
/ V; `9 R: u1 ^( G& b; U$ W: v3 ]
Date: 2014-12-14
" I" V9 S5 m6 {& d) L$ f
Language: Python2.7.5
; r* F8 I! j0 S. a/ M
Editor: Sublime Text2
2 e- V" a! |: h( e5 c$ _- ^+ j
"""
: L2 |2 d/ o7 R( G& }1 V& M
+ T) ]2 u1 j& l' q
import urllib2, re, string
" Q( P1 z- L$ L0 a7 v
import threading, Queue, time
. Q& V2 ^( ~6 k
import sys
' K5 x4 ?) w P1 U+ Q% ~
import os
; ?3 j/ l8 t" D3 a: x
from bs4 import BeautifulSoup
( l" k6 w: y" U3 d, m& L
#from pprint import pprint
4 N. P. D% ~9 j9 [& G2 m4 N
2 K/ U( U3 U7 M4 ]$ q' J* s
reload(sys)
% ]2 O2 S% @. _, h8 ^- {
sys.setdefaultencoding('utf8')
8 [) {& `( r% i5 f- e& s; p( K1 j3 Q$ C
_DATA = []
/ t+ }6 `- _& a+ o7 P( J# @: l
FILE_LOCK = threading.Lock()
: P$ ^# [2 z2 ?+ @9 y' L
SHARE_Q = Queue.Queue() #构造一个不限制大小的的队列
$ K) S4 P9 G+ F4 F% X
_WORKER_THREAD_NUM = 3 #设置线程的个数
2 ]7 E3 _2 k6 z+ I4 L
, y, N2 G' _7 g
_Num = 0 #总条数
0 q$ N8 o- P2 T
class MyThread(threading.Thread) :
6 Q7 m4 b& I9 A4 G9 `! X1 P
3 Z# z2 V3 u5 M4 p2 `
def __init__(self, func,num) :
' S+ ~# i) ]" U
super(MyThread, self).__init__() #调用父类的构造函数
# h# e" I$ m e% |
self.func = func #传入线程函数逻辑
+ c0 D; i1 e. [# S, B" K) W. D" v$ r
self.thread_num = num
7 C1 t* P D* C+ G2 b- t5 j
def run(self) :
7 `0 [9 l% Z: `& ~/ r
self.func()
+ ^- S/ r5 Z h/ o) b* T0 |; b
#print u'线程ID:',self.thread_num
% X5 J9 e0 h6 @# ]4 o3 M( J
. R" e: z8 B; X; H
def worker() :
: v- E9 s! M& v; F2 F
global SHARE_Q
. u: ^ A# Q6 ]# x* T: d3 w
while not SHARE_Q.empty():
2 @% S- y0 o2 u9 p+ h* r+ |, r( ? i
url = SHARE_Q.get() #获得任务
3 I; ~ D& g1 i5 y1 b
my_page = get_page(url)
$ ~0 } F4 u L* x9 D# X$ ~
find_data(my_page) #获得当前页面的数据
$ ?7 v* j- U+ _* K' ?
#write_into_file(temp_data)
0 K6 X( f+ I5 L0 i2 k
time.sleep(1)
/ ]: j% X3 I, w3 s: o
SHARE_Q.task_done()
0 }$ D# N, x3 m5 N# a
/ W) m2 [$ L- t8 w
def get_page(url) :
2 \" D8 i! Z4 Q: S
"""
: }3 J( j3 n# k. G( B- h
根据所给的url爬取网页HTML
& Z. L# R, A7 R! K( b
Args:
" ^% V. q9 r2 J) {# u
url: 表示当前要爬取页面的url
* t$ N6 N5 c5 M3 ^2 h
Returns:
* X4 F* ?: |( I
返回抓取到整个页面的HTML(unicode编码)
& K, u: {: r6 I* y' g
Raises:
) z" z# p- ?" e0 r" S) x4 m+ i$ r
URLError:url引发的异常
S! ^' M! o, V, t9 B
"""
8 A2 Y: u Z8 R% M! f
try :
( F) ^0 O# E; c
html = urllib2.urlopen(url).read()
6 V. W- e2 q; o x
my_page = html.decode("gbk",'ignore')
5 C$ ]$ Y0 l+ l9 A3 @
#my_page = unicode(html,'utf-8','ignore').encode('utf-8','ignore')
1 y9 P( r/ T* b. u2 }9 k
#my_page = urllib2.urlopen(url).read().decode("utf8")
2 o) _4 O- @ p& W9 K
except urllib2.URLError, e :
' E+ j" W6 n. d% r8 e" x, v
if hasattr(e, "code"):
: C7 D( M4 j" o* u" X8 A
print "The server couldn't fulfill the request."
5 s+ o# l' N& Z$ C& U# j5 d
print "Error code: %s" % e.code
% M6 X# \/ G; _5 ?' _/ ^
elif hasattr(e, "reason"):
7 |5 _" p. ?( A: g' d) [) S
print "We failed to reach a server. Please check your url and read the Reason"
& s" }. d( C5 H, G9 c2 d4 e
print "Reason: %s" % e.reason
5 E5 A/ [& w7 w j3 b' ]: G# Q1 f
return my_page
. S% o3 v* K# w8 b6 F
/ L4 [5 c' ^3 m
def find_data(my_page) :
3 ? w4 d& a' p/ b" z$ W
"""
* z' A1 L* V/ L# z( N
通过返回的整个网页HTML, 正则匹配名称
9 C6 I$ t1 A6 i @1 b* J
]6 k* l* U" S5 ]$ v! @) k8 T
Args:
3 ]5 h; D; o+ [+ [
my_page: 传入页面的HTML文本用于正则匹配
& f1 O, a7 y$ T& M/ Q
"""
( G) o9 _+ \* E: A5 P8 `4 ~
global _Num
2 A+ k g ?6 r7 Z" ^. q; u/ G" r
temp_data = []
2 j, W8 J- j: m! U3 q
items = BeautifulSoup(my_page).find_all("div", style="width:96%;margin:10px;border-bottom:1px #CCC dashed;padding-bottom:10px;")
2 i" P0 P9 Y( k+ |- P: Y
for index, item in enumerate(items) :
- w$ m, w3 j2 S0 t4 ^
#print item
# D. G: r; \9 c0 J2 S; S
#print item.h1
+ k U; G( o: k
#print h.group()
9 L* E1 q1 Y+ r, I. a3 p
#temp_data.append(item)
. W2 o$ H' Q7 }, X+ [: l r1 _
#print item.find(re.compile("^a"))
& E4 t2 E# h5 N
href = item.find(re.compile("^a"))
( l, y5 E& f2 t, Q5 O5 e C& s9 B
#soup = BeautifulSoup(item)
: J& `- c) b# T/ {6 x X, S2 H1 h
#公司名称
7 N/ i9 @; A4 r( m3 L
if item.a:
$ F/ \1 d" t1 W6 ~; R+ t* E
data = item.a.string.encode("gbk","ignore")
) y- _: `1 r. o1 ~9 }! {
print data
. N! N' ]) _: S0 Q( m8 T
temp_data.append(data)
" R2 A$ C3 y. u. J: w0 E
& Q# o# p0 y8 T8 M, }1 e9 ^6 C. `
goods = item.find_all("div", style="font-size:12px;")
8 T( B b, _' f! Y1 Y
- r4 n; N% {' C9 [ a: T
#经营产品与联系方式
/ I6 i+ i9 s* @7 n! w" m e$ j* f
for i in goods:
& M+ J, P. c( w
data = i.get_text().encode("gbk","ignore")
! f! B$ G5 c$ e& `5 m1 i6 V/ S$ H
temp_data.append(data)
; T: X( ~: m' `% q
print data
5 B9 I; q1 D% f8 ]+ A
#b = item.find_all("b")
& u) ^- j; [9 _* ?
#print b
2 C$ } Q) m: C& X6 d& @
#链接地址
, m' B h* F. ~) W0 J9 p
pat = re.compile(r'href="([^"]*)"')
/ G9 w d, K7 C1 ]6 v4 k
h = pat.search(str(item))
! q/ V4 v1 n) A* ^8 ?( h& u' x9 {
if h:
% p! F" m4 y, G9 ?% ~
#print h.group(0)
( R/ S u2 H$ D& l
href = h.group(1)
) x4 d" ?* L+ [
print href
$ _/ K( ~2 \. a! C
temp_data.append(h.group(1))
8 ^% v8 n7 S6 V0 q( e. g. z9 W- z+ M
; ~* y! N9 ]. K$ k
_Num += 1
! c5 f$ Q: k2 M
#b = item.find_all(text=re.compile("Dormouse"))
" {8 ^7 o" }* u
#pprint(goods)
9 W. a' {0 F/ a. i: h6 W- T
#print href
& b, I X8 r% h! |, h0 V9 q
#pat = re.compile(r'title="([^"]*)"')
3 N7 @6 R3 G3 ~$ i' F
#h = pat.search(str(href))
! W4 P; \& P5 l9 |7 `) h
#if h:
& d) C7 x* G( g' P1 f
#print h.group(1)
# ?& Z% N/ M+ a7 O& h& N
#temp_data.append(h.group(1))
$ R; R/ e! O3 G- Z
_DATA.append(temp_data)
7 W K1 _: t- e
& ~3 E8 s/ U* c; s8 U' U0 L
#headers = {'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}##浏览器请求头(大部分网站没有这个请求头会报错、请务必加上哦)
9 e9 N7 A/ H: u; n; u1 j ~2 I4 `4 P
#all_url = 'http://www.mzitu.com/all' ##开始的URL地址
: p$ c. b" q0 o/ y; R3 G% w% H
#start_html = requests.get(all_url, headers=headers) ##使用requests中的get方法来获取all_url(就是:http://www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释
5 {/ l2 i, c" _* w* {, }* c% c+ L- r
#print(start_html.text) ##打印出start_html (请注意,concent是二进制的数据,一般用于下载图片、视频、音频、等多媒体内容是才使用concent, 对于打印网页内容请使用text)
5 L* l9 h2 h8 @# g) J7 f- j; m. P
5 s/ z2 Y. B4 |7 h3 i& T( _+ `* ~ T
def main() :
: R# m I0 ^ X
global SHARE_Q
. `+ E* `4 D3 J4 ^0 c+ w6 [) B
threads = []
2 W$ [! z w0 n) e) ]
start = time.clock()
: } j1 ~7 e& ~: c
douban_url = "http://company.yktworld.com/comapny_search.asp?page={page}"
9 B* c b/ N3 C
#向队列中放入任务, 真正使用时, 应该设置为可持续的放入任务
( ?4 \! |9 q ?5 f z1 f6 K w
for index in xrange(20) :
6 Y1 V8 C$ p8 m$ a1 l4 ^5 k! I9 y
SHARE_Q.put(douban_url.format(page = index * 1))
) c5 B6 r& p: o& B
for i in xrange(_WORKER_THREAD_NUM) :
- I) g) w" _& S& i( U! V! P
thread = MyThread(worker,i)
* o* {' w$ p1 c' U) O+ h( c+ V( J
thread.start() #线程开始处理任务
% I) \6 d4 M# c' W4 I/ K, N
6 y/ Y9 H& @4 S$ f
threads.append(thread)
& E$ _# ~% ~4 M; [) ^# r1 e
for thread in threads :
k! _9 ]7 H- k8 Z: `: J+ j
thread.join()
J7 ?, u; L, h! Q, z1 E) `! v: T
SHARE_Q.join()
3 S7 f* @& r7 l
i = 0
+ h* H0 Z5 F% ?& }3 X" o) F T! Y$ s
with open("down.txt", "w+") as my_file :
9 j! \' H& h4 j+ Z. w' z
for page in _DATA :
. s- n* a% V3 O$ f! B4 J# ]
i += 1
/ O+ M" A: a0 J% I1 _- s. ^
for name in page:
7 T4 j: U" N6 Y1 @# O0 W
my_file.write(name + "\n")
4 K& Y/ N" b% P: j6 V
9 R4 G6 f" V% A) }0 g
print "Spider Successful!!!"
. H; ~( j6 g* w; @8 l1 @, q( X& N
end = time.clock()
$ a4 K+ g& \% t5 ~$ ?2 W. [. T2 V: r
print u'抓取完成!'
9 z. o4 k& Z6 b. ]/ n6 }: W9 M
print u'总页数:',i
4 w% Y/ `3 H! o
print u'总条数:',_Num
: c9 {& O7 @5 n& O) P
print u'一共用时:',end-start,u'秒'
" W4 K) G+ r% w% w! J) k# {4 k
1 v' S6 V+ F5 I! w" {
if __name__ == '__main__':
1 e+ E0 G, W' d7 p6 @' {
main()
7 c% o. A( }/ |. v0 S$ g* ]
5 L, V' p4 U( a% A/ j% P
6 \ M; a; t1 a
欢迎光临 数学建模社区-数学中国 (http://www.madio.net/)
Powered by Discuz! X2.5