- 在线时间
- 481 小时
- 最后登录
- 2026-8-25
- 注册时间
- 2023-7-11
- 听众数
- 4
- 收听数
- 0
- 能力
- 0 分
- 体力
- 7859 点
- 威望
- 0 点
- 阅读权限
- 255
- 积分
- 2946
- 相册
- 0
- 日志
- 0
- 记录
- 0
- 帖子
- 1177
- 主题
- 1192
- 精华
- 0
- 分享
- 0
- 好友
- 1
该用户从未签到
 |
#!/usr/bin/env python
9 o( P- i: x ^/ w# -*- coding:utf-8 -*-7 A8 R% j; V3 {9 p$ c
"""
7 S6 c9 L& ^7 A: g$ J# k+ SPython爬虫,抓取一卡通相关企业信息
% d# w" |; K; A5 C% P/ ~. S) M" wAnthor: yangyongzhen' S$ |& g- P5 j7 z2 J j2 h
Version: 0.0.2 R0 ]; [, \6 `; y% j5 K
Date: 2014-12-14
7 Y+ ?0 \' ?. v+ HLanguage: Python2.7.5
# k+ A9 m5 N5 u: q/ N3 r6 SEditor: Sublime Text2$ ^1 b+ k1 x+ q7 N; y b
"""" j+ u8 o+ O; t4 U5 x/ q
|# \9 D( B% Q: Y; Y7 |: dimport urllib2, re, string
. }* ]) j9 i( r6 limport threading, Queue, time9 r7 C) C1 }4 d# ~! Y3 z- o# f7 C, O
import sys/ t( Z* P$ X, M) e
import os, n( b' R$ A* ]" j
from bs4 import BeautifulSoup, X5 O4 _' x0 ^4 u( }9 u
#from pprint import pprint& j0 }( x1 e" \4 r- f& h" f! J ? j
! m# l( U3 b* J4 o1 T, d: Z
reload(sys): L- }) ]% U8 t; U) c- G# {
sys.setdefaultencoding('utf8'); \5 o% ]0 p7 j* r) j( c
_DATA = []
* T. u# Z+ t5 l5 QFILE_LOCK = threading.Lock()
4 f. Z L! N; G- _SHARE_Q = Queue.Queue() #构造一个不限制大小的的队列
& u- g1 A% B0 B5 z. p1 Q1 C_WORKER_THREAD_NUM = 3 #设置线程的个数( P3 Q7 \8 ~, J4 p8 \5 @9 X6 t
2 Z. b# z, X2 D: g9 E2 Y" i( ?6 O- `5 k_Num = 0 #总条数
7 i) {; a K# s' ^* k9 T% F" W6 Y2 Dclass MyThread(threading.Thread) :/ D6 k& S+ A- E( E* }! _3 ?
# a5 r) }" r) V/ U w: w' Q
def __init__(self, func,num) :
( q1 A- w! f a1 _0 G# Y- v% s; O super(MyThread, self).__init__() #调用父类的构造函数
* d5 W, V, K! L I: v; @& x, u self.func = func #传入线程函数逻辑/ j7 m9 h. ?$ H. k
self.thread_num = num
9 H8 D- G4 X/ o% v& V: b( v def run(self) :
- k* H" J# c6 [ self.func()
/ _0 a n7 ?& T/ y5 r- a3 G p+ v #print u'线程ID:',self.thread_num! G' P' C* l7 o) U o
) F) F, C" H) l/ x) Y% f9 J
def worker() :1 m8 d0 H3 k: l* E d6 t9 l
global SHARE_Q
$ X/ K& n# ]# F/ h while not SHARE_Q.empty():4 P; A- M8 \! j0 y/ b3 Z; a* g
url = SHARE_Q.get() #获得任务
! U8 d( g# q9 m5 u: w% t my_page = get_page(url)
4 g, U9 P, A+ C0 M find_data(my_page) #获得当前页面的数据
( A9 B: ]8 p {# A, l* y* d5 o #write_into_file(temp_data)/ ^, i1 K: r. r. E5 k) c8 B0 k. u
time.sleep(1)7 Z% m( b! e/ q- D# o$ m
SHARE_Q.task_done()
, Z5 h; f: s: `/ ?- N( x$ s
3 m% J% B* G' [9 t Ndef get_page(url) :
9 X( S, A3 S; X( F """4 K! P5 G0 B4 {2 ?; O0 f
根据所给的url爬取网页HTML
& I+ K% `+ |+ P/ L/ F: q9 V7 O Args: 2 E2 s) p2 [) K; x! y2 ?
url: 表示当前要爬取页面的url$ E6 x( P# j- z% ~- q; M
Returns:
" M: @% a; D+ O4 u! w# `1 ]6 Z 返回抓取到整个页面的HTML(unicode编码)
( ^7 O- W4 `# V+ b- R. y+ M Raises:- a5 D3 C" Z8 p- \
URLError:url引发的异常5 m% {! ^8 H% d
"""( a1 W0 a6 M+ g \: Y# u: I4 _
try :/ ~: r8 w0 f9 ?" O( i" [4 I- K
html = urllib2.urlopen(url).read()
! {- z4 g0 F, i my_page = html.decode("gbk",'ignore')
7 u" a0 M3 K2 x- @1 m/ L3 v #my_page = unicode(html,'utf-8','ignore').encode('utf-8','ignore')+ X2 h' B6 I; r) B5 j' r
#my_page = urllib2.urlopen(url).read().decode("utf8")# f2 C- ` j3 }( m- |7 g2 p8 C
except urllib2.URLError, e :
, d/ l, Y8 o |- ]5 ~2 z+ V( E if hasattr(e, "code"):
& f, w- O3 p. `9 D; q6 b1 l print "The server couldn't fulfill the request.": ~) ^- k* P( A) P
print "Error code: %s" % e.code* {* ?7 ~% T) Q9 M/ \& f
elif hasattr(e, "reason"):- u8 Y% q: |! f$ }' h9 V p: D
print "We failed to reach a server. Please check your url and read the Reason"2 @& [0 g. Q/ `# d2 q
print "Reason: %s" % e.reason, x) Q4 ], {; Z) ^5 l. C& s
return my_page
$ q9 m: U) Y, W% x6 A% X, Q$ u3 a: }6 L; c
def find_data(my_page) :
3 t# J4 ]( k9 _) O/ S$ \ """( `% P$ `/ U$ I( R! h5 [
通过返回的整个网页HTML, 正则匹配名称
. V2 x, q$ U3 p1 b. \
% C0 y2 H8 i3 @4 z. p6 @8 c0 A% e7 [ Args:' r- U1 N" r) h8 g4 `5 D- L
my_page: 传入页面的HTML文本用于正则匹配, d" E2 P) K# ^& X
"""
" L W( K) K8 W6 L4 H7 V; Z global _Num- m {+ Q% \4 }
temp_data = []
& \. Y" ^& h; h( Q' h% `0 z items = BeautifulSoup(my_page).find_all("div", style="width:96%;margin:10px;border-bottom:1px #CCC dashed;padding-bottom:10px;")- N: }" E7 ]: {6 x& [
for index, item in enumerate(items) :2 {/ q8 r* a' ~ q
#print item Y- ~3 i, j5 I8 U8 G9 I
#print item.h1
! G5 }/ G1 m5 X& S% _ #print h.group()
/ G4 [: t' b8 L2 H& V- C0 p #temp_data.append(item)! n5 i& ~) v! n5 U+ S# j
#print item.find(re.compile("^a"))
; I0 X( g/ V F' x3 u* D href = item.find(re.compile("^a"))
* K1 S! |5 _+ K: ~ #soup = BeautifulSoup(item)
2 c& ]9 m+ n5 H- e #公司名称* E0 @2 S+ v8 d, X
if item.a:
; Q5 g& M( f$ Q0 ] data = item.a.string.encode("gbk","ignore")2 W" O" g2 W1 T! z" g, `) ]
print data7 O& T; ^+ ^( c0 G/ r) U
temp_data.append(data)
% j" U& ^ o# X( G4 ~# B& a2 e1 q) v/ S: X
goods = item.find_all("div", style="font-size:12px;")
9 l# C6 k4 u& E6 H( d. { , G& T- q; v5 @3 `0 s
#经营产品与联系方式
0 `: m1 y l3 y for i in goods:1 k2 W% {' W, m! {
data = i.get_text().encode("gbk","ignore")
/ z+ {4 K$ x+ f temp_data.append(data)# P: F4 b; s, a" [! k. _0 p
print data
# v4 d9 V- d* l0 U! s% Q- n$ w. l" a) h2 O #b = item.find_all("b")
0 B2 n( I/ o8 m3 s+ l* u% Z #print b( v1 H- x6 V4 C
#链接地址
# V) M% J0 m9 K% t& ] pat = re.compile(r'href="([^"]*)"')
j1 {9 {- _0 Q5 K h = pat.search(str(item))
+ N, O; w. Z9 E8 j( P* R* ^ if h:
/ t8 c7 ~8 X1 _% j #print h.group(0); R/ e9 ~' h9 ?9 k4 f* Q2 L
href = h.group(1)
' |( Q' I0 l$ z4 i: N print href
3 \2 e0 K% h; Q7 _ temp_data.append(h.group(1))
' j$ a5 [8 R& q5 [( {1 m/ s
6 b! { c0 ?0 j M _Num += 1* L0 @! o. r& w* r
#b = item.find_all(text=re.compile("Dormouse"))
3 E4 D. i2 e5 p: B- t #pprint(goods)" p4 L4 f! d7 d/ u& k8 i
#print href
3 O0 c' W! g, A* A6 v0 S! t! B #pat = re.compile(r'title="([^"]*)"')% [3 a+ K/ z% s/ a7 s
#h = pat.search(str(href))
% g/ S! n4 S' }( e! B #if h:
; f& A* ~) q- U #print h.group(1)! x, O( }+ r Q, n0 @
#temp_data.append(h.group(1))
( |8 W3 _4 s) S+ K2 o8 H1 l2 U: B _DATA.append(temp_data)
: c" x+ I/ E, Z- R9 Y( |: v0 A, J6 i2 z: C
#headers = {'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}##浏览器请求头(大部分网站没有这个请求头会报错、请务必加上哦)
2 m2 v9 B) I! _; Y5 I3 F# B) T#all_url = 'http://www.mzitu.com/all' ##开始的URL地址
+ T% s* B! `, L' G. S#start_html = requests.get(all_url, headers=headers) ##使用requests中的get方法来获取all_url(就是:http://www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释) ~* F2 I$ u8 Z9 j) S
#print(start_html.text) ##打印出start_html (请注意,concent是二进制的数据,一般用于下载图片、视频、音频、等多媒体内容是才使用concent, 对于打印网页内容请使用text)- C& Y" j; J; o* f
) u' J* V2 w8 k! ?8 X) k
def main() :
( ^# m3 H3 E! U# n! G( w global SHARE_Q
1 i3 A) H8 P: P- e g# i# g threads = []
9 z- y8 c8 y, x' Z& ~& T start = time.clock()8 Z$ E" Y7 E4 ~( ], m6 H, A3 z: H6 {
douban_url = "http://company.yktworld.com/comapny_search.asp?page={page}"
1 ~9 t$ N- C% F' J0 B9 n #向队列中放入任务, 真正使用时, 应该设置为可持续的放入任务
& b8 x. X, E9 S" R for index in xrange(20) :
; N6 m# L1 b7 P4 f/ z6 i4 l SHARE_Q.put(douban_url.format(page = index * 1))
& [1 K" X* H! m6 e, j for i in xrange(_WORKER_THREAD_NUM) :
8 X% G1 |: {/ d I# r# L thread = MyThread(worker,i), {. f1 }& Z: Q. E# s& O
thread.start() #线程开始处理任务3 j, l( Y4 o C; x
- j& R4 r9 M) _- i7 Q9 \ threads.append(thread)9 _! T$ {; D) W. [5 [& |
for thread in threads :
1 A5 H; |7 E" j }3 {. p thread.join()
7 y# M2 s% @4 [& { s) u SHARE_Q.join()
5 E1 j2 h. `, _! a& } i = 0
/ j! `6 y, V0 [3 @- a) l5 d9 b H with open("down.txt", "w+") as my_file :
+ C) F. |. k( Y) p) T+ _ d% z for page in _DATA :
: g; D4 e7 Y5 K) A6 Z) n i += 1
! ~) C" i, }6 ~9 r' s" m- K( t for name in page:, t" |- A R# p1 M8 G
my_file.write(name + "\n")
. J; G2 X) {1 a Q( o1 g# _5 k p
3 x! c j' `% P) g4 B5 V( t. Y$ H print "Spider Successful!!!"3 ~" O" X$ _* V0 T" _
end = time.clock()
. l" G: P% O' G2 U: d9 V- Z print u'抓取完成!'0 o5 K+ y& @$ |# Z" P! {7 f# H
print u'总页数:',i. M4 J( B$ \. z1 t+ y f5 |
print u'总条数:',_Num
( D: D" p; v0 q) _& K# T6 v! _: w1 o print u'一共用时:',end-start,u'秒'
% g2 f% Y7 T4 P" B: M$ G" j/ l% g9 P# I; W& }1 e
if __name__ == '__main__':% |4 o& s" L. P8 g. l) N5 D; g
main()6 o- m! y8 ]' _( T' h& M) e
! U! ?, J# t$ @& A" h& D0 ]) I' _, [$ v) R2 d1 O; A
|
zan
|