- 在线时间
- 481 小时
- 最后登录
- 2026-8-25
- 注册时间
- 2023-7-11
- 听众数
- 4
- 收听数
- 0
- 能力
- 0 分
- 体力
- 7859 点
- 威望
- 0 点
- 阅读权限
- 255
- 积分
- 2946
- 相册
- 0
- 日志
- 0
- 记录
- 0
- 帖子
- 1177
- 主题
- 1192
- 精华
- 0
- 分享
- 0
- 好友
- 1
该用户从未签到
 |
#!/usr/bin/env python
# d i4 U8 M4 s4 r# -*- coding:utf-8 -*-, c+ n' Q: |* |# e0 R6 P
"""
; _% c+ {) T! U& T$ q1 wPython爬虫,抓取一卡通相关企业信息
$ ?7 D2 Y- q6 ]/ w3 iAnthor: yangyongzhen8 K. X$ ]. ?$ n) b
Version: 0.0.2
6 w) B3 K* M+ qDate: 2014-12-14( Z- B& D. [4 v; u( {- j
Language: Python2.7.5- n9 M& X% T! z
Editor: Sublime Text2
4 m$ `; t) \( T"""
) x2 A' }4 I! ]- Q, d( b) i- K
% g ?! |& t+ \3 u3 D+ S" Cimport urllib2, re, string$ S* D! X1 |: L
import threading, Queue, time
; k$ [# m! @ u9 I% }; a4 r! gimport sys
: d1 [( I; z. [' bimport os) H' F: p) p& U1 Q, X7 |1 k
from bs4 import BeautifulSoup% N& ~5 L* U2 [1 k' z
#from pprint import pprint
, B; o1 L9 j. [0 D4 `. u- {9 w# Q) C* A$ u
reload(sys)* P' V" S& h& C" w/ Q
sys.setdefaultencoding('utf8')7 n8 M5 A, F- P: a% t
_DATA = []
( m5 M$ O% F% @* p/ E7 SFILE_LOCK = threading.Lock()
* f: k1 Q5 ]' {: p) YSHARE_Q = Queue.Queue() #构造一个不限制大小的的队列
% K9 R1 V" T' N$ S% I/ q d2 b5 G_WORKER_THREAD_NUM = 3 #设置线程的个数9 z, ^4 ]- B+ ^2 n) O1 ~8 ~: u" z
( g) S! D( O0 f( d8 V! a# T& W_Num = 0 #总条数3 z; o, o# U& J' _' _
class MyThread(threading.Thread) :; ~# |3 e& E, i8 |4 W! e9 n
5 E8 _2 l" E, \8 T+ o+ I: h/ M" r! z
def __init__(self, func,num) :, m2 Q9 N' |5 R) S- R" g+ r
super(MyThread, self).__init__() #调用父类的构造函数3 J7 c! J) a4 y# i$ l
self.func = func #传入线程函数逻辑" {& j# l o$ [0 H1 U, |+ l
self.thread_num = num . ]8 X% a. V! C, O6 {
def run(self) :
# u9 }. g) z3 x+ M self.func()4 o( l6 g4 w4 P4 A# P; O: [( J; v
#print u'线程ID:',self.thread_num- _) ]3 c6 \2 p! A3 b4 j* K' c
( p# o0 L. N D2 y
def worker() :2 D. a! K+ M9 Y4 P/ O
global SHARE_Q1 m* r& q7 W! |, k1 U& F; |% D& d
while not SHARE_Q.empty():2 J1 u( V' f; e/ M |# T0 d3 s2 c
url = SHARE_Q.get() #获得任务
; Y( Z' D& E' ^ my_page = get_page(url) X9 n, O; |! n9 G
find_data(my_page) #获得当前页面的数据
& E! Z7 k! B6 @7 {# P #write_into_file(temp_data): J% A9 Y& Z& V1 y2 @
time.sleep(1). H9 N5 I% l7 x. J( T
SHARE_Q.task_done(). o8 s. m, g0 F2 ?" X
! |- L/ a- Q; Y9 w% T! d
def get_page(url) :1 V0 A ?# |4 w' {, q
"""
0 J0 x- d; f; S* n) r 根据所给的url爬取网页HTML; ^9 G: t/ C& a2 a" ~4 h* u
Args: ' b( l# E3 O( H3 f- y/ B
url: 表示当前要爬取页面的url; I& Y( `$ x: \) V6 x3 @
Returns:/ D" c! I6 ]- o ~) L5 K8 u7 w' [
返回抓取到整个页面的HTML(unicode编码)
- k0 P1 s& m3 M Z" x: a) K Raises:
' g! _$ c# Y. f# ^* K" g URLError:url引发的异常0 u8 F. A. c( R0 A& `& ? M
"""
) y Z" B' H7 T: w' Y( j& A- G$ n try :
9 v3 g, H6 K3 y* d# T html = urllib2.urlopen(url).read()
/ U B! y2 U2 P1 y my_page = html.decode("gbk",'ignore')3 @0 F! B" T o; Q% S, T
#my_page = unicode(html,'utf-8','ignore').encode('utf-8','ignore'). _% H; V4 u) G. X6 @+ a$ ?
#my_page = urllib2.urlopen(url).read().decode("utf8")( E3 i+ I1 r: P9 |( [6 H
except urllib2.URLError, e : K7 p6 r+ v5 _( W2 J% Z* y$ t
if hasattr(e, "code"):+ R6 D* f# P- L9 s# Y" ^1 e* B
print "The server couldn't fulfill the request."
2 f: D# d) y9 B M4 W1 ?% p1 H) M print "Error code: %s" % e.code
R1 ^9 K0 L! v+ x# c0 f6 F# l1 U elif hasattr(e, "reason"):
+ Z$ C, K/ V/ t+ X8 F4 E0 z print "We failed to reach a server. Please check your url and read the Reason"
1 K3 _& t) W1 X print "Reason: %s" % e.reason7 s' K: K8 b8 X* y: D- h
return my_page2 e% r. @" @, }% c9 @) w9 V
) p# ]8 h# {. t) M {9 I7 b% M) ddef find_data(my_page) :% C/ E8 w* Y. h+ f! t1 C4 F
"""
C, V8 c* z0 P9 `% | ] 通过返回的整个网页HTML, 正则匹配名称
/ V7 U& d( a% m2 `8 ]2 ]% S; J4 p# t8 k
Args:
$ f8 q8 v9 \* v; t9 q! W my_page: 传入页面的HTML文本用于正则匹配6 P/ \7 B; z% W9 x$ h7 W2 {1 j( B
""" ^4 \( S% d) T; k
global _Num7 _+ a/ X, f: g j t
temp_data = []
2 L% u, h3 R4 ?2 q: o items = BeautifulSoup(my_page).find_all("div", style="width:96%;margin:10px;border-bottom:1px #CCC dashed;padding-bottom:10px;")1 G- J2 [* H% K
for index, item in enumerate(items) :, `0 K) M. s+ O& C
#print item/ b: }0 C: t. W( n6 n/ ^
#print item.h1/ Q2 R/ X3 f' ~8 [+ r
#print h.group(): `! g) f7 j4 T3 j7 L2 Q9 o
#temp_data.append(item)
" g- q+ [% h6 t# }0 X$ G$ s #print item.find(re.compile("^a"))
) f! h" E" {/ D d$ c; J, A href = item.find(re.compile("^a"))
& n5 o; [) ~: O' N# c #soup = BeautifulSoup(item)9 i% l( l% r$ B* f
#公司名称) M$ m9 D8 X4 h' Z
if item.a:
9 n2 P6 [' _ e* \. s data = item.a.string.encode("gbk","ignore")6 Y" c3 R1 u |5 `1 O. }- `
print data
, Q0 ^: T' Q7 W; n temp_data.append(data)
8 a5 f- n. K9 y1 T" p, c( K7 V
, n- J! Y7 x8 @ goods = item.find_all("div", style="font-size:12px;"); f8 K1 `9 X* X# Q3 c1 j
4 \8 T/ ^. ?& F3 {: x
#经营产品与联系方式/ r& g$ H2 {9 U$ _* j, f
for i in goods:# F, j# @/ a* N$ Q
data = i.get_text().encode("gbk","ignore") J+ a2 y9 J1 s7 V6 U2 o) |3 \
temp_data.append(data): D. v3 ?8 E" v Y0 f# n \) d6 f
print data1 T% p% l; ^: e; Z
#b = item.find_all("b")
* z; q6 T3 _# t+ i# j% h6 b8 G! B #print b
" \0 l$ X. f4 z6 U/ f- ]+ v #链接地址0 w2 E2 ]4 H5 R& f+ l/ Y# {
pat = re.compile(r'href="([^"]*)"')
7 i5 Z% h7 `, `7 U& d h = pat.search(str(item))0 A% H+ _! k: t& h8 S3 r
if h:
; }7 q' h! C' K: N' U5 K0 | #print h.group(0)
* v& x3 A( }+ w% S: ] href = h.group(1)
* E! `1 C/ A4 T n print href1 s9 k. k$ ^: x4 l0 L5 x
temp_data.append(h.group(1))
/ }1 F" t0 O# u g
0 h6 ^$ m1 ~- ~, I5 ^ _Num += 1# |+ J! q* @( D( A$ J5 E
#b = item.find_all(text=re.compile("Dormouse"))% Q7 J# S9 e: H' }* A* r @) g
#pprint(goods)
( O' t, k. \* N3 c9 U #print href
2 O' {4 D1 ~( Y #pat = re.compile(r'title="([^"]*)"')3 K/ A0 J' @: I' \; h
#h = pat.search(str(href))4 P, e+ k+ N1 Z2 [9 t
#if h:
% U! X7 Z' b9 v/ A* y6 X- v8 k* v #print h.group(1)" [" \1 x8 s. m
#temp_data.append(h.group(1)) r" R' {) ]8 M
_DATA.append(temp_data)
$ E. Q! V# i' j2 X! i6 t/ q
, v2 g0 @; {: M) B* H" b! j* `6 A#headers = {'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}##浏览器请求头(大部分网站没有这个请求头会报错、请务必加上哦)5 W% v+ B- o! j
#all_url = 'http://www.mzitu.com/all' ##开始的URL地址; V1 h$ w0 M5 l8 N9 }& `
#start_html = requests.get(all_url, headers=headers) ##使用requests中的get方法来获取all_url(就是:http://www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释- M3 Q( T5 k5 j0 V( G( ~3 l
#print(start_html.text) ##打印出start_html (请注意,concent是二进制的数据,一般用于下载图片、视频、音频、等多媒体内容是才使用concent, 对于打印网页内容请使用text)
: J U; M' c& V0 g1 e" F! S: D7 r
7 L- m* B, l1 ^" n; }. C3 v2 O; Tdef main() :, }7 x* g5 f. M& y- }5 F
global SHARE_Q
# x1 z& o9 G) A% A threads = []
2 n$ l' r0 o# l start = time.clock()
- i1 b9 | ^2 r: ]# q douban_url = "http://company.yktworld.com/comapny_search.asp?page={page}"
+ I6 v2 j! b: W& i #向队列中放入任务, 真正使用时, 应该设置为可持续的放入任务
- ~# s3 z4 J8 ^3 z$ N7 S( W for index in xrange(20) : % l* P( i/ J+ L& V/ F* f
SHARE_Q.put(douban_url.format(page = index * 1))5 }) w6 L% i' H: c% C/ k
for i in xrange(_WORKER_THREAD_NUM) :
% V/ } h4 x/ N! x1 }$ p thread = MyThread(worker,i)4 }& ^% {0 y% [2 Q/ d* F0 e- H
thread.start() #线程开始处理任务
% t: `! O* x! P4 ~( H, T1 t
. |, e9 ?4 S2 Y$ T4 `* h1 s7 z threads.append(thread)
1 x6 R P. N* c2 N2 o4 w for thread in threads :
: I; {! ~& b9 x2 i thread.join()
3 T. D7 |& V& K% z8 M SHARE_Q.join()
l+ C) s5 ?6 H! z, T i = 0
- ?; r2 X$ L" K& o with open("down.txt", "w+") as my_file :" K* K! W) t5 L6 i$ n# F$ C5 B
for page in _DATA :
: O4 b8 ]- w/ J* v: Y$ P. W/ F i += 18 @) C" B) t& j" [, u
for name in page:: R l/ w) ?8 O% P z
my_file.write(name + "\n")
" n$ t: x3 i; V" O+ _2 b9 n9 C. _. K* |6 r, i
print "Spider Successful!!!"+ R* C4 H$ D# r A$ F/ ` L
end = time.clock()
- v2 M5 K/ u5 X. Q print u'抓取完成!'
' B! J0 c+ V) C3 b print u'总页数:',i
5 o; N! |* J7 O print u'总条数:',_Num0 r& F1 y3 V' F6 u |; }5 C H0 ~
print u'一共用时:',end-start,u'秒'
H7 {1 V4 u6 ~' c$ E7 u6 _1 M5 G6 Y4 X1 a1 Y; P7 K
if __name__ == '__main__':$ F4 r$ v+ s) j& i# P
main()
9 A0 l; N' R* K2 f: x Y5 _* |! X- u! E$ c4 y
, k H' q. L) c W |
zan
|