- 在线时间
- 481 小时
- 最后登录
- 2026-8-25
- 注册时间
- 2023-7-11
- 听众数
- 4
- 收听数
- 0
- 能力
- 0 分
- 体力
- 7859 点
- 威望
- 0 点
- 阅读权限
- 255
- 积分
- 2946
- 相册
- 0
- 日志
- 0
- 记录
- 0
- 帖子
- 1177
- 主题
- 1192
- 精华
- 0
- 分享
- 0
- 好友
- 1
该用户从未签到
 |
#!/usr/bin/env python* c$ C* R0 D+ e* |
# -*- coding:utf-8 -*-+ c: u2 u1 }* }8 M. Z# g9 @
"""7 z8 A( C0 p4 A9 e% |" d* u/ |2 N
Python爬虫,抓取一卡通相关企业信息
' C0 i: Q4 i9 {Anthor: yangyongzhen
: {! _# F* z- U" C7 _( C' ]Version: 0.0.2
1 h% V% u- B3 B7 H$ r- ]9 ADate: 2014-12-14" J- L- j; S% `7 G4 h$ G+ i c, l4 z
Language: Python2.7.5. Y+ h8 x' P" N5 O
Editor: Sublime Text2
2 ~5 ~; S7 X( A2 J"""
4 ~! F7 v) @) o5 w$ n# A, Z8 r9 [0 |# `' E/ D4 A4 L
import urllib2, re, string
1 t6 \6 B$ |1 s7 F* yimport threading, Queue, time
6 z. N7 h9 |/ c) x3 E3 X6 a4 _( x. mimport sys# t# _3 i' L' U8 p; v2 _
import os
& v0 Y8 l' K% a. O; L4 Bfrom bs4 import BeautifulSoup1 {8 j; C% `; O9 g) o% ~$ h
#from pprint import pprint/ `# n) D% y$ d3 B! y
6 i z$ t6 V' V9 I& d( f! Xreload(sys) W+ l; y1 P h
sys.setdefaultencoding('utf8')$ B9 Y* c5 a. T2 l" G& P
_DATA = []8 s3 ?! `. I* G# x4 s" ?9 ^
FILE_LOCK = threading.Lock()6 g9 }5 d, {' K- I
SHARE_Q = Queue.Queue() #构造一个不限制大小的的队列% Z% c( ^& L' ]$ w
_WORKER_THREAD_NUM = 3 #设置线程的个数* t# }4 y2 B: Q$ ~. `
4 C* Z! t+ |. N# L4 e5 @
_Num = 0 #总条数
" q X8 N& J$ h: lclass MyThread(threading.Thread) :
3 B$ D: @7 Y- k. {4 Q7 ]6 x+ y2 a' D5 l
def __init__(self, func,num) :
$ J+ R0 ~" G! ]: [5 x super(MyThread, self).__init__() #调用父类的构造函数
: s, @% a+ N8 l; Y& q5 c self.func = func #传入线程函数逻辑
u8 ^$ s" \1 _ self.thread_num = num ; B" F# `- W% J# X1 D9 V/ C
def run(self) :
1 }6 t, _/ H7 Y% m* V self.func()
4 O9 n9 D5 k0 @* C' l" ~+ d' j #print u'线程ID:',self.thread_num
8 ^& X2 f% m0 o I
) m' c3 O$ I7 E& V$ I2 d6 h) a- kdef worker() :
2 C( m7 w- R& h; ]4 g* o global SHARE_Q- p5 |) |) s2 b3 z7 p
while not SHARE_Q.empty():- G/ y [ y. J( B p. L1 m8 r$ L
url = SHARE_Q.get() #获得任务' Z) J; V& L+ o
my_page = get_page(url)
; n( u) c" ?/ Z6 V; Q; U find_data(my_page) #获得当前页面的数据
6 e0 e* Y' ~# m- `/ ?8 c0 O5 j1 g& [ #write_into_file(temp_data)+ @7 s% ~8 _# q5 Z
time.sleep(1)
) ~; f0 ^) o# |" x3 @$ u SHARE_Q.task_done()7 `( x% c; {. L8 y! E9 \/ @$ e" g
6 t. L* A3 k" n* b( f6 x
def get_page(url) :
* s, I9 X. p+ Q' v/ a6 @ w4 y C# f """
, f2 F/ I1 y# _$ q 根据所给的url爬取网页HTML# N% @& `9 z1 f% F
Args:
) O7 E% O9 X% w9 ]: ? url: 表示当前要爬取页面的url
8 a( c$ Y/ J% q. A6 b Returns:
) ?: n% F& W( z2 n 返回抓取到整个页面的HTML(unicode编码)
7 S9 t% V0 \$ _. j5 d& T8 ~3 J! f Raises:
9 N& J( `. I4 k8 N* V( Q! _ URLError:url引发的异常
/ [( V Z& @3 ~ """
! U4 a% q) g" F8 t% |2 W try :
& a# B% @* X% a/ t" |% Q& @ html = urllib2.urlopen(url).read()5 y9 e; b/ x6 y
my_page = html.decode("gbk",'ignore')
1 n5 K7 S0 \) |5 v #my_page = unicode(html,'utf-8','ignore').encode('utf-8','ignore')
8 w6 ^4 n+ p- R+ o #my_page = urllib2.urlopen(url).read().decode("utf8")2 Q. W5 D" D! |! _( U h8 q
except urllib2.URLError, e :: W1 Y; S8 K% K# D& V
if hasattr(e, "code"):6 Z9 x, c B; J1 K& S' E! G! `; S1 D
print "The server couldn't fulfill the request." e1 G+ `: ]+ }
print "Error code: %s" % e.code) _' [( c" [& c6 p* x9 @7 p+ ?9 O
elif hasattr(e, "reason"):
* P. P5 V' {' y+ z, Q7 }* | print "We failed to reach a server. Please check your url and read the Reason"9 f/ ]; v& _4 w; X8 S, }3 z
print "Reason: %s" % e.reason
5 ]6 \) \$ ^# Z! J: H return my_page
. L0 h5 M" H1 i- h: k; F- v- `9 I5 X* o8 e+ D& \
def find_data(my_page) :+ |( h$ B; \6 q' H0 W
"""
" e) K8 p& }+ P* w/ n. B% J 通过返回的整个网页HTML, 正则匹配名称
, {# ~7 ?$ Y& c! i. q& S+ c' P% u$ K1 C1 o
Args: O7 [& \; @6 Y" C7 k7 ` R) p
my_page: 传入页面的HTML文本用于正则匹配2 y# A8 M/ N2 M# Z% F
"""+ ~+ n$ r6 W6 h3 ~
global _Num h3 {) ?% ]9 M4 D8 x s2 e2 B
temp_data = []* d" U* \" t1 X9 p5 c4 G7 F. @, D$ Z
items = BeautifulSoup(my_page).find_all("div", style="width:96%;margin:10px;border-bottom:1px #CCC dashed;padding-bottom:10px;"); p' M8 V: @# b9 x5 M( F
for index, item in enumerate(items) :7 ^( C# e: q/ s, s! K
#print item; [& p: \- g! i: F) Y h
#print item.h19 l9 s) Z5 [9 |, y# p( @5 K ~% ]- X( D
#print h.group()
o3 p) E5 g3 s1 y' `7 o #temp_data.append(item)4 f3 t9 N& ?; \' S
#print item.find(re.compile("^a"))
2 R* ~5 V$ y$ e0 A href = item.find(re.compile("^a"))
6 ~! P- d7 X) U #soup = BeautifulSoup(item)
g* t e2 g" Q6 D1 p! N #公司名称* q f8 M1 C) f' J. S+ ^
if item.a:+ m2 w( C3 i" T, n: M- f) r- v F
data = item.a.string.encode("gbk","ignore")2 S& d& z3 x$ o, m0 Q
print data; L" Q, w1 Q( B2 R+ u; E
temp_data.append(data)% Z- }7 {2 Q% T7 _! q3 z
5 f9 ~ h7 H8 H. |* S
goods = item.find_all("div", style="font-size:12px;")
& ?. `! U+ F& T & R+ c( F8 r; ]
#经营产品与联系方式
5 ?( K( r- c' _6 M4 h for i in goods:* I2 ?/ h+ S- r |- L
data = i.get_text().encode("gbk","ignore")4 r( ?4 I+ |) Z# U& c/ p
temp_data.append(data)9 t/ B( k j4 l+ y5 B; j
print data
7 {( @; t( p: s- o9 W #b = item.find_all("b")) b+ Y7 M/ o1 k; ?) y" o" j
#print b8 X2 V5 C, p3 z0 }- M. r
#链接地址( ^, c2 q3 y% m; M1 g* s
pat = re.compile(r'href="([^"]*)"')& p7 I; H$ y4 l- v( ~* \
h = pat.search(str(item))( R3 |1 q4 ?4 S9 R
if h:
. k: X* Q8 g1 e, J9 @7 r #print h.group(0)- n- K& l/ u% X) Z# B# l
href = h.group(1)
# `0 Y/ x! f( ^4 o; T print href
, K+ p7 v2 }9 K: ^; f- ]' r temp_data.append(h.group(1))
8 f" o3 ^3 l$ A2 @7 M0 C; ]% c) ]* t. W4 y1 G& Z# o
_Num += 1& d$ |+ n0 c2 O$ L
#b = item.find_all(text=re.compile("Dormouse"))
/ {: n" d# c: W #pprint(goods)1 Z" k% r# o( I S+ x: @& n, k
#print href" H, _5 x y, W' P- R& O+ @0 ~- r2 u
#pat = re.compile(r'title="([^"]*)"')) v" ]4 @* q7 z
#h = pat.search(str(href))
, D4 G4 \7 Y5 s' ^$ h# {; O #if h:1 e! z4 u( x# K! j. M1 f! x6 t
#print h.group(1)
7 H b1 Z) L1 l' \0 s #temp_data.append(h.group(1))
3 O% P& X& e& ~% T B _DATA.append(temp_data)0 I' \7 U9 D! ^9 H6 r) u
# M. {2 G# Z6 D, {: E" k
#headers = {'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}##浏览器请求头(大部分网站没有这个请求头会报错、请务必加上哦)" O/ A& o8 ^, ~6 t
#all_url = 'http://www.mzitu.com/all' ##开始的URL地址+ ]% ?* Q. w" \8 o( {% N
#start_html = requests.get(all_url, headers=headers) ##使用requests中的get方法来获取all_url(就是:http://www.mzitu.com/all这个地址)的内容 headers为上面设置的请求头、请务必参考requests官方文档解释; E* m$ {4 Z. A0 M! }
#print(start_html.text) ##打印出start_html (请注意,concent是二进制的数据,一般用于下载图片、视频、音频、等多媒体内容是才使用concent, 对于打印网页内容请使用text), v- i q3 z o6 h( t s4 h9 \% G
/ k9 @4 B1 X! f" i% i2 Jdef main() :
1 a- p- N4 h$ V2 ~: T global SHARE_Q) Y$ C' ^: S7 U7 [& i
threads = []
2 i) X! d1 p/ \( J, { start = time.clock()
, L) S! O; B* P1 a; W douban_url = "http://company.yktworld.com/comapny_search.asp?page={page}"6 z1 N- z9 l& ^ J c6 G( K
#向队列中放入任务, 真正使用时, 应该设置为可持续的放入任务
( Q* o$ l' ^2 g, ^' l for index in xrange(20) : , m+ _# F+ E3 M2 u* _' k& X% g' ?
SHARE_Q.put(douban_url.format(page = index * 1))
* U* F9 e. `5 B% u- n# P# e) I( J for i in xrange(_WORKER_THREAD_NUM) :: ~3 X/ V" ~+ Z! j; M
thread = MyThread(worker,i)
. u% v% f- I( ~0 X: N4 ~7 f thread.start() #线程开始处理任务
2 [) p- ~. L/ S" P; g8 z9 i5 a9 x: A
threads.append(thread)7 ^! F x: h- d
for thread in threads :, u4 g/ U0 ?9 E
thread.join()
( [8 e$ ?' i& t1 c SHARE_Q.join()
& M) z3 x2 l& Y6 S& ^9 E# v i = 01 j5 x" c/ ~* c Y9 y
with open("down.txt", "w+") as my_file :8 W! I( u/ W6 l
for page in _DATA :
3 ]. \ ^) n1 k0 v$ k# O+ K i += 1! y- C5 F8 U6 C" L* h
for name in page:- R9 b( {0 s3 J$ h8 n9 \% Y6 d
my_file.write(name + "\n"), @, N* n5 M) \+ O+ m6 @! |( d
1 w. u& f& ]* @+ w" H print "Spider Successful!!!"9 A" y5 T: d! u. g! v7 F
end = time.clock()
% T/ C' o0 z Q8 M7 u: M$ I print u'抓取完成!'
- S+ i5 A0 ]$ r; i8 j print u'总页数:',i' A) ?8 |+ Z" q4 i: N
print u'总条数:',_Num
) g5 ^: M1 k$ L. F print u'一共用时:',end-start,u'秒'* v! \0 X2 _! n9 f& t
, }1 }: a- O" o, s6 H: a: ]- G8 l- Jif __name__ == '__main__':& f1 F; q5 q% q, e) a/ B4 X4 h- H
main()8 l6 R I' i9 c4 c$ S
+ r5 |0 m2 H7 E2 l K( l7 H8 h7 T( o, o4 u( F X9 C( {$ }" g
|
zan
|