多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python' H9 j6 {* c, R4 O
   
2. # -*- coding: utf-8 -*-
   , i0 F. X, b8 f
3. # filename: paxel.py7 L\" I9 ]: H, z, G+ w
   
4. 
   8 f9 C$ F! F5 o6 e7 ~
5. '''It is a multi-thread downloading tool
   $ }5 u0 a: `5 W9 c# W
6. - B% k, L4 Q, C- G- Y
   
7.     It was developed follow axel.
   \" b+ F2 Q6 _\" c
8.         Author: volans  O8 e- P4 j2 B/ C. ~
   
9.         E-mail: volansw [at] gmail.com
   ( `/ X; ^+ C5 N
10. '''
    * G8 L9 z% [, Z3 u3 O# {7 ~; [
11. 
    ; _5 V, c. [6 k( H+ x/ _
12. import sys
    ; p; b8 x- ]  _  s
13. import os, Z9 n% Y# i& b: `6 v
    
14. import time1 r7 T4 S0 E( N8 P& C
    
15. import urllib
    ; e# t\" [, ]7 }, n
16. from threading import Thread) G1 d9 k' O! j( B: {) v
    
17. 
    ( _: V0 K) O& m
18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    - U- `. C2 o1 V6 |. O
19. 
    ( B. w* H! x7 N
20. class AxelPython(Thread, urllib.FancyURLopener):
    ( G. d8 E9 |5 e% C! y; S
21.     '''Multi-thread downloading class.3 ~5 q7 T, _1 p* ~
    
22. 
    \" ~* r/ r, A: t. t
23.         run() is a vitural method of Thread.
    $ R  v* [6 f. }# s! j
24.     '''# M4 V; D9 Q2 Z9 B
    
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    % j$ E  Z9 t( S7 @; y# H1 N
26.         Thread.__init__(self, name=threadname): Q; l5 o9 l# }\" U
    
27.         urllib.FancyURLopener.__init__(self, proxies)
    5 L9 L  i0 J1 i2 R, D8 m3 s
28.         self.name = threadname) @: S5 T1 y3 q/ E
    
29.         self.url = url
    % O2 U# S. W3 f
30.         self.filename = filename
    6 {  ^5 q6 r) }$ W3 Y5 \
31.         self.ranges = ranges
    ) p6 r' ~2 s- C* ]; P. ~
32.         self.downloaded = 0( l  A5 u; Q4 u3 z! n- m
    
33. 
    \" ^, u% G7 r+ O+ u' s, Q0 \
34.     def run(self):
    ' W0 h\" X' I. D& {6 F
35.         '''vertual function in Thread'''! ?9 U( x# \+ B' v
    
36.         try:. m* n: w0 j( @# s
    
37.             self.downloaded = os.path.getsize( self.filename )
    $ y, O# m; v( p
38.         except OSError:' M) |6 H# U/ C2 W6 R) W  Q
    
39.             #print 'never downloaded'
    . T6 u; `) ^7 @' F& h& b, x
40.             self.downloaded = 0$ [, Y) K$ x; \& K
    
41. ! b# B* {) N4 U9 }- o1 i! b
    
42.         # rebuild start poind
    / p. N% J! C; u9 ]3 @4 q
43.         self.startpoint = self.ranges[0] + self.downloaded8 v2 s\" l+ |. t+ \
    
44.          
    9 e: I; V, V) b6 b
45.         # This part is completed* j5 ]' z, h5 E- X7 u
    
46.         if self.startpoint >= self.ranges[1]:$ O* f( E% D7 J) d! T4 Q3 x
    
47.             print 'Part %s has been downloaded over.' % self.filename0 }6 g/ w. |6 a: x2 m$ v% H. T- o
    
48.             return6 G1 |1 J6 @. j; |+ G+ l
    
49.          
    ! y7 }) q$ U+ S+ N
50.         self.oneTimeSize = 16384 #16kByte/time- G& Q- Y\" m' U3 C& r
    
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])\" ^9 w' h& V0 u5 r8 T3 m) ]
    
52. \" t8 ~4 j\" N# M+ h
    
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))- |0 r: U8 a# W' [; c$ _, j
    
54.             
    % Z$ b' V4 M\" |4 M
55.         self.urlhandle = self.open( self.url )
    . K6 @5 D- Z) D
56. 
    % j% y1 D) V8 w) x
57.         data = self.urlhandle.read( self.oneTimeSize ); t: G/ d* T% l' X  `
    
58.         while data:
    4 ?* y3 p: N- \7 c
59.             filehandle = open( self.filename, 'ab+' ), d: m\" U) P! b! E6 n! n5 `
    
60.             filehandle.write( data )' P& w/ N! T  C( C  j) ?( p
    
61.             filehandle.close()
    \" E5 q4 z- M\" i# }
62. % c3 f3 i& O; ]  ^: g  E  ]
    
63.             self.downloaded += len( data )
    7 I- Y  U% ]& X) ^
64.             #print "%s" % (self.name); }# T\" P8 B: L\" W
    
65.             #progress = u'\r...'  v  c3 h( ~  r
    
66. 
    5 ^) y, u  f5 m- ?
67.             data = self.urlhandle.read( self.oneTimeSize )
    + _2 B\" l/ a- Y: l& @
68.          
      {- g. \3 Z: P7 {\" ^4 f
69. def GetUrlFileSize(url, proxies={}):
    4 U/ M! m- H3 N1 b
70.     urlHandler = urllib.urlopen( url, proxies=proxies )* P8 o8 g5 a5 {  l
    
71.     headers = urlHandler.info().headers! }6 Q  q$ g* v8 A6 j\" k
    
72.     length = 0, d3 S) U* @7 q* w
    
73.     for header in headers:; _1 D; [, D+ p8 ]( l& b/ Q: O
    
74.         if header.find('Length') != -1:
      }/ |8 [. c+ r, _- i
75.             length = header.split(':')[-1].strip()& z, t& _$ g4 [
    
76.             length = int(length): ?0 X8 d+ @4 D! Y) l# Y
    
77.     return length; H0 _# g# O( o! B5 z# C( `
    
78. 
    / y% r  F/ l* ?1 s
79. def SpliteBlocks(totalsize, blocknumber):
    6 D  X1 [: ~7 Z9 l
80.     blocksize = totalsize/blocknumber
    \" {3 c3 o, ~% e
81.     ranges = []5 B: x8 S0 n! ?7 C' i. {- M
    
82.     for i in range(0, blocknumber-1):
    1 |% d, R$ S3 V* _/ y9 A. Q
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1)); m* q6 i8 |* j& Y3 ^8 ~. s) O$ L7 i
    
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
    ! z+ S  C, Y\" z5 ^/ z( P5 x8 X9 @
85. 
    0 A- p7 J0 q' ]$ h( G& k6 _7 Q
86.     return ranges4 w7 z% s# n- {3 @% E0 \
    
87. def islive(tasks):
    * O; Z# l% L* f* O  S1 {
88.     for task in tasks:0 `1 X4 z' f! u8 }; A% A
    
89.         if task.isAlive():  o$ l8 m! ]2 a; Y4 H8 ~
    
90.             return True; g  K% j7 D9 W- C  R( n
    
91.     return False
    % u1 @) @, E\" G9 e% ~4 d5 c3 F
92. ( R9 G  z7 Q/ {1 C1 m
    
93. def paxel(url, output, blocks=6, proxies=local_proxies):' D, Y3 @$ t: @# v: U
    
94.     ''' paxel
    ; ]! q$ E8 h% C* u# Z8 ?/ q, l6 O- y
95.     '''
    $ K8 V6 W5 ?; O* d
96.     size = GetUrlFileSize( url, proxies ). d7 Q' z) ~! [\" @) V% K( A
    
97.     ranges = SpliteBlocks( size, blocks )
    0 p  g\" M& D' g( y
98. 
    \" t5 Y/ j! `/ }# m8 H. e
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]5 J8 w7 c4 a1 W
    
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]+ A$ ?: ?1 n, u3 X$ ~
     
101.    7 B$ g2 U1 p2 j
     
102.     tasks = []
     % K\" i) R: Q$ D, t% r* T\" o
103.     for i in range(0,blocks):3 A9 P: t# X! W( H3 V
     
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )7 J0 m, D1 \+ {8 k6 s
     
105.         task.setDaemon( True )$ {0 U) U9 E8 g\" j, p6 R, g2 d; u
     
106.         task.start()) f( b/ a' d% `
     
107.         tasks.append( task ), n7 ?# J& n. J6 f
     
108.          
     ; [  r- b5 k6 B4 X1 Y( q+ G
109.     time.sleep( 2 )  z/ q) R! {9 {% i. f5 W
     
110.     while islive(tasks):
     0 V. {. ^% z' L0 t: V# q+ @9 g7 }
111.         downloaded = sum( [task.downloaded for task in tasks] )4 \0 Y4 c  R9 y* [1 v# O& }+ w
     
112.         process = downloaded/float(size)*100' x5 \8 L& b3 U, F' U
     
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
     % p, P$ Q2 H4 ~2 }* q
114.         sys.stdout.write(show)
     1 J0 G+ a& n7 s5 g
115.         sys.stdout.flush()3 `0 \' s8 q\" u5 ^
     
116.         time.sleep( 0.5 )! N* }9 D1 M( I  I0 i; w5 ?/ G
     
117.              3 @5 V& z+ e  \1 G
     
118.     filehandle = open( output, 'wb+' )
     ; F$ @# `% Z+ [% Q6 B5 r% y0 `
119.     for i in filename:
     4 G8 l' I: @' l8 q6 X' K1 F
120.         f = open( i, 'rb' )
       e7 w- K) I3 k4 l+ `
121.         filehandle.write( f.read() )
     1 L7 y$ B' `# Z7 F+ R
122.         f.close()1 p& ^4 r4 F7 m* o2 N
     
123.         try:
     - e8 z! L/ c$ P1 U
124.             os.remove(i)1 ^. a8 p: i5 G& z- R
     
125.             pass5 C6 ^6 P: e2 b6 E
     
126.         except:
     ! X/ b) L( t0 j( t  r: o
127.             pass
     , m4 s, ?% E4 A; Q8 v
128. \" T) v4 |6 ?- T' v
     
129.     filehandle.close()- D1 p& Q% a* \6 P
     
130. ' @2 E7 E9 h: t7 C3 C
     
131. if __name__ == '__main__':
     * M; `* \7 G* K+ m4 ^
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg", q1 O/ b( B# o
     
133.     output = '001.jpg'
     1 q8 {: }$ O\" i) E7 v+ v& f
134.     paxel( url, output, blocks=4, proxies={} )

复制代码

; P" E7 F( K5 z6 l( X$ F