多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python
   9 Y# x: }: E* B6 Y! e) P
2. # -*- coding: utf-8 -*-
   + t. P8 z+ r$ W) n% [- L& X
3. # filename: paxel.py$ N- z5 @7 h2 Z4 V) m$ g
   
4. 9 E- Y- P+ }/ l$ E: `1 b; z
   
5. '''It is a multi-thread downloading tool\" m- _9 t* }4 E! I3 y' w
   
6. 1 B% C* G' i+ O9 P
   
7.     It was developed follow axel.
   , X  O$ T7 G' g- E0 u; M
8.         Author: volans
   + C& i4 t) ~7 k/ S4 _7 L7 w
9.         E-mail: volansw [at] gmail.com
   $ {! [2 H1 r9 K: ?- J
10. '''
    2 Y( Z  z\" @3 G+ F, p# {
11. 4 h' f( t% X  E/ ~! x
    
12. import sys
    ; R8 v  [+ |  Y7 J, v! J
13. import os
    3 S5 k5 a! U; n2 [
14. import time8 e3 s\" Z5 \  h( E2 e2 c
    
15. import urllib
    # o. |1 K0 B% `
16. from threading import Thread( [; G; j% I6 X1 ]) C
    
17. 5 v; o; I3 b- N6 n( M
    
18. local_proxies = {'http': 'http://131.139.58.200:8080'}2 H- ]% G1 @1 b4 v/ g) a) s
    
19. 
    # o8 {3 S# F8 q6 x7 v  h6 A
20. class AxelPython(Thread, urllib.FancyURLopener):/ ?2 M! C0 g' ?
    
21.     '''Multi-thread downloading class.! W6 C( z9 z7 ^, e* j$ k# e
    
22. 
      _4 e3 f  l0 ~) ~! n
23.         run() is a vitural method of Thread.  Z) r% Y: c/ F3 S7 r/ m
    
24.     '''
    & I1 T+ R/ d3 J# w3 P
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    \" n& U& D8 k9 l9 N  ?. ?
26.         Thread.__init__(self, name=threadname)& {0 S/ h/ J! o  F/ k\" w
    
27.         urllib.FancyURLopener.__init__(self, proxies)
    + `8 S\" c/ g& {( i
28.         self.name = threadname
    3 C! z& q2 }6 W: x$ J
29.         self.url = url
    , q+ e0 D  c/ C; ^% z5 S9 f1 D
30.         self.filename = filename; L5 g8 `6 M, [: i6 J' L. j+ n5 S
    
31.         self.ranges = ranges4 h\" F, ^$ i) a2 M# s. w$ l+ U
    
32.         self.downloaded = 0  @% r, J5 i$ _: z
    
33. 
      g) a6 I% X* f
34.     def run(self):$ R/ a3 M% z6 \
    
35.         '''vertual function in Thread''': E8 y9 Y( L% Z\" a
    
36.         try:
    ) U8 f( t! Z) m& X  p$ t
37.             self.downloaded = os.path.getsize( self.filename )
    ( V\" [+ A+ ], @$ e. z
38.         except OSError:
    ' c& s6 U! O+ w% z) Y$ h& f
39.             #print 'never downloaded'
    ! j8 R4 N& r6 }
40.             self.downloaded = 0
    # m8 S5 o5 c! y: o: }( Q
41. & h9 ~$ @7 F; L: p9 l1 m
    
42.         # rebuild start poind
      {6 J\" N8 U$ G% [8 v
43.         self.startpoint = self.ranges[0] + self.downloaded
    0 z9 O- @8 r; H  C3 }
44.          ! `% S/ F2 f3 E\" g, Q0 A0 t
    
45.         # This part is completed
    3 l' c8 n/ l; ?6 V\" p
46.         if self.startpoint >= self.ranges[1]:& i9 S5 s* j\" [6 y! _  {
    
47.             print 'Part %s has been downloaded over.' % self.filename/ W3 ]+ J( h7 |- f% V; b' Z
    
48.             return) ^: F3 S\" @, l, q( E/ x
    
49.          
    ; ?6 `- R: u8 W; D0 a, ~
50.         self.oneTimeSize = 16384 #16kByte/time
    0 J( W& `/ @  N
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])* ^3 S0 ]3 [: G* M
    
52. 
    , @8 A9 i7 E0 P4 s/ W2 p
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))7 A+ Z; }. g2 H% r# B
    
54.              ) p2 S\" O3 y7 L: P
    
55.         self.urlhandle = self.open( self.url )9 q! @# Y8 H# [1 {3 d3 \; L$ o
    
56. + ]) }  F: }( ^2 J6 H7 B
    
57.         data = self.urlhandle.read( self.oneTimeSize )
    7 _, a2 ?2 _8 v* r
58.         while data:
    : |7 m( V/ n9 b' Q- r+ G
59.             filehandle = open( self.filename, 'ab+' )\" H; E3 w9 @# C% h) [2 l6 T
    
60.             filehandle.write( data )
    , L# ^2 Z\" Z! q4 T2 ?\" c; |/ U
61.             filehandle.close()) A6 X! z3 U8 \0 u3 U; i; P9 m
    
62. ; j9 R/ m1 K3 B  r2 N: X; D
    
63.             self.downloaded += len( data )
    * {6 m: R+ `8 ~
64.             #print "%s" % (self.name)
    / Z0 b4 w# ]0 l% l. H0 _
65.             #progress = u'\r...'7 o% t: c' ^: g
    
66. ) t  A$ d2 p, c2 v
    
67.             data = self.urlhandle.read( self.oneTimeSize )0 Q) a6 i  i( F, K# k) M
    
68.          2 l$ n+ s4 a  A2 O2 I\" p
    
69. def GetUrlFileSize(url, proxies={}):
    9 t  s' d( g% F4 l
70.     urlHandler = urllib.urlopen( url, proxies=proxies )7 u8 G4 n1 X\" A; i% U+ {0 Q
    
71.     headers = urlHandler.info().headers  V& L; d; r5 F0 k6 B
    
72.     length = 0+ M! E0 q) s6 b, c$ X$ z! M
    
73.     for header in headers:$ B+ G( ?, L4 z* _1 @
    
74.         if header.find('Length') != -1:
    3 o0 f. M- B( s4 N# _3 t0 p
75.             length = header.split(':')[-1].strip()7 `5 W$ q' {+ Y5 h. k
    
76.             length = int(length)6 y+ d+ A2 m) D. ~+ h
    
77.     return length
    $ k6 i- c1 ~9 ^  t' I1 e
78. 
    , M* r, A1 O6 c# E6 [) A$ d
79. def SpliteBlocks(totalsize, blocknumber):. [- y. H- G1 F/ f# L. B; J' M( J
    
80.     blocksize = totalsize/blocknumber
    : g! i+ C: N. d7 y
81.     ranges = []5 a, Z% h  z, u) t2 J1 u  M
    
82.     for i in range(0, blocknumber-1):
    + }7 |. m1 h/ L% u* u
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    + @# Y' l! E+ }/ s8 p# x7 F
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))& C0 r  s! ?! j/ I0 Z# [. c
    
85. 
    \" V& d  u2 k/ y  L
86.     return ranges
    ' k, B; z5 G! E! t  H; s
87. def islive(tasks):! E\" |, M7 R* F  Y6 E1 E$ m
    
88.     for task in tasks:
    \" E& V! t3 L2 a4 o
89.         if task.isAlive():/ @\" I\" U% C# r- U- |+ s) S9 E
    
90.             return True
    ( X1 L' t2 N: P: F0 o
91.     return False
    * @) a5 Y5 k% d2 b6 z- z
92. 6 K. G% L  |( m\" H& T
    
93. def paxel(url, output, blocks=6, proxies=local_proxies):
    . N4 f' D! e  K- d5 \6 m: U: V1 C
94.     ''' paxel. H. O) K& }( i. ]# e
    
95.     '''
    + n$ e/ \) b4 Z; H& i- O
96.     size = GetUrlFileSize( url, proxies )
    $ p: w' q1 Y$ Q- _! P1 }
97.     ranges = SpliteBlocks( size, blocks )
    ; \' N3 R  }\" F\" y! [* t
98. : v3 R/ J2 V/ ?  F: }! Z\" @
    
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]2 s3 j; W$ @! j% h1 W0 L% _6 @: M7 m* d
    
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]! e6 \! [, l! V- Y3 g  D
     
101.    ' R# V, t: G: d
     
102.     tasks = []. ?6 n+ T5 M! g! K* Z
     
103.     for i in range(0,blocks):, ^9 s6 ?7 ~! c* |: z( Y) S4 I9 n
     
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )/ k. ?% ?$ H. o' ]8 C\" g
     
105.         task.setDaemon( True )9 J\" k6 Q4 B( J, R& |# A
     
106.         task.start()4 z& y  N! _1 c, M$ F# s; ~' Q# p
     
107.         tasks.append( task )! f: }3 G! w1 R3 O+ m
     
108.          8 O$ ]' V3 N0 R6 N6 }\" c  B4 R0 G
     
109.     time.sleep( 2 )
     : H+ q3 f# c. q+ r
110.     while islive(tasks):
     % a) F$ s6 b- |. W
111.         downloaded = sum( [task.downloaded for task in tasks] )2 f1 B* g2 J! }8 @\" i) P+ ^
     
112.         process = downloaded/float(size)*100
     1 q1 |, X/ s  U\" k
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
     6 o: ^- s$ ?% a0 }\" X( A
114.         sys.stdout.write(show): N1 ]5 x; }2 a+ c7 m
     
115.         sys.stdout.flush()+ a9 H0 T) `, T; k& M) c) A; C
     
116.         time.sleep( 0.5 ). [/ x( G' o! z& B/ j6 J
     
117.              7 v9 G) e: M! \# f
     
118.     filehandle = open( output, 'wb+' )\" a) M( a1 d$ b9 ]
     
119.     for i in filename:' `9 U$ V, {5 j$ a\" Q\" p- W
     
120.         f = open( i, 'rb' )
     ; M4 E* r. x5 x, K% m) u; R: W
121.         filehandle.write( f.read() )
     * |1 r6 Y% g, V5 e8 V
122.         f.close()8 @; u) ^+ M9 ~\" }2 f- \5 G
     
123.         try:
     / D) m- O5 L+ i8 u4 ?
124.             os.remove(i)( s$ w\" ~0 r. ]2 G! D
     
125.             pass- ]+ z) F/ E; M3 ^, ]& t: e
     
126.         except:\" _/ n$ e: R# t2 Z\" j0 h
     
127.             pass
     3 a# W, ^; @$ ]$ h
128. 
     4 E( j+ O  R, Q8 Y
129.     filehandle.close()
     & X3 A; l6 O) m2 M9 j& d& Z: _
130. 
     / C: D( A; P) T3 j
131. if __name__ == '__main__':
     ( k\" @) f% M4 ~6 V, |3 C, ]' ?
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"3 P& L) i6 ?6 ?3 O: T8 f1 |
     
133.     output = '001.jpg'7 H9 v  G3 ]8 B' z) _- ]7 A8 E
     
134.     paxel( url, output, blocks=4, proxies={} )

复制代码

+ l: R+ R0 o. r; U8 Q, ~