多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python
   , I+ U( U( M! J7 P# I9 Y- C* D0 f
2. # -*- coding: utf-8 -*-
   $ g( r7 j( ?) n  z( @+ m
3. # filename: paxel.py
   ; u) h9 p* _1 m+ F- `5 ~
4. 
   7 n& a* ]$ e8 [/ U0 ~4 V  [
5. '''It is a multi-thread downloading tool8 Q+ p7 _9 |) s0 K  S/ v$ h' e
   
6. 
   9 x$ C4 D+ U: ^) G& j
7.     It was developed follow axel.; L& c8 ?\" A4 U$ M2 W
   
8.         Author: volans
   # ?) N1 e8 L) d* [\" e
9.         E-mail: volansw [at] gmail.com% x! N% ]* b* \3 H
   
10. '''
    5 c3 ~) h6 w% `
11. 
    + b. x8 X6 E5 {7 g/ L& p
12. import sys& a2 C2 K& H, @3 X. F1 w
    
13. import os% `6 ]! n; a, |  C' t( m
    
14. import time4 I! n7 Y9 E0 B: p/ N2 b: M, l* L
    
15. import urllib; l# M7 h' X' y; ]5 r
    
16. from threading import Thread
    1 ?& @+ Q) h; b( m
17. - a' K3 d% j+ H* E4 }. `
    
18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    9 T& I+ s+ B* u0 i: p7 Q+ m
19. 
    9 L\" b/ v; X: E; N' g
20. class AxelPython(Thread, urllib.FancyURLopener):
    9 H' T* B' Y- u3 U: v0 T0 r6 n
21.     '''Multi-thread downloading class.
    ' a; s# k) h: v  O! E# i0 W. |
22. 
    7 w, m3 T( d1 u5 L' M# r' q
23.         run() is a vitural method of Thread.$ ]\" b- X8 p- x1 L
    
24.     '''  @; \6 j! h8 J0 P( y
    
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    + f' E9 {* K* d0 |) j2 N, H6 j
26.         Thread.__init__(self, name=threadname)
    5 @- x& p3 ^! D& R/ ~2 t
27.         urllib.FancyURLopener.__init__(self, proxies)
    ' ]2 n\" a. @5 i$ N/ I; {0 U9 |) R
28.         self.name = threadname; {( g\" B! F6 k' C/ G1 ^/ _4 v' P
    
29.         self.url = url* m  u8 P( [5 Y
    
30.         self.filename = filename
      K1 C- \( Z3 f  j' U
31.         self.ranges = ranges# K/ h: F8 V, p* j
    
32.         self.downloaded = 0! i3 ?8 i6 G* o, m2 y, M
    
33. % T; u6 ~8 v+ ]
    
34.     def run(self):
    \" y+ z% q3 R4 L6 t
35.         '''vertual function in Thread'''
    $ }  _3 k8 O1 e# t7 `7 S1 J+ H
36.         try:
    ' v( g/ Z& O- @! |\" }8 n, t
37.             self.downloaded = os.path.getsize( self.filename )
    0 s6 }% B  Y6 G* e7 G
38.         except OSError:
    / X8 V$ d- r! u9 P0 c6 T, C0 t
39.             #print 'never downloaded'
    ; m, [) z- {\" G- M  x; |
40.             self.downloaded = 0
    : ^1 b0 `8 G8 U: G1 J! ]
41. 7 N; z' q! z\" m! [8 v: X
    
42.         # rebuild start poind
    1 Q4 w$ a$ X& Y
43.         self.startpoint = self.ranges[0] + self.downloaded: C* _# i9 H& w, p! z+ b
    
44.          
    3 @! C7 ]. o3 X' k% L5 |) ?! Q' i
45.         # This part is completed
    ) w1 ~/ w* N3 [0 i( ?* J
46.         if self.startpoint >= self.ranges[1]:
    7 T; v4 O5 Y2 x\" T( l4 r5 u# K3 v
47.             print 'Part %s has been downloaded over.' % self.filename: o- G; |5 N4 f+ j1 R
    
48.             return\" X) {2 Q1 k. R. d4 _$ J3 V0 ~
    
49.          
    , A* n2 }; l) I6 ^, e- z6 T) L
50.         self.oneTimeSize = 16384 #16kByte/time; M( e- |\" N# _0 X; C
    
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
    6 H% G+ i  i4 k0 w& P( m/ x% P' W
52. \" M% D8 \/ [% P) }; J* Y  Q
    
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
    5 h: D- T, C# c' M. q1 j1 s6 s
54.             
    1 R% a: J\" c3 ^& l$ V
55.         self.urlhandle = self.open( self.url )
    \" j. d# b% E! c6 S
56. - \1 B. ^! W8 r1 u* f
    
57.         data = self.urlhandle.read( self.oneTimeSize )5 F\" G, d* O0 s- k, K- u
    
58.         while data:+ ^& f1 \, Y2 Y
    
59.             filehandle = open( self.filename, 'ab+' )) I$ E2 e) f# G. R\" i
    
60.             filehandle.write( data )9 \/ v4 m. D! ~( R
    
61.             filehandle.close()& ~: A+ M, H2 S- v
    
62. 
    : s% }2 o( `$ U0 N# L' O7 J' U- m3 G
63.             self.downloaded += len( data )
    0 ?/ k+ Q4 w  o
64.             #print "%s" % (self.name)/ k7 K7 L4 s\" H0 e\" e' E+ C4 m: J* K
    
65.             #progress = u'\r...'9 k8 M0 |1 z/ J+ n9 j* p; a
    
66. / d, B) d+ P2 K# f; T
    
67.             data = self.urlhandle.read( self.oneTimeSize )
    4 }! h) J$ x1 J
68.          
    ( z4 \2 q\" e# P. u$ o( E
69. def GetUrlFileSize(url, proxies={}):# n6 w5 T6 z4 ?, E
    
70.     urlHandler = urllib.urlopen( url, proxies=proxies )
    ' o( l: ?  l$ r' b\" F- @) R
71.     headers = urlHandler.info().headers- m2 c9 d+ u. l; s\" a8 |
    
72.     length = 08 R; p8 l; x) {
    
73.     for header in headers:! J$ ?7 W0 j+ T2 n# Q$ O; N
    
74.         if header.find('Length') != -1:
    \" a* {% r0 s' J
75.             length = header.split(':')[-1].strip()4 A7 k1 L$ F  ^3 a2 h2 q
    
76.             length = int(length)- z+ E* I& K5 d
    
77.     return length# m# q9 z7 S' C+ e
    
78. $ V: R$ g2 u& S# s: W, b\" y. Q# G
    
79. def SpliteBlocks(totalsize, blocknumber):
    8 G\" N$ x0 i: _\" F3 _\" U
80.     blocksize = totalsize/blocknumber
    1 c$ i% V3 S% l) \0 S* m
81.     ranges = []
    - f  F. v7 K* O5 Z\" D* l
82.     for i in range(0, blocknumber-1):
    \" x5 m$ ?5 y- ]) ~4 L
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))9 M3 Q. u8 [# m/ I7 K
    
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
    ; `) W* y4 r4 `: F0 x/ S' i6 [
85. 
    + F) e5 o8 Z7 m2 z- q9 f4 _
86.     return ranges
    ' b8 w* d6 o7 l& g7 f$ P
87. def islive(tasks):
    / o5 w9 t5 k' H
88.     for task in tasks:9 s1 |) `( W3 {3 L
    
89.         if task.isAlive():: I$ ]- q, `# W, I
    
90.             return True
    \" \/ B% X0 L7 ^/ T  n
91.     return False, \4 |: s( _7 Y2 r/ @, j
    
92. 
      ?& d$ F$ P# o7 r# g' w  G0 v8 V
93. def paxel(url, output, blocks=6, proxies=local_proxies):1 l7 L: R1 H% R3 H( ~
    
94.     ''' paxel
      J* W6 C9 r3 A6 Y4 W
95.     '''
    ; K: g1 p+ M+ N2 X' H1 Z) T
96.     size = GetUrlFileSize( url, proxies )
    ! M9 n  g; ?' G\" i) Y\" `& o/ x8 G
97.     ranges = SpliteBlocks( size, blocks )
    ) m- m6 J: |3 p& D* Z# C4 m8 \$ K
98. 
    * |' r1 j- p; ?# y3 V
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]; i% h/ @\" L; i% D3 W$ `
    
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
     % l/ v/ m9 b, t+ \  b( e
101.    
     ; A% `9 S! ~9 K8 _% N\" g
102.     tasks = []. [% [6 K8 K! B7 A
     
103.     for i in range(0,blocks):
     & b1 V5 A- o, c& O* R\" ]6 W, A
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )5 ?' o' I9 O  l/ s/ ]& _$ I/ Y
     
105.         task.setDaemon( True )
     ) u& F7 O; [; K1 ]: ^, E
106.         task.start()  A  z) Y\" W) r# G& x
     
107.         tasks.append( task )  j! m( p. z  F\" a
     
108.          * ]4 w' P; |  ~% M1 ]. Q
     
109.     time.sleep( 2 )
     8 ?, [) {$ Z$ r0 u% c, c; j
110.     while islive(tasks):# l9 j1 G+ w9 l' L5 J/ w* [
     
111.         downloaded = sum( [task.downloaded for task in tasks] )
     & a) j( V- a) x\" }) \
112.         process = downloaded/float(size)*100
     5 M; l3 W6 ]9 o\" b. h  f! j
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
     . v# A0 Q( x* [% P
114.         sys.stdout.write(show)+ ]1 n- F: J* j8 ?
     
115.         sys.stdout.flush()2 }( t- b8 ?4 X5 {6 {2 G4 B* w
     
116.         time.sleep( 0.5 )
     - q9 V! k! |& N9 r% @, g% I1 ^. H
117.             
     0 B- e: ^2 }8 }6 |: g. {
118.     filehandle = open( output, 'wb+' )) v1 W7 u) C, Y7 |$ s) r2 M
     
119.     for i in filename:: [& S# k4 l/ j4 U, A
     
120.         f = open( i, 'rb' )
     ( A9 c0 a1 d+ D; o  \
121.         filehandle.write( f.read() )
     : D  }9 G* T. N0 E
122.         f.close()
     + O& u) e0 S/ J8 w
123.         try:$ W4 ], P2 i) M7 [) C- i6 ~9 H9 y. d* W
     
124.             os.remove(i)/ x' f9 C( N3 b\" T+ k- |$ V. v
     
125.             pass
     ; y1 e4 C3 n8 P1 @, S\" d
126.         except:
     ' a1 R, |8 ~1 V, D
127.             pass
     ) J4 B9 N1 V0 T
128. 
     5 w3 Y9 @4 t8 J) ^; l5 ~7 q% B* E
129.     filehandle.close()4 K. I\" \( n: _( q) l8 w
     
130. 
     / g- P7 Y( P8 O3 c9 H- d
131. if __name__ == '__main__':\" a, ]/ w# N9 b6 h$ Y
     
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"# L7 h8 [5 t0 T  }( m: O
     
133.     output = '001.jpg'+ I3 y\" l, `' Y& C8 n' }
     
134.     paxel( url, output, blocks=4, proxies={} )

复制代码