多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python
   & U% _- |; t5 K  w
2. # -*- coding: utf-8 -*-
   8 Y/ n6 t  s$ w& }9 u$ n) Q) U
3. # filename: paxel.py, X  J/ ]$ N. P  o
   
4. ' ~5 V- l! `, f
   
5. '''It is a multi-thread downloading tool' E% R- s: q, U: P$ {
   
6. 
   & N+ L\" _1 d$ P) a. Y) f
7.     It was developed follow axel.
   % L$ E2 |6 c% n0 Y
8.         Author: volans5 [3 A% C+ r0 T+ U0 h
   
9.         E-mail: volansw [at] gmail.com
   ' l3 @; o4 t, H8 u( S& n
10. '''6 i1 C5 `1 v0 p9 V0 c) p
    
11. 
    : h& `  h% c9 Q  o& E, x
12. import sys
    . d  B. b0 Q& W: C; M1 ^2 i
13. import os4 P' @- n# G' K0 P
    
14. import time
    , ?4 L- q$ z; P: u$ s
15. import urllib
    ; c: L0 b+ s& P* b/ \
16. from threading import Thread) D- |% Q4 z) t% ^, M0 M
    
17. 9 @4 x# G  `9 F
    
18. local_proxies = {'http': 'http://131.139.58.200:8080'}) ^- l# @% W; o& d3 P+ E' a5 m
    
19. 
    . ]! Z! q9 V. _9 ?: c5 a
20. class AxelPython(Thread, urllib.FancyURLopener):\" J\" Z5 Y4 {6 Y$ u+ [
    
21.     '''Multi-thread downloading class.  ^- P- r1 ]\" _6 A0 F* X' W+ o
    
22. 
    ! R4 m- L; H6 b% a. |& m
23.         run() is a vitural method of Thread.$ }\" e6 E4 H8 V$ ^
    
24.     ''': }  X# j+ F' [6 D1 T2 J
    
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    : b7 X3 V- X/ _0 c$ k$ [! {
26.         Thread.__init__(self, name=threadname)4 g9 X0 J! W1 t' i/ \8 S
    
27.         urllib.FancyURLopener.__init__(self, proxies)) B( v/ I8 Q\" g; [\" M& v
    
28.         self.name = threadname
    + S* ?6 ]9 p& w9 \% j+ F
29.         self.url = url
    . j7 y$ k1 c* O4 T% T\" `+ Y; r4 O
30.         self.filename = filename2 F- _! k7 A5 o, v( M9 ?
    
31.         self.ranges = ranges
    # U  b$ h* w6 n, ~: A
32.         self.downloaded = 0/ W4 L4 M1 V' W' D$ u2 x1 f
    
33. # j8 f! N  k* u; `2 Y
    
34.     def run(self):
    6 n3 I, s- |; v5 t\" ^! g
35.         '''vertual function in Thread''': B0 T# V* W+ n0 G
    
36.         try:* r- k+ r- J; V  _/ f
    
37.             self.downloaded = os.path.getsize( self.filename )2 }+ b\" ]3 e$ n( u
    
38.         except OSError:+ _/ z! e  ]8 N. Y3 Z% u( k
    
39.             #print 'never downloaded') q& s- h4 S, Q8 e& r* F
    
40.             self.downloaded = 0\" J8 x% S2 s2 ~) m# S
    
41. ( x8 r' V8 |. k9 t\" D- o
    
42.         # rebuild start poind
    5 z' s, w' i% a: Z
43.         self.startpoint = self.ranges[0] + self.downloaded
    ! C6 v! Y3 c* K. I
44.          
    ! Y# y1 Y& l$ W
45.         # This part is completed# Z9 Z- f* C0 j\" C' h* z
    
46.         if self.startpoint >= self.ranges[1]:
    7 A% ]( B/ y5 X5 }
47.             print 'Part %s has been downloaded over.' % self.filename2 w3 T4 m& m, m& T' U3 R  E! d
    
48.             return5 K3 W9 }6 q, v/ ~+ _  d. g
    
49.          \" K  T+ H( F; {: [
    
50.         self.oneTimeSize = 16384 #16kByte/time: ~1 ]: }3 e9 V* S- p
    
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
    3 H% ^0 j+ r! @
52. 
    9 H1 v! n# q4 g
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
    # a& N: g  l( n. J0 F  Z# \
54.              & N4 v: g: r  H& ^0 Y) v* N
    
55.         self.urlhandle = self.open( self.url )6 @& c( @; w) V/ I3 w% q
    
56. ) }! G8 p+ M4 L7 e
    
57.         data = self.urlhandle.read( self.oneTimeSize ). u$ }+ W$ _- W3 a2 h7 r* H$ f0 ^
    
58.         while data:' B' }3 }  |1 `- D
    
59.             filehandle = open( self.filename, 'ab+' )
    4 o. t% n4 n3 X5 c( [9 s2 ]0 u
60.             filehandle.write( data )
    ; t( w0 ^+ q4 G4 J; b
61.             filehandle.close()- {$ [\" d/ J4 `7 a. c) T
    
62. 
    ! H+ v& }) w9 {5 Q% v
63.             self.downloaded += len( data )
    \" t5 Q/ H9 l+ x1 `* y
64.             #print "%s" % (self.name)
    ' C) ]8 M: `/ r1 d* q\" D6 }; N% i9 I1 [# Z
65.             #progress = u'\r...'' f. A! o$ \4 G
    
66. 
    ' T# M& r  J+ Y$ b2 k
67.             data = self.urlhandle.read( self.oneTimeSize )0 q  [/ S+ t% x' S: C
    
68.          
    3 G5 ?8 [3 M7 K\" {' A. @2 C5 c
69. def GetUrlFileSize(url, proxies={}):
    - S/ _! m  E5 y3 ~) T
70.     urlHandler = urllib.urlopen( url, proxies=proxies )
    % @7 d6 W/ [9 g& I4 G$ a! {8 F
71.     headers = urlHandler.info().headers
    : m0 }+ l# N9 W6 z) }
72.     length = 0
    2 E, s1 O* N% u; u. h) h
73.     for header in headers:; f( C) {  P0 g) x0 \/ w\" G
    
74.         if header.find('Length') != -1:
    - T/ s0 l; F# Y\" ^. O2 f5 f
75.             length = header.split(':')[-1].strip()
      R3 X' `% g9 L- K7 l2 q6 @$ F+ O
76.             length = int(length)
    - S\" x# ^2 o  d8 M' Y
77.     return length
      d. k* }0 }8 X, L$ ?
78. 5 R( ~: a1 H\" n4 o. t9 R
    
79. def SpliteBlocks(totalsize, blocknumber):
    . y9 c6 n  v. }% w# V; |
80.     blocksize = totalsize/blocknumber
    2 L9 Z2 Q4 m# ]$ ~4 G, i
81.     ranges = []
    $ U7 ^+ i( e( l- O, d
82.     for i in range(0, blocknumber-1):
    0 T9 g0 p( I) }7 |2 t( P4 I: o
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))9 \9 ^& i3 s; h9 f
    
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
    5 y3 I) E5 T; l+ Z) k
85. 
    + ~. F\" v2 D3 O1 `3 t4 e
86.     return ranges
    # Y2 F. A) ]* B4 T/ e$ E$ m. N
87. def islive(tasks):
    / q' d: _. q/ I- z
88.     for task in tasks:7 P4 D+ e3 |. j9 J4 z5 ?
    
89.         if task.isAlive():
    ! k7 o/ k  h' D, [* _
90.             return True
    / P7 h2 e/ ]% w  k3 [: f2 l
91.     return False( h9 H% V; P' y
    
92. 2 x- f4 o! z' Q5 X- l+ M5 m
    
93. def paxel(url, output, blocks=6, proxies=local_proxies):- \\" w# l) w6 }
    
94.     ''' paxel/ `: l- z8 b  u1 Z4 i& ~
    
95.     '''& v6 B1 B$ @/ n  p8 O, W8 n
    
96.     size = GetUrlFileSize( url, proxies ); L0 B) R2 B: H' j0 F* F* ?
    
97.     ranges = SpliteBlocks( size, blocks )
    1 }/ V* c  X- Y( ]
98. . H$ v0 A9 t; x\" P8 O  }9 g
    
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]# H- B0 L\" m; g\" `4 y8 l
    
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
     . B! v! s9 Y1 b# t
101.    * N+ X, o5 X0 E( f/ W: r
     
102.     tasks = []+ d3 s7 o& q8 R% u2 }+ T$ G- @) t
     
103.     for i in range(0,blocks):0 k) T7 |# H1 z) D1 N
     
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] ); o- @8 X# a* x
     
105.         task.setDaemon( True )/ B1 A6 E- [3 {; v8 G
     
106.         task.start()$ ]. m, w5 w- p- l( M& r% z1 x$ K
     
107.         tasks.append( task )3 G( p; F# t0 d6 l
     
108.          + o+ n) u; p8 y: Q9 V2 p0 k
     
109.     time.sleep( 2 )7 T; @# Z' t  I6 H
     
110.     while islive(tasks):, Q# z- g; |: L6 E1 ^! H
     
111.         downloaded = sum( [task.downloaded for task in tasks] )
     ; t5 P9 W; H. A2 E0 @5 ~
112.         process = downloaded/float(size)*1006 {, O; ^; f' R5 W1 Z; J  m8 f
     
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
     : ?8 q* t  @6 M) }2 [! h8 f  h
114.         sys.stdout.write(show)
     : Z) j9 G\" I  l4 B
115.         sys.stdout.flush()9 B  Q\" e2 q* H* _
     
116.         time.sleep( 0.5 )& h( o9 n5 R6 {\" U) L! x
     
117.              0 D3 }9 h2 S! d; y9 `
     
118.     filehandle = open( output, 'wb+' )
     0 [3 v  I, n) a- c
119.     for i in filename:( |* P2 \& M3 s9 w; v# e
     
120.         f = open( i, 'rb' ); ~( ^3 M9 b% v* c  K  _\" s
     
121.         filehandle.write( f.read() )5 x$ O6 O  @6 M3 ?0 p
     
122.         f.close()\" r9 A0 X7 S, N% g' g6 d' O1 L  `7 [
     
123.         try:
     4 Q1 @3 Z5 F0 M/ Q( `* J7 I3 D
124.             os.remove(i)
     / T* u! @5 G% R
125.             pass
     . f% |! W/ N$ C$ Y. }: d
126.         except:  j$ S$ C) y. P& B& m* O
     
127.             pass
     7 D$ r2 V) ]( x\" R% V
128.   u8 Y4 Q7 y0 C' E
     
129.     filehandle.close()
     1 }6 ^/ ]) Q% H  W0 \
130. 
     9 Y# D( W. ?8 Y, V$ v( V8 s
131. if __name__ == '__main__':
     , G1 l, v& O2 ^0 r
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"
     # `8 E3 n( V0 E. x: w! P) M, S+ K
133.     output = '001.jpg'3 w) ~) {( g\" q# X. J
     
134.     paxel( url, output, blocks=4, proxies={} )

复制代码