多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python
   7 f& d, P$ k  B, b
2. # -*- coding: utf-8 -*-7 }! c4 [4 @% k
   
3. # filename: paxel.py
   ) J/ R* ~3 f! ^3 L
4. \" c( n+ {: L: l
   
5. '''It is a multi-thread downloading tool
   1 a% l$ q3 \4 f\" f! L( c- m
6. / n8 L8 f7 H: q! l1 f
   
7.     It was developed follow axel.
   ' \6 C$ W% n* u$ {9 K) C$ C! @
8.         Author: volans
   # N0 G  E$ I- s$ [8 t4 [0 s  q$ l! |
9.         E-mail: volansw [at] gmail.com. q: {! r0 V1 p
   
10. '''6 V/ s( }5 @& \/ U
    
11. 
    2 y( D  H6 ~- v
12. import sys
    % G' {: _6 E4 [, U: P+ E: ?
13. import os' W) H  f% Y. z2 c\" L' T
    
14. import time
    % G3 `7 N# s& \; D4 L\" w1 N
15. import urllib
    & C* p& y) w6 X) y8 D6 A
16. from threading import Thread( S0 ]9 q& q+ Q( G( p' w: M# f. q
    
17. 
      }7 Q1 I. I5 w$ r
18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    3 Q2 U% c& h  Q) ^
19. 
    ! }) j) I: D/ G3 Z& W* A1 N
20. class AxelPython(Thread, urllib.FancyURLopener):
    4 J9 B$ G3 S' u4 @
21.     '''Multi-thread downloading class.+ |7 U; A; ^  P+ B. u4 V
    
22. 
    8 @! Q, }9 ]% Q
23.         run() is a vitural method of Thread.
    ; d\" ^, J$ f  @% g0 U6 v
24.     '''
    . s! ?) ?& M* ~/ j5 P
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    ; F+ \: x6 x6 U
26.         Thread.__init__(self, name=threadname)* Z( e& X5 O) j' O2 \
    
27.         urllib.FancyURLopener.__init__(self, proxies)2 x' G4 ?  J6 ]\" ~
    
28.         self.name = threadname! m4 s5 [& U- v5 I; r& p
    
29.         self.url = url
    - [) Q/ \/ y( }* h. Y: ?' i
30.         self.filename = filename
    7 D# z* V\" k+ }6 o, r2 r) c* M
31.         self.ranges = ranges
    $ V8 O6 `' R# a+ F. U4 [* T
32.         self.downloaded = 0+ L\" q7 j6 ~9 x# ~
    
33. 
      \, t# c4 U+ \6 K3 V
34.     def run(self):
    % i- U4 W5 ~7 c( B# [- o) g4 i
35.         '''vertual function in Thread'''3 J+ d) h4 r) `5 a2 V+ ?
    
36.         try:
    ; E' l2 D0 x2 c4 y
37.             self.downloaded = os.path.getsize( self.filename )  a. s6 U+ F, ]
    
38.         except OSError:0 @* G9 ~* M- {* L4 {
    
39.             #print 'never downloaded'- E  z0 \- q& h6 {. [- L! Q6 r
    
40.             self.downloaded = 0
    & M1 w\" g+ A, ]2 J/ j( `8 z
41. % E$ ^8 s4 |\" o) ]! g) z
    
42.         # rebuild start poind
    6 ^+ F/ a; J& b2 d
43.         self.startpoint = self.ranges[0] + self.downloaded
    ( N9 v1 e3 K0 K\" h3 q. [5 k( e
44.          3 @$ O' j' k8 M3 G
    
45.         # This part is completed
    % Z5 b7 s$ G0 q: a
46.         if self.startpoint >= self.ranges[1]:
    5 [+ }3 w2 A$ F9 l8 @
47.             print 'Part %s has been downloaded over.' % self.filename& b) w- b, b/ g$ q4 U
    
48.             return
    3 _) ?, |- ~8 u! ]% q
49.          9 f$ ^0 K6 d) N* y& `! ?5 Y
    
50.         self.oneTimeSize = 16384 #16kByte/time$ |\" a0 {  F* G# \* B
    
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
    + q4 U' G% u* V. g/ A
52. 
    , Y2 ]$ k; {0 G! c+ Q
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))4 J$ [. O* u9 r) N: @2 K) K9 @
    
54.             
    0 m/ [! z! u9 B2 _- n6 Y( t) f7 _
55.         self.urlhandle = self.open( self.url )
    * L1 u* ~) t1 V1 ]* z1 F- Z1 K+ o
56. 4 {  M  j* Q% v. J% M0 \5 X  a0 I
    
57.         data = self.urlhandle.read( self.oneTimeSize )
    ( i) g8 L& C2 g( f- d& e( Y% k; N# Z
58.         while data:0 n$ G: k' s  _& ?& w8 _( L# Y
    
59.             filehandle = open( self.filename, 'ab+' )\" m/ z, H$ |# g
    
60.             filehandle.write( data )
    7 Z% M& l# A  k* D5 g* _\" G& m; m
61.             filehandle.close()4 z# d1 l# P( S; b- u
    
62. 
    / u- W( D/ J2 f5 Z# N
63.             self.downloaded += len( data )
    \" A  W; \9 R( i, l; `1 l/ B9 A
64.             #print "%s" % (self.name)3 a6 p0 |7 g; Q
    
65.             #progress = u'\r...'
    3 \9 P- X9 _: }- i\" Y
66. $ Q& e$ |% d) o1 f1 N( I
    
67.             data = self.urlhandle.read( self.oneTimeSize )4 r8 e0 e9 B; _! _% \2 u
    
68.          
    : t# Q6 `0 l# a9 [' a
69. def GetUrlFileSize(url, proxies={}):# C1 v, W\" r# W+ h7 Q/ U& h. Z& n
    
70.     urlHandler = urllib.urlopen( url, proxies=proxies )1 t4 k8 P' z, G6 o2 S5 L& N
    
71.     headers = urlHandler.info().headers! b  r  C  H3 Z8 `) e  z/ K
    
72.     length = 0. Z+ |' R5 ^6 V- m0 w6 k1 C6 M5 W
    
73.     for header in headers:
    # C# [: ~7 Q: `$ @3 r\" c+ {
74.         if header.find('Length') != -1:
    ( k9 E! ]5 ?! E- d# K  R
75.             length = header.split(':')[-1].strip()
    : j0 v& k9 \* F5 s4 o% C5 L& a
76.             length = int(length)
    ' B5 ]& M% ?- x& f# m  {& {; c
77.     return length  u& r7 z  c\" t
    
78. ! J5 x5 k\" D# [6 \
    
79. def SpliteBlocks(totalsize, blocknumber):
    7 R/ n% l6 k/ t6 c
80.     blocksize = totalsize/blocknumber' Q$ q8 o8 u3 \4 \
    
81.     ranges = []
    # g  H( ^: \. p% {) g; q
82.     for i in range(0, blocknumber-1):
    7 W9 M% n/ d# R
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    $ Y4 r2 m6 S* |  o# D, d
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))& ^, X3 R4 _1 q4 E* P
    
85. 7 d3 ~' P& b& M1 g/ z# X9 ?/ `
    
86.     return ranges: z6 I+ Y( @8 Y\" a2 n
    
87. def islive(tasks):1 ?0 B7 E  [& l( w/ j7 w1 @) m2 k
    
88.     for task in tasks:& }0 C5 ^! V4 N+ d
    
89.         if task.isAlive():
    9 j6 C- @: m( z8 C! t
90.             return True
    2 n\" E: }8 p$ o0 t
91.     return False& R: P3 d  T4 B  r/ E# I$ a: T
    
92. 
      @7 B( o) s- M$ ?8 N: C
93. def paxel(url, output, blocks=6, proxies=local_proxies):
    \" I1 b3 g' s4 O* f; X
94.     ''' paxel/ q' K5 }\" ?' N0 Y8 P
    
95.     '''
    $ u5 o7 q( ^& I; P0 R+ N
96.     size = GetUrlFileSize( url, proxies )
    % W! r: k' O  ?+ f4 X  l8 V6 J; n
97.     ranges = SpliteBlocks( size, blocks )
    + r' k8 m$ }5 l/ S5 X
98. 
      J# b; e5 V7 ]5 k
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]/ E! \4 G6 _/ x: T& F
    
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
     2 K$ C2 A/ B$ j' q! I9 W# b
101.    
     5 J1 B+ m9 R- D3 M
102.     tasks = []
     4 ?# F% l6 A\" {  K! W
103.     for i in range(0,blocks):
     - u/ |2 q/ H\" k5 Z% |% m
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] ); @! F2 b8 P' A$ S* a( O' L
     
105.         task.setDaemon( True )
     6 }7 u- F! I+ F' i
106.         task.start()- Q. f, y. H( `
     
107.         tasks.append( task ); V% f, x9 q; U; ~& B, g; f8 r
     
108.          
     6 ]. ^3 }- F4 }( A/ n7 |- x
109.     time.sleep( 2 )- v% _, G& K0 r
     
110.     while islive(tasks):
     5 h9 N# E' k: ]
111.         downloaded = sum( [task.downloaded for task in tasks] )\" f$ x7 q\" _, I  T) R
     
112.         process = downloaded/float(size)*100
     9 A, j3 [# ?; B  U( v  f
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)  t7 {, u: L5 W+ I- b/ W3 U7 v
     
114.         sys.stdout.write(show)4 {. x7 M( o( Q# R
     
115.         sys.stdout.flush()
     ( h. g/ j' w+ X2 C
116.         time.sleep( 0.5 )
     + c$ _6 f- C; U- `# Q5 [0 K\" y' z
117.             
     5 E  U\" i: D5 p
118.     filehandle = open( output, 'wb+' )
     # v' ^1 s9 |& a! @
119.     for i in filename:
     3 k8 x* j, j; o/ P3 r\" }
120.         f = open( i, 'rb' )
     2 ^0 ~\" e5 F+ c
121.         filehandle.write( f.read() )1 l8 ]) w0 S  v' g6 Y
     
122.         f.close()
       [5 o3 Z$ F$ W. t
123.         try:
     ( a# o# g9 o- O6 g& X+ `9 h
124.             os.remove(i)
     & o, Z0 {9 u0 g& J3 z  }( Y
125.             pass5 \. \. s& u1 }# X, H! E4 ^
     
126.         except:
     # j! J  ]+ ^$ U3 G+ g- a8 [7 C% \
127.             pass- ~$ p2 e5 Z+ Z; p; g/ p/ a
     
128. 
     - x! }\" q+ Q; H6 B8 d2 t$ d
129.     filehandle.close()2 X' F2 A9 p: [9 C6 v; e
     
130. 
     2 l' o# H+ a% P7 G
131. if __name__ == '__main__':
     8 g9 t1 I/ G- l+ b4 \
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"$ s! R6 _: q5 O+ Z
     
133.     output = '001.jpg'
     . K: a/ y' q. q4 v7 F$ f9 e5 k) N
134.     paxel( url, output, blocks=4, proxies={} )

复制代码

- S# s- L6 i! h