多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python
   , f% m( W; {, i) Y% G( G+ q
2. # -*- coding: utf-8 -*-
   8 j: E- O+ w6 m* e, K/ P$ A
3. # filename: paxel.py8 w' f7 H: V: i' L' k\" u, @+ \
   
4. # Z- d$ K: r$ z% o  ?; G& C
   
5. '''It is a multi-thread downloading tool  Z& C3 T\" a* o7 G2 M& B% a7 o
   
6. - {# j4 m\" R  u$ K  m% p
   
7.     It was developed follow axel.
     s; i: P7 X; g) E\" ], f: ~
8.         Author: volans/ H) y+ z8 `& t  e
   
9.         E-mail: volansw [at] gmail.com( P8 I# g8 R) h\" U9 z
   
10. '''9 S# g- q6 y  c
    
11.   U, }+ {& |\" o7 x* ?% X
    
12. import sys
    6 Y\" f. x  Y' G. R
13. import os
    8 v/ O* K; S$ G6 q\" P( `7 o8 z
14. import time5 W\" U  C. m. M3 b' H
    
15. import urllib
    \" T$ O& m) e1 c% \& B- u# K
16. from threading import Thread
    1 W! P0 l; K  B4 F1 k
17. 
    * u\" l. L' z5 B6 ]! h6 o( j/ i; w6 x
18. local_proxies = {'http': 'http://131.139.58.200:8080'}/ d/ X  |- b) O
    
19. 
    0 v% x. g7 e8 \9 `& N\" M& R9 g
20. class AxelPython(Thread, urllib.FancyURLopener):) u: X% u( t3 K\" C. o4 T6 Y
    
21.     '''Multi-thread downloading class.
    ' l% C$ o7 U- [: _1 \2 Z; {
22. 
    8 t- b8 r. F$ D
23.         run() is a vitural method of Thread.
    ) j0 E3 C3 o$ C7 T6 |; |6 o
24.     '''
    \" g4 l6 H\" v: y
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    & K  h+ A7 n/ E1 O; ~/ W1 a7 X\" @
26.         Thread.__init__(self, name=threadname)  r: K, z( J& E  k) R* V; D
    
27.         urllib.FancyURLopener.__init__(self, proxies)
    8 x; |  m5 f+ C( r% ^1 P0 N
28.         self.name = threadname
    8 s0 s\" D2 f3 n2 e7 h\" m1 R( [
29.         self.url = url
    9 r- y$ @% \: h) @; t
30.         self.filename = filename
    * O, ^2 I9 s) X; A2 y; s\" a
31.         self.ranges = ranges! Y% y9 m; Q, Y. S2 w8 i. B
    
32.         self.downloaded = 08 s- t; x+ h! D  L
    
33. 3 F+ b- {, I; F6 R, \\" z3 S
    
34.     def run(self):
    3 }2 K$ ?2 O5 L4 F
35.         '''vertual function in Thread'''/ v: ^7 _6 O8 R- v' l! [
    
36.         try:\" D2 L9 C\" V( a/ C
    
37.             self.downloaded = os.path.getsize( self.filename )
    ! Z\" F( l* p' o) B9 T2 Q  u
38.         except OSError:; a# R7 f; z: W! e
    
39.             #print 'never downloaded'+ n8 l$ S. v! j8 [/ B3 C# C5 X& j& ^
    
40.             self.downloaded = 0- P+ F' |! ~$ s( X+ y
    
41. & f; w. [$ f' v3 {; Y- L; t$ R3 X
    
42.         # rebuild start poind5 n9 f: y: X$ E
    
43.         self.startpoint = self.ranges[0] + self.downloaded$ H8 G; n. f5 Y& c0 @1 H9 L
    
44.          , P' X) H( M: B) f0 n
    
45.         # This part is completed
    ; M& Y& c1 f6 ~- a2 `( t
46.         if self.startpoint >= self.ranges[1]:
    4 b5 A\" l) E+ E' ~7 Q% E$ A
47.             print 'Part %s has been downloaded over.' % self.filename9 K' `8 z\" [% u- _0 |5 z+ I5 g
    
48.             return& s; b& X1 ?  H1 d7 ^
    
49.          0 p. B& v2 }5 A5 X1 A1 G0 Q\" z
    
50.         self.oneTimeSize = 16384 #16kByte/time! ]: Z; Q& o5 h6 @+ \
    
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])7 e: i# C6 v6 T6 d1 V
    
52. * X: A( r3 h/ h+ P- v  T9 l
    
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
    5 H# f\" H2 J+ R/ k1 S
54.             
    8 @' i2 e9 X3 Z' O6 @* Y4 k
55.         self.urlhandle = self.open( self.url )- }, Y  u: l3 s+ H5 _: R: D
    
56. 
    ) L! O, \* A& \* {4 {- e$ _\" F
57.         data = self.urlhandle.read( self.oneTimeSize )
    $ l) I; u% l7 {' |
58.         while data:  e/ T- g3 ]  o6 i
    
59.             filehandle = open( self.filename, 'ab+' )/ g- c6 w8 Y+ c; i
    
60.             filehandle.write( data )9 e0 n. b* P* J( `  B* t9 L$ |\" V8 w
    
61.             filehandle.close(). w, i; \/ I$ A1 {! h, w: ]
    
62. 3 k; J. Z4 G1 O5 O- ]) s7 h, R  R
    
63.             self.downloaded += len( data )7 I& _4 J! Z  a  ]
    
64.             #print "%s" % (self.name)5 u0 t* p. @) p. g1 Z\" ^7 S0 B4 [
    
65.             #progress = u'\r...') ]8 ]5 ~9 F% e9 f3 q8 y
    
66. 1 Q2 _+ u# {, T% P, D7 t
    
67.             data = self.urlhandle.read( self.oneTimeSize )2 i6 K/ k# g6 D2 y& H- Y2 M+ C# H0 G
    
68.          
    ; Z) q8 f. y1 h5 @
69. def GetUrlFileSize(url, proxies={}):
    3 D4 w  r/ ^# I- \
70.     urlHandler = urllib.urlopen( url, proxies=proxies )* T& B! ^) q8 b' Z/ p& h
    
71.     headers = urlHandler.info().headers
    + }2 r, [; K/ X) V/ z  g
72.     length = 00 m+ \\" j5 R6 b# ^9 P1 G4 m
    
73.     for header in headers:
    ! E. h+ Y, e( C) n- U
74.         if header.find('Length') != -1:
    ' W8 u\" E& z$ m/ D6 v6 h& p
75.             length = header.split(':')[-1].strip()
    ) Q. m; E9 Q0 ~% @* t
76.             length = int(length)
    4 {1 d. p; H- d' P
77.     return length
    3 g1 I- ?1 r- w
78. 3 N0 L! F4 J' r) M% q
    
79. def SpliteBlocks(totalsize, blocknumber):* ?: p# K5 E+ F5 G
    
80.     blocksize = totalsize/blocknumber
    8 V$ e) O) q+ T; {; p
81.     ranges = []# u4 V/ K\" O\" w# T
    
82.     for i in range(0, blocknumber-1):2 k* @' {7 n; L2 F6 U) U$ z
    
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))) H6 O3 U( n/ O0 M' X' H
    
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))' h' k6 r+ C# K+ F
    
85. 
    9 B8 S* R; l$ f% v# v' h
86.     return ranges
    1 R$ n5 M5 S! w& J0 N
87. def islive(tasks):! Q4 K& [' B+ j5 y
    
88.     for task in tasks:. D9 Z: H9 y1 d' j  X1 ?+ y4 V
    
89.         if task.isAlive():) U1 Y+ J' l% n$ `# I7 e5 J\" a$ R
    
90.             return True
    2 d3 b  M0 \2 n
91.     return False\" l6 s- E0 t% O( I6 r; K
    
92. 
    9 R3 q- [/ v& ?- y+ D# @
93. def paxel(url, output, blocks=6, proxies=local_proxies):: {- `- Y) [& z8 n; B
    
94.     ''' paxel
    , E3 n' A# x& N* \% h; `
95.     '''9 e9 K7 h! L' s. c- H6 l! a; v
    
96.     size = GetUrlFileSize( url, proxies )3 @( b/ V$ w: i
    
97.     ranges = SpliteBlocks( size, blocks )
    ; e5 d) f+ k! H1 j( p( _7 J
98. - }6 q% h2 h) P9 E7 C* S' h
    
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]
    8 Q' x# R% ^3 x% |2 ~
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
     , ^* a( V* S5 S6 Q, Y5 z
101.    ; B1 k. ~! ~# _) s
     
102.     tasks = []# |\" u3 y4 p1 H' Y4 x2 z# t
     
103.     for i in range(0,blocks):
     9 e\" _( k% G: Z- t6 X; B0 A& j
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )+ K( i8 H- z$ P\" M/ Q( c0 G
     
105.         task.setDaemon( True )
       H7 ^; I( Q: h! H1 @3 [
106.         task.start()
     6 p! S$ \$ d' E+ R- ]: R& X
107.         tasks.append( task ). ^3 z9 P) k, `8 A$ v! ^
     
108.          2 N9 Q, _: S+ A9 [
     
109.     time.sleep( 2 )& D( S1 e+ R/ [
     
110.     while islive(tasks):
     - @5 P) w5 v# i; s+ E7 P, `
111.         downloaded = sum( [task.downloaded for task in tasks] )& s& ~+ ~2 x, c2 q, n: L
     
112.         process = downloaded/float(size)*100& `4 O. C) m0 B4 K* z1 r
     
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)5 }0 i. N9 W' O: L- X' D7 B
     
114.         sys.stdout.write(show)
     0 o7 Q, H$ ]5 `2 m! ?, R& \2 W) ^
115.         sys.stdout.flush(). P# g* p( A/ X7 Z9 d% `
     
116.         time.sleep( 0.5 )\" T: F1 T0 k4 B; p8 }
     
117.             
     ! K# h0 q  @+ c5 i# N9 Y8 a5 o
118.     filehandle = open( output, 'wb+' )
     8 A' J4 V! H8 m2 j8 @
119.     for i in filename:- g& g9 y  s. m0 p! V( E
     
120.         f = open( i, 'rb' )2 B, u& B7 t) a
     
121.         filehandle.write( f.read() )( t\" g! e- ~0 U
     
122.         f.close()
     # E  h8 ^0 j: x, K
123.         try:* p, i' V4 ?) @\" \
     
124.             os.remove(i)
     / ^) A9 y. x\" }2 v$ t* r
125.             pass\" p$ T' }0 k/ c$ N+ i; b
     
126.         except:( u) @( D  P, Q5 a/ N( B
     
127.             pass
     4 F! o4 n6 F\" K/ R' y( S0 R0 a
128. 
     * i! d$ H\" U/ P+ [' o3 {
129.     filehandle.close()
     , o& y. ?: F, j! w, s
130. 
     : T: q5 Z$ J. f3 J\" d6 T8 r
131. if __name__ == '__main__':
     ' B\" H' _( r: Y# _7 b# c1 ]/ `
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"
     8 m3 }; K5 _) i  Z( x
133.     output = '001.jpg'8 B  s' U7 `  D/ w0 A
     
134.     paxel( url, output, blocks=4, proxies={} )

复制代码

3 t& {9 {% \: y; ~! o% H