多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python
   / ~. d) f. Z, n& o/ C
2. # -*- coding: utf-8 -*-# I' \6 z1 }' q+ H) G/ l  r
   
3. # filename: paxel.py0 F5 f! a( a: `' o4 h; h3 B+ a
   
4. 
     i# ^/ ?8 j0 A
5. '''It is a multi-thread downloading tool/ H) o% o, y9 r0 @8 [0 J1 x
   
6. 
     u( y. g# U\" F) N# Y
7.     It was developed follow axel.8 x9 [, L* h; g9 Z! \( N1 F8 h
   
8.         Author: volans2 Q, B4 r- S% J# D# `) Q& a
   
9.         E-mail: volansw [at] gmail.com
   ) C7 P! \0 C/ C% Q! w9 W
10. '''2 g; D% j1 U9 l# P9 q
    
11. 
    * ?0 p( p# P# W. [2 G\" p
12. import sys' S4 r\" E$ V  l6 u7 g: I
    
13. import os2 E6 c# S' L. J, H8 g4 c
    
14. import time
    8 T; v0 k* ~5 n& z$ T' \$ T# X
15. import urllib
    9 _* f+ W) P: ]: d8 a
16. from threading import Thread
      Q# @1 e8 I4 ?7 L& M' F2 r
17. 
    / i  Q% {% f3 g
18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    6 U: k3 L& }\" w1 X5 S/ p
19. ; ?/ A' _9 A  l1 A9 r+ }3 ?
    
20. class AxelPython(Thread, urllib.FancyURLopener):
    ; ~: ]# s$ ^( z. b2 ?) p
21.     '''Multi-thread downloading class.
    ) Q( g\" P# i% x
22. 
    7 ^+ d) |3 {) H$ }1 ~
23.         run() is a vitural method of Thread.
    6 Y: Q7 w; t/ g( c; }
24.     '''2 W4 Q3 t8 P1 [% D/ {
    
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    4 l8 i5 A+ T3 G% ~
26.         Thread.__init__(self, name=threadname)
    : [* \1 X: W& k
27.         urllib.FancyURLopener.__init__(self, proxies)
    : V+ M. f1 _( x& U2 g# S3 E+ }& J
28.         self.name = threadname
    $ Z1 d: s- @4 P- G1 O% H' E
29.         self.url = url! \2 k. l% r/ t6 K\" R3 O
    
30.         self.filename = filename- t7 K) L7 B( G) P% {
    
31.         self.ranges = ranges
    & ^2 Q: k8 w! d* [; D\" ]5 f$ m
32.         self.downloaded = 0
    % w8 m9 v0 O+ g, a/ Q
33. ) U9 n% v' J* r3 Z& I* u8 W
    
34.     def run(self):7 ?) K* d2 W% H! P4 X, m& k
    
35.         '''vertual function in Thread'''3 D8 t& S0 w; `; m% z' \1 v
    
36.         try:
    6 J( A& U& s6 W( H( _
37.             self.downloaded = os.path.getsize( self.filename )2 F\" x+ K: x- \2 j  k7 n8 L1 M
    
38.         except OSError:0 S. g6 ~6 D$ u0 G\" D$ d' j
    
39.             #print 'never downloaded'
    6 J6 T9 d0 w4 B6 U, i
40.             self.downloaded = 0- `5 N3 a2 e\" n0 f0 W# N
    
41. 
    / |  G% J. c# x- l1 O6 i4 I* x
42.         # rebuild start poind
    ! `) x9 U* p* W# \8 b0 t( P6 O3 Z
43.         self.startpoint = self.ranges[0] + self.downloaded% `# D0 g/ T* Z* Y7 W+ z. k! a- o
    
44.          
    # T) z/ t8 M\" ^% e\" d+ W+ m
45.         # This part is completed; t; D/ P) S9 T/ }
    
46.         if self.startpoint >= self.ranges[1]:
    $ j: ~6 P  K. b3 a- J
47.             print 'Part %s has been downloaded over.' % self.filename/ @( B: M! l' x
    
48.             return; G) F9 ]) u; i8 `! c* a/ ^* V5 u2 s
    
49.          
    : P4 M: W; C# K
50.         self.oneTimeSize = 16384 #16kByte/time
    ' ~+ k9 H$ o( e\" s+ P4 z( O- M
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])' |\" c\" @% d- I+ h) t( T7 V
    
52. : \5 o6 U0 K) _4 _. o6 Y
    
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))& t1 o) u. E# Z; D8 A% C  ]
    
54.              8 C# T  v& z+ b3 Z5 q) ?
    
55.         self.urlhandle = self.open( self.url )4 [2 j, L* G4 h8 m; I$ g% G
    
56. 9 G8 Z, I' y0 U0 l( _+ ~
    
57.         data = self.urlhandle.read( self.oneTimeSize )
    : v4 F$ g2 i. t6 n\" }% p
58.         while data:
    0 u$ m1 b3 ^7 |
59.             filehandle = open( self.filename, 'ab+' )
    ! J\" l: Z/ L1 K% O
60.             filehandle.write( data )/ }; [. ?6 M) W  b4 |9 Q
    
61.             filehandle.close()0 z( o\" u/ p5 \5 |' |
    
62. 
    \" g) J* R& P# e* h1 ]& z
63.             self.downloaded += len( data ). _7 P; l/ o\" m) S
    
64.             #print "%s" % (self.name)2 Y# r$ V3 B3 d6 c
    
65.             #progress = u'\r...'  M' f# L/ @2 F. F
    
66. ( J$ e% @& i# t' b* |4 G
    
67.             data = self.urlhandle.read( self.oneTimeSize )
    : }/ n' j9 ]$ ]& T  _' e. O( A5 b
68.          
    \" ?7 v% z9 P  `7 d5 J
69. def GetUrlFileSize(url, proxies={}):0 H+ ]: Y- v; y; E
    
70.     urlHandler = urllib.urlopen( url, proxies=proxies )8 e- ^\" A5 S1 T0 S, I; D
    
71.     headers = urlHandler.info().headers: s2 f& I* r\" ?\" K, B
    
72.     length = 0
    7 U4 u1 u5 b- j2 M# a
73.     for header in headers:
    2 ~7 n9 e5 A9 G; H: r\" |: m0 f3 ]8 D
74.         if header.find('Length') != -1:
    ' r6 f0 G# B& i7 b1 [* F# J% E  g7 x
75.             length = header.split(':')[-1].strip()
    8 V0 i# Z1 M* {( n, V8 ~9 w
76.             length = int(length)9 C% ]; R* R/ k/ g* T) ]$ w* y\" R
    
77.     return length3 F( o) f8 C4 U  t' b, {
    
78. 
    ) o' B! e% h6 N1 q/ f
79. def SpliteBlocks(totalsize, blocknumber):\" p( r3 }% Z# \# n9 p/ H
    
80.     blocksize = totalsize/blocknumber# c- `/ `2 a5 ~1 ]1 T. ~6 m3 x: s
    
81.     ranges = []
    ; H; `7 n\" b. W4 K, _6 e
82.     for i in range(0, blocknumber-1):/ D3 f/ R$ V\" {/ ?
    
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    + z0 J5 r/ E5 g
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
    , I8 h6 u9 y. D) g# J( A
85. 
    # N- L& A$ m; b& s! N6 F, J
86.     return ranges/ k/ G  S) F. O! H
    
87. def islive(tasks):
    $ p\" a9 O; {5 p/ q6 D  C: E9 i
88.     for task in tasks:$ `6 K# B) |7 r9 h, k$ ^, t& e
    
89.         if task.isAlive():
    # M6 _5 c/ L, n! J# x& G
90.             return True+ H- Q\" [- R' e. e
    
91.     return False
    9 U, M/ S: [+ ~5 e8 Z/ x# Y
92. % p8 s# ~2 L' e: g5 E! s: A
    
93. def paxel(url, output, blocks=6, proxies=local_proxies):5 G& |6 y\" S- a
    
94.     ''' paxel( B* W  u# [0 A$ ]7 Z  L! v- o3 d
    
95.     '''9 B/ m3 q5 Y' n& h
    
96.     size = GetUrlFileSize( url, proxies )
      P: @% J0 B' ~+ l+ H: j/ ~4 c7 E
97.     ranges = SpliteBlocks( size, blocks )7 C( E! H! r' l4 s. L1 L
    
98.   P) S; {+ g' y3 t, l! d$ [
    
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]4 ]6 z8 U4 N& |; r
    
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]  _2 Q& x2 l0 J/ r4 s
     
101.    
     ( U0 w\" R: ?3 U. k9 k  F
102.     tasks = []3 E\" G$ J  w; o+ k9 b2 F1 [
     
103.     for i in range(0,blocks):
     \" h2 z0 q% y5 ^- }+ _
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )3 h7 M( j5 z7 ^8 o* F6 R
     
105.         task.setDaemon( True )\" t# ~- j& ]0 @5 |
     
106.         task.start(), u- Y# V/ X7 z8 [! J1 k$ h' m
     
107.         tasks.append( task )1 r: t+ ?7 m/ m! e: e\" T
     
108.          
     4 |/ v- [0 a\" O$ W' y
109.     time.sleep( 2 )! ~! `9 X$ g- w0 ?
     
110.     while islive(tasks):
     # @) S) Z/ h; w0 o\" o
111.         downloaded = sum( [task.downloaded for task in tasks] )
       Z& {: W3 N* u! N+ }% {9 W
112.         process = downloaded/float(size)*100$ Q* Z$ F. h- i( K% a\" j
     
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)* l1 C5 v  x/ ~' ^6 `& S7 M
     
114.         sys.stdout.write(show)* M' ^5 O; q) W) b0 ]) A
     
115.         sys.stdout.flush()
     9 C' U1 ]' G. c! a3 ~
116.         time.sleep( 0.5 )
     6 g7 |9 m6 A& ^& e0 n
117.             
     , [* L& B* g9 H
118.     filehandle = open( output, 'wb+' )* W. p# X/ j5 ?: d
     
119.     for i in filename:
     7 o; V: `) V' b! M
120.         f = open( i, 'rb' )- Q0 Y) E4 j1 n0 R( L  m
     
121.         filehandle.write( f.read() )
     4 |; f/ {6 t/ I/ _
122.         f.close()
     # p* k' m9 q) a
123.         try:
     5 i% i3 g) g9 p+ w' c
124.             os.remove(i)6 a' U* v+ A4 _0 O2 Y- a$ _: }
     
125.             pass: Q1 _+ q( \, q' L: |+ p
     
126.         except:
     + u\" x0 Z7 N' A6 v6 J0 Q  A7 }& D9 G0 [
127.             pass
     ( x' G: I6 y. k& e
128. + {8 J( r- T( Z
     
129.     filehandle.close()# T) L$ q. d1 ^1 Z
     
130. 4 R( b* R$ A, {$ P* g/ R% b1 @
     
131. if __name__ == '__main__':7 y\" f/ i' @) A% M3 y4 I/ C- z
     
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"
     - Y, K( d4 ~6 U  V( I: [1 [
133.     output = '001.jpg'! a) z/ T& L\" M/ j0 J
     
134.     paxel( url, output, blocks=4, proxies={} )

复制代码

9 b( J" [9 W  S: o6 }. C+ S$ @