多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python
   % F, s) v6 _9 {+ H* o& E6 {# b
2. # -*- coding: utf-8 -*-
   ( f# K# N& J' z2 J
3. # filename: paxel.py
     ?% t7 v1 W! |9 \4 Y& }( F
4. 1 t6 a' X' b2 |% K8 H% c
   
5. '''It is a multi-thread downloading tool
   7 E\" o, K% Y7 l; ^, d# l/ M9 p
6. 4 q2 S$ H, f  O9 h
   
7.     It was developed follow axel.( e! Q. i4 Y. D5 u
   
8.         Author: volans
   ) `) n/ ]3 g0 Q4 V3 `
9.         E-mail: volansw [at] gmail.com
   & @- K2 J2 S/ \- D7 n, {  f4 r
10. '''
      o: @5 {: I7 ]- S3 s0 a
11. 
    $ I- o$ R9 y( S3 K# s, c( d
12. import sys
    7 L& p' g0 P) z( ~
13. import os
    & J# r* `, K) V: J
14. import time7 {0 D- ~: n3 R4 V4 \  @
    
15. import urllib3 |9 ?3 a) Z* L
    
16. from threading import Thread/ [' J4 h' s' d/ U4 s' M
    
17. \" ]. z9 r, {/ x3 S- n
    
18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    5 x' w5 _. s' f! e0 j
19. 8 X5 _# L% c8 @; I
    
20. class AxelPython(Thread, urllib.FancyURLopener):\" z7 Q7 L7 D3 }1 m  C* ~) R
    
21.     '''Multi-thread downloading class.9 m) R\" E0 X0 ~# f) D0 K) v4 D. U+ `$ O
    
22. 9 g  ~. _  [! k- b- A6 |
    
23.         run() is a vitural method of Thread.
    & ]  S. s0 ]! j* [
24.     '''
    - T2 ]9 x9 _7 `\" @3 j2 C/ M' V
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):& ]4 b# _- r' ]  w2 d  V+ V$ S) ?4 V
    
26.         Thread.__init__(self, name=threadname)1 d; V& b5 n( K8 w& K8 ]
    
27.         urllib.FancyURLopener.__init__(self, proxies)
    3 l& I2 Q1 p9 O, F/ r
28.         self.name = threadname
    , {* b( S# ^0 h3 t% j
29.         self.url = url2 y& E1 A5 z\" s
    
30.         self.filename = filename\" G$ s2 R' s5 z+ i
    
31.         self.ranges = ranges  H1 P! c, I5 ~
    
32.         self.downloaded = 0! y\" t0 L/ ?0 w+ y8 p# j) E
    
33. 
    ; T* `$ I6 O0 I5 `1 V+ ~# G
34.     def run(self):
    ' q) q- i\" }\" B5 {* O+ v3 p4 w
35.         '''vertual function in Thread'''
    ) ?; s5 @$ {/ _$ d
36.         try:\" c8 Z  x8 Q2 a\" _& K0 C) n$ f: L
    
37.             self.downloaded = os.path.getsize( self.filename )# y2 x$ h7 f4 P% o( T, V2 D/ _3 U
    
38.         except OSError:3 Q# F, P- p% S( r7 H5 g5 A8 U/ S
    
39.             #print 'never downloaded'
    * u; G3 L8 ^7 L9 E
40.             self.downloaded = 0
    1 l1 l3 e' K\" W: Q' X. I! c
41. 
    \" C# z. a/ M7 E\" W6 j! v
42.         # rebuild start poind4 u9 z; J  @& Z- j& n
    
43.         self.startpoint = self.ranges[0] + self.downloaded
    \" q) v7 l4 @- {  j/ G0 j
44.          3 I, j\" Y6 _4 @3 ~! h
    
45.         # This part is completed) D7 C4 w9 G) C- J\" W0 B
    
46.         if self.startpoint >= self.ranges[1]:
    4 P6 I* i4 V6 r\" e! ~3 ~, k
47.             print 'Part %s has been downloaded over.' % self.filename  s7 j9 R* Q) E* f# C0 W
    
48.             return
    % j. N& v5 D) u8 A6 Y2 [
49.          4 }2 p% A9 m! I7 W# h
    
50.         self.oneTimeSize = 16384 #16kByte/time1 t7 m0 z; w4 R\" v\" g3 K0 N
    
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])\" d' a6 D\" o7 {3 L
    
52. $ _2 U+ o* b5 ^9 J5 s
    
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1])); q: [& z& C% @; M# f# ]
    
54.             
    9 P8 r1 p6 y1 M, Y
55.         self.urlhandle = self.open( self.url )% m* x* J( Z: K5 y$ g% Q
    
56. 
    % ~7 \+ Y5 h- Y9 e2 i6 t+ |5 }8 X
57.         data = self.urlhandle.read( self.oneTimeSize ). [, U\" X: t  s
    
58.         while data:
    7 r0 H9 G8 A$ C- T: ]\" L
59.             filehandle = open( self.filename, 'ab+' )
    : _5 x$ W( ?5 v3 u# O- @\" G
60.             filehandle.write( data )
    + c8 i! [8 t. a# Q
61.             filehandle.close(): J( f, W% h+ t5 K  Q5 X
    
62. % ^% ?& k. W% w- u4 H4 g( M
    
63.             self.downloaded += len( data )
    \" o, E% S7 n( }  N' \  R& U7 v
64.             #print "%s" % (self.name)
    ; j- N+ C9 f0 x/ h9 [
65.             #progress = u'\r...'4 I6 g% a7 Y: y' |6 @  a
    
66. 4 l* c: N* u6 b
    
67.             data = self.urlhandle.read( self.oneTimeSize )
    ; ]; J7 V1 x\" ~# j* c/ N+ A
68.          0 o  |  L7 I. @7 d# p
    
69. def GetUrlFileSize(url, proxies={}):6 E% T3 m+ Z% h8 {4 c, X& T
    
70.     urlHandler = urllib.urlopen( url, proxies=proxies )
    ; y4 m% A0 B: W+ f& X6 w2 k# i
71.     headers = urlHandler.info().headers
    # T+ b\" K( T7 I/ L5 k1 n
72.     length = 0
    & m2 m1 ^9 _! \8 P9 E6 x$ P
73.     for header in headers:
    ' E3 U! C- {( H9 `4 X- Q' R7 M
74.         if header.find('Length') != -1:+ I+ D! x5 k/ D4 P! R' s! g* Z
    
75.             length = header.split(':')[-1].strip()- ^! A  t, J$ Q
    
76.             length = int(length)
    . t1 m+ C! O) L7 @( s. Y$ l/ h
77.     return length
    7 V7 l! F\" m% B' G4 i/ C
78. 
    . \; I1 `* z4 z\" z5 j- T
79. def SpliteBlocks(totalsize, blocknumber):
    , r# F/ K  T: T2 y6 n\" `
80.     blocksize = totalsize/blocknumber
    0 k7 V2 ~$ ~6 ?3 {* h# W; U
81.     ranges = []
    , g& O( B' q) T0 C) X% a6 i) z
82.     for i in range(0, blocknumber-1):
      r* i) d* u! m- \\" g. I, q
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    # d/ l: t- v8 T: E: K. o
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 )): {! j5 |. e9 }/ F. u; K
    
85. 
    ( O& {- z1 M5 n
86.     return ranges/ {( |9 M1 T- R6 d: N4 G# V* ?
    
87. def islive(tasks):
    % Z' W; V. u2 {; {; }\" R5 @
88.     for task in tasks:: w3 Y1 }0 p/ |2 ^1 ^
    
89.         if task.isAlive():
    ) u- Q' E6 r4 X0 |2 r
90.             return True
    3 m  u  @3 v* M) y
91.     return False5 N% h/ k\" G' r( A9 @# X! s
    
92. + ~0 S1 x' l% I
    
93. def paxel(url, output, blocks=6, proxies=local_proxies):
    * T1 Z/ T/ H+ w2 `% ]
94.     ''' paxel
    0 ^/ z, Y: W8 f7 w; s$ V8 H
95.     '''
    4 }; l\" u. E8 B: E- m' c4 S
96.     size = GetUrlFileSize( url, proxies )
    . x: P! {5 V1 R5 a' R; \
97.     ranges = SpliteBlocks( size, blocks )
    , z\" c: R, M( E7 B; f0 Y
98. ! d/ ~9 D# V# S\" l
    
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ], Y4 K( x1 c: W% H
    
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]- s4 O4 x+ s' Z8 C+ r9 T5 M
     
101.    
     + L$ ?* }; z- `) A
102.     tasks = []7 J0 t. l! X- J
     
103.     for i in range(0,blocks):
       H  s% u$ t; L# v) z\" N' \
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )
       ^7 w$ v6 V  ?& d
105.         task.setDaemon( True )1 H, @. W1 V  s8 b% v
     
106.         task.start()
     & p\" M  ?' n; g2 A1 \
107.         tasks.append( task )) C- Q2 o0 o\" J9 b0 H8 Y/ b
     
108.          
     9 K5 `2 A4 P5 ^' P* H3 b
109.     time.sleep( 2 )
     ' D2 c/ m) B* o0 i1 j( ~& a( g: Q
110.     while islive(tasks):
     2 u# ?  W: z$ C9 n\" e/ p
111.         downloaded = sum( [task.downloaded for task in tasks] ): I3 }/ A: G. @4 K
     
112.         process = downloaded/float(size)*100' s5 J; j& h( _7 R4 ~
     
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process); a) w\" `: |: `. ?$ z$ Q
     
114.         sys.stdout.write(show)
     ! e) B+ d' |( ?) {) v+ j' b! h; `
115.         sys.stdout.flush()
     , j, R\" C/ b\" t  u1 l
116.         time.sleep( 0.5 )4 C3 X7 O! |1 K% X* v\" i7 C
     
117.              , ~& q, C% {; b5 _: m/ s
     
118.     filehandle = open( output, 'wb+' )) e7 c- D\" `) O, i; a* f, L
     
119.     for i in filename:
     ( l3 q( ~. w1 }  K\" z4 s3 b
120.         f = open( i, 'rb' )
     6 F9 K: r% s4 k* X\" H6 B\" A6 e
121.         filehandle.write( f.read() )
     . e- C# }+ k! v
122.         f.close()8 K3 P) u8 @$ A* ]% V# X
     
123.         try:
     ! R  y' ]7 k. t5 v! m4 d9 u\" ~1 t/ [+ s
124.             os.remove(i). |' r0 i; _4 u7 {5 e( O5 t: ^
     
125.             pass+ e% o0 ]- \$ g0 Y7 C! Y
     
126.         except:
     , N* g9 R1 N2 z  j
127.             pass
     ; t1 n2 k% ^& x# h
128. 
     & ]' k) W4 X  s
129.     filehandle.close()& }& t( Q. k# W* F
     
130. ) L. Z- `+ }) j: M3 D2 i9 {
     
131. if __name__ == '__main__':
     / z, P& ^3 L$ r4 w! S9 w+ J5 u
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"4 Z' X' G& H  j, l% W4 P: }; `
     
133.     output = '001.jpg'
     % q# N; ^1 L. e% W5 z
134.     paxel( url, output, blocks=4, proxies={} )

复制代码