多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python; f\" J2 O7 Q! x
   
2. # -*- coding: utf-8 -*-
   8 |- u: `. N! @2 Q0 s
3. # filename: paxel.py
   \" E: ]9 ]. R2 V
4. 
   3 f& S$ ^\" b( O# d5 {7 k
5. '''It is a multi-thread downloading tool' y- g& k0 w6 i: G* m% ]
   
6. 
   ! f& ^' ^/ K' l% Z7 g
7.     It was developed follow axel.
   1 k8 i! W: H2 z9 b( R/ S
8.         Author: volans% S+ f5 J9 e+ w5 n& k5 W; }' Q
   
9.         E-mail: volansw [at] gmail.com
   * t5 O. r! @, x* q/ N
10. '''4 y1 I- `: h! p% ~$ D  N+ L# N# X; o
    
11. 
    * x, [1 a\" y9 M3 c& M7 N0 H7 q
12. import sys
    ; ~) m( k  k# @5 a
13. import os4 g: l% B\" E: ?& X( t  e# B' B
    
14. import time# Q% ?$ v5 a3 O# l
    
15. import urllib% _6 E0 D7 F) \' u: x: S
    
16. from threading import Thread
    ! H) J, v- ?0 F! n& \: [3 o
17. 7 ^9 t* j2 j* D, j
    
18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    $ m- w) A& m- ^' Q, q
19. 3 q1 q3 g$ V/ t+ j7 _
    
20. class AxelPython(Thread, urllib.FancyURLopener):
    ) A. B: ]$ {& c# ?
21.     '''Multi-thread downloading class.
    3 Y, u/ H# Y* w; k. ]9 O3 y! x* M
22. 
    & Y7 w+ Y/ L! n
23.         run() is a vitural method of Thread.9 b: h- |9 W- Z( K& C/ J
    
24.     '''\" Z1 k- Z# a; M/ l- m# F; C
    
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):7 ^4 j( I: W\" H\" X; K1 b# D$ K
    
26.         Thread.__init__(self, name=threadname)
    ' N& _8 P- u2 a\" z2 N
27.         urllib.FancyURLopener.__init__(self, proxies)
    & K- b6 E' m) o! j
28.         self.name = threadname\" f& R; R; z# Q3 H8 p& L% I; x- k
    
29.         self.url = url\" |: |+ q4 y, @0 a
    
30.         self.filename = filename
    # E1 D  ^! j* t1 D' l
31.         self.ranges = ranges; G; x  g/ s8 f5 r! @/ ~5 H
    
32.         self.downloaded = 0  ]6 }9 c3 u# `! O: N
    
33. 
    * t0 t2 V( O6 `! o, C
34.     def run(self):
    ( T1 m3 q4 L4 _/ q
35.         '''vertual function in Thread'''
    ; O& U6 {5 p  S. m! h( T; J3 g: s
36.         try:
    0 V  X& i- O' f
37.             self.downloaded = os.path.getsize( self.filename ), y- u0 K/ s0 L8 I+ a
    
38.         except OSError:
    % a3 V9 [. i! s7 _; |4 b2 p% a* ?
39.             #print 'never downloaded'
    * `, V- k  Y. p( k
40.             self.downloaded = 0! V) W5 J- W( B- @- ]& h
    
41. 
    2 k2 Q* J+ S. o3 ^
42.         # rebuild start poind
    4 l; L2 p) ?+ t2 b2 _% v7 R
43.         self.startpoint = self.ranges[0] + self.downloaded8 q4 ^7 K1 {& L& j; ~* x' _
    
44.          + x0 y+ B* L# ]. r7 A
    
45.         # This part is completed
    / p\" ~\" C! W6 x+ Q/ L5 R1 k# x
46.         if self.startpoint >= self.ranges[1]:# N# R. F/ ?' {. D/ w  G
    
47.             print 'Part %s has been downloaded over.' % self.filename2 G5 y$ ~7 H2 p4 v
    
48.             return- d, o\" [; g: _. [5 t5 m* s
    
49.          
    ( t! ^$ I6 ]9 |* F7 ], L
50.         self.oneTimeSize = 16384 #16kByte/time
    % o* L; v/ ?+ d; C
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
    * b0 B8 r, w% A+ c4 _6 O0 f8 X
52. 
    2 ~! d9 X/ d6 _
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))' R: @! o% ?2 I9 d( ?- P8 t
    
54.             
    : o, ]! s) K& q4 m( ~! l
55.         self.urlhandle = self.open( self.url )2 V7 V9 `* x  ?. C, s3 O: h# D1 k  F1 w
    
56. 
    9 N  {3 |$ o7 I
57.         data = self.urlhandle.read( self.oneTimeSize )1 E5 h5 B, q2 Z+ ^! u; m; N
    
58.         while data:
    7 e6 n# [0 Y* R4 V/ X
59.             filehandle = open( self.filename, 'ab+' )2 i5 k. C6 b# b# b) G  H. F' k
    
60.             filehandle.write( data )) {4 a5 Y7 h' P\" q\" ]3 P/ a/ {
    
61.             filehandle.close(). }6 A0 C) Z# M( R
    
62. 
    - q7 v6 b( ]1 {3 r2 C# l. P
63.             self.downloaded += len( data )8 R7 [; |$ R( G8 z\" w6 L
    
64.             #print "%s" % (self.name)3 O1 \  N- ~) w4 A# C4 m- Q
    
65.             #progress = u'\r...'1 W) _- X8 `+ o, l- H
    
66. 8 `6 J* x0 e) H3 e  Q* T
    
67.             data = self.urlhandle.read( self.oneTimeSize )* w7 @+ k/ T( x4 r8 p, m; y
    
68.          
    * ~2 {* r0 e2 @2 w
69. def GetUrlFileSize(url, proxies={}):' S# \: _; w9 Q8 a
    
70.     urlHandler = urllib.urlopen( url, proxies=proxies )
    4 G+ [2 R% Z6 V: x! r$ c' s
71.     headers = urlHandler.info().headers
    : j% {: N1 ~' _
72.     length = 0
    - \6 I3 C0 c0 R8 {& y1 C
73.     for header in headers:
    : C( C, ~% B$ n0 l* Z. W: x% n
74.         if header.find('Length') != -1:3 U3 \0 G' ?! `- \5 y
    
75.             length = header.split(':')[-1].strip()
    ( M$ X& [, w3 C2 [, N\" o4 a
76.             length = int(length); _1 m' ^: J9 ?+ f, M* }
    
77.     return length0 k! `& G4 k; d( e
    
78. , ]0 O; f$ y9 @& ^4 C\" |' D/ b
    
79. def SpliteBlocks(totalsize, blocknumber):8 {( o\" ~: H  n' q% }  a) N
    
80.     blocksize = totalsize/blocknumber
    . u  q1 e1 ]) A\" x
81.     ranges = []
    ! z\" ~# H) }4 m% O\" F0 ^( t
82.     for i in range(0, blocknumber-1):
    , o\" `1 f0 ?& Y9 j& o- u
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))5 h% B- I1 C( E4 u9 y
    
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
    5 |) K0 Z. c( c5 l& H# {
85. 
    2 L2 X+ l4 l$ D. w6 h# h
86.     return ranges
    & k$ l! E) V& i6 ]\" Q4 t
87. def islive(tasks):2 d4 D- X/ J* e8 T1 d
    
88.     for task in tasks:6 Y! l2 Q- i3 b; t
    
89.         if task.isAlive():3 T/ F/ h; Z1 k5 L
    
90.             return True
    3 v\" F+ _. W# Y+ h\" }$ P
91.     return False$ P5 R\" m9 I; }( G1 O% T
    
92. 
    : v, J4 V  R6 T& J
93. def paxel(url, output, blocks=6, proxies=local_proxies):& L$ j- \! D! N. r
    
94.     ''' paxel
    ( u  L$ u# l: A
95.     '''
    - \3 M3 G2 K$ h1 ^6 c7 E1 i* C
96.     size = GetUrlFileSize( url, proxies )- L- E$ s  ~9 T) Z8 v$ L
    
97.     ranges = SpliteBlocks( size, blocks )
    % r6 J' ?) P. i$ F! x1 L1 l* Y
98. 
    6 Y: d1 V6 {  B* A7 E) V. a+ v
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]1 D9 K0 M4 r+ Q1 t
    
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]& C3 z+ z  x% }3 m0 \- g9 A
     
101.    
       I, G+ X: A5 N7 a
102.     tasks = []3 Z+ J/ a7 A3 v3 R7 T% t7 p' S
     
103.     for i in range(0,blocks):
     9 O' ~5 G3 L) A
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )
     * H) N. I' R. @4 ~0 r
105.         task.setDaemon( True )
     6 H, w9 V  u0 U3 H6 {( ~+ o
106.         task.start()/ ]9 s- w: S; }6 m: {
     
107.         tasks.append( task )
     : t+ Y9 ~% _5 F
108.          
     : a3 f! G( j* p/ V\" _9 t* v! }% S2 t
109.     time.sleep( 2 )
       C9 K4 T5 \6 [5 u3 e* b
110.     while islive(tasks):
     5 N) d$ I: T$ I  k! }
111.         downloaded = sum( [task.downloaded for task in tasks] )
     4 _. ]: _. r# Z* N/ L' m
112.         process = downloaded/float(size)*1001 Y+ Z  O9 s6 h6 ~4 @
     
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process). p0 T# G; c! d  b\" R; ?$ ?
     
114.         sys.stdout.write(show)! E9 N- w( S/ l2 e# c* R$ W9 Z
     
115.         sys.stdout.flush()
     1 K& ^$ c8 M; W4 V' x- o, t
116.         time.sleep( 0.5 )7 J8 A- P, u5 o( R, f
     
117.             
     # D9 Q7 d  `: E# `) p- T4 D% Z5 y
118.     filehandle = open( output, 'wb+' )
     $ ~. [7 y& @- B
119.     for i in filename:
     4 B2 _3 Y1 a( m. t
120.         f = open( i, 'rb' )
     0 p) h; Y, W  G5 ]) [
121.         filehandle.write( f.read() )
     # ?& t' a& o1 d0 K8 }
122.         f.close()) F8 Y/ l: G7 B5 F
     
123.         try:
     # E* F; f6 m, [. e; ]# h1 }( e
124.             os.remove(i)
     0 @! q4 E9 T8 O1 _# K6 n. n
125.             pass% y1 N# }) Z. N  T' R) k
     
126.         except:; A. D  e+ X5 ~, f% Q/ T
     
127.             pass6 M. _1 F& s* |) H0 h+ R+ X/ d1 X
     
128. 8 M! I0 x1 w$ X$ g
     
129.     filehandle.close()& s$ `+ x& ?/ h6 p& o
     
130. 3 e' I: h2 \9 J\" Y4 {
     
131. if __name__ == '__main__':/ s: g6 I! B: _! z6 e6 O7 R6 r7 j
     
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"# u6 H: n' c$ ?9 B% k+ f
     
133.     output = '001.jpg'
     * T6 t7 O4 U/ \% n
134.     paxel( url, output, blocks=4, proxies={} )

复制代码