多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python
   0 g7 R' d6 p+ Q) K0 z& d( Q
2. # -*- coding: utf-8 -*-
   ' E+ [: M% H! i; p\" d0 a3 y
3. # filename: paxel.py8 |( l9 J+ I/ V8 c\" z
   
4. 
   % ?( B\" {$ F\" N4 f- O5 G
5. '''It is a multi-thread downloading tool8 W1 {& \) P; g9 E9 ?\" H' Y
   
6. % S\" R  ]+ @5 D& U$ i: p! Y
   
7.     It was developed follow axel.
   5 U! `( l. l8 z- u8 n9 Y& ^: w
8.         Author: volans% U\" Y, ~- ?& B, Z7 z
   
9.         E-mail: volansw [at] gmail.com
   4 Y1 w* b( l1 [0 g. `
10. '''
    * ?3 h2 r+ P$ |\" |
11. 
      m& A8 c( V( z7 {
12. import sys
    & X( k3 `: `* U6 ^7 r5 Q4 v, i
13. import os  ^5 x1 u( }1 G, T' `2 x
    
14. import time. o* @+ G* \- B  \2 E/ E) B
    
15. import urllib' u7 I9 S4 B( T( V4 Z; L
    
16. from threading import Thread: r  o+ ~9 G; j4 c. ?
    
17. 
    : s* S- Q* {+ \
18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    0 E6 `$ X3 \\" K
19. * d1 \+ D' [# D3 F
    
20. class AxelPython(Thread, urllib.FancyURLopener):
    , w$ g5 I8 D( T% J
21.     '''Multi-thread downloading class.
    \" o* B$ X' y% Y# G/ I! R8 m
22. 
    : H# u9 t# K+ S  t
23.         run() is a vitural method of Thread.( P& {5 r- r/ b# {# y2 b\" W# p
    
24.     '''
    4 S# |7 S, d8 x8 h( M) x7 t6 _6 V
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):4 l% N: f2 |* a: R' F, V
    
26.         Thread.__init__(self, name=threadname)
    % G3 M. d4 |0 a' K: N\" j( T
27.         urllib.FancyURLopener.__init__(self, proxies)! T\" m$ U0 T. C9 G
    
28.         self.name = threadname
    \" F: ^( F- G& T. Z) J) A
29.         self.url = url
    8 T9 U$ \\" s; D4 w2 P+ s7 y; m
30.         self.filename = filename
      z; X- M* p9 s3 y  Z
31.         self.ranges = ranges; K; N\" l! X2 ?* ?6 f
    
32.         self.downloaded = 0
    & e% Z# R$ m+ r$ R
33. , H, _3 X, q- V+ W3 o
    
34.     def run(self):8 o+ g+ z* l  ]5 [
    
35.         '''vertual function in Thread'''! E& e  b' x+ i$ K, G
    
36.         try:5 j. O- ?) x' N3 C% Q4 _8 H: k
    
37.             self.downloaded = os.path.getsize( self.filename )
    - u( p# W- m\" ^0 s/ s# y6 V
38.         except OSError:, `# D- C2 G. p4 O: ~
    
39.             #print 'never downloaded'
    8 {3 L* p, \& V1 N! m' `
40.             self.downloaded = 0
    . P% N  {( w8 S( I  m! R6 c% T
41. # g: G6 \6 E) }& S+ S+ U* Z: ]
    
42.         # rebuild start poind) r: t- H$ s7 Y* ]
    
43.         self.startpoint = self.ranges[0] + self.downloaded) ~. Z: W: c: M) W
    
44.          
    \" ?/ L  \( o  h
45.         # This part is completed
    : Z( w. B7 G! f/ `# L; M# y* n& Z
46.         if self.startpoint >= self.ranges[1]:
    4 _7 _' e0 l  _% d6 V, H
47.             print 'Part %s has been downloaded over.' % self.filename0 e- b& B% A  p\" c4 g- @: J+ d6 T
    
48.             return
    4 n5 s) t! x+ l; Z
49.          8 O7 i- `( z6 L/ [
    
50.         self.oneTimeSize = 16384 #16kByte/time
    8 y5 e4 l) P% s' \, L
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])2 y; G1 k* V, `  K+ H
    
52. + Q# Q: H* A( Y
    
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))5 d! y- }2 q. L4 P4 ^$ A
    
54.              , H* {7 R. [$ u/ Q
    
55.         self.urlhandle = self.open( self.url )6 x, u5 j8 _# m: R7 j) [
    
56. 
    & e/ t: @( T8 r- B, z
57.         data = self.urlhandle.read( self.oneTimeSize )
    , x( a; _1 u8 w# X\" i, H6 |
58.         while data:7 J/ i0 d& {5 [$ H5 u  q$ Y) R7 ]9 h
    
59.             filehandle = open( self.filename, 'ab+' )
    % ^- _9 R$ j: ^9 _! W) x7 Y6 ^6 C) n
60.             filehandle.write( data )
    4 G+ {$ W5 ]8 p+ r+ q
61.             filehandle.close()& }9 G; s# p/ M
    
62. 0 e1 _3 ], m! A. r) ]& x
    
63.             self.downloaded += len( data )2 a( B6 R% m! P) h% t0 s
    
64.             #print "%s" % (self.name)* o  `; h; Y' @7 I5 C0 j
    
65.             #progress = u'\r...'/ j3 m0 f: G6 b+ L\" h
    
66. : U5 Z' v( v) X2 `\" Y# u; I: ^, c
    
67.             data = self.urlhandle.read( self.oneTimeSize )
    6 z\" `& x) |' G  R+ A& z
68.          
    # T  e7 {: G\" |9 o
69. def GetUrlFileSize(url, proxies={}):2 t( T$ ]$ ?2 ]9 p- E
    
70.     urlHandler = urllib.urlopen( url, proxies=proxies )
    4 x4 @) ]1 |. M/ B( U/ u
71.     headers = urlHandler.info().headers
    ' c/ K2 f  I, j9 h
72.     length = 01 _, B' \* h: b8 ^- E5 H5 f
    
73.     for header in headers:2 q4 T$ e8 Y' A$ ^9 L! X. x' I
    
74.         if header.find('Length') != -1:7 e- R- t: F$ w( G3 c) W1 t
    
75.             length = header.split(':')[-1].strip()
    + `& V: U. n3 `9 B7 r' w- [+ g# Z: O
76.             length = int(length)
    . l7 f0 J$ s) {! s4 _, ]! I/ @9 m
77.     return length, c- |2 U; i; g! M$ |
    
78. 6 P7 p; h4 c4 p+ ~7 x' G
    
79. def SpliteBlocks(totalsize, blocknumber):$ s/ G2 D3 q7 j5 {
    
80.     blocksize = totalsize/blocknumber: e, @! x2 K! n4 Y
    
81.     ranges = []
    ! N) j4 d  ]; Y0 m
82.     for i in range(0, blocknumber-1):/ x! H- r5 \- C) }
    
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    6 u* ?+ A7 f\" G5 y
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))4 d  L% h) S/ _
    
85. ( q4 h+ ^. j3 O: a. }) z1 i& |4 N
    
86.     return ranges
    ' W& g2 E* s* b( ]
87. def islive(tasks):
    $ p7 E8 c% `' V' L8 T: c% P
88.     for task in tasks:
    $ K) c, t* L  M5 b5 ?
89.         if task.isAlive():
    ) F* i9 c9 D7 Z$ ?: e+ p/ i
90.             return True
    7 S1 q2 r* O! Q3 ^( B
91.     return False
    - Q: t: U# R/ R
92. 1 v, S; [# b9 o
    
93. def paxel(url, output, blocks=6, proxies=local_proxies):7 B, \- ^; t, O( Z) b6 L0 f4 Y
    
94.     ''' paxel
    ; A  `$ {( d* Z: S% A
95.     '''
    ( O. x! j8 r( ]* I7 W8 U
96.     size = GetUrlFileSize( url, proxies )
    , d# ]) A7 Y7 L\" x* J
97.     ranges = SpliteBlocks( size, blocks ): m6 j2 o: k% b& I% i$ m8 o\" w' Q
    
98. 
    - L7 t# i) U* T/ n4 Q$ ?5 ?
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]# n$ W6 Q* r0 f, a* O
    
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]% O+ D% Y/ V, C  Y! {$ _) L8 W
     
101.    
     / z. J# R* N) Y, P7 d6 Z8 j
102.     tasks = []3 y\" \& p4 g' O, F5 C! ~
     
103.     for i in range(0,blocks):
     2 g9 N0 A: M. l
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )6 f+ Z1 T5 D, Y
     
105.         task.setDaemon( True )' E% K! Q7 m1 a/ I  I8 T! J
     
106.         task.start()
     ) L! ]# w! C! b
107.         tasks.append( task )# `' K$ v$ H. |$ C
     
108.          
       W$ T/ C3 G: q. q. o' L
109.     time.sleep( 2 )0 g: R5 R1 z+ [
     
110.     while islive(tasks):/ ]; r: P) m: G. m
     
111.         downloaded = sum( [task.downloaded for task in tasks] ). O2 ]8 f( A- f
     
112.         process = downloaded/float(size)*100
     2 g6 A( U\" P. D\" M: O  l) z
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
     2 n2 C  l$ m% I$ R, g; c
114.         sys.stdout.write(show)2 Q# v) X. K7 P% ]. X( x
     
115.         sys.stdout.flush()
     / E1 m9 r; r! R% z* ?1 `
116.         time.sleep( 0.5 )% G, I: C& ^0 r2 g/ n1 e
     
117.              & ~6 Y! Z5 f! ]/ K1 w5 P& P! |
     
118.     filehandle = open( output, 'wb+' )( ^- G# v- f5 a6 ?% C
     
119.     for i in filename:* q% T5 f8 [% B0 P9 W, B6 D
     
120.         f = open( i, 'rb' )
     1 w  i( ]1 _% C) R0 `  |8 c
121.         filehandle.write( f.read() )! ?' v. l0 `4 O( s  P
     
122.         f.close()0 \0 e% c* \# j: h/ Z. N6 D. u! ?$ r
     
123.         try:; o1 C' g5 _) `2 }, |) V. M
     
124.             os.remove(i)
       c6 D' z$ I6 |  h& u
125.             pass
     & d6 j) X$ I* _, b; c
126.         except:8 o$ _/ \9 ]6 t9 n: \  D
     
127.             pass
     5 ]. I0 G8 @: w\" E* D% a
128. 9 R& B3 _+ K4 \) j
     
129.     filehandle.close()
     4 {# [3 A. m( i. ]2 i2 l
130. 
     * E% u5 C4 S+ h+ V4 Q) x# G
131. if __name__ == '__main__':* w& N& H8 l. c0 z
     
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"
     8 e4 s+ q( j\" e2 {' j9 ]! m
133.     output = '001.jpg'1 Z0 S$ [+ t, Z
     
134.     paxel( url, output, blocks=4, proxies={} )

复制代码