多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python
   ' s) {8 J0 D( e0 V
2. # -*- coding: utf-8 -*-
   . n* y. u& a( A7 b6 F* X: ]7 _' `
3. # filename: paxel.py, o2 [, v- [. r/ `7 x& Y
   
4. 5 I# @2 _0 t$ n* I1 ?
   
5. '''It is a multi-thread downloading tool* T1 a/ {. N! f2 G: J7 C' l7 B( ^\" u
   
6. 
   ) |# t1 P2 ?& q8 N
7.     It was developed follow axel.
   1 ?) ?$ _) U1 x5 P! }4 f
8.         Author: volans& @* g/ c. L3 P% e. x3 i  l  k
   
9.         E-mail: volansw [at] gmail.com
   % T' {0 u$ z$ v7 S$ y- O' K4 u5 N
10. '''( J3 n, M5 Y- W2 s4 y7 Y
    
11. 
    / l2 }8 w5 b- e9 r! y1 c( S
12. import sys6 u7 T6 ]\" e; a' K  F# O+ v\" K
    
13. import os
    : V  f. b9 C. D# G3 W3 f# T9 B
14. import time
    ) X* q8 Y) v+ g/ l8 a1 K  V8 }
15. import urllib
    & G5 ~0 F( o* S
16. from threading import Thread
    $ w9 J\" q) u* x
17. ' X+ s- V$ U\" e; F) j9 u3 m
    
18. local_proxies = {'http': 'http://131.139.58.200:8080'}' |1 Y  ^( _* Q- Z4 B* V9 J
    
19.   g% s# R\" k) d# K\" e
    
20. class AxelPython(Thread, urllib.FancyURLopener):
    % h$ I. ]1 j& \; ^1 e/ @
21.     '''Multi-thread downloading class.& l/ Q8 W9 P$ k2 m\" e; w
    
22. 
    3 O& r3 @; x# }( \, G5 F+ s
23.         run() is a vitural method of Thread.
    - E  h) T# ~# r  f
24.     '''1 q1 Q5 V; J, v& p( y
    
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):: c/ n5 N# E) V: L
    
26.         Thread.__init__(self, name=threadname)
    7 \9 _# }% P+ _* p0 T8 ]. T
27.         urllib.FancyURLopener.__init__(self, proxies)
    8 {) \) e9 Y) L8 h8 \' M
28.         self.name = threadname
    - `7 l6 [\" ~, j
29.         self.url = url: o- P6 ~9 m$ i) y) A\" j5 A
    
30.         self.filename = filename
    2 s5 D3 f( q3 {
31.         self.ranges = ranges5 i: W% M+ Q4 R3 A
    
32.         self.downloaded = 05 N- |, Z; }% V; q( i8 r+ G7 f
    
33. 
    : z# X. E; ]1 ~8 n: L0 v/ C- T
34.     def run(self):
    $ @  \1 f4 a9 i, f/ a& I
35.         '''vertual function in Thread'''
    2 k- z6 R: l# W  ]- S
36.         try:
      M4 {' i$ a) y  b: h2 {9 r
37.             self.downloaded = os.path.getsize( self.filename )0 O7 C. D\" e! ^8 n$ T
    
38.         except OSError:
    7 W2 m2 {( k\" p, Q! w
39.             #print 'never downloaded'4 b# z% m' z/ I2 m
    
40.             self.downloaded = 0\" _/ F; O. U9 V2 q: }4 R
    
41. 
    0 ^! J4 k5 }' H+ B3 X
42.         # rebuild start poind) A- f\" B# |* q# V
    
43.         self.startpoint = self.ranges[0] + self.downloaded8 Y- d\" y8 }# o' ?+ Z
    
44.          ! p' {* b( R4 |9 p# C7 }
    
45.         # This part is completed+ |8 Z; q) L' a+ @8 z5 U
    
46.         if self.startpoint >= self.ranges[1]:
    ) W! Q( `9 Z4 J\" N* _- O
47.             print 'Part %s has been downloaded over.' % self.filename
      F/ _( x' `0 l! k8 b% b
48.             return2 k; n' S0 U3 c1 Y\" h3 c& R
    
49.          
    8 w8 V6 ~\" k. d$ Z
50.         self.oneTimeSize = 16384 #16kByte/time2 D( u4 f- ?* m1 T6 ~
    
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
    6 d& R% r; v# l8 X\" F- j0 N
52. ! d! c0 g8 L8 Y2 \$ J
    
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))4 f  E( t6 u1 V2 K# T9 J- [. X
    
54.             
    , E6 k' w5 c( S5 Z$ q1 }5 A( [1 T
55.         self.urlhandle = self.open( self.url )
    # }# ~1 T; A* u! m
56. 
    / W! W# k9 G8 U
57.         data = self.urlhandle.read( self.oneTimeSize )3 |! i4 Z0 N, @* `1 F
    
58.         while data:
    5 X) v' u$ e\" R/ J3 q
59.             filehandle = open( self.filename, 'ab+' )
    - v; {9 |( J6 L. q& D( G: G, h
60.             filehandle.write( data )
    ( s& s# G9 k8 T0 E; r
61.             filehandle.close()
    5 j( P0 ?\" F( W+ d9 f# @+ N6 p& O
62. 
      o. x/ G! e% [\" z: S9 s
63.             self.downloaded += len( data )3 v, Q, o' [3 Y; ^  C
    
64.             #print "%s" % (self.name), }# u$ O* w! R% B
    
65.             #progress = u'\r...'
    . C4 n: W( i9 D  z- h7 g5 y
66. 
    + N, N& o- k9 q1 w# E
67.             data = self.urlhandle.read( self.oneTimeSize )0 L$ Z. }7 Q( x4 R
    
68.          . p! \) I+ e& x+ R) |
    
69. def GetUrlFileSize(url, proxies={}):+ D% s: ^5 G- F  y' E/ h7 J) Z
    
70.     urlHandler = urllib.urlopen( url, proxies=proxies )9 z/ [9 i6 z4 m& J
    
71.     headers = urlHandler.info().headers: r6 J4 F, _9 E\" J# B- Y. |
    
72.     length = 0
    ) b1 C  I$ p' r+ r9 c- F
73.     for header in headers:9 W- B3 M9 m4 N/ y; f, y; B( _
    
74.         if header.find('Length') != -1:* f  f4 A. X8 b% i
    
75.             length = header.split(':')[-1].strip()
    ; }* c' g' t, s
76.             length = int(length)
    6 v! @4 O6 R% U
77.     return length7 o4 r0 M. A- n8 |; O  V; ]
    
78. ) {. G. s: P! X. K* f! r
    
79. def SpliteBlocks(totalsize, blocknumber):
    ' r\" V) Q$ \9 o: x2 C
80.     blocksize = totalsize/blocknumber+ U; v8 \+ I- h\" N& n7 _7 j+ \+ a
    
81.     ranges = []/ B, u4 f1 U5 _
    
82.     for i in range(0, blocknumber-1):
    , U. H1 L& r7 B' e% f9 x& g: R\" Y* ~9 i2 Q
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
      k# C% q% h* g
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
    0 [( G) N, n* K+ A
85. 
      w/ k3 |7 v8 o6 l$ i
86.     return ranges; q# [- ^6 R6 [% V& X\" u
    
87. def islive(tasks):1 ^  M' \5 k# V
    
88.     for task in tasks:/ x. P+ g, o1 ^0 T! }
    
89.         if task.isAlive():1 i& G9 e' e) F3 N
    
90.             return True. o0 O7 w, y* z3 p5 j
    
91.     return False
      Z. N5 c( Q$ l, C7 t6 `* G. @7 q
92. & G6 F$ b# s5 k1 `+ D
    
93. def paxel(url, output, blocks=6, proxies=local_proxies):
    3 y/ S9 \- z) `/ X% O+ v0 p+ E
94.     ''' paxel
    ( I0 B$ l2 u( W9 J8 `
95.     '''4 M* N: @9 y% S# K3 q& Y4 }
    
96.     size = GetUrlFileSize( url, proxies )2 @; r; ?( l! U. `; F2 I  E8 @2 r
    
97.     ranges = SpliteBlocks( size, blocks )
    + b0 W  ]$ P# g; [, C7 p- e5 x
98. 
    % n8 U5 |8 ^1 `3 W2 e' g
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]
    $ m1 V. r1 G% ^& |  {
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
     1 o\" h' a5 d6 ?* X) X! E6 ?* x; S
101.    
     8 ]- x1 m; Y8 M  H& I. Y' d+ r
102.     tasks = []* L( `0 x5 p- @- P+ W
     
103.     for i in range(0,blocks):# I2 f0 `& v4 A/ a\" Z( t
     
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )
     , ?$ j; v) \9 q9 G1 U\" Z# K& L
105.         task.setDaemon( True ), i3 E, j- O* D
     
106.         task.start()
     2 J8 k3 L1 \0 z. w9 a/ a4 J
107.         tasks.append( task )
     + n. X- A, d% A' \
108.          & v$ z6 U- x5 F6 B$ g3 ~
     
109.     time.sleep( 2 )
     ) d' S5 E3 P6 ~  A
110.     while islive(tasks):
     1 I/ A& G7 }/ \& x4 g
111.         downloaded = sum( [task.downloaded for task in tasks] )% F% y! i8 l7 e+ {\" E: X  o
     
112.         process = downloaded/float(size)*100
     5 U# M, a5 q\" p. Z! t
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process): J! {2 i  I! w* ^- t* O
     
114.         sys.stdout.write(show)! b+ K0 T/ {+ f  Y9 X! r0 d
     
115.         sys.stdout.flush()
     . _5 c4 V; t% z& U5 ?% S
116.         time.sleep( 0.5 )
     2 b( B+ c: ~3 U6 k; k
117.              * {5 y3 Q, S7 I) z# A
     
118.     filehandle = open( output, 'wb+' )$ b% B4 L, m/ C# R& t# N
     
119.     for i in filename:1 ]; w) h* O- ~& k
     
120.         f = open( i, 'rb' )3 [\" Z- Q; M& p- t, \1 Q
     
121.         filehandle.write( f.read() ). M, a- ~& y) @
     
122.         f.close()
     * H: g8 q) h7 L6 B' x
123.         try:
     * Q9 m3 ^$ H# z2 r! ^
124.             os.remove(i)
     6 h% Z\" ^7 F$ b7 R7 q
125.             pass
     + |; N' c  h/ C1 j6 p
126.         except:
     . j- d1 e\" _% r( ^9 Z
127.             pass
     5 L/ o/ ^3 U6 Z  [8 c
128. 
     2 X/ E( d  t' x! e
129.     filehandle.close()0 D( \) Q# A- H2 |) }  L' T
     
130. ' \) f: ^4 f6 G4 @
     
131. if __name__ == '__main__':8 K5 d; I7 j6 U& t% I
     
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"
     . Q' F6 G6 r- |: K6 ]
133.     output = '001.jpg'\" c7 K  O\" s\" i  Y5 }
     
134.     paxel( url, output, blocks=4, proxies={} )

复制代码

: {9 g( m4 T7 `1 s! }