多线程下载图集

2744557306

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

1. #!/usr/bin/python2 B# c: T# m) b8 b
   
2. # -*- coding: utf-8 -*-6 V+ v7 z, U\" O\" t9 M7 I' `6 x
   
3. # filename: paxel.py
   \" M3 ^- E6 g* l0 \: |, l+ T% C3 Q
4. % N0 s2 U9 `+ ~- _; F
   
5. '''It is a multi-thread downloading tool
   9 G. l7 |1 v$ r
6. 
   9 R# n* _6 i# x( I: W
7.     It was developed follow axel.( u- A1 j: ]3 b5 E
   
8.         Author: volans
   4 J7 d* g% m' A$ {
9.         E-mail: volansw [at] gmail.com3 H8 k% g' w+ b- d, Y
   
10. '''
    % B% p\" j8 e7 u0 F
11. 
    ' ?& c2 k' ~8 Y- O( K2 t5 z1 C0 x5 e
12. import sys' M7 ?/ S9 [% D/ T
    
13. import os
    0 a) `7 @+ w' }8 J; `6 K
14. import time
    2 E/ a& r3 I& o# K  d$ E
15. import urllib
    2 O6 q& O) i% d/ q0 g+ Y
16. from threading import Thread+ k\" h2 C$ M5 N- K% I& N; k. J  R
    
17. . B; Q7 L5 x( W# [7 f/ D
    
18. local_proxies = {'http': 'http://131.139.58.200:8080'}( S# ?6 i( b7 i+ a9 ~, y
    
19. 
    + K+ h1 Y7 G0 B# x( t' A
20. class AxelPython(Thread, urllib.FancyURLopener):
    \" ~! _4 V- q3 W: Z, S) n/ u6 z
21.     '''Multi-thread downloading class.
    ( U! ?$ W0 P! ~: `4 p
22. 
      e; j8 G1 ]\" b; ~  F$ G+ G
23.         run() is a vitural method of Thread., D2 Y4 a% j1 t' n6 {, |/ @\" s4 v
    
24.     '''
    . N5 V) a% U: n, B# w
25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):3 U' e' ?0 }' ]& p9 V+ Y( y$ O2 c) P
    
26.         Thread.__init__(self, name=threadname), E4 ~; [4 |/ v
    
27.         urllib.FancyURLopener.__init__(self, proxies)* E( G1 k0 [5 J( F6 |
    
28.         self.name = threadname
    , u) `' L# |. d\" F+ L. U. r
29.         self.url = url1 ^* Y0 ~2 Z* d* O# O! s
    
30.         self.filename = filename+ e/ ]1 d4 U/ J2 c
    
31.         self.ranges = ranges  i0 I: Y\" [0 _\" q# J
    
32.         self.downloaded = 0
    & p' h& [2 R9 s4 q; c4 f
33. 
    ; ]: ^3 {: x2 Q; o5 n2 z
34.     def run(self):% c( Q  C\" L. R2 b2 u7 g# N
    
35.         '''vertual function in Thread'''
    . I\" v6 J* Q( j7 v+ }3 z
36.         try:
    6 Q2 j# I9 _/ I\" J- N& c\" @. ?
37.             self.downloaded = os.path.getsize( self.filename )) [! A! w. [0 M\" o4 ^
    
38.         except OSError:, e3 j) z\" P8 }. f+ e
    
39.             #print 'never downloaded'
    6 k' h, T  ^$ [* l0 h9 _) q& }8 {
40.             self.downloaded = 08 l5 v  ?# f  N* }; L
    
41. 
    ) L5 L. P* C! E8 ~- b
42.         # rebuild start poind/ z+ K/ `% T0 z\" ~4 T9 u$ j& {
    
43.         self.startpoint = self.ranges[0] + self.downloaded
    - h: ~- F/ ?. y4 [3 |0 E8 s/ Y% ]
44.          
    & K  z, n4 m3 E5 i; ]  z% w
45.         # This part is completed
    : S& p6 T) P' ~\" ~7 H
46.         if self.startpoint >= self.ranges[1]:. a6 b/ K- k( \# k
    
47.             print 'Part %s has been downloaded over.' % self.filename
    ( A3 n8 ^* Q! C' D! t
48.             return
    8 S4 ~# e  i5 W# r' T5 t. }
49.          
    9 F: T3 m; N6 y+ X  c5 Q4 L* M
50.         self.oneTimeSize = 16384 #16kByte/time
    # D& J! }$ F* x  ~4 w5 n
51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
    . s. n$ n) P7 ]
52. 0 ~4 \3 L) J9 l( f7 o
    
53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))5 G8 T/ t6 B$ `
    
54.              4 q' }- I5 _( I  v$ t8 C
    
55.         self.urlhandle = self.open( self.url )
    . K3 Q2 ]9 i+ Y
56. 
    & ?1 e$ {9 w$ _7 A
57.         data = self.urlhandle.read( self.oneTimeSize )% H6 s1 n8 J) |5 v' Y7 x2 M) h
    
58.         while data:
    3 ~2 {0 H( ]5 u; @% D+ O  R6 d/ d
59.             filehandle = open( self.filename, 'ab+' )
    * t' }' d! Y  q) L2 t6 E# n
60.             filehandle.write( data )4 \8 H4 A1 J( J: B* d& W' d8 c  i
    
61.             filehandle.close()
    . p; U) n3 v% E3 W& B6 `9 I0 U
62. 
    \" d+ x6 z: p0 b( A  ?
63.             self.downloaded += len( data )
    4 \% g1 P( |* n3 F9 L* Y\" Y+ I
64.             #print "%s" % (self.name)+ {$ l/ ]; ^1 Y. i% w. [# x2 r
    
65.             #progress = u'\r...'; Z/ v0 y( E0 i
    
66. 
    $ R/ P\" b+ K( m8 L  L+ u* [5 A
67.             data = self.urlhandle.read( self.oneTimeSize )
    ! t# \$ h3 g; Q3 ^% P. U
68.          
    1 p9 u9 N: Q6 {* S2 a  w) L
69. def GetUrlFileSize(url, proxies={}):  e) W7 x3 b* K6 Z, _, h2 U! C# ]8 U
    
70.     urlHandler = urllib.urlopen( url, proxies=proxies )% z9 ]5 c\" p0 i) i\" z
    
71.     headers = urlHandler.info().headers
    ! t/ t: f! ]% m  m4 x6 L: Q) |. ]- u
72.     length = 0
    . F5 H2 M7 X/ ^2 Z+ y$ w/ r
73.     for header in headers:$ Z- Y# P1 ~% c  E' W\" o/ p. A
    
74.         if header.find('Length') != -1:
    + I/ {7 q6 n: }- r
75.             length = header.split(':')[-1].strip()
    4 ^+ n8 z2 ^  X, H# u- E  ?
76.             length = int(length)
    * ^* |- ?1 b8 p- _8 s
77.     return length- g8 |6 g, w3 c  D3 |
    
78. 9 \2 o% X& M' a8 E! Z  Z- i
    
79. def SpliteBlocks(totalsize, blocknumber):* _  [% B* V7 Z( n
    
80.     blocksize = totalsize/blocknumber7 N8 k1 L8 F0 ^/ V9 G. H7 s
    
81.     ranges = []
    5 R# P4 F( j- a7 ~/ b: x7 e
82.     for i in range(0, blocknumber-1):& [1 U( M+ Q) V  i$ ]3 h5 j
    
83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))$ P% s% \; r) D8 W2 d% b
    
84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
    ; q( i4 s7 K8 k3 {! \& U
85. 
    # C, O2 c$ J5 `* R# a. b
86.     return ranges* R& i% Z! L; b9 W; W9 q
    
87. def islive(tasks):
    ) b3 k\" t8 {: S, X1 N1 i8 C
88.     for task in tasks:% M. v: R' C! `- P5 ?
    
89.         if task.isAlive():( D! ~4 ?  \9 c+ n8 }6 m
    
90.             return True) i5 k# I+ c5 W2 z
    
91.     return False
    2 N, x6 |0 |( \  X  S7 m6 [3 X
92. 
    9 s3 T) [! K0 E) D$ P4 `9 Y
93. def paxel(url, output, blocks=6, proxies=local_proxies):/ p0 U, [3 G( }1 K8 L; ]& @& y
    
94.     ''' paxel
    9 W2 ~& W8 D4 i0 B( L7 L
95.     '''
    , R! D\" C- A2 y) @, M
96.     size = GetUrlFileSize( url, proxies )0 e4 |5 X9 h$ ^/ |
    
97.     ranges = SpliteBlocks( size, blocks )0 j8 v8 t) r3 |& E
    
98. 1 |9 Y% c+ q7 f; J# s+ ~
    
99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]
    : C' P+ T! K, A2 e\" g4 p: e) D5 O' `
100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]/ L9 E/ w# @$ _0 m. c
     
101.    
     6 Q% E: W' B2 {- e, U
102.     tasks = []8 f- r6 |) P6 N& \1 G; g; j, D
     
103.     for i in range(0,blocks):
     \" J9 t: c' U9 a) O1 g0 v6 l
104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] ); V: C6 }1 S: A- i: L
     
105.         task.setDaemon( True )
     5 S+ S+ j# j& J# z) b2 v
106.         task.start()$ Z3 ^$ A( X5 N6 ~0 b: G9 `
     
107.         tasks.append( task )
     1 G% ~6 n* d\" P# W) n) {' I; A, x
108.          % c' L0 z! Z2 A+ S6 C. ^+ }
     
109.     time.sleep( 2 )5 A/ N& m' _! V( y) v8 v
     
110.     while islive(tasks):2 V& O; d3 y/ ~2 G
     
111.         downloaded = sum( [task.downloaded for task in tasks] )( \' F8 n7 M- a  e) k
     
112.         process = downloaded/float(size)*100( u' W9 W! i+ d  o, L, R7 }1 E7 E
     
113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
     ' b$ R  x+ x. Q8 C- R
114.         sys.stdout.write(show)
     : ]7 D/ E5 D3 M  @8 M$ f
115.         sys.stdout.flush()
     ' X9 X) \. f, \) R  l# k
116.         time.sleep( 0.5 )7 \# ^, S: V  k+ I
     
117.              ( }! w( p4 i; k
     
118.     filehandle = open( output, 'wb+' )
     3 p  i+ Y\" z* s: w: N* S) r) w8 y
119.     for i in filename:
     ' W\" o  h\" L- |# ~
120.         f = open( i, 'rb' )
     2 U. }& c! a0 P
121.         filehandle.write( f.read() )
     1 K& Z. Y5 n6 u6 C
122.         f.close()
     $ P/ ~) C& J! H4 d; `
123.         try:- ?' ]; y! V  L
     
124.             os.remove(i)
       F3 Z; R8 K& s' B2 {2 H$ w
125.             pass
     - i# s- z0 r4 C! l3 g
126.         except:$ z* }8 p) c- `+ k1 a/ ^
     
127.             pass, D  o& [* T6 ?5 i) v) }* }
     
128. 3 [6 U# D. m) C% y\" \! |7 |0 b
     
129.     filehandle.close()9 M: m# {, T. Q* m1 ]( t# d0 j; ^
     
130. * b* k; _# b: l8 Y+ ^. ^& E
     
131. if __name__ == '__main__':
     - G0 |' d8 w+ Y
132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"+ K  B4 y0 W- N( [! x
     
133.     output = '001.jpg'
     ! k9 p$ W, o% Y& S- D
134.     paxel( url, output, blocks=4, proxies={} )

复制代码