数学建模社区-数学中国

标题: 多线程下载图集 [打印本页]
作者: 2744557306    时间: 2024-3-31 17:18
标题: 多线程下载图集
网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。
  1. #!/usr/bin/python( r# E3 Q3 m8 v+ z$ N) k  G
  2. # -*- coding: utf-8 -*-9 G) ]+ A% C5 t0 G, G7 u  h" V+ f: |
  3. # filename: paxel.py# u3 ^' Q3 E% R* t8 S6 k

  4. 1 X/ Z+ y4 u$ L2 U0 ?% }
  5. '''It is a multi-thread downloading tool
    3 V# E* Z" a3 T

  6. " o( U/ x. y& n
  7.     It was developed follow axel.+ ~8 T& I$ g& G4 _9 j8 \& l0 |
  8.         Author: volans
    # s8 A3 f/ ~! X" l0 C+ f8 N. r
  9.         E-mail: volansw [at] gmail.com
    6 y, {# g1 W& w6 c0 z
  10. '''
    + D) k7 [' E$ g# g

  11. 3 }  a% t. W* b* e2 M- `1 B
  12. import sys
    . B% [1 o5 c- v' i; @
  13. import os
    6 K& ?6 [$ v$ }/ R9 o" V
  14. import time  X8 A: C# i) V, V/ K! a5 {- z
  15. import urllib9 T+ n0 F/ q; h) a
  16. from threading import Thread  q4 f) U4 D1 \! ~! e2 j
  17. ; [4 ?, }: ?- \( A' n, f; ?3 P: P
  18. local_proxies = {'http': 'http://131.139.58.200:8080'}/ z* M1 {$ h7 ^. m
  19. 3 ?; H7 H: a6 G7 ], u
  20. class AxelPython(Thread, urllib.FancyURLopener):) n: z1 S/ w+ A/ h  y. v6 |2 [" }' E
  21.     '''Multi-thread downloading class." V8 X. U$ v) j. ~0 A
  22. 7 p( b9 s3 w6 N; K7 U9 x8 M
  23.         run() is a vitural method of Thread.
    " N7 c& t2 H% p( s  w
  24.     '''
    7 C% z; i$ }, D. g9 P1 t" w
  25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    9 a$ H  ]# w4 c' |
  26.         Thread.__init__(self, name=threadname)
    " q# S; G  U) n- I+ J. E
  27.         urllib.FancyURLopener.__init__(self, proxies)0 ]$ o  W1 J! O$ q- y& v; F% v
  28.         self.name = threadname
    . @* d$ y) c. [* O0 A
  29.         self.url = url
    , d+ K0 q- U3 k; `8 Q* ^9 S5 X
  30.         self.filename = filename
    7 _0 F" x  [4 v) t
  31.         self.ranges = ranges
    0 X% a' L1 H6 z2 a, r! _
  32.         self.downloaded = 0
    . o- E' s. t. d& l0 V0 J8 `

  33. ! n) b. s8 m  B& [
  34.     def run(self):
    1 \9 ]8 x5 _$ P8 R8 y
  35.         '''vertual function in Thread'''* R/ q$ V3 v9 y; B8 m9 V' I0 _
  36.         try:
    6 i  }# }! y0 k: a" y
  37.             self.downloaded = os.path.getsize( self.filename ): J0 k! h. r% b1 W
  38.         except OSError:0 p3 T& r9 f$ r' p% m5 H
  39.             #print 'never downloaded'
    6 W3 B, @4 a* X  J
  40.             self.downloaded = 0
    4 L9 `) ]  b; {1 z8 k  z+ H( Q. |

  41. , ~9 H/ P# F& O" E: N9 [( n/ b
  42.         # rebuild start poind& l" @2 ?3 M. \5 k% p1 y5 ?! w# p* o
  43.         self.startpoint = self.ranges[0] + self.downloaded
    4 x- j7 N. R7 c4 n
  44.          
    . d, B1 f% N6 N6 O
  45.         # This part is completed/ P6 ?& |' ]" }! e% u8 P
  46.         if self.startpoint >= self.ranges[1]:: D4 J+ i1 y" c1 L% l; f0 G+ A
  47.             print 'Part %s has been downloaded over.' % self.filename
    6 ^+ t# [$ t6 X1 Z$ _
  48.             return+ |2 K3 z' ^0 K4 y" Z) [
  49.          " m. r; y" J0 Y$ n5 w
  50.         self.oneTimeSize = 16384 #16kByte/time; L3 r% `0 e( n2 m" }- P
  51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
    & O. s3 T& j: e2 Y) p3 ^

  52. : [. v. n( _! k$ L, _% m
  53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
    7 G1 T9 \" l4 d/ Y2 A  _* z! [: i
  54.              8 j8 p) G* G2 e9 {
  55.         self.urlhandle = self.open( self.url )
    * U4 j3 w( N8 h. I0 y: {
  56. : Y: x3 }8 v/ Z! Q  `3 H
  57.         data = self.urlhandle.read( self.oneTimeSize )5 A4 ?0 @2 P6 r6 F
  58.         while data:
    . B% i8 B3 R& G8 x" |! t
  59.             filehandle = open( self.filename, 'ab+' )8 e! u; q& U% `4 C- l! V# `
  60.             filehandle.write( data )0 i; b. Z5 c( ~
  61.             filehandle.close()
    7 h8 T) E( k7 u5 E0 U8 M% _. W  ?; T

  62. % T7 ]& _3 N: r& D: e
  63.             self.downloaded += len( data )
    & M, e) F; S# z/ z) ^8 P
  64.             #print "%s" % (self.name)
    + F4 y% w/ y- Z! i: q
  65.             #progress = u'\r...'
    + u( M4 q, I: |( p
  66. 8 i% g4 e$ x. D1 f5 Z
  67.             data = self.urlhandle.read( self.oneTimeSize )
    & X& e$ f9 S) h1 L5 f
  68.          
    7 d0 z! b6 c2 ^! _; o* h
  69. def GetUrlFileSize(url, proxies={}):& l4 H" d* X0 S% W7 {9 W1 o/ A
  70.     urlHandler = urllib.urlopen( url, proxies=proxies )
    2 R, n5 Z" b. M% D" c+ R
  71.     headers = urlHandler.info().headers
    8 Y" o+ E* c7 o/ M$ d$ Y
  72.     length = 0
    # I( a; a4 g6 w4 V! A2 ^2 I
  73.     for header in headers:
    / N) P5 M2 m8 z5 ~; a- y# ~5 U
  74.         if header.find('Length') != -1:9 ]; X! F3 ?. R/ a
  75.             length = header.split(':')[-1].strip()4 p" g& Z" f' M& D
  76.             length = int(length)
    & |( B$ W/ Y6 N0 t3 s  a) K
  77.     return length
    1 E0 \7 Y$ e/ a% c: s: [! S

  78. 2 z2 R: I4 E. n. r
  79. def SpliteBlocks(totalsize, blocknumber):: J5 r8 P9 \! S
  80.     blocksize = totalsize/blocknumber" L! g, V% s7 h, H, K
  81.     ranges = []8 @  \) h6 ?6 z$ O
  82.     for i in range(0, blocknumber-1):
    - j: H$ e' r, {; z
  83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))* Q: p3 A9 h: a- @, T
  84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))" e! z/ U+ V$ C0 y
  85. 6 V; O2 i5 c/ ^1 `4 m# W
  86.     return ranges
    & u* ^& O, `6 w$ ~' x4 v6 o
  87. def islive(tasks):
    2 c6 M4 Z) R: m: u/ P' p7 N+ f: I- ^
  88.     for task in tasks:. |: j* U1 x. b4 O8 b9 f
  89.         if task.isAlive():
    * o1 h) f: J# c: u7 h# @, J6 y
  90.             return True
    ) f4 ]1 |8 B$ U0 M  t
  91.     return False: f, M. Q# x* C: M# a
  92. $ p; T2 \0 w' p! B4 Z! Q
  93. def paxel(url, output, blocks=6, proxies=local_proxies):- O" y8 D) S' a, ]
  94.     ''' paxel/ i" B4 Y! J4 w) i( g
  95.     '''6 y/ V' |% t" Y$ L5 K2 S
  96.     size = GetUrlFileSize( url, proxies )3 q8 @* E! [6 J3 c. w
  97.     ranges = SpliteBlocks( size, blocks )9 b/ _# G3 ]5 h7 q, c
  98. 5 J$ s. s: }0 W
  99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]
    + \1 ^4 ?4 L1 H# t3 r' p& O+ E
  100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]* b- N: i1 q1 G3 B2 l
  101.    # x( e6 f  K0 `' L$ |+ \( A
  102.     tasks = []7 s, z. X: T: o* z3 V
  103.     for i in range(0,blocks):
    / f4 r6 Y' x. J8 S# O9 t0 w0 c
  104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )
    ! M8 }' |2 @) ?4 ?5 s2 N0 q
  105.         task.setDaemon( True )
    5 K% \" h/ n( |
  106.         task.start()
    9 m9 f! @# P# c& C8 c3 Y, S" c
  107.         tasks.append( task ). R+ m3 I0 o# `) d& k
  108.          * }7 L1 |  ~' f5 j
  109.     time.sleep( 2 )
    . ]: c& t+ v" O4 K. M
  110.     while islive(tasks):
    ! [, ~& q9 E- W& T" U
  111.         downloaded = sum( [task.downloaded for task in tasks] )
    ( d1 P; U+ {, @2 F( L1 O
  112.         process = downloaded/float(size)*1006 ~$ u& r: a9 `9 A
  113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)8 S4 R5 a- P3 y3 d/ e: F
  114.         sys.stdout.write(show)
    ; l! L! E8 Z* M  e
  115.         sys.stdout.flush()6 i2 X4 S1 C6 e) g+ C" D
  116.         time.sleep( 0.5 )
    : x/ a: l1 L4 N2 g* o% Y$ d
  117.              / n0 c, N6 V% Y9 \
  118.     filehandle = open( output, 'wb+' )/ Z# ?0 u0 G# U# Q7 Z
  119.     for i in filename:
    2 u- K  \2 Y: m+ V1 K$ U0 F) t" B
  120.         f = open( i, 'rb' )
    4 i" r0 Z( S- O2 s
  121.         filehandle.write( f.read() )
      S0 D: M" ]9 h+ [; R4 z
  122.         f.close()
    . Y' @- n- S" i) G( v1 I2 i# E' _" n
  123.         try:
    ( @. _) Y5 H  b0 y0 ~
  124.             os.remove(i)
    1 o' M& q- D4 O0 q6 ]; h
  125.             pass2 R6 N8 B" U' j8 t5 {
  126.         except:
      p9 B7 t1 \  ^% `
  127.             pass4 P! i/ D/ ^; {5 \

  128. & B% L2 ^0 T, G1 `9 B2 x) V
  129.     filehandle.close()
    4 s6 a9 X: P6 Z6 w6 p; S

  130. 1 k5 N" ^* _1 e6 p
  131. if __name__ == '__main__':4 ~! B! w; b9 T4 G* p: V$ a) P
  132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"
    ) p+ c# p# S6 R
  133.     output = '001.jpg'
    1 q& g; U0 t( X# @/ W) M0 p3 K
  134.     paxel( url, output, blocks=4, proxies={} )
复制代码
6 Y9 C+ D; ~# r8 O: e1 z2 j




欢迎光临 数学建模社区-数学中国 (http://www.madio.net/) Powered by Discuz! X2.5