数学建模社区-数学中国

标题: 多线程下载图集 [打印本页]
作者: 2744557306    时间: 2024-3-31 17:18
标题: 多线程下载图集
网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。
  1. #!/usr/bin/python2 h$ `0 P) A: b3 d; E' M
  2. # -*- coding: utf-8 -*-; [% d" S9 |: g! i/ U
  3. # filename: paxel.py* X. i+ V( N- J9 N: D$ s# B; g
  4. * N: f5 Y% l* b, |% W1 D0 ]
  5. '''It is a multi-thread downloading tool+ w0 g  v! y; j- f

  6. / k- Y3 ^" m* Q: u
  7.     It was developed follow axel.  C6 _8 ^8 c$ a3 q, J- L, `; F: `& l
  8.         Author: volans
    + j& q3 n" z3 D2 h
  9.         E-mail: volansw [at] gmail.com
    / s! V5 }9 a- n) @2 x& D0 b% ^
  10. ''': P! ]) C- r/ y

  11. 7 i  m+ R" F+ N* I; r* H+ T4 o
  12. import sys
    2 N/ k5 K, V4 @2 ?
  13. import os
    + O. h! P: j9 V5 Z/ D( u
  14. import time
    % l1 C- L6 Q; _( _8 v" s; t
  15. import urllib
    ( ~# v0 y- I! e. n$ U  J  ?+ A6 ~
  16. from threading import Thread
    + W) J( l4 y3 Y5 i! F2 f
  17.   i: h! v! c( g  \
  18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    ) u2 q2 T: T, ~9 q

  19. / |$ H$ }: X& F/ f* G
  20. class AxelPython(Thread, urllib.FancyURLopener):
    4 `2 q9 u# I1 n# L
  21.     '''Multi-thread downloading class.- ]4 C+ R+ o+ I( @' m& T

  22. 8 o8 D+ D! V( y, k; b/ f
  23.         run() is a vitural method of Thread., b5 I0 j% Z7 T7 L
  24.     '''
    - d. @# S( l2 \9 \- Q
  25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):5 D) b$ _: N1 [7 Q' Z/ K: k$ ]8 I- J
  26.         Thread.__init__(self, name=threadname)8 l' ?( Y* p, j' `3 ~8 x+ x$ _' V8 K: a: Z
  27.         urllib.FancyURLopener.__init__(self, proxies)
    ' L& d6 ?' [+ Y$ [
  28.         self.name = threadname
    & i/ i. f8 d6 Z# n
  29.         self.url = url
    8 _4 Y  T; J; V$ |8 O) A
  30.         self.filename = filename( u* }' O$ J) Z
  31.         self.ranges = ranges5 @; n6 G( L2 d6 l$ Y1 c  y+ F4 j; f
  32.         self.downloaded = 0: o, [" o3 Q' ~( Q$ S" @: b

  33. : ?8 ^/ l+ k8 ~2 \9 |& u1 e
  34.     def run(self):
    5 [# P* D9 [/ F+ H9 n
  35.         '''vertual function in Thread'''
    ( [" h# @* k; C6 K! h! h
  36.         try:
    , S) N! G4 [+ Y: O9 z2 e2 G; v
  37.             self.downloaded = os.path.getsize( self.filename )5 v. b2 F5 F* }0 w" Q
  38.         except OSError:! f1 q' c0 J6 C# i9 n. k
  39.             #print 'never downloaded'  ^- E% \0 m- W& H" W9 P; d+ a3 Q
  40.             self.downloaded = 0
    / M5 x, ^( f# H5 w

  41. 1 ?8 t& c* D0 a1 r
  42.         # rebuild start poind5 L: e* h  `3 a& {, l  |" f( B
  43.         self.startpoint = self.ranges[0] + self.downloaded
    8 j) X) j7 o8 k) `4 K1 h4 O
  44.          3 X1 a6 }$ e+ d  {* Y
  45.         # This part is completed7 d/ [# @& L; b; E0 v
  46.         if self.startpoint >= self.ranges[1]:( L' @& F( t4 R$ i! k( L$ `% t
  47.             print 'Part %s has been downloaded over.' % self.filename2 K7 L- C2 q* P
  48.             return
    ; f$ M9 C  |  O. E
  49.          % V0 f) ?$ H9 y$ \% r# v
  50.         self.oneTimeSize = 16384 #16kByte/time$ V7 }0 B! N+ e' E; S* {
  51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])! [% T) r+ K8 y
  52. 4 |, r0 E( P- n
  53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))4 A/ E. t' |7 U5 d% M
  54.              8 \* i* @! m% S9 V
  55.         self.urlhandle = self.open( self.url )
    " P4 P/ T, ]6 ?2 x9 s8 l/ s4 D
  56. # \$ O' ~+ J1 C! t; J& |1 W
  57.         data = self.urlhandle.read( self.oneTimeSize )
    , X5 z" H, a. C  s9 m
  58.         while data:
    ( H- o: h+ H/ b7 e; u+ W
  59.             filehandle = open( self.filename, 'ab+' )
    2 P1 P8 k& E0 \( H, m" I
  60.             filehandle.write( data )
    2 O; G2 t3 o5 g6 g6 [
  61.             filehandle.close()
    / M" I; s" w0 M  d4 b
  62. $ @# y# w( v: [5 U
  63.             self.downloaded += len( data )
    # R2 O3 b9 S% s/ q7 Y% L
  64.             #print "%s" % (self.name)& k2 ]1 l8 j- U
  65.             #progress = u'\r...'
    + ^- L+ S) s2 @# P5 v

  66. 1 z/ ^. n9 u0 ]; H
  67.             data = self.urlhandle.read( self.oneTimeSize )3 o2 D, \3 W- l4 }$ x9 P5 L! o) v/ O
  68.          
    6 {; t  q$ u! \  T/ Y
  69. def GetUrlFileSize(url, proxies={}):: R, o  q/ J2 N+ f  d
  70.     urlHandler = urllib.urlopen( url, proxies=proxies )& y1 n* ^- \8 b# Y: F
  71.     headers = urlHandler.info().headers1 l/ n% p/ E' f7 D" x# [
  72.     length = 0
    ! l! ?" N8 P: b
  73.     for header in headers:6 W! a( ?3 q7 d' a
  74.         if header.find('Length') != -1:
    & Y; R  k, A, d, x7 X  B
  75.             length = header.split(':')[-1].strip()
    & V" S6 t  |8 y/ ]1 w
  76.             length = int(length)
    7 Y  o2 l9 J7 D4 H& l: [$ n, `& v
  77.     return length" G9 K. H2 F' l
  78. : B& U3 X- R; T4 c8 R
  79. def SpliteBlocks(totalsize, blocknumber):9 @% q* b* `3 c1 D
  80.     blocksize = totalsize/blocknumber; b* @( x1 D# G4 Z  o8 ^
  81.     ranges = []
    . n& Z3 [4 h- e" ], K  k
  82.     for i in range(0, blocknumber-1):
    6 L2 \# o# {& e, s" _8 Q
  83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    # A) L1 |% U& O" o. d
  84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
    3 z2 `9 s5 J+ C: l

  85. * r" c$ X" {& f7 r
  86.     return ranges
    ; }  ]  C3 m. l
  87. def islive(tasks):
    0 \( |. p  l" }3 d8 r1 a
  88.     for task in tasks:0 E) N8 Y' j" a2 K
  89.         if task.isAlive():
    & q; N  Y" {( {* e
  90.             return True2 k" y" c3 t0 X  B# W' X% d
  91.     return False
    ! K; h: C! v, {

  92. , S! j( h: S; D. O& L
  93. def paxel(url, output, blocks=6, proxies=local_proxies):8 K1 H: q. Y7 }# k- X& z
  94.     ''' paxel
    , d7 n9 d& Y5 r& q  S6 N
  95.     '''0 i$ B- Q8 V  n  U$ W/ {' o6 n& C. i
  96.     size = GetUrlFileSize( url, proxies )
    / b, e8 G% A9 T  M6 o. I
  97.     ranges = SpliteBlocks( size, blocks )" |, q( o) R6 H* R0 @7 @) D" ^) ]2 s* }

  98. ; t/ t" M2 k, O' P- n- \
  99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]6 f; D6 {- Q2 t" ~, b
  100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]3 |4 {6 k  L- S5 q  E
  101.    % N5 }5 ~- V: E) n  j& g
  102.     tasks = []; H3 K" _; M1 }$ W7 g. |; Q4 Y
  103.     for i in range(0,blocks):+ o" j2 H& T! z% @8 ~+ R/ z
  104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )
    7 j& d+ H* D& D: L% n. e! K+ ]
  105.         task.setDaemon( True )
    6 N3 ?0 G* Y0 }" p5 @" i8 o
  106.         task.start()! Y3 ^1 e' u, \+ O  C" D1 j/ [* r) K/ U
  107.         tasks.append( task )
    $ m$ T+ }) Z8 D' e3 y
  108.          ( z3 k$ X* I2 K9 Q$ i
  109.     time.sleep( 2 )% |. [0 `4 f% Y1 \
  110.     while islive(tasks):$ l, v9 U6 K3 P0 Y
  111.         downloaded = sum( [task.downloaded for task in tasks] )" _& r6 Z! S7 g1 D% M
  112.         process = downloaded/float(size)*1009 }9 o! z) h3 V, K5 ~0 ~0 B) C4 x+ O
  113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
    % E5 f( D' l9 N$ `8 j0 K8 L$ V
  114.         sys.stdout.write(show)) A; a. w; W  n0 b
  115.         sys.stdout.flush()
    3 i5 K$ Y/ z- Q, _! I2 O0 D
  116.         time.sleep( 0.5 )
    ; o; V6 D$ v; u7 v3 E# R9 U
  117.              ( D2 a# E9 Y9 _8 D
  118.     filehandle = open( output, 'wb+' )
    * o6 g; \  s6 H( p0 L3 \/ k; J
  119.     for i in filename:
    * w5 x. U& w2 Z2 ]- j
  120.         f = open( i, 'rb' )
    8 o- x2 b0 N+ L# q
  121.         filehandle.write( f.read() ). g8 w, i! o+ i% `* w" W" Z% k, x
  122.         f.close()7 G( F1 M& P9 g6 M; N- N& H
  123.         try:
    & }- o0 D7 s3 ]; A
  124.             os.remove(i)
    - `" l# R9 |# `6 N, m
  125.             pass0 B- b: t! a0 m5 K6 @) J
  126.         except:
    7 v# _6 f: m* s% U( i0 ]8 N) Y9 |
  127.             pass
    $ e  v& w% ~5 [: h0 y' [
  128. / \5 R/ q: O0 Y# q" }1 f
  129.     filehandle.close()/ @+ H0 @( X9 \" h
  130. + q: ?6 H4 F. s* J- N/ b
  131. if __name__ == '__main__':' `) g: I+ E) d+ ?
  132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"
    ; l! S' w: E4 C( R0 Z2 Y
  133.     output = '001.jpg') ]& J  f: }( c" M/ a
  134.     paxel( url, output, blocks=4, proxies={} )
复制代码

) |5 T) I9 A" N# y! r



欢迎光临 数学建模社区-数学中国 (http://www.madio.net/) Powered by Discuz! X2.5