QQ登录

只需要一步,快速开始

 注册地址  找回密码
查看: 2269|回复: 0
打印 上一主题 下一主题

多线程下载图集

[复制链接]
字体大小: 正常 放大

1189

主题

4

听众

2934

积分

该用户从未签到

跳转到指定楼层
1#
发表于 2024-3-31 17:18 |只看该作者 |倒序浏览
|招呼Ta 关注Ta
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。
  1. #!/usr/bin/python
    9 Y# x: }: E* B6 Y! e) P
  2. # -*- coding: utf-8 -*-
    + t. P8 z+ r$ W) n% [- L& X
  3. # filename: paxel.py$ N- z5 @7 h2 Z4 V) m$ g
  4. 9 E- Y- P+ }/ l$ E: `1 b; z
  5. '''It is a multi-thread downloading tool\" m- _9 t* }4 E! I3 y' w
  6. 1 B% C* G' i+ O9 P
  7.     It was developed follow axel.
    , X  O$ T7 G' g- E0 u; M
  8.         Author: volans
    + C& i4 t) ~7 k/ S4 _7 L7 w
  9.         E-mail: volansw [at] gmail.com
    $ {! [2 H1 r9 K: ?- J
  10. '''
    2 Y( Z  z\" @3 G+ F, p# {
  11. 4 h' f( t% X  E/ ~! x
  12. import sys
    ; R8 v  [+ |  Y7 J, v! J
  13. import os
    3 S5 k5 a! U; n2 [
  14. import time8 e3 s\" Z5 \  h( E2 e2 c
  15. import urllib
    # o. |1 K0 B% `
  16. from threading import Thread( [; G; j% I6 X1 ]) C
  17. 5 v; o; I3 b- N6 n( M
  18. local_proxies = {'http': 'http://131.139.58.200:8080'}2 H- ]% G1 @1 b4 v/ g) a) s

  19. # o8 {3 S# F8 q6 x7 v  h6 A
  20. class AxelPython(Thread, urllib.FancyURLopener):/ ?2 M! C0 g' ?
  21.     '''Multi-thread downloading class.! W6 C( z9 z7 ^, e* j$ k# e

  22.   _4 e3 f  l0 ~) ~! n
  23.         run() is a vitural method of Thread.  Z) r% Y: c/ F3 S7 r/ m
  24.     '''
    & I1 T+ R/ d3 J# w3 P
  25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    \" n& U& D8 k9 l9 N  ?. ?
  26.         Thread.__init__(self, name=threadname)& {0 S/ h/ J! o  F/ k\" w
  27.         urllib.FancyURLopener.__init__(self, proxies)
    + `8 S\" c/ g& {( i
  28.         self.name = threadname
    3 C! z& q2 }6 W: x$ J
  29.         self.url = url
    , q+ e0 D  c/ C; ^% z5 S9 f1 D
  30.         self.filename = filename; L5 g8 `6 M, [: i6 J' L. j+ n5 S
  31.         self.ranges = ranges4 h\" F, ^$ i) a2 M# s. w$ l+ U
  32.         self.downloaded = 0  @% r, J5 i$ _: z

  33.   g) a6 I% X* f
  34.     def run(self):$ R/ a3 M% z6 \
  35.         '''vertual function in Thread''': E8 y9 Y( L% Z\" a
  36.         try:
    ) U8 f( t! Z) m& X  p$ t
  37.             self.downloaded = os.path.getsize( self.filename )
    ( V\" [+ A+ ], @$ e. z
  38.         except OSError:
    ' c& s6 U! O+ w% z) Y$ h& f
  39.             #print 'never downloaded'
    ! j8 R4 N& r6 }
  40.             self.downloaded = 0
    # m8 S5 o5 c! y: o: }( Q
  41. & h9 ~$ @7 F; L: p9 l1 m
  42.         # rebuild start poind
      {6 J\" N8 U$ G% [8 v
  43.         self.startpoint = self.ranges[0] + self.downloaded
    0 z9 O- @8 r; H  C3 }
  44.          ! `% S/ F2 f3 E\" g, Q0 A0 t
  45.         # This part is completed
    3 l' c8 n/ l; ?6 V\" p
  46.         if self.startpoint >= self.ranges[1]:& i9 S5 s* j\" [6 y! _  {
  47.             print 'Part %s has been downloaded over.' % self.filename/ W3 ]+ J( h7 |- f% V; b' Z
  48.             return) ^: F3 S\" @, l, q( E/ x
  49.          
    ; ?6 `- R: u8 W; D0 a, ~
  50.         self.oneTimeSize = 16384 #16kByte/time
    0 J( W& `/ @  N
  51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])* ^3 S0 ]3 [: G* M

  52. , @8 A9 i7 E0 P4 s/ W2 p
  53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))7 A+ Z; }. g2 H% r# B
  54.              ) p2 S\" O3 y7 L: P
  55.         self.urlhandle = self.open( self.url )9 q! @# Y8 H# [1 {3 d3 \; L$ o
  56. + ]) }  F: }( ^2 J6 H7 B
  57.         data = self.urlhandle.read( self.oneTimeSize )
    7 _, a2 ?2 _8 v* r
  58.         while data:
    : |7 m( V/ n9 b' Q- r+ G
  59.             filehandle = open( self.filename, 'ab+' )\" H; E3 w9 @# C% h) [2 l6 T
  60.             filehandle.write( data )
    , L# ^2 Z\" Z! q4 T2 ?\" c; |/ U
  61.             filehandle.close()) A6 X! z3 U8 \0 u3 U; i; P9 m
  62. ; j9 R/ m1 K3 B  r2 N: X; D
  63.             self.downloaded += len( data )
    * {6 m: R+ `8 ~
  64.             #print "%s" % (self.name)
    / Z0 b4 w# ]0 l% l. H0 _
  65.             #progress = u'\r...'7 o% t: c' ^: g
  66. ) t  A$ d2 p, c2 v
  67.             data = self.urlhandle.read( self.oneTimeSize )0 Q) a6 i  i( F, K# k) M
  68.          2 l$ n+ s4 a  A2 O2 I\" p
  69. def GetUrlFileSize(url, proxies={}):
    9 t  s' d( g% F4 l
  70.     urlHandler = urllib.urlopen( url, proxies=proxies )7 u8 G4 n1 X\" A; i% U+ {0 Q
  71.     headers = urlHandler.info().headers  V& L; d; r5 F0 k6 B
  72.     length = 0+ M! E0 q) s6 b, c$ X$ z! M
  73.     for header in headers:$ B+ G( ?, L4 z* _1 @
  74.         if header.find('Length') != -1:
    3 o0 f. M- B( s4 N# _3 t0 p
  75.             length = header.split(':')[-1].strip()7 `5 W$ q' {+ Y5 h. k
  76.             length = int(length)6 y+ d+ A2 m) D. ~+ h
  77.     return length
    $ k6 i- c1 ~9 ^  t' I1 e

  78. , M* r, A1 O6 c# E6 [) A$ d
  79. def SpliteBlocks(totalsize, blocknumber):. [- y. H- G1 F/ f# L. B; J' M( J
  80.     blocksize = totalsize/blocknumber
    : g! i+ C: N. d7 y
  81.     ranges = []5 a, Z% h  z, u) t2 J1 u  M
  82.     for i in range(0, blocknumber-1):
    + }7 |. m1 h/ L% u* u
  83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    + @# Y' l! E+ }/ s8 p# x7 F
  84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))& C0 r  s! ?! j/ I0 Z# [. c

  85. \" V& d  u2 k/ y  L
  86.     return ranges
    ' k, B; z5 G! E! t  H; s
  87. def islive(tasks):! E\" |, M7 R* F  Y6 E1 E$ m
  88.     for task in tasks:
    \" E& V! t3 L2 a4 o
  89.         if task.isAlive():/ @\" I\" U% C# r- U- |+ s) S9 E
  90.             return True
    ( X1 L' t2 N: P: F0 o
  91.     return False
    * @) a5 Y5 k% d2 b6 z- z
  92. 6 K. G% L  |( m\" H& T
  93. def paxel(url, output, blocks=6, proxies=local_proxies):
    . N4 f' D! e  K- d5 \6 m: U: V1 C
  94.     ''' paxel. H. O) K& }( i. ]# e
  95.     '''
    + n$ e/ \) b4 Z; H& i- O
  96.     size = GetUrlFileSize( url, proxies )
    $ p: w' q1 Y$ Q- _! P1 }
  97.     ranges = SpliteBlocks( size, blocks )
    ; \' N3 R  }\" F\" y! [* t
  98. : v3 R/ J2 V/ ?  F: }! Z\" @
  99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]2 s3 j; W$ @! j% h1 W0 L% _6 @: M7 m* d
  100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]! e6 \! [, l! V- Y3 g  D
  101.    ' R# V, t: G: d
  102.     tasks = []. ?6 n+ T5 M! g! K* Z
  103.     for i in range(0,blocks):, ^9 s6 ?7 ~! c* |: z( Y) S4 I9 n
  104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )/ k. ?% ?$ H. o' ]8 C\" g
  105.         task.setDaemon( True )9 J\" k6 Q4 B( J, R& |# A
  106.         task.start()4 z& y  N! _1 c, M$ F# s; ~' Q# p
  107.         tasks.append( task )! f: }3 G! w1 R3 O+ m
  108.          8 O$ ]' V3 N0 R6 N6 }\" c  B4 R0 G
  109.     time.sleep( 2 )
    : H+ q3 f# c. q+ r
  110.     while islive(tasks):
    % a) F$ s6 b- |. W
  111.         downloaded = sum( [task.downloaded for task in tasks] )2 f1 B* g2 J! }8 @\" i) P+ ^
  112.         process = downloaded/float(size)*100
    1 q1 |, X/ s  U\" k
  113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
    6 o: ^- s$ ?% a0 }\" X( A
  114.         sys.stdout.write(show): N1 ]5 x; }2 a+ c7 m
  115.         sys.stdout.flush()+ a9 H0 T) `, T; k& M) c) A; C
  116.         time.sleep( 0.5 ). [/ x( G' o! z& B/ j6 J
  117.              7 v9 G) e: M! \# f
  118.     filehandle = open( output, 'wb+' )\" a) M( a1 d$ b9 ]
  119.     for i in filename:' `9 U$ V, {5 j$ a\" Q\" p- W
  120.         f = open( i, 'rb' )
    ; M4 E* r. x5 x, K% m) u; R: W
  121.         filehandle.write( f.read() )
    * |1 r6 Y% g, V5 e8 V
  122.         f.close()8 @; u) ^+ M9 ~\" }2 f- \5 G
  123.         try:
    / D) m- O5 L+ i8 u4 ?
  124.             os.remove(i)( s$ w\" ~0 r. ]2 G! D
  125.             pass- ]+ z) F/ E; M3 ^, ]& t: e
  126.         except:\" _/ n$ e: R# t2 Z\" j0 h
  127.             pass
    3 a# W, ^; @$ ]$ h

  128. 4 E( j+ O  R, Q8 Y
  129.     filehandle.close()
    & X3 A; l6 O) m2 M9 j& d& Z: _

  130. / C: D( A; P) T3 j
  131. if __name__ == '__main__':
    ( k\" @) f% M4 ~6 V, |3 C, ]' ?
  132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"3 P& L) i6 ?6 ?3 O: T8 f1 |
  133.     output = '001.jpg'7 H9 v  G3 ]8 B' z) _- ]7 A8 E
  134.     paxel( url, output, blocks=4, proxies={} )
复制代码

+ l: R+ R0 o. r; U8 Q, ~
zan
转播转播0 分享淘帖0 分享分享0 收藏收藏0 支持支持0 反对反对0 微信微信
您需要登录后才可以回帖 登录 | 注册地址

qq
收缩
  • 电话咨询

  • 04714969085
fastpost

关于我们| 联系我们| 诚征英才| 对外合作| 产品服务| QQ

手机版|Archiver| |繁體中文 手机客户端  

蒙公网安备 15010502000194号

Powered by Discuz! X2.5   © 2001-2013 数学建模网-数学中国 ( 蒙ICP备14002410号-3 蒙BBS备-0002号 )     论坛法律顾问:王兆丰

GMT+8, 2026-7-18 12:44 , Processed in 0.446568 second(s), 50 queries .

回顶部