QQ登录

只需要一步,快速开始

 注册地址  找回密码
查看: 2292|回复: 0
打印 上一主题 下一主题

多线程下载图集

[复制链接]
字体大小: 正常 放大

1192

主题

4

听众

2946

积分

该用户从未签到

跳转到指定楼层
1#
发表于 2024-3-31 17:18 |只看该作者 |倒序浏览
|招呼Ta 关注Ta
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。
  1. #!/usr/bin/python
    3 \0 S4 r1 A1 `# T9 ?/ B+ t\" [6 m
  2. # -*- coding: utf-8 -*-  |) t  c, @. [4 I' T% x
  3. # filename: paxel.py% D4 q9 ?( E. l! c6 i+ z3 @
  4. ( q( c% v1 t% t. P2 i9 u! d9 v* F. `
  5. '''It is a multi-thread downloading tool
    + f& w4 V5 U. o, ]# u! a% n
  6. 4 |9 Y\" W* Y  T0 m: g  |
  7.     It was developed follow axel.$ B3 U; E2 K\" ^$ e
  8.         Author: volans
    ) R5 A: z! b$ k7 R) c9 t' [. F+ O
  9.         E-mail: volansw [at] gmail.com
    5 |7 X, p2 u% M/ w
  10. '''0 ]/ D+ f. l$ V/ B- l% b

  11. \" T7 k7 x0 t4 \' `
  12. import sys9 i- S+ Y7 g- @+ I
  13. import os6 h  a# K  y6 P9 O; ?
  14. import time4 {' [, @- M+ F) K0 b: a4 }4 m
  15. import urllib
    * N% l# |\" {3 @3 M9 Q( q6 y
  16. from threading import Thread; R6 L5 Y. A4 D8 r\" _8 T# h
  17. ( C: G1 e9 l- T
  18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    9 k5 M0 M5 F9 ?* B& P

  19. * w; Q& }& E$ q\" q' U1 h  k  f. K
  20. class AxelPython(Thread, urllib.FancyURLopener):
    3 Q; f* G1 }0 Y3 M
  21.     '''Multi-thread downloading class.' C  t0 ]- \  y9 U+ o9 Z

  22. + x: N\" J# l8 O
  23.         run() is a vitural method of Thread.
    . j& d3 J+ V  o* `1 @+ S& P
  24.     '''
    4 U( U) W4 h' ~\" E, G5 W; x; `+ i
  25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    + b$ \5 p# v) o0 H
  26.         Thread.__init__(self, name=threadname)
    ( P, ^+ S  ^3 g# X& I8 ~' I$ e
  27.         urllib.FancyURLopener.__init__(self, proxies)! X  O  `/ q6 a) ]9 Q4 C
  28.         self.name = threadname
    + Q# P( c7 e) S' E
  29.         self.url = url$ O6 N- \) y9 ~6 O2 x0 O
  30.         self.filename = filename
    9 h* l- v% l# N1 @' |6 S5 z% ^+ N
  31.         self.ranges = ranges
    3 d% P5 {% V\" X; D, _1 O
  32.         self.downloaded = 06 O2 g( o, w8 M: r( O* w# H
  33. 4 h; H5 M\" K* q
  34.     def run(self):
    : q1 U& Z4 f& H, j- \1 Q
  35.         '''vertual function in Thread'''$ p! Y3 w( y: m; F* F
  36.         try:
    & V/ P( Z# b0 \7 s6 Y/ d0 a4 K! B
  37.             self.downloaded = os.path.getsize( self.filename )
    * f3 Y. v* t( ~2 I
  38.         except OSError:
    $ a6 J' D* E0 r& B
  39.             #print 'never downloaded'
    ; h. I1 T0 ]* N6 n* K2 }
  40.             self.downloaded = 04 G  f% j/ }1 w* T$ R6 X2 I9 v# V

  41.   o. T\" C2 E2 `, F. m
  42.         # rebuild start poind
    % i! y9 _7 L# c$ ?2 r8 E/ [4 Q
  43.         self.startpoint = self.ranges[0] + self.downloaded
    \" I4 O8 x9 Q9 d' D6 A
  44.          0 A3 o4 h/ |3 L7 [1 [
  45.         # This part is completed+ @% ~, n& X5 n* n; C
  46.         if self.startpoint >= self.ranges[1]:
    % o2 Y* L\" F0 d& Q( h
  47.             print 'Part %s has been downloaded over.' % self.filename
    8 ]: o4 d3 R) G; U9 E
  48.             return6 |: i9 H9 Q5 x\" M$ N6 _, P7 m
  49.          5 f9 y$ {\" Y( `6 }/ K
  50.         self.oneTimeSize = 16384 #16kByte/time8 E% M/ X( S9 {
  51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])9 K! l) F! U* ^5 v7 g& ~! e
  52. $ b4 s! k$ ~- R% u* I. i
  53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
    / i0 m5 k: N* t* U9 i
  54.             
    / @3 H. ^7 ?2 @: x; s; t- d5 b
  55.         self.urlhandle = self.open( self.url )$ Z6 o  |3 l5 O7 I) g$ ?

  56. ) g+ \! V( n3 q
  57.         data = self.urlhandle.read( self.oneTimeSize ); [0 U  z+ q' b
  58.         while data:3 U' p% Q1 t5 T  u  q' J
  59.             filehandle = open( self.filename, 'ab+' )  |% Y- d2 B- w
  60.             filehandle.write( data )
    $ f5 e) r: @9 g; b( c
  61.             filehandle.close()
    7 E7 n( \4 x6 `2 D! ^4 H0 Y

  62. ! V( E\" s4 e0 c6 i
  63.             self.downloaded += len( data )* @4 M\" l- o  ]5 A% ~# H8 w& [7 [
  64.             #print "%s" % (self.name)
    9 `: y+ f) r+ J3 p
  65.             #progress = u'\r...'
    , h/ R3 d7 Q* }% V+ v\" B
  66. ' d/ v\" h6 R3 m4 T( l9 {1 X- ?' j
  67.             data = self.urlhandle.read( self.oneTimeSize )& c( d3 N* b* H8 M0 J. z: [: r: N
  68.          
    8 b) m$ L% j8 m/ a% Y
  69. def GetUrlFileSize(url, proxies={}):! x: Q# p0 V8 k+ Q) u$ L
  70.     urlHandler = urllib.urlopen( url, proxies=proxies )8 ]1 e0 _! N9 b
  71.     headers = urlHandler.info().headers
    ; B, h  ?\" `! m9 t9 C4 f
  72.     length = 0. B' ?5 p' S6 O% N
  73.     for header in headers:; B' u2 E& l9 T$ k, ^9 d
  74.         if header.find('Length') != -1:' g+ c  e7 ]1 z  S\" X
  75.             length = header.split(':')[-1].strip()' ?- r4 s0 V2 H0 }' _
  76.             length = int(length)( _3 g! [2 F) z\" v# z
  77.     return length6 a' M  e1 \0 s( h( [

  78. 5 q1 o+ i4 g3 S3 v5 v2 ^
  79. def SpliteBlocks(totalsize, blocknumber):8 R6 N$ T! k0 e  ^- f0 i
  80.     blocksize = totalsize/blocknumber
    $ I4 Y7 O5 C5 E0 A  I
  81.     ranges = []  Y3 H- q( [, q5 e
  82.     for i in range(0, blocknumber-1):
    6 t, R9 h8 Z/ O! q6 L5 P8 ~+ L
  83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    ( [+ R5 w/ E8 b( y1 d) g
  84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))9 n0 w) X! @* P$ B3 l$ z' F

  85.   {5 S/ `8 ~# k* R4 j# C
  86.     return ranges
    . }4 H' @' ?, d
  87. def islive(tasks):' J5 b) b6 X2 ^5 o, E' n& {
  88.     for task in tasks:
    ; v1 Z: J( M9 \; T0 f  b( J3 O
  89.         if task.isAlive():
    # R0 B/ F0 R/ ~; a# U0 O
  90.             return True' r0 X/ V' D3 \1 l6 s/ @
  91.     return False
    \" |& m) z1 W- H' O. P' T4 M
  92. & ~# J- f0 @2 k2 }/ L' r8 |$ t- F
  93. def paxel(url, output, blocks=6, proxies=local_proxies):
    . Z% f\" v9 }\" h3 x3 Z1 h
  94.     ''' paxel+ h8 E0 j3 l6 P) u9 n
  95.     '''
    # d1 J5 U/ J% k0 M
  96.     size = GetUrlFileSize( url, proxies )
    $ V  c! @0 _- k! S' S/ C2 Y
  97.     ranges = SpliteBlocks( size, blocks )$ y5 n/ t' F* P# N6 Q

  98. ; K! r$ C: Y5 E: W7 V+ u; b
  99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]
    * U$ s6 G+ {) P* [' K& P6 q( U; i
  100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]9 I  t# c& l: {$ ]
  101.    - O% g9 v\" s9 X+ d$ W\" a: |
  102.     tasks = []# I- p8 Y* Y) f
  103.     for i in range(0,blocks):1 a( L4 V8 ]7 Y' C% u8 z
  104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )+ p4 I; T, N. p5 K
  105.         task.setDaemon( True )
    * o) X5 \6 \0 \' I4 V* W
  106.         task.start()+ M, s\" A. [& ], c: Z& f
  107.         tasks.append( task )
    8 j2 J2 r/ N0 Y5 M: u
  108.          9 o: b( l2 ~! U+ [: A% J/ ^4 y$ J' B
  109.     time.sleep( 2 )6 w9 w4 ]5 a, ]2 q\" e
  110.     while islive(tasks):7 M+ T1 Q3 R8 a: Y
  111.         downloaded = sum( [task.downloaded for task in tasks] )9 s6 V! I* M\" X7 o6 [  y
  112.         process = downloaded/float(size)*100
    $ n2 ~+ t  b3 j2 Z* [$ i2 G0 X7 U
  113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)& B8 \6 d% A- \# z$ j4 Y, S4 g
  114.         sys.stdout.write(show)
    8 b. C0 g9 p! F  {) {5 f5 w+ r) i
  115.         sys.stdout.flush()
    * ]6 g; m: [( |6 F8 |( b% H  z2 L
  116.         time.sleep( 0.5 )
    5 ?\" V  c  i3 Y: C* r
  117.                n( y) T% A3 {& N+ U! ^
  118.     filehandle = open( output, 'wb+' )
    3 w  h0 @: [5 [6 |5 G- z$ }\" f
  119.     for i in filename:
    $ s; `! S& ]0 Y) n* k
  120.         f = open( i, 'rb' )
    5 q& W9 _8 f6 k1 ]\" f
  121.         filehandle.write( f.read() )* t! u4 N1 T# Q, B. a
  122.         f.close()( _! R! z, N3 p, p
  123.         try:- i' e+ Q% A, U6 G
  124.             os.remove(i)
    2 L) r\" b8 D5 b- }: s4 I
  125.             pass
    ' V# K% f  i$ `\" D, D9 C! m+ Q
  126.         except:\" h5 B8 c+ D& m; z$ v
  127.             pass, d) o6 c6 P\" v
  128. 9 d/ y& B0 `. _% a5 C9 t1 q* @
  129.     filehandle.close()1 e# }+ u/ A7 \7 ]- B
  130. 6 _# N- a9 V/ {
  131. if __name__ == '__main__':9 Q9 S\" S9 r( U5 Y1 v
  132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"
    ! X7 _0 p4 x, A- y( v
  133.     output = '001.jpg'
    1 C  G( x  L2 m& T! }8 P
  134.     paxel( url, output, blocks=4, proxies={} )
复制代码

. I) x0 t0 k  f$ G4 ?
zan
转播转播0 分享淘帖0 分享分享0 收藏收藏0 支持支持0 反对反对0 微信微信
您需要登录后才可以回帖 登录 | 注册地址

qq
收缩
  • 电话咨询

  • 04714969085
fastpost

关于我们| 联系我们| 诚征英才| 对外合作| 产品服务| QQ

手机版|Archiver| |繁體中文 手机客户端  

蒙公网安备 15010502000194号

Powered by Discuz! X2.5   © 2001-2013 数学建模网-数学中国 ( 蒙ICP备14002410号-3 蒙BBS备-0002号 )     论坛法律顾问:王兆丰

GMT+8, 2026-9-2 17:23 , Processed in 0.472727 second(s), 50 queries .

回顶部