QQ登录

只需要一步,快速开始

 注册地址  找回密码
查看: 2288|回复: 0
打印 上一主题 下一主题

多线程下载图集

[复制链接]
字体大小: 正常 放大

1192

主题

4

听众

2946

积分

该用户从未签到

跳转到指定楼层
1#
发表于 2024-3-31 17:18 |只看该作者 |倒序浏览
|招呼Ta 关注Ta
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。
  1. #!/usr/bin/python
    7 f& d, P$ k  B, b
  2. # -*- coding: utf-8 -*-7 }! c4 [4 @% k
  3. # filename: paxel.py
    ) J/ R* ~3 f! ^3 L
  4. \" c( n+ {: L: l
  5. '''It is a multi-thread downloading tool
    1 a% l$ q3 \4 f\" f! L( c- m
  6. / n8 L8 f7 H: q! l1 f
  7.     It was developed follow axel.
    ' \6 C$ W% n* u$ {9 K) C$ C! @
  8.         Author: volans
    # N0 G  E$ I- s$ [8 t4 [0 s  q$ l! |
  9.         E-mail: volansw [at] gmail.com. q: {! r0 V1 p
  10. '''6 V/ s( }5 @& \/ U

  11. 2 y( D  H6 ~- v
  12. import sys
    % G' {: _6 E4 [, U: P+ E: ?
  13. import os' W) H  f% Y. z2 c\" L' T
  14. import time
    % G3 `7 N# s& \; D4 L\" w1 N
  15. import urllib
    & C* p& y) w6 X) y8 D6 A
  16. from threading import Thread( S0 ]9 q& q+ Q( G( p' w: M# f. q

  17.   }7 Q1 I. I5 w$ r
  18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    3 Q2 U% c& h  Q) ^

  19. ! }) j) I: D/ G3 Z& W* A1 N
  20. class AxelPython(Thread, urllib.FancyURLopener):
    4 J9 B$ G3 S' u4 @
  21.     '''Multi-thread downloading class.+ |7 U; A; ^  P+ B. u4 V

  22. 8 @! Q, }9 ]% Q
  23.         run() is a vitural method of Thread.
    ; d\" ^, J$ f  @% g0 U6 v
  24.     '''
    . s! ?) ?& M* ~/ j5 P
  25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):
    ; F+ \: x6 x6 U
  26.         Thread.__init__(self, name=threadname)* Z( e& X5 O) j' O2 \
  27.         urllib.FancyURLopener.__init__(self, proxies)2 x' G4 ?  J6 ]\" ~
  28.         self.name = threadname! m4 s5 [& U- v5 I; r& p
  29.         self.url = url
    - [) Q/ \/ y( }* h. Y: ?' i
  30.         self.filename = filename
    7 D# z* V\" k+ }6 o, r2 r) c* M
  31.         self.ranges = ranges
    $ V8 O6 `' R# a+ F. U4 [* T
  32.         self.downloaded = 0+ L\" q7 j6 ~9 x# ~

  33.   \, t# c4 U+ \6 K3 V
  34.     def run(self):
    % i- U4 W5 ~7 c( B# [- o) g4 i
  35.         '''vertual function in Thread'''3 J+ d) h4 r) `5 a2 V+ ?
  36.         try:
    ; E' l2 D0 x2 c4 y
  37.             self.downloaded = os.path.getsize( self.filename )  a. s6 U+ F, ]
  38.         except OSError:0 @* G9 ~* M- {* L4 {
  39.             #print 'never downloaded'- E  z0 \- q& h6 {. [- L! Q6 r
  40.             self.downloaded = 0
    & M1 w\" g+ A, ]2 J/ j( `8 z
  41. % E$ ^8 s4 |\" o) ]! g) z
  42.         # rebuild start poind
    6 ^+ F/ a; J& b2 d
  43.         self.startpoint = self.ranges[0] + self.downloaded
    ( N9 v1 e3 K0 K\" h3 q. [5 k( e
  44.          3 @$ O' j' k8 M3 G
  45.         # This part is completed
    % Z5 b7 s$ G0 q: a
  46.         if self.startpoint >= self.ranges[1]:
    5 [+ }3 w2 A$ F9 l8 @
  47.             print 'Part %s has been downloaded over.' % self.filename& b) w- b, b/ g$ q4 U
  48.             return
    3 _) ?, |- ~8 u! ]% q
  49.          9 f$ ^0 K6 d) N* y& `! ?5 Y
  50.         self.oneTimeSize = 16384 #16kByte/time$ |\" a0 {  F* G# \* B
  51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
    + q4 U' G% u* V. g/ A

  52. , Y2 ]$ k; {0 G! c+ Q
  53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))4 J$ [. O* u9 r) N: @2 K) K9 @
  54.             
    0 m/ [! z! u9 B2 _- n6 Y( t) f7 _
  55.         self.urlhandle = self.open( self.url )
    * L1 u* ~) t1 V1 ]* z1 F- Z1 K+ o
  56. 4 {  M  j* Q% v. J% M0 \5 X  a0 I
  57.         data = self.urlhandle.read( self.oneTimeSize )
    ( i) g8 L& C2 g( f- d& e( Y% k; N# Z
  58.         while data:0 n$ G: k' s  _& ?& w8 _( L# Y
  59.             filehandle = open( self.filename, 'ab+' )\" m/ z, H$ |# g
  60.             filehandle.write( data )
    7 Z% M& l# A  k* D5 g* _\" G& m; m
  61.             filehandle.close()4 z# d1 l# P( S; b- u

  62. / u- W( D/ J2 f5 Z# N
  63.             self.downloaded += len( data )
    \" A  W; \9 R( i, l; `1 l/ B9 A
  64.             #print "%s" % (self.name)3 a6 p0 |7 g; Q
  65.             #progress = u'\r...'
    3 \9 P- X9 _: }- i\" Y
  66. $ Q& e$ |% d) o1 f1 N( I
  67.             data = self.urlhandle.read( self.oneTimeSize )4 r8 e0 e9 B; _! _% \2 u
  68.          
    : t# Q6 `0 l# a9 [' a
  69. def GetUrlFileSize(url, proxies={}):# C1 v, W\" r# W+ h7 Q/ U& h. Z& n
  70.     urlHandler = urllib.urlopen( url, proxies=proxies )1 t4 k8 P' z, G6 o2 S5 L& N
  71.     headers = urlHandler.info().headers! b  r  C  H3 Z8 `) e  z/ K
  72.     length = 0. Z+ |' R5 ^6 V- m0 w6 k1 C6 M5 W
  73.     for header in headers:
    # C# [: ~7 Q: `$ @3 r\" c+ {
  74.         if header.find('Length') != -1:
    ( k9 E! ]5 ?! E- d# K  R
  75.             length = header.split(':')[-1].strip()
    : j0 v& k9 \* F5 s4 o% C5 L& a
  76.             length = int(length)
    ' B5 ]& M% ?- x& f# m  {& {; c
  77.     return length  u& r7 z  c\" t
  78. ! J5 x5 k\" D# [6 \
  79. def SpliteBlocks(totalsize, blocknumber):
    7 R/ n% l6 k/ t6 c
  80.     blocksize = totalsize/blocknumber' Q$ q8 o8 u3 \4 \
  81.     ranges = []
    # g  H( ^: \. p% {) g; q
  82.     for i in range(0, blocknumber-1):
    7 W9 M% n/ d# R
  83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    $ Y4 r2 m6 S* |  o# D, d
  84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))& ^, X3 R4 _1 q4 E* P
  85. 7 d3 ~' P& b& M1 g/ z# X9 ?/ `
  86.     return ranges: z6 I+ Y( @8 Y\" a2 n
  87. def islive(tasks):1 ?0 B7 E  [& l( w/ j7 w1 @) m2 k
  88.     for task in tasks:& }0 C5 ^! V4 N+ d
  89.         if task.isAlive():
    9 j6 C- @: m( z8 C! t
  90.             return True
    2 n\" E: }8 p$ o0 t
  91.     return False& R: P3 d  T4 B  r/ E# I$ a: T

  92.   @7 B( o) s- M$ ?8 N: C
  93. def paxel(url, output, blocks=6, proxies=local_proxies):
    \" I1 b3 g' s4 O* f; X
  94.     ''' paxel/ q' K5 }\" ?' N0 Y8 P
  95.     '''
    $ u5 o7 q( ^& I; P0 R+ N
  96.     size = GetUrlFileSize( url, proxies )
    % W! r: k' O  ?+ f4 X  l8 V6 J; n
  97.     ranges = SpliteBlocks( size, blocks )
    + r' k8 m$ }5 l/ S5 X

  98.   J# b; e5 V7 ]5 k
  99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]/ E! \4 G6 _/ x: T& F
  100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
    2 K$ C2 A/ B$ j' q! I9 W# b
  101.    
    5 J1 B+ m9 R- D3 M
  102.     tasks = []
    4 ?# F% l6 A\" {  K! W
  103.     for i in range(0,blocks):
    - u/ |2 q/ H\" k5 Z% |% m
  104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] ); @! F2 b8 P' A$ S* a( O' L
  105.         task.setDaemon( True )
    6 }7 u- F! I+ F' i
  106.         task.start()- Q. f, y. H( `
  107.         tasks.append( task ); V% f, x9 q; U; ~& B, g; f8 r
  108.          
    6 ]. ^3 }- F4 }( A/ n7 |- x
  109.     time.sleep( 2 )- v% _, G& K0 r
  110.     while islive(tasks):
    5 h9 N# E' k: ]
  111.         downloaded = sum( [task.downloaded for task in tasks] )\" f$ x7 q\" _, I  T) R
  112.         process = downloaded/float(size)*100
    9 A, j3 [# ?; B  U( v  f
  113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)  t7 {, u: L5 W+ I- b/ W3 U7 v
  114.         sys.stdout.write(show)4 {. x7 M( o( Q# R
  115.         sys.stdout.flush()
    ( h. g/ j' w+ X2 C
  116.         time.sleep( 0.5 )
    + c$ _6 f- C; U- `# Q5 [0 K\" y' z
  117.             
    5 E  U\" i: D5 p
  118.     filehandle = open( output, 'wb+' )
    # v' ^1 s9 |& a! @
  119.     for i in filename:
    3 k8 x* j, j; o/ P3 r\" }
  120.         f = open( i, 'rb' )
    2 ^0 ~\" e5 F+ c
  121.         filehandle.write( f.read() )1 l8 ]) w0 S  v' g6 Y
  122.         f.close()
      [5 o3 Z$ F$ W. t
  123.         try:
    ( a# o# g9 o- O6 g& X+ `9 h
  124.             os.remove(i)
    & o, Z0 {9 u0 g& J3 z  }( Y
  125.             pass5 \. \. s& u1 }# X, H! E4 ^
  126.         except:
    # j! J  ]+ ^$ U3 G+ g- a8 [7 C% \
  127.             pass- ~$ p2 e5 Z+ Z; p; g/ p/ a

  128. - x! }\" q+ Q; H6 B8 d2 t$ d
  129.     filehandle.close()2 X' F2 A9 p: [9 C6 v; e

  130. 2 l' o# H+ a% P7 G
  131. if __name__ == '__main__':
    8 g9 t1 I/ G- l+ b4 \
  132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"$ s! R6 _: q5 O+ Z
  133.     output = '001.jpg'
    . K: a/ y' q. q4 v7 F$ f9 e5 k) N
  134.     paxel( url, output, blocks=4, proxies={} )
复制代码

- S# s- L6 i! h
zan
转播转播0 分享淘帖0 分享分享0 收藏收藏0 支持支持0 反对反对0 微信微信
您需要登录后才可以回帖 登录 | 注册地址

qq
收缩
  • 电话咨询

  • 04714969085
fastpost

关于我们| 联系我们| 诚征英才| 对外合作| 产品服务| QQ

手机版|Archiver| |繁體中文 手机客户端  

蒙公网安备 15010502000194号

Powered by Discuz! X2.5   © 2001-2013 数学建模网-数学中国 ( 蒙ICP备14002410号-3 蒙BBS备-0002号 )     论坛法律顾问:王兆丰

GMT+8, 2026-9-2 03:16 , Processed in 0.432067 second(s), 50 queries .

回顶部