QQ登录

只需要一步,快速开始

 注册地址  找回密码
查看: 2268|回复: 0
打印 上一主题 下一主题

多线程下载图集

[复制链接]
字体大小: 正常 放大

1189

主题

4

听众

2934

积分

该用户从未签到

跳转到指定楼层
1#
发表于 2024-3-31 17:18 |只看该作者 |倒序浏览
|招呼Ta 关注Ta
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。
  1. #!/usr/bin/python
    % F, s) v6 _9 {+ H* o& E6 {# b
  2. # -*- coding: utf-8 -*-
    ( f# K# N& J' z2 J
  3. # filename: paxel.py
      ?% t7 v1 W! |9 \4 Y& }( F
  4. 1 t6 a' X' b2 |% K8 H% c
  5. '''It is a multi-thread downloading tool
    7 E\" o, K% Y7 l; ^, d# l/ M9 p
  6. 4 q2 S$ H, f  O9 h
  7.     It was developed follow axel.( e! Q. i4 Y. D5 u
  8.         Author: volans
    ) `) n/ ]3 g0 Q4 V3 `
  9.         E-mail: volansw [at] gmail.com
    & @- K2 J2 S/ \- D7 n, {  f4 r
  10. '''
      o: @5 {: I7 ]- S3 s0 a

  11. $ I- o$ R9 y( S3 K# s, c( d
  12. import sys
    7 L& p' g0 P) z( ~
  13. import os
    & J# r* `, K) V: J
  14. import time7 {0 D- ~: n3 R4 V4 \  @
  15. import urllib3 |9 ?3 a) Z* L
  16. from threading import Thread/ [' J4 h' s' d/ U4 s' M
  17. \" ]. z9 r, {/ x3 S- n
  18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    5 x' w5 _. s' f! e0 j
  19. 8 X5 _# L% c8 @; I
  20. class AxelPython(Thread, urllib.FancyURLopener):\" z7 Q7 L7 D3 }1 m  C* ~) R
  21.     '''Multi-thread downloading class.9 m) R\" E0 X0 ~# f) D0 K) v4 D. U+ `$ O
  22. 9 g  ~. _  [! k- b- A6 |
  23.         run() is a vitural method of Thread.
    & ]  S. s0 ]! j* [
  24.     '''
    - T2 ]9 x9 _7 `\" @3 j2 C/ M' V
  25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):& ]4 b# _- r' ]  w2 d  V+ V$ S) ?4 V
  26.         Thread.__init__(self, name=threadname)1 d; V& b5 n( K8 w& K8 ]
  27.         urllib.FancyURLopener.__init__(self, proxies)
    3 l& I2 Q1 p9 O, F/ r
  28.         self.name = threadname
    , {* b( S# ^0 h3 t% j
  29.         self.url = url2 y& E1 A5 z\" s
  30.         self.filename = filename\" G$ s2 R' s5 z+ i
  31.         self.ranges = ranges  H1 P! c, I5 ~
  32.         self.downloaded = 0! y\" t0 L/ ?0 w+ y8 p# j) E

  33. ; T* `$ I6 O0 I5 `1 V+ ~# G
  34.     def run(self):
    ' q) q- i\" }\" B5 {* O+ v3 p4 w
  35.         '''vertual function in Thread'''
    ) ?; s5 @$ {/ _$ d
  36.         try:\" c8 Z  x8 Q2 a\" _& K0 C) n$ f: L
  37.             self.downloaded = os.path.getsize( self.filename )# y2 x$ h7 f4 P% o( T, V2 D/ _3 U
  38.         except OSError:3 Q# F, P- p% S( r7 H5 g5 A8 U/ S
  39.             #print 'never downloaded'
    * u; G3 L8 ^7 L9 E
  40.             self.downloaded = 0
    1 l1 l3 e' K\" W: Q' X. I! c

  41. \" C# z. a/ M7 E\" W6 j! v
  42.         # rebuild start poind4 u9 z; J  @& Z- j& n
  43.         self.startpoint = self.ranges[0] + self.downloaded
    \" q) v7 l4 @- {  j/ G0 j
  44.          3 I, j\" Y6 _4 @3 ~! h
  45.         # This part is completed) D7 C4 w9 G) C- J\" W0 B
  46.         if self.startpoint >= self.ranges[1]:
    4 P6 I* i4 V6 r\" e! ~3 ~, k
  47.             print 'Part %s has been downloaded over.' % self.filename  s7 j9 R* Q) E* f# C0 W
  48.             return
    % j. N& v5 D) u8 A6 Y2 [
  49.          4 }2 p% A9 m! I7 W# h
  50.         self.oneTimeSize = 16384 #16kByte/time1 t7 m0 z; w4 R\" v\" g3 K0 N
  51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])\" d' a6 D\" o7 {3 L
  52. $ _2 U+ o* b5 ^9 J5 s
  53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1])); q: [& z& C% @; M# f# ]
  54.             
    9 P8 r1 p6 y1 M, Y
  55.         self.urlhandle = self.open( self.url )% m* x* J( Z: K5 y$ g% Q

  56. % ~7 \+ Y5 h- Y9 e2 i6 t+ |5 }8 X
  57.         data = self.urlhandle.read( self.oneTimeSize ). [, U\" X: t  s
  58.         while data:
    7 r0 H9 G8 A$ C- T: ]\" L
  59.             filehandle = open( self.filename, 'ab+' )
    : _5 x$ W( ?5 v3 u# O- @\" G
  60.             filehandle.write( data )
    + c8 i! [8 t. a# Q
  61.             filehandle.close(): J( f, W% h+ t5 K  Q5 X
  62. % ^% ?& k. W% w- u4 H4 g( M
  63.             self.downloaded += len( data )
    \" o, E% S7 n( }  N' \  R& U7 v
  64.             #print "%s" % (self.name)
    ; j- N+ C9 f0 x/ h9 [
  65.             #progress = u'\r...'4 I6 g% a7 Y: y' |6 @  a
  66. 4 l* c: N* u6 b
  67.             data = self.urlhandle.read( self.oneTimeSize )
    ; ]; J7 V1 x\" ~# j* c/ N+ A
  68.          0 o  |  L7 I. @7 d# p
  69. def GetUrlFileSize(url, proxies={}):6 E% T3 m+ Z% h8 {4 c, X& T
  70.     urlHandler = urllib.urlopen( url, proxies=proxies )
    ; y4 m% A0 B: W+ f& X6 w2 k# i
  71.     headers = urlHandler.info().headers
    # T+ b\" K( T7 I/ L5 k1 n
  72.     length = 0
    & m2 m1 ^9 _! \8 P9 E6 x$ P
  73.     for header in headers:
    ' E3 U! C- {( H9 `4 X- Q' R7 M
  74.         if header.find('Length') != -1:+ I+ D! x5 k/ D4 P! R' s! g* Z
  75.             length = header.split(':')[-1].strip()- ^! A  t, J$ Q
  76.             length = int(length)
    . t1 m+ C! O) L7 @( s. Y$ l/ h
  77.     return length
    7 V7 l! F\" m% B' G4 i/ C

  78. . \; I1 `* z4 z\" z5 j- T
  79. def SpliteBlocks(totalsize, blocknumber):
    , r# F/ K  T: T2 y6 n\" `
  80.     blocksize = totalsize/blocknumber
    0 k7 V2 ~$ ~6 ?3 {* h# W; U
  81.     ranges = []
    , g& O( B' q) T0 C) X% a6 i) z
  82.     for i in range(0, blocknumber-1):
      r* i) d* u! m- \\" g. I, q
  83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    # d/ l: t- v8 T: E: K. o
  84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 )): {! j5 |. e9 }/ F. u; K

  85. ( O& {- z1 M5 n
  86.     return ranges/ {( |9 M1 T- R6 d: N4 G# V* ?
  87. def islive(tasks):
    % Z' W; V. u2 {; {; }\" R5 @
  88.     for task in tasks:: w3 Y1 }0 p/ |2 ^1 ^
  89.         if task.isAlive():
    ) u- Q' E6 r4 X0 |2 r
  90.             return True
    3 m  u  @3 v* M) y
  91.     return False5 N% h/ k\" G' r( A9 @# X! s
  92. + ~0 S1 x' l% I
  93. def paxel(url, output, blocks=6, proxies=local_proxies):
    * T1 Z/ T/ H+ w2 `% ]
  94.     ''' paxel
    0 ^/ z, Y: W8 f7 w; s$ V8 H
  95.     '''
    4 }; l\" u. E8 B: E- m' c4 S
  96.     size = GetUrlFileSize( url, proxies )
    . x: P! {5 V1 R5 a' R; \
  97.     ranges = SpliteBlocks( size, blocks )
    , z\" c: R, M( E7 B; f0 Y
  98. ! d/ ~9 D# V# S\" l
  99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ], Y4 K( x1 c: W% H
  100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]- s4 O4 x+ s' Z8 C+ r9 T5 M
  101.    
    + L$ ?* }; z- `) A
  102.     tasks = []7 J0 t. l! X- J
  103.     for i in range(0,blocks):
      H  s% u$ t; L# v) z\" N' \
  104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )
      ^7 w$ v6 V  ?& d
  105.         task.setDaemon( True )1 H, @. W1 V  s8 b% v
  106.         task.start()
    & p\" M  ?' n; g2 A1 \
  107.         tasks.append( task )) C- Q2 o0 o\" J9 b0 H8 Y/ b
  108.          
    9 K5 `2 A4 P5 ^' P* H3 b
  109.     time.sleep( 2 )
    ' D2 c/ m) B* o0 i1 j( ~& a( g: Q
  110.     while islive(tasks):
    2 u# ?  W: z$ C9 n\" e/ p
  111.         downloaded = sum( [task.downloaded for task in tasks] ): I3 }/ A: G. @4 K
  112.         process = downloaded/float(size)*100' s5 J; j& h( _7 R4 ~
  113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process); a) w\" `: |: `. ?$ z$ Q
  114.         sys.stdout.write(show)
    ! e) B+ d' |( ?) {) v+ j' b! h; `
  115.         sys.stdout.flush()
    , j, R\" C/ b\" t  u1 l
  116.         time.sleep( 0.5 )4 C3 X7 O! |1 K% X* v\" i7 C
  117.              , ~& q, C% {; b5 _: m/ s
  118.     filehandle = open( output, 'wb+' )) e7 c- D\" `) O, i; a* f, L
  119.     for i in filename:
    ( l3 q( ~. w1 }  K\" z4 s3 b
  120.         f = open( i, 'rb' )
    6 F9 K: r% s4 k* X\" H6 B\" A6 e
  121.         filehandle.write( f.read() )
    . e- C# }+ k! v
  122.         f.close()8 K3 P) u8 @$ A* ]% V# X
  123.         try:
    ! R  y' ]7 k. t5 v! m4 d9 u\" ~1 t/ [+ s
  124.             os.remove(i). |' r0 i; _4 u7 {5 e( O5 t: ^
  125.             pass+ e% o0 ]- \$ g0 Y7 C! Y
  126.         except:
    , N* g9 R1 N2 z  j
  127.             pass
    ; t1 n2 k% ^& x# h

  128. & ]' k) W4 X  s
  129.     filehandle.close()& }& t( Q. k# W* F
  130. ) L. Z- `+ }) j: M3 D2 i9 {
  131. if __name__ == '__main__':
    / z, P& ^3 L$ r4 w! S9 w+ J5 u
  132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"4 Z' X' G& H  j, l% W4 P: }; `
  133.     output = '001.jpg'
    % q# N; ^1 L. e% W5 z
  134.     paxel( url, output, blocks=4, proxies={} )
复制代码
: r" M3 \/ I  s5 [+ H" ^2 ^* b8 d- n
zan
转播转播0 分享淘帖0 分享分享0 收藏收藏0 支持支持0 反对反对0 微信微信
您需要登录后才可以回帖 登录 | 注册地址

qq
收缩
  • 电话咨询

  • 04714969085
fastpost

关于我们| 联系我们| 诚征英才| 对外合作| 产品服务| QQ

手机版|Archiver| |繁體中文 手机客户端  

蒙公网安备 15010502000194号

Powered by Discuz! X2.5   © 2001-2013 数学建模网-数学中国 ( 蒙ICP备14002410号-3 蒙BBS备-0002号 )     论坛法律顾问:王兆丰

GMT+8, 2026-7-18 11:01 , Processed in 0.427634 second(s), 51 queries .

回顶部