QQ登录

只需要一步,快速开始

 注册地址  找回密码
查看: 2293|回复: 0
打印 上一主题 下一主题

多线程下载图集

[复制链接]
字体大小: 正常 放大

1192

主题

4

听众

2946

积分

该用户从未签到

跳转到指定楼层
1#
发表于 2024-3-31 17:18 |只看该作者 |倒序浏览
|招呼Ta 关注Ta
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。
  1. #!/usr/bin/python
    0 g7 R' d6 p+ Q) K0 z& d( Q
  2. # -*- coding: utf-8 -*-
    ' E+ [: M% H! i; p\" d0 a3 y
  3. # filename: paxel.py8 |( l9 J+ I/ V8 c\" z

  4. % ?( B\" {$ F\" N4 f- O5 G
  5. '''It is a multi-thread downloading tool8 W1 {& \) P; g9 E9 ?\" H' Y
  6. % S\" R  ]+ @5 D& U$ i: p! Y
  7.     It was developed follow axel.
    5 U! `( l. l8 z- u8 n9 Y& ^: w
  8.         Author: volans% U\" Y, ~- ?& B, Z7 z
  9.         E-mail: volansw [at] gmail.com
    4 Y1 w* b( l1 [0 g. `
  10. '''
    * ?3 h2 r+ P$ |\" |

  11.   m& A8 c( V( z7 {
  12. import sys
    & X( k3 `: `* U6 ^7 r5 Q4 v, i
  13. import os  ^5 x1 u( }1 G, T' `2 x
  14. import time. o* @+ G* \- B  \2 E/ E) B
  15. import urllib' u7 I9 S4 B( T( V4 Z; L
  16. from threading import Thread: r  o+ ~9 G; j4 c. ?

  17. : s* S- Q* {+ \
  18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    0 E6 `$ X3 \\" K
  19. * d1 \+ D' [# D3 F
  20. class AxelPython(Thread, urllib.FancyURLopener):
    , w$ g5 I8 D( T% J
  21.     '''Multi-thread downloading class.
    \" o* B$ X' y% Y# G/ I! R8 m

  22. : H# u9 t# K+ S  t
  23.         run() is a vitural method of Thread.( P& {5 r- r/ b# {# y2 b\" W# p
  24.     '''
    4 S# |7 S, d8 x8 h( M) x7 t6 _6 V
  25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):4 l% N: f2 |* a: R' F, V
  26.         Thread.__init__(self, name=threadname)
    % G3 M. d4 |0 a' K: N\" j( T
  27.         urllib.FancyURLopener.__init__(self, proxies)! T\" m$ U0 T. C9 G
  28.         self.name = threadname
    \" F: ^( F- G& T. Z) J) A
  29.         self.url = url
    8 T9 U$ \\" s; D4 w2 P+ s7 y; m
  30.         self.filename = filename
      z; X- M* p9 s3 y  Z
  31.         self.ranges = ranges; K; N\" l! X2 ?* ?6 f
  32.         self.downloaded = 0
    & e% Z# R$ m+ r$ R
  33. , H, _3 X, q- V+ W3 o
  34.     def run(self):8 o+ g+ z* l  ]5 [
  35.         '''vertual function in Thread'''! E& e  b' x+ i$ K, G
  36.         try:5 j. O- ?) x' N3 C% Q4 _8 H: k
  37.             self.downloaded = os.path.getsize( self.filename )
    - u( p# W- m\" ^0 s/ s# y6 V
  38.         except OSError:, `# D- C2 G. p4 O: ~
  39.             #print 'never downloaded'
    8 {3 L* p, \& V1 N! m' `
  40.             self.downloaded = 0
    . P% N  {( w8 S( I  m! R6 c% T
  41. # g: G6 \6 E) }& S+ S+ U* Z: ]
  42.         # rebuild start poind) r: t- H$ s7 Y* ]
  43.         self.startpoint = self.ranges[0] + self.downloaded) ~. Z: W: c: M) W
  44.          
    \" ?/ L  \( o  h
  45.         # This part is completed
    : Z( w. B7 G! f/ `# L; M# y* n& Z
  46.         if self.startpoint >= self.ranges[1]:
    4 _7 _' e0 l  _% d6 V, H
  47.             print 'Part %s has been downloaded over.' % self.filename0 e- b& B% A  p\" c4 g- @: J+ d6 T
  48.             return
    4 n5 s) t! x+ l; Z
  49.          8 O7 i- `( z6 L/ [
  50.         self.oneTimeSize = 16384 #16kByte/time
    8 y5 e4 l) P% s' \, L
  51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])2 y; G1 k* V, `  K+ H
  52. + Q# Q: H* A( Y
  53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))5 d! y- }2 q. L4 P4 ^$ A
  54.              , H* {7 R. [$ u/ Q
  55.         self.urlhandle = self.open( self.url )6 x, u5 j8 _# m: R7 j) [

  56. & e/ t: @( T8 r- B, z
  57.         data = self.urlhandle.read( self.oneTimeSize )
    , x( a; _1 u8 w# X\" i, H6 |
  58.         while data:7 J/ i0 d& {5 [$ H5 u  q$ Y) R7 ]9 h
  59.             filehandle = open( self.filename, 'ab+' )
    % ^- _9 R$ j: ^9 _! W) x7 Y6 ^6 C) n
  60.             filehandle.write( data )
    4 G+ {$ W5 ]8 p+ r+ q
  61.             filehandle.close()& }9 G; s# p/ M
  62. 0 e1 _3 ], m! A. r) ]& x
  63.             self.downloaded += len( data )2 a( B6 R% m! P) h% t0 s
  64.             #print "%s" % (self.name)* o  `; h; Y' @7 I5 C0 j
  65.             #progress = u'\r...'/ j3 m0 f: G6 b+ L\" h
  66. : U5 Z' v( v) X2 `\" Y# u; I: ^, c
  67.             data = self.urlhandle.read( self.oneTimeSize )
    6 z\" `& x) |' G  R+ A& z
  68.          
    # T  e7 {: G\" |9 o
  69. def GetUrlFileSize(url, proxies={}):2 t( T$ ]$ ?2 ]9 p- E
  70.     urlHandler = urllib.urlopen( url, proxies=proxies )
    4 x4 @) ]1 |. M/ B( U/ u
  71.     headers = urlHandler.info().headers
    ' c/ K2 f  I, j9 h
  72.     length = 01 _, B' \* h: b8 ^- E5 H5 f
  73.     for header in headers:2 q4 T$ e8 Y' A$ ^9 L! X. x' I
  74.         if header.find('Length') != -1:7 e- R- t: F$ w( G3 c) W1 t
  75.             length = header.split(':')[-1].strip()
    + `& V: U. n3 `9 B7 r' w- [+ g# Z: O
  76.             length = int(length)
    . l7 f0 J$ s) {! s4 _, ]! I/ @9 m
  77.     return length, c- |2 U; i; g! M$ |
  78. 6 P7 p; h4 c4 p+ ~7 x' G
  79. def SpliteBlocks(totalsize, blocknumber):$ s/ G2 D3 q7 j5 {
  80.     blocksize = totalsize/blocknumber: e, @! x2 K! n4 Y
  81.     ranges = []
    ! N) j4 d  ]; Y0 m
  82.     for i in range(0, blocknumber-1):/ x! H- r5 \- C) }
  83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))
    6 u* ?+ A7 f\" G5 y
  84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))4 d  L% h) S/ _
  85. ( q4 h+ ^. j3 O: a. }) z1 i& |4 N
  86.     return ranges
    ' W& g2 E* s* b( ]
  87. def islive(tasks):
    $ p7 E8 c% `' V' L8 T: c% P
  88.     for task in tasks:
    $ K) c, t* L  M5 b5 ?
  89.         if task.isAlive():
    ) F* i9 c9 D7 Z$ ?: e+ p/ i
  90.             return True
    7 S1 q2 r* O! Q3 ^( B
  91.     return False
    - Q: t: U# R/ R
  92. 1 v, S; [# b9 o
  93. def paxel(url, output, blocks=6, proxies=local_proxies):7 B, \- ^; t, O( Z) b6 L0 f4 Y
  94.     ''' paxel
    ; A  `$ {( d* Z: S% A
  95.     '''
    ( O. x! j8 r( ]* I7 W8 U
  96.     size = GetUrlFileSize( url, proxies )
    , d# ]) A7 Y7 L\" x* J
  97.     ranges = SpliteBlocks( size, blocks ): m6 j2 o: k% b& I% i$ m8 o\" w' Q

  98. - L7 t# i) U* T/ n4 Q$ ?5 ?
  99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]# n$ W6 Q* r0 f, a* O
  100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]% O+ D% Y/ V, C  Y! {$ _) L8 W
  101.    
    / z. J# R* N) Y, P7 d6 Z8 j
  102.     tasks = []3 y\" \& p4 g' O, F5 C! ~
  103.     for i in range(0,blocks):
    2 g9 N0 A: M. l
  104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )6 f+ Z1 T5 D, Y
  105.         task.setDaemon( True )' E% K! Q7 m1 a/ I  I8 T! J
  106.         task.start()
    ) L! ]# w! C! b
  107.         tasks.append( task )# `' K$ v$ H. |$ C
  108.          
      W$ T/ C3 G: q. q. o' L
  109.     time.sleep( 2 )0 g: R5 R1 z+ [
  110.     while islive(tasks):/ ]; r: P) m: G. m
  111.         downloaded = sum( [task.downloaded for task in tasks] ). O2 ]8 f( A- f
  112.         process = downloaded/float(size)*100
    2 g6 A( U\" P. D\" M: O  l) z
  113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
    2 n2 C  l$ m% I$ R, g; c
  114.         sys.stdout.write(show)2 Q# v) X. K7 P% ]. X( x
  115.         sys.stdout.flush()
    / E1 m9 r; r! R% z* ?1 `
  116.         time.sleep( 0.5 )% G, I: C& ^0 r2 g/ n1 e
  117.              & ~6 Y! Z5 f! ]/ K1 w5 P& P! |
  118.     filehandle = open( output, 'wb+' )( ^- G# v- f5 a6 ?% C
  119.     for i in filename:* q% T5 f8 [% B0 P9 W, B6 D
  120.         f = open( i, 'rb' )
    1 w  i( ]1 _% C) R0 `  |8 c
  121.         filehandle.write( f.read() )! ?' v. l0 `4 O( s  P
  122.         f.close()0 \0 e% c* \# j: h/ Z. N6 D. u! ?$ r
  123.         try:; o1 C' g5 _) `2 }, |) V. M
  124.             os.remove(i)
      c6 D' z$ I6 |  h& u
  125.             pass
    & d6 j) X$ I* _, b; c
  126.         except:8 o$ _/ \9 ]6 t9 n: \  D
  127.             pass
    5 ]. I0 G8 @: w\" E* D% a
  128. 9 R& B3 _+ K4 \) j
  129.     filehandle.close()
    4 {# [3 A. m( i. ]2 i2 l

  130. * E% u5 C4 S+ h+ V4 Q) x# G
  131. if __name__ == '__main__':* w& N& H8 l. c0 z
  132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"
    8 e4 s+ q( j\" e2 {' j9 ]! m
  133.     output = '001.jpg'1 Z0 S$ [+ t, Z
  134.     paxel( url, output, blocks=4, proxies={} )
复制代码
6 k3 P, M9 h6 E3 T% W/ t8 k5 m0 u
zan
转播转播0 分享淘帖0 分享分享0 收藏收藏0 支持支持0 反对反对0 微信微信
您需要登录后才可以回帖 登录 | 注册地址

qq
收缩
  • 电话咨询

  • 04714969085
fastpost

关于我们| 联系我们| 诚征英才| 对外合作| 产品服务| QQ

手机版|Archiver| |繁體中文 手机客户端  

蒙公网安备 15010502000194号

Powered by Discuz! X2.5   © 2001-2013 数学建模网-数学中国 ( 蒙ICP备14002410号-3 蒙BBS备-0002号 )     论坛法律顾问:王兆丰

GMT+8, 2026-9-2 17:41 , Processed in 0.422619 second(s), 50 queries .

回顶部