QQ登录

只需要一步,快速开始

 注册地址  找回密码
查看: 2290|回复: 0
打印 上一主题 下一主题

多线程下载图集

[复制链接]
字体大小: 正常 放大

1192

主题

4

听众

2946

积分

该用户从未签到

跳转到指定楼层
1#
发表于 2024-3-31 17:18 |只看该作者 |正序浏览
|招呼Ta 关注Ta
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。
  1. #!/usr/bin/python; f\" J2 O7 Q! x
  2. # -*- coding: utf-8 -*-
    8 |- u: `. N! @2 Q0 s
  3. # filename: paxel.py
    \" E: ]9 ]. R2 V

  4. 3 f& S$ ^\" b( O# d5 {7 k
  5. '''It is a multi-thread downloading tool' y- g& k0 w6 i: G* m% ]

  6. ! f& ^' ^/ K' l% Z7 g
  7.     It was developed follow axel.
    1 k8 i! W: H2 z9 b( R/ S
  8.         Author: volans% S+ f5 J9 e+ w5 n& k5 W; }' Q
  9.         E-mail: volansw [at] gmail.com
    * t5 O. r! @, x* q/ N
  10. '''4 y1 I- `: h! p% ~$ D  N+ L# N# X; o

  11. * x, [1 a\" y9 M3 c& M7 N0 H7 q
  12. import sys
    ; ~) m( k  k# @5 a
  13. import os4 g: l% B\" E: ?& X( t  e# B' B
  14. import time# Q% ?$ v5 a3 O# l
  15. import urllib% _6 E0 D7 F) \' u: x: S
  16. from threading import Thread
    ! H) J, v- ?0 F! n& \: [3 o
  17. 7 ^9 t* j2 j* D, j
  18. local_proxies = {'http': 'http://131.139.58.200:8080'}
    $ m- w) A& m- ^' Q, q
  19. 3 q1 q3 g$ V/ t+ j7 _
  20. class AxelPython(Thread, urllib.FancyURLopener):
    ) A. B: ]$ {& c# ?
  21.     '''Multi-thread downloading class.
    3 Y, u/ H# Y* w; k. ]9 O3 y! x* M

  22. & Y7 w+ Y/ L! n
  23.         run() is a vitural method of Thread.9 b: h- |9 W- Z( K& C/ J
  24.     '''\" Z1 k- Z# a; M/ l- m# F; C
  25.     def __init__(self, threadname, url, filename, ranges=0, proxies={}):7 ^4 j( I: W\" H\" X; K1 b# D$ K
  26.         Thread.__init__(self, name=threadname)
    ' N& _8 P- u2 a\" z2 N
  27.         urllib.FancyURLopener.__init__(self, proxies)
    & K- b6 E' m) o! j
  28.         self.name = threadname\" f& R; R; z# Q3 H8 p& L% I; x- k
  29.         self.url = url\" |: |+ q4 y, @0 a
  30.         self.filename = filename
    # E1 D  ^! j* t1 D' l
  31.         self.ranges = ranges; G; x  g/ s8 f5 r! @/ ~5 H
  32.         self.downloaded = 0  ]6 }9 c3 u# `! O: N

  33. * t0 t2 V( O6 `! o, C
  34.     def run(self):
    ( T1 m3 q4 L4 _/ q
  35.         '''vertual function in Thread'''
    ; O& U6 {5 p  S. m! h( T; J3 g: s
  36.         try:
    0 V  X& i- O' f
  37.             self.downloaded = os.path.getsize( self.filename ), y- u0 K/ s0 L8 I+ a
  38.         except OSError:
    % a3 V9 [. i! s7 _; |4 b2 p% a* ?
  39.             #print 'never downloaded'
    * `, V- k  Y. p( k
  40.             self.downloaded = 0! V) W5 J- W( B- @- ]& h

  41. 2 k2 Q* J+ S. o3 ^
  42.         # rebuild start poind
    4 l; L2 p) ?+ t2 b2 _% v7 R
  43.         self.startpoint = self.ranges[0] + self.downloaded8 q4 ^7 K1 {& L& j; ~* x' _
  44.          + x0 y+ B* L# ]. r7 A
  45.         # This part is completed
    / p\" ~\" C! W6 x+ Q/ L5 R1 k# x
  46.         if self.startpoint >= self.ranges[1]:# N# R. F/ ?' {. D/ w  G
  47.             print 'Part %s has been downloaded over.' % self.filename2 G5 y$ ~7 H2 p4 v
  48.             return- d, o\" [; g: _. [5 t5 m* s
  49.          
    ( t! ^$ I6 ]9 |* F7 ], L
  50.         self.oneTimeSize = 16384 #16kByte/time
    % o* L; v/ ?+ d; C
  51.         print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
    * b0 B8 r, w% A+ c4 _6 O0 f8 X

  52. 2 ~! d9 X/ d6 _
  53.         self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))' R: @! o% ?2 I9 d( ?- P8 t
  54.             
    : o, ]! s) K& q4 m( ~! l
  55.         self.urlhandle = self.open( self.url )2 V7 V9 `* x  ?. C, s3 O: h# D1 k  F1 w

  56. 9 N  {3 |$ o7 I
  57.         data = self.urlhandle.read( self.oneTimeSize )1 E5 h5 B, q2 Z+ ^! u; m; N
  58.         while data:
    7 e6 n# [0 Y* R4 V/ X
  59.             filehandle = open( self.filename, 'ab+' )2 i5 k. C6 b# b# b) G  H. F' k
  60.             filehandle.write( data )) {4 a5 Y7 h' P\" q\" ]3 P/ a/ {
  61.             filehandle.close(). }6 A0 C) Z# M( R

  62. - q7 v6 b( ]1 {3 r2 C# l. P
  63.             self.downloaded += len( data )8 R7 [; |$ R( G8 z\" w6 L
  64.             #print "%s" % (self.name)3 O1 \  N- ~) w4 A# C4 m- Q
  65.             #progress = u'\r...'1 W) _- X8 `+ o, l- H
  66. 8 `6 J* x0 e) H3 e  Q* T
  67.             data = self.urlhandle.read( self.oneTimeSize )* w7 @+ k/ T( x4 r8 p, m; y
  68.          
    * ~2 {* r0 e2 @2 w
  69. def GetUrlFileSize(url, proxies={}):' S# \: _; w9 Q8 a
  70.     urlHandler = urllib.urlopen( url, proxies=proxies )
    4 G+ [2 R% Z6 V: x! r$ c' s
  71.     headers = urlHandler.info().headers
    : j% {: N1 ~' _
  72.     length = 0
    - \6 I3 C0 c0 R8 {& y1 C
  73.     for header in headers:
    : C( C, ~% B$ n0 l* Z. W: x% n
  74.         if header.find('Length') != -1:3 U3 \0 G' ?! `- \5 y
  75.             length = header.split(':')[-1].strip()
    ( M$ X& [, w3 C2 [, N\" o4 a
  76.             length = int(length); _1 m' ^: J9 ?+ f, M* }
  77.     return length0 k! `& G4 k; d( e
  78. , ]0 O; f$ y9 @& ^4 C\" |' D/ b
  79. def SpliteBlocks(totalsize, blocknumber):8 {( o\" ~: H  n' q% }  a) N
  80.     blocksize = totalsize/blocknumber
    . u  q1 e1 ]) A\" x
  81.     ranges = []
    ! z\" ~# H) }4 m% O\" F0 ^( t
  82.     for i in range(0, blocknumber-1):
    , o\" `1 f0 ?& Y9 j& o- u
  83.         ranges.append((i*blocksize, i*blocksize +blocksize - 1))5 h% B- I1 C( E4 u9 y
  84.     ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
    5 |) K0 Z. c( c5 l& H# {

  85. 2 L2 X+ l4 l$ D. w6 h# h
  86.     return ranges
    & k$ l! E) V& i6 ]\" Q4 t
  87. def islive(tasks):2 d4 D- X/ J* e8 T1 d
  88.     for task in tasks:6 Y! l2 Q- i3 b; t
  89.         if task.isAlive():3 T/ F/ h; Z1 k5 L
  90.             return True
    3 v\" F+ _. W# Y+ h\" }$ P
  91.     return False$ P5 R\" m9 I; }( G1 O% T

  92. : v, J4 V  R6 T& J
  93. def paxel(url, output, blocks=6, proxies=local_proxies):& L$ j- \! D! N. r
  94.     ''' paxel
    ( u  L$ u# l: A
  95.     '''
    - \3 M3 G2 K$ h1 ^6 c7 E1 i* C
  96.     size = GetUrlFileSize( url, proxies )- L- E$ s  ~9 T) Z8 v$ L
  97.     ranges = SpliteBlocks( size, blocks )
    % r6 J' ?) P. i$ F! x1 L1 l* Y

  98. 6 Y: d1 V6 {  B* A7 E) V. a+ v
  99.     threadname = [ "thread_%d" % i for i in range(0, blocks) ]1 D9 K0 M4 r+ Q1 t
  100.     filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]& C3 z+ z  x% }3 m0 \- g9 A
  101.    
      I, G+ X: A5 N7 a
  102.     tasks = []3 Z+ J/ a7 A3 v3 R7 T% t7 p' S
  103.     for i in range(0,blocks):
    9 O' ~5 G3 L) A
  104.         task = AxelPython( threadname[i], url, filename[i], ranges[i] )
    * H) N. I' R. @4 ~0 r
  105.         task.setDaemon( True )
    6 H, w9 V  u0 U3 H6 {( ~+ o
  106.         task.start()/ ]9 s- w: S; }6 m: {
  107.         tasks.append( task )
    : t+ Y9 ~% _5 F
  108.          
    : a3 f! G( j* p/ V\" _9 t* v! }% S2 t
  109.     time.sleep( 2 )
      C9 K4 T5 \6 [5 u3 e* b
  110.     while islive(tasks):
    5 N) d$ I: T$ I  k! }
  111.         downloaded = sum( [task.downloaded for task in tasks] )
    4 _. ]: _. r# Z* N/ L' m
  112.         process = downloaded/float(size)*1001 Y+ Z  O9 s6 h6 ~4 @
  113.         show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process). p0 T# G; c! d  b\" R; ?$ ?
  114.         sys.stdout.write(show)! E9 N- w( S/ l2 e# c* R$ W9 Z
  115.         sys.stdout.flush()
    1 K& ^$ c8 M; W4 V' x- o, t
  116.         time.sleep( 0.5 )7 J8 A- P, u5 o( R, f
  117.             
    # D9 Q7 d  `: E# `) p- T4 D% Z5 y
  118.     filehandle = open( output, 'wb+' )
    $ ~. [7 y& @- B
  119.     for i in filename:
    4 B2 _3 Y1 a( m. t
  120.         f = open( i, 'rb' )
    0 p) h; Y, W  G5 ]) [
  121.         filehandle.write( f.read() )
    # ?& t' a& o1 d0 K8 }
  122.         f.close()) F8 Y/ l: G7 B5 F
  123.         try:
    # E* F; f6 m, [. e; ]# h1 }( e
  124.             os.remove(i)
    0 @! q4 E9 T8 O1 _# K6 n. n
  125.             pass% y1 N# }) Z. N  T' R) k
  126.         except:; A. D  e+ X5 ~, f% Q/ T
  127.             pass6 M. _1 F& s* |) H0 h+ R+ X/ d1 X
  128. 8 M! I0 x1 w$ X$ g
  129.     filehandle.close()& s$ `+ x& ?/ h6 p& o
  130. 3 e' I: h2 \9 J\" Y4 {
  131. if __name__ == '__main__':/ s: g6 I! B: _! z6 e6 O7 R6 r7 j
  132.     url = "http://xz1.mm667.com/xz84/images/001.jpg"# u6 H: n' c$ ?9 B% k+ f
  133.     output = '001.jpg'
    * T6 t7 O4 U/ \% n
  134.     paxel( url, output, blocks=4, proxies={} )
复制代码
- M' y  E# V6 s
zan
转播转播0 分享淘帖0 分享分享0 收藏收藏0 支持支持0 反对反对0 微信微信
您需要登录后才可以回帖 登录 | 注册地址

qq
收缩
  • 电话咨询

  • 04714969085
fastpost

关于我们| 联系我们| 诚征英才| 对外合作| 产品服务| QQ

手机版|Archiver| |繁體中文 手机客户端  

蒙公网安备 15010502000194号

Powered by Discuz! X2.5   © 2001-2013 数学建模网-数学中国 ( 蒙ICP备14002410号-3 蒙BBS备-0002号 )     论坛法律顾问:王兆丰

GMT+8, 2026-9-2 13:40 , Processed in 4.711403 second(s), 51 queries .

回顶部