- 在线时间
- 481 小时
- 最后登录
- 2026-8-25
- 注册时间
- 2023-7-11
- 听众数
- 4
- 收听数
- 0
- 能力
- 0 分
- 体力
- 7859 点
- 威望
- 0 点
- 阅读权限
- 255
- 积分
- 2946
- 相册
- 0
- 日志
- 0
- 记录
- 0
- 帖子
- 1177
- 主题
- 1192
- 精华
- 0
- 分享
- 0
- 好友
- 1
该用户从未签到
 |
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。- #!/usr/bin/python
. W! ~( Y, S\" s - # -*- coding: utf-8 -*-- ^, L; x: d( s- k. |. h
- # filename: paxel.py' {6 A7 @; p5 S- Q! p( S
- 5 v' g+ i9 @4 A) I
- '''It is a multi-thread downloading tool
& ^7 T1 Q4 n9 h$ d( m1 K4 M% z -
; j3 [0 y; x4 E9 r; U\" J - It was developed follow axel.2 o/ X' S+ N* E2 r& s/ c
- Author: volans
) T$ m* I# c0 }/ a2 H# K - E-mail: volansw [at] gmail.com
/ F+ J( i2 d# p, n - '''+ U\" O c; h& o& |2 Z; l# [. y) ]
- 2 v5 C. E' S( f# r! |) }
- import sys3 R/ K\" M+ T; t9 a
- import os6 S+ W! _6 J2 H
- import time
1 ]& [3 q3 e7 a9 p/ A# ] - import urllib
& c* Q5 r6 l8 [ - from threading import Thread) q' `) `2 A1 R0 L0 j
- & |1 e4 @3 u& |2 F. S
- local_proxies = {'http': 'http://131.139.58.200:8080'}/ w. p, d5 L4 B- H' ^! |
-
5 o' P( B) f5 X# [* e+ F( n - class AxelPython(Thread, urllib.FancyURLopener):
: n) P6 a7 X/ D1 l - '''Multi-thread downloading class.' j+ d; h8 H4 ~0 t1 _% R H
- 2 k y' `3 o2 m U/ ]+ D# r- e
- run() is a vitural method of Thread.
8 m) X5 W( n; J- h. s T\" G - '''7 J# ^+ I0 M1 D( k
- def __init__(self, threadname, url, filename, ranges=0, proxies={}):
8 E/ i* ?' E: r: r; y - Thread.__init__(self, name=threadname)
+ n/ d8 A3 _. r0 L5 Y) d* M - urllib.FancyURLopener.__init__(self, proxies) a9 ~0 S6 \. O
- self.name = threadname
4 }1 l! B& D5 H) u - self.url = url: q. l, `, A* U4 T
- self.filename = filename& S( G- f3 d* f\" v b
- self.ranges = ranges, } u. _8 N3 }
- self.downloaded = 0- d0 ^4 J$ d9 R6 i, n' W f
- # {3 f4 [; _8 H B# k: `; G
- def run(self):1 O/ C\" I) W# k5 E) g1 D H+ ?
- '''vertual function in Thread'''7 F/ J. k) r; M: d
- try:
- \' s\" r/ A, ?' \ - self.downloaded = os.path.getsize( self.filename )
2 A7 g! R% U r - except OSError:' L3 f& J' o- }
- #print 'never downloaded'# z+ i. s, f! q7 Y
- self.downloaded = 0
; k, A* s5 ~# u - 6 @- H! f) f8 I! ~+ m
- # rebuild start poind
+ s2 Q$ `7 r+ V0 } - self.startpoint = self.ranges[0] + self.downloaded
: W: A0 k% I+ o, Y% P3 L1 j -
/ `: s3 e8 \0 ]: n4 ?4 O - # This part is completed
/ T9 {! V: y W( L - if self.startpoint >= self.ranges[1]:
% i4 R9 \- u1 } - print 'Part %s has been downloaded over.' % self.filename
9 f) J1 V! N( M+ _9 k5 f7 y W - return
% }7 Q! B\" K6 m6 c/ U -
+ _9 ]\" _7 `9 g - self.oneTimeSize = 16384 #16kByte/time
I+ Y1 o1 P* a. w' ~\" t - print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
2 c7 z9 s( b: ~6 _5 O$ H: b -
7 Z5 f; ], m9 f* Q5 E' g/ Q - self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))3 M! y5 s* l( P/ n# V
- 9 b6 L4 {+ C' S, T, F' I) @6 \. `
- self.urlhandle = self.open( self.url )2 W$ z& p! f\" n. E
- ) ^1 I0 ~: _6 j7 C, }+ L
- data = self.urlhandle.read( self.oneTimeSize )
1 | [0 M# m% e: A8 [/ N+ p - while data:' i2 Y1 h% D3 y. x1 o& w, b3 a
- filehandle = open( self.filename, 'ab+' )) a8 w8 d: G; L% I+ ^
- filehandle.write( data )) L. D$ P0 v* Z0 v B8 E; n3 F/ B
- filehandle.close()
$ {6 G/ H6 O* y) M5 g -
]8 e\" }8 }/ d0 _# @ b - self.downloaded += len( data )
3 e5 l6 S\" b) y. ?( R* {- ? - #print "%s" % (self.name)
6 p\" ?$ K3 S% z/ w3 ~) R - #progress = u'\r...'
* s2 y5 v1 y; n# r7 M1 j7 y - u g0 t! q. p% b6 @
- data = self.urlhandle.read( self.oneTimeSize )& l# y3 j6 Q& l* o' l, [) P8 d; T
- # _- c# R; a6 G! d2 F
- def GetUrlFileSize(url, proxies={}):
7 I' j J% V% S0 b g% n* a - urlHandler = urllib.urlopen( url, proxies=proxies )& a8 i. t# E3 e) p1 @! k
- headers = urlHandler.info().headers N7 v7 z. G6 A' K. x
- length = 0, r# w\" P& b2 m6 t, r
- for header in headers:$ F& {1 |# \4 }2 o
- if header.find('Length') != -1:
4 ~% p9 d( m5 m - length = header.split(':')[-1].strip()
. s; o! n- ]4 c# s% E - length = int(length)0 i. _% F/ X4 q% t4 t' w7 b! a
- return length
1 k: v X2 ^: j& F: P -
# ~$ K* } S/ v) m% d# P - def SpliteBlocks(totalsize, blocknumber):- c0 V: ?; r; C/ Y; c6 e\" W7 d
- blocksize = totalsize/blocknumber
8 }( o+ F3 H9 s9 K( \ - ranges = []9 H\" e$ B8 C6 ?+ J6 x
- for i in range(0, blocknumber-1):, G9 n, P9 j( [1 E# v5 r
- ranges.append((i*blocksize, i*blocksize +blocksize - 1))' T; p# |) q: Q5 o& U
- ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))) ?: q- Y& j' f8 Z8 |( t# C
-
$ V. A `( y' @ - return ranges
5 l, V% d! G B- b - def islive(tasks):
4 K, _. S& p6 ^, D: ?/ V - for task in tasks:
' K+ n/ u0 h* l. K) q - if task.isAlive():
6 w\" I) y q5 g9 m' ~7 p& p, M - return True
$ l2 ^' j* [: j - return False4 v: c6 j. F- L. j5 e3 J
-
: g; |5 n& g/ ]' _% i - def paxel(url, output, blocks=6, proxies=local_proxies):
! J; c# x- S, V( W% u( J; z0 y - ''' paxel
. p3 R( s+ r' q' e( f! Q% O - '''5 F5 o- p( Y( C( F
- size = GetUrlFileSize( url, proxies )
' F) i% H8 E4 `& l - ranges = SpliteBlocks( size, blocks )5 d+ ]5 e2 c% b9 @, l- S
-
U! x* p- i4 N# u& z - threadname = [ "thread_%d" % i for i in range(0, blocks) ]
/ E9 X, f( w; J - filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]9 e4 T' E7 k. p, U' g+ i6 Z
- , T- D. _( L/ j4 X, l' n& T, o
- tasks = []\" \$ S/ {4 i' B9 L2 e& G- Q
- for i in range(0,blocks):9 V$ \4 M9 V5 d! B1 K
- task = AxelPython( threadname[i], url, filename[i], ranges[i] )5 }9 v0 Q Z: n' y\" c2 b
- task.setDaemon( True )
. ?' C- P u5 ~( L7 N1 X - task.start()& @. g) k- a! d$ I
- tasks.append( task )
* {6 L4 _6 d$ @, r -
6 R9 }\" f+ l1 X. n3 |2 G% j8 I - time.sleep( 2 )
: u+ Q( i) i' K\" `- W - while islive(tasks):
* H0 W, W' K! I - downloaded = sum( [task.downloaded for task in tasks] )
; Q: N% ?# g9 Q9 |9 @6 ^5 x - process = downloaded/float(size)*1000 F* W/ ?3 x% }& O. v5 _
- show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
+ x, L# |& S8 F - sys.stdout.write(show)5 I' v# `5 m! m' H) v
- sys.stdout.flush()0 e8 w1 P' A\" [* G\" ^4 J
- time.sleep( 0.5 )* u9 e8 p# Q0 _ ?+ y& ?8 Y: ~
- A: M1 I# V\" i0 V8 ?. H6 g$ G9 A
- filehandle = open( output, 'wb+' )0 ~; P c: j) f6 d- g
- for i in filename:
, h. w1 \: [* p* [) U/ B - f = open( i, 'rb' )
, t4 y* m) S+ {7 b% I - filehandle.write( f.read() )
v4 g! G( c5 d: \( b8 G, n0 I. L - f.close(); P& ^/ f$ d1 p4 x' s# L. h
- try:2 x2 m) N' V) ^8 }; e y, x6 \/ T
- os.remove(i)5 @# m' \% h4 w# J/ L7 H
- pass
0 o! Y! i0 [( \# ^& w0 Y+ g6 r - except:
6 G A% F; O2 K% I/ O8 O$ D - pass. [# L2 l& _5 `# b
-
; b1 Z& N& n/ r\" S. q! U: w' n' P - filehandle.close(). p+ J, d7 }2 i6 q, v0 L/ b
-
7 Z$ ^# e2 E! X8 H& a; l1 W - if __name__ == '__main__':
: ^ J( b& {\" U' S# X# ~) k% @ - url = "http://xz1.mm667.com/xz84/images/001.jpg"
, u* X9 I# @8 X4 r\" p6 j3 X - output = '001.jpg'& v# B y& H& B5 p0 b' d
- paxel( url, output, blocks=4, proxies={} )
复制代码
* X1 W, _* F9 i1 ]0 [3 c0 a |
zan
|