- 在线时间
- 481 小时
- 最后登录
- 2026-8-25
- 注册时间
- 2023-7-11
- 听众数
- 4
- 收听数
- 0
- 能力
- 0 分
- 体力
- 7859 点
- 威望
- 0 点
- 阅读权限
- 255
- 积分
- 2946
- 相册
- 0
- 日志
- 0
- 记录
- 0
- 帖子
- 1177
- 主题
- 1192
- 精华
- 0
- 分享
- 0
- 好友
- 1
该用户从未签到
 |
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。- #!/usr/bin/python6 K6 Q' p# ^2 c
- # -*- coding: utf-8 -*-
D\" i/ Q* F5 C- r, G - # filename: paxel.py
7 a9 v5 o F, Q+ ?' e. e: g -
1 D; f7 O, H* Y, D' G - '''It is a multi-thread downloading tool- r7 h9 T\" M6 \3 H4 z( G
- 6 U5 ^6 T' M/ O2 _0 p# P) F
- It was developed follow axel.
, f9 e% C' y( Y - Author: volans& p' b! y. _5 @4 x( N
- E-mail: volansw [at] gmail.com6 H+ @8 m+ M J# w; h* l7 E
- '''9 o4 @- T+ c! r+ p- G- _9 [
- 5 r2 x @; Q [& ^: p
- import sys' x' j7 w2 R\" l% j: q
- import os/ y6 {% R* y+ R, o( N, c
- import time q1 x, ^) \1 L7 c
- import urllib
4 x6 _2 b' O2 [# M- }* ~ - from threading import Thread1 d! F% G2 N' q: c6 {6 r
- 3 X( ~: m8 Q/ f! L. S; u
- local_proxies = {'http': 'http://131.139.58.200:8080'}% I. R' E\" z/ W5 E$ Y- a) |
- 9 s\" b# d4 ?7 J7 b3 o
- class AxelPython(Thread, urllib.FancyURLopener):
' a! p8 Z\" C\" h& x; W - '''Multi-thread downloading class.) S( k! D) U+ _$ [\" e+ T
- + n: n) A, v) L1 \+ r\" f) _* m4 N
- run() is a vitural method of Thread.$ e) B, D9 C- _5 p: W) H0 z$ J* E
- '''
* i$ \+ o1 h3 s\" P0 U5 V - def __init__(self, threadname, url, filename, ranges=0, proxies={}):
* F. ~& k J1 x$ [\" E2 h0 i+ } - Thread.__init__(self, name=threadname)
% B. m+ Q4 M6 M0 d3 x, M, W2 y3 g - urllib.FancyURLopener.__init__(self, proxies); k! B; H4 @: M( [- g
- self.name = threadname% A3 p$ m6 j- b0 ]\" h; W, t
- self.url = url( I, m( y6 Q4 P: J
- self.filename = filename
2 [3 x5 Q7 U8 w& ^: i - self.ranges = ranges$ z\" s3 f* r: p( @1 O
- self.downloaded = 0
3 y. t n. ~2 y; F0 c: ~ - 8 }, c6 H+ T) }6 S
- def run(self):
: |( u. T* z* ]9 A - '''vertual function in Thread'''% j+ M+ W. f9 e* ]1 M) a, W% P
- try:6 p4 z) Z- C5 {8 J* L
- self.downloaded = os.path.getsize( self.filename )+ f3 @- t5 o9 e% f2 r; R
- except OSError:( s( E* N+ g: ?5 ?6 ~ U
- #print 'never downloaded'1 o8 z9 {3 M6 E, n\" P+ W
- self.downloaded = 0! h2 A8 M9 U: X* f
-
8 K1 m9 {7 [/ ?8 ?, A1 j - # rebuild start poind7 r\" L! H+ {5 a; q, e
- self.startpoint = self.ranges[0] + self.downloaded# R6 K* V: p( Q9 d6 T+ M
-
9 W# T+ y9 k5 l. n - # This part is completed. u4 o, z8 a/ `7 P4 O7 J0 Z
- if self.startpoint >= self.ranges[1]:
# k% w, [, I1 @$ F9 _) u& C' f - print 'Part %s has been downloaded over.' % self.filename2 L& I. ?& @6 j! p* u
- return
# p2 e N Y4 P, g0 r& z - 2 {0 X( Q3 Q, D7 S, s+ G- T: B' H
- self.oneTimeSize = 16384 #16kByte/time
A0 v* f; c6 E; R3 f4 V - print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])# F1 h* l7 B\" K \. B\" N. w
- ) c* c: V& `4 Q/ t
- self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))3 r2 [. K1 P, _% x! ]/ z
- 7 p\" ~0 G4 ]' v! X\" J& h6 ?\" s
- self.urlhandle = self.open( self.url )
9 P! Z8 e, h% c- ^ - - }1 l* J- M/ v\" Z3 x9 t
- data = self.urlhandle.read( self.oneTimeSize )
1 A2 q f# P: {6 k5 A - while data:9 x1 x4 T6 \4 O\" t' S( _) h) F/ `% c4 d
- filehandle = open( self.filename, 'ab+' )' q2 I) _- {2 {* g9 y3 g9 I
- filehandle.write( data )( ]( e J\" Y( f0 Q; m- \
- filehandle.close()
$ v. J% U' e2 u% ]0 M -
& \( r# q7 q B. H\" L - self.downloaded += len( data )# E; x, J, y# C; e6 K! Z
- #print "%s" % (self.name). |5 c! X# W1 D- R! ~
- #progress = u'\r...'* }! ~. `) h, u' M; x
- : y6 O; G% u. F Y/ T. p# i
- data = self.urlhandle.read( self.oneTimeSize )( W7 n/ w* e+ J6 H7 _1 o) e8 H
- ; g0 W( w6 t( P; C2 P
- def GetUrlFileSize(url, proxies={}):! V\" f1 h8 a* l8 N) ]6 {/ l
- urlHandler = urllib.urlopen( url, proxies=proxies )5 s5 z* y. M6 ]: u+ `
- headers = urlHandler.info().headers
* r& k, X4 t, D* g - length = 0
* _2 {8 x* u% b9 _/ m( w- O# T5 Y - for header in headers:( i1 \* z0 }6 e. ?: N
- if header.find('Length') != -1:
h; Y# r! w8 e: g; N: y2 v - length = header.split(':')[-1].strip()
% _% N7 ~5 j2 @; k8 T - length = int(length)5 V! M3 _8 C! {0 X4 C
- return length
3 m) w: y& A& r7 t; b -
) U, o0 C/ e7 v& a) i - def SpliteBlocks(totalsize, blocknumber):
8 c/ y) U' M, h - blocksize = totalsize/blocknumber
6 l+ Y. O7 \2 P1 i: t- f% u) J - ranges = []
; w# y$ m+ j3 k( v: c6 l - for i in range(0, blocknumber-1):
- @7 X L! f$ |/ W Q! F1 i. h* \ - ranges.append((i*blocksize, i*blocksize +blocksize - 1)); L( h4 c& M9 s# P f& f% G! t2 l q3 o. p
- ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))* V! w0 Q5 ]1 c/ k; U! k3 U
-
4 Z2 q1 O9 q( x/ C# y - return ranges
+ H$ I3 Y. ?' T' Z; O7 h3 X - def islive(tasks):9 J2 @- {( y( X4 w* h o6 Q
- for task in tasks:
# j+ e) N+ }5 J. g2 q( M# s - if task.isAlive():! i\" o! Z9 L& f# p: b% t\" \8 g
- return True4 H. K! [* T\" h/ ^
- return False) }& l) `3 a% u- v. x
- 6 F, M\" W. I2 t1 b) ]7 \/ T9 [( i5 W
- def paxel(url, output, blocks=6, proxies=local_proxies):
5 D. o1 \- ?. I4 w - ''' paxel
8 z ` p6 N; R* `4 o9 m q, f - '''\" e6 l; w3 ^& C0 c
- size = GetUrlFileSize( url, proxies )\" | W- O& `; M: r. B
- ranges = SpliteBlocks( size, blocks )+ ^ {7 Q e/ S w
- % c- @5 ^2 r$ \9 }; D: f- m
- threadname = [ "thread_%d" % i for i in range(0, blocks) ]
1 t( F\" p5 @( _9 P5 T* b$ O - filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
7 k0 F9 X4 u! P7 Q6 J -
& U0 p\" [6 [: x$ Y% w/ }4 I - tasks = []
; L$ K3 n/ d, {# } - for i in range(0,blocks):( s% j8 z\" R8 D e6 k( l' T1 \\" b
- task = AxelPython( threadname[i], url, filename[i], ranges[i] )
+ r2 s6 u6 _ l9 z4 K) U8 k5 m - task.setDaemon( True )6 L5 l1 D7 R8 ], `& r+ C2 B
- task.start()
b\" X) p1 e \; s - tasks.append( task )
) M4 H# j- C. Q0 E, d - ; Y# d& \ L% z& u7 s; p
- time.sleep( 2 )
# x x6 v# v' ]6 I* H6 |4 G( o - while islive(tasks):4 \ i1 D; x6 h$ h
- downloaded = sum( [task.downloaded for task in tasks] )% D' Z( N' Q0 i2 ~' a' N7 X
- process = downloaded/float(size)*100% b5 L; \) [2 `, G3 l. }
- show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)' I8 c, N1 G) G6 o o* W+ f% x
- sys.stdout.write(show)
; c: S# |3 V* u( p* [- D$ h% N - sys.stdout.flush()1 i1 `2 \ ?% Z- Z\" ]% @4 D
- time.sleep( 0.5 )
6 z+ ^( t* T& f0 @. L+ g0 y - + t7 C3 ]4 X o/ W
- filehandle = open( output, 'wb+' )% \( o( i! I R2 h3 x# }3 d6 I
- for i in filename:
- g: D, R9 r$ W* D2 W - f = open( i, 'rb' ), c% U/ _/ e3 O5 p L
- filehandle.write( f.read() )
4 G\" j) G; }8 u+ ^) z; ^# U - f.close()3 N7 ~, S- O\" Q\" Q
- try:
$ U, y& A- q9 ^9 `0 p: {6 }9 q' ~) ~2 i - os.remove(i)
' M/ K; D- R J9 ^3 ]/ V& l' B2 s - pass& W* z8 A+ o5 l. ]# G, h6 q
- except:3 ]3 X6 ]8 T2 ]' t* f
- pass$ K. T: Y* c\" N! T
- 3 a& f: V) B( J! E. }
- filehandle.close()\" a$ k% G& w/ }6 ~
- 1 K* Q; m' I+ F
- if __name__ == '__main__':0 H8 f( Z\" ~4 t5 m6 E0 ^1 p
- url = "http://xz1.mm667.com/xz84/images/001.jpg"5 B% ]4 E ^5 F) R& b% m! E5 l
- output = '001.jpg'. g U& ?6 y( F% D8 E3 c4 N% G3 K& T
- paxel( url, output, blocks=4, proxies={} )
复制代码
5 B# c8 ]# _% d% f( v9 n6 }2 q |
zan
|