- 在线时间
- 481 小时
- 最后登录
- 2026-8-25
- 注册时间
- 2023-7-11
- 听众数
- 4
- 收听数
- 0
- 能力
- 0 分
- 体力
- 7859 点
- 威望
- 0 点
- 阅读权限
- 255
- 积分
- 2946
- 相册
- 0
- 日志
- 0
- 记录
- 0
- 帖子
- 1177
- 主题
- 1192
- 精华
- 0
- 分享
- 0
- 好友
- 1
该用户从未签到
 |
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。- #!/usr/bin/python/ X9 T8 o, k& b- a' a0 k
- # -*- coding: utf-8 -*-
\" z2 }\" @- q/ \5 o3 Y - # filename: paxel.py
- v Y) f\" n ` -
8 m. g8 \7 Z& M. `; l o - '''It is a multi-thread downloading tool3 m7 c; f: z\" n1 m& X p+ n K* c
- ) j; q2 b6 R% X8 \
- It was developed follow axel.1 I, Q+ s\" j+ i' h8 T
- Author: volans
, H8 p$ D3 ]8 D) I d8 }* {& q - E-mail: volansw [at] gmail.com1 J: h# `% M# r; C
- '''
# ~* x! G9 s6 r1 V\" o -
* f& \. J8 @8 b. { - import sys
: c' ~' \: S( R - import os
\" V4 n1 z' \& f6 j - import time
* e\" @2 q+ n& `) Y7 d\" x - import urllib2 p: P$ d6 }6 K+ B& S) N* R; R
- from threading import Thread
4 g: [. E9 j% o5 n - 7 f2 B- d3 t% b3 @
- local_proxies = {'http': 'http://131.139.58.200:8080'}
1 m8 y( K6 A* b1 G0 \2 N- P - + r0 H7 s7 I1 [\" i& {8 p
- class AxelPython(Thread, urllib.FancyURLopener):
& Y4 T' }6 E0 y/ c$ f2 g - '''Multi-thread downloading class.
n3 S! S6 ]% g- o9 T0 k -
7 ~, [. k! N! V- r - run() is a vitural method of Thread.
. _/ P$ \) a# l0 Z - '''' A/ y6 }: _( Q: {
- def __init__(self, threadname, url, filename, ranges=0, proxies={}):* }& q' z7 k( h7 ^; i+ H
- Thread.__init__(self, name=threadname)
( d4 l# A8 d1 H [1 r - urllib.FancyURLopener.__init__(self, proxies)
0 S' G$ e8 [) i P8 a9 x. l% w' \( m - self.name = threadname
0 C$ V% Z% V, }2 Y\" o - self.url = url
* a& W; X# @& {0 F& P - self.filename = filename0 T' h3 T6 H1 `4 D |% N
- self.ranges = ranges; n# L6 }; J( E d3 H D
- self.downloaded = 0. G' \6 D& f3 R+ }3 \- j) R$ c! c. l
-
- M) w0 J5 a; l# H: \9 R - def run(self):
6 f: C1 g\" G; E - '''vertual function in Thread'''
( U: ^( A. e( C - try:
* }! T( X+ B! k' D+ Y - self.downloaded = os.path.getsize( self.filename )! C1 t. E5 W5 N9 I, U
- except OSError:# Q8 U5 g' F& |+ @' I) i4 U2 ]; ?
- #print 'never downloaded'7 I* q1 U9 o: \
- self.downloaded = 0: s2 ~* J! @$ V0 J; A6 w
-
! @5 j+ X\" I4 ]0 i3 \6 G6 _9 ^# i - # rebuild start poind
* {/ d+ J; G% B0 I/ }' ? - self.startpoint = self.ranges[0] + self.downloaded3 c& }. F' a! R& N
- - Q7 V1 i/ x5 \! H: ?, m\" e
- # This part is completed
8 I9 F$ \+ s# b3 a- m# y6 G& n - if self.startpoint >= self.ranges[1]:1 a! t8 [8 a$ F6 a1 }
- print 'Part %s has been downloaded over.' % self.filename
9 g9 f! R8 a! q1 X - return5 M) a4 p* q7 d @- }
- 4 d4 l9 f/ l6 _1 J, m- T4 T {
- self.oneTimeSize = 16384 #16kByte/time
. }( E3 b6 B- ]2 K - print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
3 c6 J- r4 M/ l) d5 t - ) D% U) k8 h. @' V
- self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))/ f7 \$ `) k1 G! x+ b% T
-
1 Z0 d% D/ d3 `, h5 [ - self.urlhandle = self.open( self.url )+ I7 v) u5 b: `6 }1 X3 P0 \
- - W0 G+ K1 T3 Y& q b. h5 ^
- data = self.urlhandle.read( self.oneTimeSize )
. T1 s. v0 u5 S) m - while data:0 |( W' J7 s& ? U1 _ A0 F1 E- T/ }
- filehandle = open( self.filename, 'ab+' )0 l, W% P5 y0 G. S2 }7 m4 k5 k
- filehandle.write( data )
) _ E3 [# R2 R. A: W) L7 f% u9 _ - filehandle.close(). @) O& u+ X6 O/ \8 a% `6 _$ b! H
- + I# E0 c4 W, D, H8 f# N) N
- self.downloaded += len( data )
0 o. C F# h( C - #print "%s" % (self.name)8 q4 h; \$ q: g/ `' v9 d
- #progress = u'\r...'/ x, u\" u# g% E' @( m$ v x
- 6 z D$ l/ l6 p/ o6 z: |
- data = self.urlhandle.read( self.oneTimeSize )3 v$ h# R: C2 O/ U& K& t6 j' ]& g
-
/ `2 ~/ L) W1 Y8 s& g+ n0 V- Z% q# ^ - def GetUrlFileSize(url, proxies={}):! {3 o# [& K) G2 W/ k% Q' q
- urlHandler = urllib.urlopen( url, proxies=proxies )
n1 W: ]7 g# I - headers = urlHandler.info().headers
/ F' D: }* K, [, H - length = 0
/ Z( l7 l\" H+ |9 Z' r4 v$ O7 L L9 g - for header in headers:! r9 ^, M4 {8 a# A# ]
- if header.find('Length') != -1:
8 Q0 E6 L5 c( q: C$ V0 y- S! _/ O - length = header.split(':')[-1].strip()- s9 J& ^. O2 U' d! Z
- length = int(length)
H$ g3 a$ n% }# B3 [2 \ - return length
( l9 s4 r9 O& r. h2 J* N u1 V -
9 q( r# {8 e) G3 k9 q1 Q' d4 ]9 A - def SpliteBlocks(totalsize, blocknumber):. G5 l* z' \3 L9 c5 U! {0 ]3 K+ w
- blocksize = totalsize/blocknumber& C+ I, T/ j5 w' S\" Z
- ranges = []
% O- _! Y# n6 v - for i in range(0, blocknumber-1):
* s8 ^: c. N1 o7 N0 q& p) m' C - ranges.append((i*blocksize, i*blocksize +blocksize - 1))
\" S0 H1 R! ?. D' Z( @6 T2 d& M - ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))7 H G\" G9 u0 Y0 Y/ R2 ?
- 3 g6 A. R. @% M! @$ o! ~6 }. k
- return ranges
( o) a\" K1 M) ?5 o e/ A1 M - def islive(tasks):
, W D! R) F* b$ ~/ X' h M+ u - for task in tasks:
+ P, A+ U! u% J/ m& t6 J6 s - if task.isAlive():8 q; X# o, g5 }7 m1 O/ V
- return True. N. s( e# o\" ^4 \: N! \8 z
- return False7 m; n2 v- Y' |; @# w9 {
- 5 U Q/ B( x) n3 t
- def paxel(url, output, blocks=6, proxies=local_proxies):
. m& B- Z2 n( g+ s - ''' paxel. Q/ o: P$ k9 w7 n% p
- '''
. Q+ q7 u' T2 }; n% b% k: \ - size = GetUrlFileSize( url, proxies )1 L5 w7 Z; x. I9 u: J; k+ X
- ranges = SpliteBlocks( size, blocks )
d! [, y! k/ a - , T8 i# z2 B* m N) L8 |- s1 ?# \
- threadname = [ "thread_%d" % i for i in range(0, blocks) ]
$ Q( a5 f1 t$ @9 } - filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]' D$ m4 r( ]. T, \0 W. l
- 0 b) o1 |9 o# q\" O, d
- tasks = []2 A, | y- |5 n/ X/ n- J
- for i in range(0,blocks):+ {# O- F. z2 v7 F$ A5 ^7 {( X
- task = AxelPython( threadname[i], url, filename[i], ranges[i] )
+ J6 C3 ]\" p: j. N6 b' j- t2 G - task.setDaemon( True ): K( \6 t3 z\" B9 z% Y4 }\" q, \
- task.start()
) T! c8 T0 H1 T - tasks.append( task )( {, U7 T9 l9 _3 D0 p
- + j1 X* u- Z! M' T3 o9 V
- time.sleep( 2 )
+ |/ j' |( o0 B& d* X9 C. t( @ - while islive(tasks):9 T7 f8 {+ V\" T! w
- downloaded = sum( [task.downloaded for task in tasks] )' Q3 [\" _$ v$ X0 y# K* t) l, b
- process = downloaded/float(size)*100
\" z% @, n8 p# S\" o0 J2 X5 h0 } - show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)! K& ~9 ^5 f2 c3 V( r% y$ ^8 X
- sys.stdout.write(show)1 `+ j1 ^, X6 E1 @1 `
- sys.stdout.flush()
$ E. P7 j0 l9 H, F - time.sleep( 0.5 )2 M, s' l; b. x( ^
- / j3 b\" T/ P( T3 l0 S% r
- filehandle = open( output, 'wb+' )% ?6 T2 S( o\" V
- for i in filename:5 u) [8 N; D _7 G- S* P* P4 b
- f = open( i, 'rb' )+ V% b. v5 K( {0 m8 H
- filehandle.write( f.read() )
6 t B$ Q. |2 {- b - f.close()1 l( ^4 }# Q/ I0 X0 H' l6 T( d
- try:
0 b6 u8 t# {, j- s, l4 n6 ? - os.remove(i)
5 Y$ y* Y0 F5 }6 J& i& r - pass
! |+ T, h1 s- ?( a- T - except:
. d4 |* G9 D) H( r; V. V+ [ - pass9 b1 r\" U* y) f: D
-
* C; D\" S6 R$ f+ h\" A - filehandle.close(). }& d% Q# g0 M8 ? a3 L
- 0 K y) }3 l, w! \4 y6 W
- if __name__ == '__main__':
8 U+ C; E2 ~5 Z4 G# [8 h - url = "http://xz1.mm667.com/xz84/images/001.jpg"0 y0 B* C0 l# K1 L
- output = '001.jpg'
& y. r$ p7 e9 P - paxel( url, output, blocks=4, proxies={} )
复制代码 ! P' ^6 u# |$ ~1 @) w4 x% E
|
zan
|