多线程下载图集

[复制链接]

字体大小: 正常放大

2744557306

1192 主题	4 听众	2946 积分

该用户从未签到

电梯直达

1^#

发表于 2024-3-31 17:18 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

#!/usr/bin/python6 K6 Q' p# ^2 c
# -*- coding: utf-8 -*-
D\" i/ Q* F5 C- r, G
# filename: paxel.py
7 a9 v5 o F, Q+ ?' e. e: g
1 D; f7 O, H* Y, D' G
'''It is a multi-thread downloading tool- r7 h9 T\" M6 \3 H4 z( G
6 U5 ^6 T' M/ O2 _0 p# P) F
It was developed follow axel.
, f9 e% C' y( Y
Author: volans& p' b! y. _5 @4 x( N
E-mail: volansw [at] gmail.com6 H+ @8 m+ M J# w; h* l7 E
'''9 o4 @- T+ c! r+ p- G- _9 [
5 r2 x @; Q [& ^: p
import sys' x' j7 w2 R\" l% j: q
import os/ y6 {% R* y+ R, o( N, c
import time q1 x, ^) \1 L7 c
import urllib
4 x6 _2 b' O2 [# M- }* ~
from threading import Thread1 d! F% G2 N' q: c6 {6 r
3 X( ~: m8 Q/ f! L. S; u
local_proxies = {'http': 'http://131.139.58.200:8080'}% I. R' E\" z/ W5 E$ Y- a) |
9 s\" b# d4 ?7 J7 b3 o
class AxelPython(Thread, urllib.FancyURLopener):
' a! p8 Z\" C\" h& x; W
'''Multi-thread downloading class.) S( k! D) U+ _$ [\" e+ T
+ n: n) A, v) L1 \+ r\" f) _* m4 N
run() is a vitural method of Thread.$ e) B, D9 C- _5 p: W) H0 z$ J* E
'''
* i$ \+ o1 h3 s\" P0 U5 V
def __init__(self, threadname, url, filename, ranges=0, proxies={}):
* F. ~& k J1 x$ [\" E2 h0 i+ }
Thread.__init__(self, name=threadname)
% B. m+ Q4 M6 M0 d3 x, M, W2 y3 g
urllib.FancyURLopener.__init__(self, proxies); k! B; H4 @: M( [- g
self.name = threadname% A3 p$ m6 j- b0 ]\" h; W, t
self.url = url( I, m( y6 Q4 P: J
self.filename = filename
2 [3 x5 Q7 U8 w& ^: i
self.ranges = ranges$ z\" s3 f* r: p( @1 O
self.downloaded = 0
3 y. t n. ~2 y; F0 c: ~
8 }, c6 H+ T) }6 S
def run(self):
: |( u. T* z* ]9 A
'''vertual function in Thread'''% j+ M+ W. f9 e* ]1 M) a, W% P
try:6 p4 z) Z- C5 {8 J* L
self.downloaded = os.path.getsize( self.filename )+ f3 @- t5 o9 e% f2 r; R
except OSError:( s( E* N+ g: ?5 ?6 ~ U
#print 'never downloaded'1 o8 z9 {3 M6 E, n\" P+ W
self.downloaded = 0! h2 A8 M9 U: X* f
8 K1 m9 {7 [/ ?8 ?, A1 j
# rebuild start poind7 r\" L! H+ {5 a; q, e
self.startpoint = self.ranges[0] + self.downloaded# R6 K* V: p( Q9 d6 T+ M
9 W# T+ y9 k5 l. n
# This part is completed. u4 o, z8 a/ `7 P4 O7 J0 Z
if self.startpoint >= self.ranges[1]:
# k% w, [, I1 @$ F9 _) u& C' f
print 'Part %s has been downloaded over.' % self.filename2 L& I. ?& @6 j! p* u
return
# p2 e N Y4 P, g0 r& z
2 {0 X( Q3 Q, D7 S, s+ G- T: B' H
self.oneTimeSize = 16384 #16kByte/time
A0 v* f; c6 E; R3 f4 V
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])# F1 h* l7 B\" K \. B\" N. w
) c* c: V& `4 Q/ t
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))3 r2 [. K1 P, _% x! ]/ z
7 p\" ~0 G4 ]' v! X\" J& h6 ?\" s
self.urlhandle = self.open( self.url )
9 P! Z8 e, h% c- ^
- }1 l* J- M/ v\" Z3 x9 t
data = self.urlhandle.read( self.oneTimeSize )
1 A2 q f# P: {6 k5 A
while data:9 x1 x4 T6 \4 O\" t' S( _) h) F/ `% c4 d
filehandle = open( self.filename, 'ab+' )' q2 I) _- {2 {* g9 y3 g9 I
filehandle.write( data )( ]( e J\" Y( f0 Q; m- \
filehandle.close()
$ v. J% U' e2 u% ]0 M
& \( r# q7 q B. H\" L
self.downloaded += len( data )# E; x, J, y# C; e6 K! Z
#print "%s" % (self.name). |5 c! X# W1 D- R! ~
#progress = u'\r...'* }! ~. `) h, u' M; x
: y6 O; G% u. F Y/ T. p# i
data = self.urlhandle.read( self.oneTimeSize )( W7 n/ w* e+ J6 H7 _1 o) e8 H
; g0 W( w6 t( P; C2 P
def GetUrlFileSize(url, proxies={}):! V\" f1 h8 a* l8 N) ]6 {/ l
urlHandler = urllib.urlopen( url, proxies=proxies )5 s5 z* y. M6 ]: u+ `
headers = urlHandler.info().headers
* r& k, X4 t, D* g
length = 0
* _2 {8 x* u% b9 _/ m( w- O# T5 Y
for header in headers:( i1 \* z0 }6 e. ?: N
if header.find('Length') != -1:
h; Y# r! w8 e: g; N: y2 v
length = header.split(':')[-1].strip()
% _% N7 ~5 j2 @; k8 T
length = int(length)5 V! M3 _8 C! {0 X4 C
return length
3 m) w: y& A& r7 t; b
) U, o0 C/ e7 v& a) i
def SpliteBlocks(totalsize, blocknumber):
8 c/ y) U' M, h
blocksize = totalsize/blocknumber
6 l+ Y. O7 \2 P1 i: t- f% u) J
ranges = []
; w# y$ m+ j3 k( v: c6 l
for i in range(0, blocknumber-1):
- @7 X L! f$ |/ W Q! F1 i. h* \
ranges.append((i*blocksize, i*blocksize +blocksize - 1)); L( h4 c& M9 s# P f& f% G! t2 l q3 o. p
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))* V! w0 Q5 ]1 c/ k; U! k3 U
4 Z2 q1 O9 q( x/ C# y
return ranges
+ H$ I3 Y. ?' T' Z; O7 h3 X
def islive(tasks):9 J2 @- {( y( X4 w* h o6 Q
for task in tasks:
# j+ e) N+ }5 J. g2 q( M# s
if task.isAlive():! i\" o! Z9 L& f# p: b% t\" \8 g
return True4 H. K! [* T\" h/ ^
return False) }& l) `3 a% u- v. x
6 F, M\" W. I2 t1 b) ]7 \/ T9 [( i5 W
def paxel(url, output, blocks=6, proxies=local_proxies):
5 D. o1 \- ?. I4 w
''' paxel
8 z ` p6 N; R* `4 o9 m q, f
'''\" e6 l; w3 ^& C0 c
size = GetUrlFileSize( url, proxies )\" | W- O& `; M: r. B
ranges = SpliteBlocks( size, blocks )+ ^ {7 Q e/ S w
% c- @5 ^2 r$ \9 }; D: f- m
threadname = [ "thread_%d" % i for i in range(0, blocks) ]
1 t( F\" p5 @( _9 P5 T* b$ O
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
7 k0 F9 X4 u! P7 Q6 J
& U0 p\" [6 [: x$ Y% w/ }4 I
tasks = []
; L$ K3 n/ d, {# }
for i in range(0,blocks):( s% j8 z\" R8 D e6 k( l' T1 \\" b
task = AxelPython( threadname[i], url, filename[i], ranges[i] )
+ r2 s6 u6 _ l9 z4 K) U8 k5 m
task.setDaemon( True )6 L5 l1 D7 R8 ], `& r+ C2 B
task.start()
b\" X) p1 e \; s
tasks.append( task )
) M4 H# j- C. Q0 E, d
; Y# d& \ L% z& u7 s; p
time.sleep( 2 )
# x x6 v# v' ]6 I* H6 |4 G( o
while islive(tasks):4 \ i1 D; x6 h$ h
downloaded = sum( [task.downloaded for task in tasks] )% D' Z( N' Q0 i2 ~' a' N7 X
process = downloaded/float(size)*100% b5 L; \) [2 `, G3 l. }
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)' I8 c, N1 G) G6 o o* W+ f% x
sys.stdout.write(show)
; c: S# |3 V* u( p* [- D$ h% N
sys.stdout.flush()1 i1 `2 \ ?% Z- Z\" ]% @4 D
time.sleep( 0.5 )
6 z+ ^( t* T& f0 @. L+ g0 y
+ t7 C3 ]4 X o/ W
filehandle = open( output, 'wb+' )% \( o( i! I R2 h3 x# }3 d6 I
for i in filename:
- g: D, R9 r$ W* D2 W
f = open( i, 'rb' ), c% U/ _/ e3 O5 p L
filehandle.write( f.read() )
4 G\" j) G; }8 u+ ^) z; ^# U
f.close()3 N7 ~, S- O\" Q\" Q
try:
$ U, y& A- q9 ^9 `0 p: {6 }9 q' ~) ~2 i
os.remove(i)
' M/ K; D- R J9 ^3 ]/ V& l' B2 s
pass& W* z8 A+ o5 l. ]# G, h6 q
except:3 ]3 X6 ]8 T2 ]' t* f
pass$ K. T: Y* c\" N! T
3 a& f: V) B( J! E. }
filehandle.close()\" a$ k% G& w/ }6 ~
1 K* Q; m' I+ F
if __name__ == '__main__':0 H8 f( Z\" ~4 t5 m6 E0 ^1 p
url = "http://xz1.mm667.com/xz84/images/001.jpg"5 B% ]4 E ^5 F) R& b% m! E5 l
output = '001.jpg'. g U& ?6 y( F% D8 E3 c4 N% G3 K& T
paxel( url, output, blocks=4, proxies={} )