多线程下载图集

[复制链接]

字体大小: 正常放大

2744557306

1192 主题	4 听众	2946 积分

该用户从未签到

电梯直达

1^#

发表于 2024-3-31 17:18 |只看该作者 |正序浏览

|招呼Ta 关注Ta

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

#!/usr/bin/python/ X9 T8 o, k& b- a' a0 k
# -*- coding: utf-8 -*-
\" z2 }\" @- q/ \5 o3 Y
# filename: paxel.py
- v Y) f\" n `
8 m. g8 \7 Z& M. `; l o
'''It is a multi-thread downloading tool3 m7 c; f: z\" n1 m& X p+ n K* c
) j; q2 b6 R% X8 \
It was developed follow axel.1 I, Q+ s\" j+ i' h8 T
Author: volans
, H8 p$ D3 ]8 D) I d8 }* {& q
E-mail: volansw [at] gmail.com1 J: h# `% M# r; C
'''
# ~* x! G9 s6 r1 V\" o
* f& \. J8 @8 b. {
import sys
: c' ~' \: S( R
import os
\" V4 n1 z' \& f6 j
import time
* e\" @2 q+ n& `) Y7 d\" x
import urllib2 p: P$ d6 }6 K+ B& S) N* R; R
from threading import Thread
4 g: [. E9 j% o5 n
7 f2 B- d3 t% b3 @
local_proxies = {'http': 'http://131.139.58.200:8080'}
1 m8 y( K6 A* b1 G0 \2 N- P
+ r0 H7 s7 I1 [\" i& {8 p
class AxelPython(Thread, urllib.FancyURLopener):
& Y4 T' }6 E0 y/ c$ f2 g
'''Multi-thread downloading class.
n3 S! S6 ]% g- o9 T0 k
7 ~, [. k! N! V- r
run() is a vitural method of Thread.
. _/ P$ \) a# l0 Z
'''' A/ y6 }: _( Q: {
def __init__(self, threadname, url, filename, ranges=0, proxies={}):* }& q' z7 k( h7 ^; i+ H
Thread.__init__(self, name=threadname)
( d4 l# A8 d1 H [1 r
urllib.FancyURLopener.__init__(self, proxies)
0 S' G$ e8 [) i P8 a9 x. l% w' \( m
self.name = threadname
0 C$ V% Z% V, }2 Y\" o
self.url = url
* a& W; X# @& {0 F& P
self.filename = filename0 T' h3 T6 H1 `4 D |% N
self.ranges = ranges; n# L6 }; J( E d3 H D
self.downloaded = 0. G' \6 D& f3 R+ }3 \- j) R$ c! c. l
- M) w0 J5 a; l# H: \9 R
def run(self):
6 f: C1 g\" G; E
'''vertual function in Thread'''
( U: ^( A. e( C
try:
* }! T( X+ B! k' D+ Y
self.downloaded = os.path.getsize( self.filename )! C1 t. E5 W5 N9 I, U
except OSError:# Q8 U5 g' F& |+ @' I) i4 U2 ]; ?
#print 'never downloaded'7 I* q1 U9 o: \
self.downloaded = 0: s2 ~* J! @$ V0 J; A6 w
! @5 j+ X\" I4 ]0 i3 \6 G6 _9 ^# i
# rebuild start poind
* {/ d+ J; G% B0 I/ }' ?
self.startpoint = self.ranges[0] + self.downloaded3 c& }. F' a! R& N
- Q7 V1 i/ x5 \! H: ?, m\" e
# This part is completed
8 I9 F$ \+ s# b3 a- m# y6 G& n
if self.startpoint >= self.ranges[1]:1 a! t8 [8 a$ F6 a1 }
print 'Part %s has been downloaded over.' % self.filename
9 g9 f! R8 a! q1 X
return5 M) a4 p* q7 d @- }
4 d4 l9 f/ l6 _1 J, m- T4 T {
self.oneTimeSize = 16384 #16kByte/time
. }( E3 b6 B- ]2 K
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
3 c6 J- r4 M/ l) d5 t
) D% U) k8 h. @' V
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))/ f7 \$ `) k1 G! x+ b% T
1 Z0 d% D/ d3 `, h5 [
self.urlhandle = self.open( self.url )+ I7 v) u5 b: `6 }1 X3 P0 \
- W0 G+ K1 T3 Y& q b. h5 ^
data = self.urlhandle.read( self.oneTimeSize )
. T1 s. v0 u5 S) m
while data:0 |( W' J7 s& ? U1 _ A0 F1 E- T/ }
filehandle = open( self.filename, 'ab+' )0 l, W% P5 y0 G. S2 }7 m4 k5 k
filehandle.write( data )
) _ E3 [# R2 R. A: W) L7 f% u9 _
filehandle.close(). @) O& u+ X6 O/ \8 a% `6 _$ b! H
+ I# E0 c4 W, D, H8 f# N) N
self.downloaded += len( data )
0 o. C F# h( C
#print "%s" % (self.name)8 q4 h; \$ q: g/ `' v9 d
#progress = u'\r...'/ x, u\" u# g% E' @( m$ v x
6 z D$ l/ l6 p/ o6 z: |
data = self.urlhandle.read( self.oneTimeSize )3 v$ h# R: C2 O/ U& K& t6 j' ]& g
/ `2 ~/ L) W1 Y8 s& g+ n0 V- Z% q# ^
def GetUrlFileSize(url, proxies={}):! {3 o# [& K) G2 W/ k% Q' q
urlHandler = urllib.urlopen( url, proxies=proxies )
n1 W: ]7 g# I
headers = urlHandler.info().headers
/ F' D: }* K, [, H
length = 0
/ Z( l7 l\" H+ |9 Z' r4 v$ O7 L L9 g
for header in headers:! r9 ^, M4 {8 a# A# ]
if header.find('Length') != -1:
8 Q0 E6 L5 c( q: C$ V0 y- S! _/ O
length = header.split(':')[-1].strip()- s9 J& ^. O2 U' d! Z
length = int(length)
H$ g3 a$ n% }# B3 [2 \
return length
( l9 s4 r9 O& r. h2 J* N u1 V
9 q( r# {8 e) G3 k9 q1 Q' d4 ]9 A
def SpliteBlocks(totalsize, blocknumber):. G5 l* z' \3 L9 c5 U! {0 ]3 K+ w
blocksize = totalsize/blocknumber& C+ I, T/ j5 w' S\" Z
ranges = []
% O- _! Y# n6 v
for i in range(0, blocknumber-1):
* s8 ^: c. N1 o7 N0 q& p) m' C
ranges.append((i*blocksize, i*blocksize +blocksize - 1))
\" S0 H1 R! ?. D' Z( @6 T2 d& M
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))7 H G\" G9 u0 Y0 Y/ R2 ?
3 g6 A. R. @% M! @$ o! ~6 }. k
return ranges
( o) a\" K1 M) ?5 o e/ A1 M
def islive(tasks):
, W D! R) F* b$ ~/ X' h M+ u
for task in tasks:
+ P, A+ U! u% J/ m& t6 J6 s
if task.isAlive():8 q; X# o, g5 }7 m1 O/ V
return True. N. s( e# o\" ^4 \: N! \8 z
return False7 m; n2 v- Y' |; @# w9 {
5 U Q/ B( x) n3 t
def paxel(url, output, blocks=6, proxies=local_proxies):
. m& B- Z2 n( g+ s
''' paxel. Q/ o: P$ k9 w7 n% p
'''
. Q+ q7 u' T2 }; n% b% k: \
size = GetUrlFileSize( url, proxies )1 L5 w7 Z; x. I9 u: J; k+ X
ranges = SpliteBlocks( size, blocks )
d! [, y! k/ a
, T8 i# z2 B* m N) L8 |- s1 ?# \
threadname = [ "thread_%d" % i for i in range(0, blocks) ]
$ Q( a5 f1 t$ @9 }
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]' D$ m4 r( ]. T, \0 W. l
0 b) o1 |9 o# q\" O, d
tasks = []2 A, | y- |5 n/ X/ n- J
for i in range(0,blocks):+ {# O- F. z2 v7 F$ A5 ^7 {( X
task = AxelPython( threadname[i], url, filename[i], ranges[i] )
+ J6 C3 ]\" p: j. N6 b' j- t2 G
task.setDaemon( True ): K( \6 t3 z\" B9 z% Y4 }\" q, \
task.start()
) T! c8 T0 H1 T
tasks.append( task )( {, U7 T9 l9 _3 D0 p
+ j1 X* u- Z! M' T3 o9 V
time.sleep( 2 )
+ |/ j' |( o0 B& d* X9 C. t( @
while islive(tasks):9 T7 f8 {+ V\" T! w
downloaded = sum( [task.downloaded for task in tasks] )' Q3 [\" _$ v$ X0 y# K* t) l, b
process = downloaded/float(size)*100
\" z% @, n8 p# S\" o0 J2 X5 h0 }
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)! K& ~9 ^5 f2 c3 V( r% y$ ^8 X
sys.stdout.write(show)1 `+ j1 ^, X6 E1 @1 `
sys.stdout.flush()
$ E. P7 j0 l9 H, F
time.sleep( 0.5 )2 M, s' l; b. x( ^
/ j3 b\" T/ P( T3 l0 S% r
filehandle = open( output, 'wb+' )% ?6 T2 S( o\" V
for i in filename:5 u) [8 N; D _7 G- S* P* P4 b
f = open( i, 'rb' )+ V% b. v5 K( {0 m8 H
filehandle.write( f.read() )
6 t B$ Q. |2 {- b
f.close()1 l( ^4 }# Q/ I0 X0 H' l6 T( d
try:
0 b6 u8 t# {, j- s, l4 n6 ?
os.remove(i)
5 Y$ y* Y0 F5 }6 J& i& r
pass
! |+ T, h1 s- ?( a- T
except:
. d4 |* G9 D) H( r; V. V+ [
pass9 b1 r\" U* y) f: D
* C; D\" S6 R$ f+ h\" A
filehandle.close(). }& d% Q# g0 M8 ? a3 L
0 K y) }3 l, w! \4 y6 W
if __name__ == '__main__':
8 U+ C; E2 ~5 Z4 G# [8 h
url = "http://xz1.mm667.com/xz84/images/001.jpg"0 y0 B* C0 l# K1 L
output = '001.jpg'
& y. r$ p7 e9 P
paxel( url, output, blocks=4, proxies={} )