多线程下载图集

[复制链接]

字体大小: 正常放大

2744557306

1192 主题	4 听众	2946 积分

该用户从未签到

电梯直达

1^#

发表于 2024-3-31 17:18 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

#!/usr/bin/python
3 \0 S4 r1 A1 `# T9 ?/ B+ t\" [6 m
# -*- coding: utf-8 -*- |) t c, @. [4 I' T% x
# filename: paxel.py% D4 q9 ?( E. l! c6 i+ z3 @
( q( c% v1 t% t. P2 i9 u! d9 v* F. `
'''It is a multi-thread downloading tool
+ f& w4 V5 U. o, ]# u! a% n
4 |9 Y\" W* Y T0 m: g |
It was developed follow axel.$ B3 U; E2 K\" ^$ e
Author: volans
) R5 A: z! b$ k7 R) c9 t' [. F+ O
E-mail: volansw [at] gmail.com
5 |7 X, p2 u% M/ w
'''0 ]/ D+ f. l$ V/ B- l% b
\" T7 k7 x0 t4 \' `
import sys9 i- S+ Y7 g- @+ I
import os6 h a# K y6 P9 O; ?
import time4 {' [, @- M+ F) K0 b: a4 }4 m
import urllib
* N% l# |\" {3 @3 M9 Q( q6 y
from threading import Thread; R6 L5 Y. A4 D8 r\" _8 T# h
( C: G1 e9 l- T
local_proxies = {'http': 'http://131.139.58.200:8080'}
9 k5 M0 M5 F9 ?* B& P
* w; Q& }& E$ q\" q' U1 h k f. K
class AxelPython(Thread, urllib.FancyURLopener):
3 Q; f* G1 }0 Y3 M
'''Multi-thread downloading class.' C t0 ]- \ y9 U+ o9 Z
+ x: N\" J# l8 O
run() is a vitural method of Thread.
. j& d3 J+ V o* `1 @+ S& P
'''
4 U( U) W4 h' ~\" E, G5 W; x; `+ i
def __init__(self, threadname, url, filename, ranges=0, proxies={}):
+ b$ \5 p# v) o0 H
Thread.__init__(self, name=threadname)
( P, ^+ S ^3 g# X& I8 ~' I$ e
urllib.FancyURLopener.__init__(self, proxies)! X O `/ q6 a) ]9 Q4 C
self.name = threadname
+ Q# P( c7 e) S' E
self.url = url$ O6 N- \) y9 ~6 O2 x0 O
self.filename = filename
9 h* l- v% l# N1 @' |6 S5 z% ^+ N
self.ranges = ranges
3 d% P5 {% V\" X; D, _1 O
self.downloaded = 06 O2 g( o, w8 M: r( O* w# H
4 h; H5 M\" K* q
def run(self):
: q1 U& Z4 f& H, j- \1 Q
'''vertual function in Thread'''$ p! Y3 w( y: m; F* F
try:
& V/ P( Z# b0 \7 s6 Y/ d0 a4 K! B
self.downloaded = os.path.getsize( self.filename )
* f3 Y. v* t( ~2 I
except OSError:
$ a6 J' D* E0 r& B
#print 'never downloaded'
; h. I1 T0 ]* N6 n* K2 }
self.downloaded = 04 G f% j/ }1 w* T$ R6 X2 I9 v# V
o. T\" C2 E2 `, F. m
# rebuild start poind
% i! y9 _7 L# c$ ?2 r8 E/ [4 Q
self.startpoint = self.ranges[0] + self.downloaded
\" I4 O8 x9 Q9 d' D6 A
0 A3 o4 h/ |3 L7 [1 [
# This part is completed+ @% ~, n& X5 n* n; C
if self.startpoint >= self.ranges[1]:
% o2 Y* L\" F0 d& Q( h
print 'Part %s has been downloaded over.' % self.filename
8 ]: o4 d3 R) G; U9 E
return6 |: i9 H9 Q5 x\" M$ N6 _, P7 m
5 f9 y$ {\" Y( `6 }/ K
self.oneTimeSize = 16384 #16kByte/time8 E% M/ X( S9 {
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])9 K! l) F! U* ^5 v7 g& ~! e
$ b4 s! k$ ~- R% u* I. i
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
/ i0 m5 k: N* t* U9 i
/ @3 H. ^7 ?2 @: x; s; t- d5 b
self.urlhandle = self.open( self.url )$ Z6 o |3 l5 O7 I) g$ ?
) g+ \! V( n3 q
data = self.urlhandle.read( self.oneTimeSize ); [0 U z+ q' b
while data:3 U' p% Q1 t5 T u q' J
filehandle = open( self.filename, 'ab+' ) |% Y- d2 B- w
filehandle.write( data )
$ f5 e) r: @9 g; b( c
filehandle.close()
7 E7 n( \4 x6 `2 D! ^4 H0 Y
! V( E\" s4 e0 c6 i
self.downloaded += len( data )* @4 M\" l- o ]5 A% ~# H8 w& [7 [
#print "%s" % (self.name)
9 `: y+ f) r+ J3 p
#progress = u'\r...'
, h/ R3 d7 Q* }% V+ v\" B
' d/ v\" h6 R3 m4 T( l9 {1 X- ?' j
data = self.urlhandle.read( self.oneTimeSize )& c( d3 N* b* H8 M0 J. z: [: r: N
8 b) m$ L% j8 m/ a% Y
def GetUrlFileSize(url, proxies={}):! x: Q# p0 V8 k+ Q) u$ L
urlHandler = urllib.urlopen( url, proxies=proxies )8 ]1 e0 _! N9 b
headers = urlHandler.info().headers
; B, h ?\" `! m9 t9 C4 f
length = 0. B' ?5 p' S6 O% N
for header in headers:; B' u2 E& l9 T$ k, ^9 d
if header.find('Length') != -1:' g+ c e7 ]1 z S\" X
length = header.split(':')[-1].strip()' ?- r4 s0 V2 H0 }' _
length = int(length)( _3 g! [2 F) z\" v# z
return length6 a' M e1 \0 s( h( [
5 q1 o+ i4 g3 S3 v5 v2 ^
def SpliteBlocks(totalsize, blocknumber):8 R6 N$ T! k0 e ^- f0 i
blocksize = totalsize/blocknumber
$ I4 Y7 O5 C5 E0 A I
ranges = [] Y3 H- q( [, q5 e
for i in range(0, blocknumber-1):
6 t, R9 h8 Z/ O! q6 L5 P8 ~+ L
ranges.append((i*blocksize, i*blocksize +blocksize - 1))
( [+ R5 w/ E8 b( y1 d) g
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))9 n0 w) X! @* P$ B3 l$ z' F
{5 S/ `8 ~# k* R4 j# C
return ranges
. }4 H' @' ?, d
def islive(tasks):' J5 b) b6 X2 ^5 o, E' n& {
for task in tasks:
; v1 Z: J( M9 \; T0 f b( J3 O
if task.isAlive():
# R0 B/ F0 R/ ~; a# U0 O
return True' r0 X/ V' D3 \1 l6 s/ @
return False
\" |& m) z1 W- H' O. P' T4 M
& ~# J- f0 @2 k2 }/ L' r8 |$ t- F
def paxel(url, output, blocks=6, proxies=local_proxies):
. Z% f\" v9 }\" h3 x3 Z1 h
''' paxel+ h8 E0 j3 l6 P) u9 n
'''
# d1 J5 U/ J% k0 M
size = GetUrlFileSize( url, proxies )
$ V c! @0 _- k! S' S/ C2 Y
ranges = SpliteBlocks( size, blocks )$ y5 n/ t' F* P# N6 Q
; K! r$ C: Y5 E: W7 V+ u; b
threadname = [ "thread_%d" % i for i in range(0, blocks) ]
* U$ s6 G+ {) P* [' K& P6 q( U; i
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]9 I t# c& l: {$ ]
- O% g9 v\" s9 X+ d$ W\" a: |
tasks = []# I- p8 Y* Y) f
for i in range(0,blocks):1 a( L4 V8 ]7 Y' C% u8 z
task = AxelPython( threadname[i], url, filename[i], ranges[i] )+ p4 I; T, N. p5 K
task.setDaemon( True )
* o) X5 \6 \0 \' I4 V* W
task.start()+ M, s\" A. [& ], c: Z& f
tasks.append( task )
8 j2 J2 r/ N0 Y5 M: u
9 o: b( l2 ~! U+ [: A% J/ ^4 y$ J' B
time.sleep( 2 )6 w9 w4 ]5 a, ]2 q\" e
while islive(tasks):7 M+ T1 Q3 R8 a: Y
downloaded = sum( [task.downloaded for task in tasks] )9 s6 V! I* M\" X7 o6 [ y
process = downloaded/float(size)*100
$ n2 ~+ t b3 j2 Z* [$ i2 G0 X7 U
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)& B8 \6 d% A- \# z$ j4 Y, S4 g
sys.stdout.write(show)
8 b. C0 g9 p! F {) {5 f5 w+ r) i
sys.stdout.flush()
* ]6 g; m: [( |6 F8 |( b% H z2 L
time.sleep( 0.5 )
5 ?\" V c i3 Y: C* r
n( y) T% A3 {& N+ U! ^
filehandle = open( output, 'wb+' )
3 w h0 @: [5 [6 |5 G- z$ }\" f
for i in filename:
$ s; `! S& ]0 Y) n* k
f = open( i, 'rb' )
5 q& W9 _8 f6 k1 ]\" f
filehandle.write( f.read() )* t! u4 N1 T# Q, B. a
f.close()( _! R! z, N3 p, p
try:- i' e+ Q% A, U6 G
os.remove(i)
2 L) r\" b8 D5 b- }: s4 I
pass
' V# K% f i$ `\" D, D9 C! m+ Q
except:\" h5 B8 c+ D& m; z$ v
pass, d) o6 c6 P\" v
9 d/ y& B0 `. _% a5 C9 t1 q* @
filehandle.close()1 e# }+ u/ A7 \7 ]- B
6 _# N- a9 V/ {
if __name__ == '__main__':9 Q9 S\" S9 r( U5 Y1 v
url = "http://xz1.mm667.com/xz84/images/001.jpg"
! X7 _0 p4 x, A- y( v
output = '001.jpg'
1 C G( x L2 m& T! }8 P
paxel( url, output, blocks=4, proxies={} )