多线程下载图集

[复制链接]

字体大小: 正常放大

2744557306

1192 主题	4 听众	2946 积分

该用户从未签到

电梯直达

1^#

发表于 2024-3-31 17:18 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

#!/usr/bin/python
. W! ~( Y, S\" s
# -*- coding: utf-8 -*-- ^, L; x: d( s- k. |. h
# filename: paxel.py' {6 A7 @; p5 S- Q! p( S
5 v' g+ i9 @4 A) I
'''It is a multi-thread downloading tool
& ^7 T1 Q4 n9 h$ d( m1 K4 M% z
; j3 [0 y; x4 E9 r; U\" J
It was developed follow axel.2 o/ X' S+ N* E2 r& s/ c
Author: volans
) T$ m* I# c0 }/ a2 H# K
E-mail: volansw [at] gmail.com
/ F+ J( i2 d# p, n
'''+ U\" O c; h& o& |2 Z; l# [. y) ]
2 v5 C. E' S( f# r! |) }
import sys3 R/ K\" M+ T; t9 a
import os6 S+ W! _6 J2 H
import time
1 ]& [3 q3 e7 a9 p/ A# ]
import urllib
& c* Q5 r6 l8 [
from threading import Thread) q' `) `2 A1 R0 L0 j
& |1 e4 @3 u& |2 F. S
local_proxies = {'http': 'http://131.139.58.200:8080'}/ w. p, d5 L4 B- H' ^! |
5 o' P( B) f5 X# [* e+ F( n
class AxelPython(Thread, urllib.FancyURLopener):
: n) P6 a7 X/ D1 l
'''Multi-thread downloading class.' j+ d; h8 H4 ~0 t1 _% R H
2 k y' `3 o2 m U/ ]+ D# r- e
run() is a vitural method of Thread.
8 m) X5 W( n; J- h. s T\" G
'''7 J# ^+ I0 M1 D( k
def __init__(self, threadname, url, filename, ranges=0, proxies={}):
8 E/ i* ?' E: r: r; y
Thread.__init__(self, name=threadname)
+ n/ d8 A3 _. r0 L5 Y) d* M
urllib.FancyURLopener.__init__(self, proxies) a9 ~0 S6 \. O
self.name = threadname
4 }1 l! B& D5 H) u
self.url = url: q. l, `, A* U4 T
self.filename = filename& S( G- f3 d* f\" v b
self.ranges = ranges, } u. _8 N3 }
self.downloaded = 0- d0 ^4 J$ d9 R6 i, n' W f
# {3 f4 [; _8 H B# k: `; G
def run(self):1 O/ C\" I) W# k5 E) g1 D H+ ?
'''vertual function in Thread'''7 F/ J. k) r; M: d
try:
- \' s\" r/ A, ?' \
self.downloaded = os.path.getsize( self.filename )
2 A7 g! R% U r
except OSError:' L3 f& J' o- }
#print 'never downloaded'# z+ i. s, f! q7 Y
self.downloaded = 0
; k, A* s5 ~# u
6 @- H! f) f8 I! ~+ m
# rebuild start poind
+ s2 Q$ `7 r+ V0 }
self.startpoint = self.ranges[0] + self.downloaded
: W: A0 k% I+ o, Y% P3 L1 j
/ `: s3 e8 \0 ]: n4 ?4 O
# This part is completed
/ T9 {! V: y W( L
if self.startpoint >= self.ranges[1]:
% i4 R9 \- u1 }
print 'Part %s has been downloaded over.' % self.filename
9 f) J1 V! N( M+ _9 k5 f7 y W
return
% }7 Q! B\" K6 m6 c/ U
+ _9 ]\" _7 `9 g
self.oneTimeSize = 16384 #16kByte/time
I+ Y1 o1 P* a. w' ~\" t
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
2 c7 z9 s( b: ~6 _5 O$ H: b
7 Z5 f; ], m9 f* Q5 E' g/ Q
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))3 M! y5 s* l( P/ n# V
9 b6 L4 {+ C' S, T, F' I) @6 \. `
self.urlhandle = self.open( self.url )2 W$ z& p! f\" n. E
) ^1 I0 ~: _6 j7 C, }+ L
data = self.urlhandle.read( self.oneTimeSize )
1 | [0 M# m% e: A8 [/ N+ p
while data:' i2 Y1 h% D3 y. x1 o& w, b3 a
filehandle = open( self.filename, 'ab+' )) a8 w8 d: G; L% I+ ^
filehandle.write( data )) L. D$ P0 v* Z0 v B8 E; n3 F/ B
filehandle.close()
$ {6 G/ H6 O* y) M5 g
]8 e\" }8 }/ d0 _# @ b
self.downloaded += len( data )
3 e5 l6 S\" b) y. ?( R* {- ?
#print "%s" % (self.name)
6 p\" ?$ K3 S% z/ w3 ~) R
#progress = u'\r...'
* s2 y5 v1 y; n# r7 M1 j7 y
u g0 t! q. p% b6 @
data = self.urlhandle.read( self.oneTimeSize )& l# y3 j6 Q& l* o' l, [) P8 d; T
# _- c# R; a6 G! d2 F
def GetUrlFileSize(url, proxies={}):
7 I' j J% V% S0 b g% n* a
urlHandler = urllib.urlopen( url, proxies=proxies )& a8 i. t# E3 e) p1 @! k
headers = urlHandler.info().headers N7 v7 z. G6 A' K. x
length = 0, r# w\" P& b2 m6 t, r
for header in headers:$ F& {1 |# \4 }2 o
if header.find('Length') != -1:
4 ~% p9 d( m5 m
length = header.split(':')[-1].strip()
. s; o! n- ]4 c# s% E
length = int(length)0 i. _% F/ X4 q% t4 t' w7 b! a
return length
1 k: v X2 ^: j& F: P
# ~$ K* } S/ v) m% d# P
def SpliteBlocks(totalsize, blocknumber):- c0 V: ?; r; C/ Y; c6 e\" W7 d
blocksize = totalsize/blocknumber
8 }( o+ F3 H9 s9 K( \
ranges = []9 H\" e$ B8 C6 ?+ J6 x
for i in range(0, blocknumber-1):, G9 n, P9 j( [1 E# v5 r
ranges.append((i*blocksize, i*blocksize +blocksize - 1))' T; p# |) q: Q5 o& U
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))) ?: q- Y& j' f8 Z8 |( t# C
$ V. A `( y' @
return ranges
5 l, V% d! G B- b
def islive(tasks):
4 K, _. S& p6 ^, D: ?/ V
for task in tasks:
' K+ n/ u0 h* l. K) q
if task.isAlive():
6 w\" I) y q5 g9 m' ~7 p& p, M
return True
$ l2 ^' j* [: j
return False4 v: c6 j. F- L. j5 e3 J
: g; |5 n& g/ ]' _% i
def paxel(url, output, blocks=6, proxies=local_proxies):
! J; c# x- S, V( W% u( J; z0 y
''' paxel
. p3 R( s+ r' q' e( f! Q% O
'''5 F5 o- p( Y( C( F
size = GetUrlFileSize( url, proxies )
' F) i% H8 E4 `& l
ranges = SpliteBlocks( size, blocks )5 d+ ]5 e2 c% b9 @, l- S
U! x* p- i4 N# u& z
threadname = [ "thread_%d" % i for i in range(0, blocks) ]
/ E9 X, f( w; J
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]9 e4 T' E7 k. p, U' g+ i6 Z
, T- D. _( L/ j4 X, l' n& T, o
tasks = []\" \$ S/ {4 i' B9 L2 e& G- Q
for i in range(0,blocks):9 V$ \4 M9 V5 d! B1 K
task = AxelPython( threadname[i], url, filename[i], ranges[i] )5 }9 v0 Q Z: n' y\" c2 b
task.setDaemon( True )
. ?' C- P u5 ~( L7 N1 X
task.start()& @. g) k- a! d$ I
tasks.append( task )
* {6 L4 _6 d$ @, r
6 R9 }\" f+ l1 X. n3 |2 G% j8 I
time.sleep( 2 )
: u+ Q( i) i' K\" `- W
while islive(tasks):
* H0 W, W' K! I
downloaded = sum( [task.downloaded for task in tasks] )
; Q: N% ?# g9 Q9 |9 @6 ^5 x
process = downloaded/float(size)*1000 F* W/ ?3 x% }& O. v5 _
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
+ x, L# |& S8 F
sys.stdout.write(show)5 I' v# `5 m! m' H) v
sys.stdout.flush()0 e8 w1 P' A\" [* G\" ^4 J
time.sleep( 0.5 )* u9 e8 p# Q0 _ ?+ y& ?8 Y: ~
A: M1 I# V\" i0 V8 ?. H6 g$ G9 A
filehandle = open( output, 'wb+' )0 ~; P c: j) f6 d- g
for i in filename:
, h. w1 \: [* p* [) U/ B
f = open( i, 'rb' )
, t4 y* m) S+ {7 b% I
filehandle.write( f.read() )
v4 g! G( c5 d: \( b8 G, n0 I. L
f.close(); P& ^/ f$ d1 p4 x' s# L. h
try:2 x2 m) N' V) ^8 }; e y, x6 \/ T
os.remove(i)5 @# m' \% h4 w# J/ L7 H
pass
0 o! Y! i0 [( \# ^& w0 Y+ g6 r
except:
6 G A% F; O2 K% I/ O8 O$ D
pass. [# L2 l& _5 `# b
; b1 Z& N& n/ r\" S. q! U: w' n' P
filehandle.close(). p+ J, d7 }2 i6 q, v0 L/ b
7 Z$ ^# e2 E! X8 H& a; l1 W
if __name__ == '__main__':
: ^ J( b& {\" U' S# X# ~) k% @
url = "http://xz1.mm667.com/xz84/images/001.jpg"
, u* X9 I# @8 X4 r\" p6 j3 X
output = '001.jpg'& v# B y& H& B5 p0 b' d
paxel( url, output, blocks=4, proxies={} )