多线程下载图集

[复制链接]

字体大小: 正常放大

2744557306

1189 主题	4 听众	2934 积分

该用户从未签到

电梯直达

1^#

发表于 2024-3-31 17:18 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

#!/usr/bin/python: ` S2 \, g2 {\" b
# -*- coding: utf-8 -*-8 W% @2 ^: `% g1 X+ j. r2 M
# filename: paxel.py8 c0 w7 E- h& U4 o8 J! \
- J& U2 ?8 s3 q0 L3 G\" ? S: y' e
'''It is a multi-thread downloading tool$ t\" U3 [9 D& O) y' ?/ q
$ L\" M; P3 g\" r9 S& p( k
It was developed follow axel.
0 O. r) T6 J7 g8 [; m\" {7 Y+ a) u
Author: volans1 u\" s' U/ Y& k/ S, ]9 @/ x
E-mail: volansw [at] gmail.com' N\" I. w1 S$ Q7 K* c. g
'''3 o/ U: p\" d$ _4 V+ a6 s
2 }+ T' U) z5 w6 E+ X+ U$ G
import sys1 k& p# c; u5 y& t
import os3 W9 Z\" E0 j7 [3 Q z
import time
+ B\" R& X& A2 x! D. I
import urllib2 c9 V; f k- a9 _, C
from threading import Thread$ Z! r! X5 {% e\" O8 ^9 J8 R
: U3 m% i z2 J8 h- k9 d$ N
local_proxies = {'http': 'http://131.139.58.200:8080'}
$ [% ~/ K. K; v2 M+ g
% o0 B\" Z0 s& {5 \* B: F
class AxelPython(Thread, urllib.FancyURLopener):
; |$ I\" a- D1 x4 k2 o
'''Multi-thread downloading class.3 e, ?/ D- ?8 Z/ k; C
: [& F7 q9 |3 ?+ X
run() is a vitural method of Thread.
0 x# s4 Z+ g6 c7 d9 Y9 s
'''
' p( b8 e& ~5 r) Z# T: k& u1 x8 x
def __init__(self, threadname, url, filename, ranges=0, proxies={}):
+ V) ~% ^/ y' Y# G* b G
Thread.__init__(self, name=threadname) T\" b; b/ t5 k* r( n9 K
urllib.FancyURLopener.__init__(self, proxies)
) Q( R% Y8 \2 b9 @+ _' ?- S
self.name = threadname
$ ~9 q4 n, z6 Q: o
self.url = url3 K- a/ k3 L7 I3 b
self.filename = filename+ V2 c$ R1 R5 a1 r& @5 e$ V0 z
self.ranges = ranges R# W/ z; ~0 \6 H% {
self.downloaded = 0* p4 t8 ?* \) C. M& R
1 v3 U9 D7 o1 S9 o$ E) ~& t
def run(self):$ Z$ d$ {' p0 i% ^' Y0 H6 f! a G& G
'''vertual function in Thread'''
' M# D# k/ v/ A
try:
3 n5 w1 W4 j6 W# p! K\" l! i\" C, _3 G
self.downloaded = os.path.getsize( self.filename )
2 d4 j. d2 z6 o/ v
except OSError:
/ Z9 @# s' @ y0 V8 S: B g1 I3 N
#print 'never downloaded' Q S/ R1 i$ k5 Z1 h4 B8 v% p
self.downloaded = 0- I& W( X* i\" U0 r( d
8 T( _8 R4 }/ w1 x w. H
# rebuild start poind2 S3 q& F4 h0 p
self.startpoint = self.ranges[0] + self.downloaded
/ p1 o' Q% ?, p# g6 q5 V0 B
8 H' n/ |4 R* j+ b. \9 D; u. ^8 l. @
# This part is completed' u- i\" A- V& X$ f2 {
if self.startpoint >= self.ranges[1]:
* D- o! r0 b: }8 M
print 'Part %s has been downloaded over.' % self.filename: @# {! N4 ?& {/ ^
return: \( V' K( u4 }; [- }* R
6 j# Y2 y( d7 A( X N1 J
self.oneTimeSize = 16384 #16kByte/time' m5 i' S3 _ f3 \5 M
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
\" S% m$ b! A) E/ J: b/ t: P\" q
- z% y3 j* x1 k; K0 ?3 M
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
6 O0 \5 Y1 n; b/ R# l! W+ c$ y
9 X; _+ I! x7 r: B8 H* P8 W6 @9 D
self.urlhandle = self.open( self.url )
3 e+ t, @& R( [
- w# \6 E+ e5 ?2 o Q5 r: I- S
data = self.urlhandle.read( self.oneTimeSize )
7 c$ m6 `4 j6 r8 O3 ?8 G, x }
while data: Y* h1 C$ a* M0 E6 v0 A
filehandle = open( self.filename, 'ab+' ); ?# s- p9 ?2 |. W3 V/ b
filehandle.write( data )
& i/ U f. G( c# S& y; `
filehandle.close()
9 O, P' { A6 \' a8 c0 o
8 ]$ }' u! g4 ?2 }( }7 r$ @
self.downloaded += len( data )0 A% Q: B9 y q
#print "%s" % (self.name)/ q# A& z\" a# |0 T' h! L7 |
#progress = u'\r...'
5 l3 s) T0 `# C4 {
* z8 A% z% [! b2 k, B& ~7 ?+ _
data = self.urlhandle.read( self.oneTimeSize )% m q- H. a. Z- x
; i ~ e# u; r' N) W
def GetUrlFileSize(url, proxies={}):
# \& C. F! c% b
urlHandler = urllib.urlopen( url, proxies=proxies )
, m# N- `0 A% [
headers = urlHandler.info().headers7 i; U4 N0 Z6 ]5 f2 [
length = 0
. k' S& B4 B B4 K\" |. Y) H
for header in headers:
0 |) A' H% E5 ?/ K+ r/ K4 L! I
if header.find('Length') != -1:$ }. y5 R% I: H* T2 x. [
length = header.split(':')[-1].strip()
, I, V7 N7 e2 d
length = int(length)
3 G& z4 k0 i L
return length
5 q; w' s9 k, [; b8 V# G! J* ~0 {) D. v
* Z0 h6 W0 Y( P1 D
def SpliteBlocks(totalsize, blocknumber):- m* \6 R& K b5 v+ [
blocksize = totalsize/blocknumber
+ o8 C8 I1 }* l E, w+ Q9 d! B! g
ranges = []/ c/ F4 |' h2 k2 q
for i in range(0, blocknumber-1):# I6 X: B9 Z3 j7 V. g
ranges.append((i*blocksize, i*blocksize +blocksize - 1))
* O: M$ _! Z+ }( P
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
6 M+ H9 s2 A) P2 m3 z\" c! e+ k+ @
8 I) ] @/ e+ v# C% k: b
return ranges+ F+ Y1 E e; y
def islive(tasks):
+ Q$ R+ ~( |3 j& S
for task in tasks:# r4 R4 F7 I\" N
if task.isAlive():
* B2 g, q6 E$ _ ^ P
return True
$ h. i* s1 s- f# `# s1 j; n9 N
return False3 a+ r8 c1 R( `2 k. `2 R: m, m. x
3 V4 l( A: y2 ~
def paxel(url, output, blocks=6, proxies=local_proxies):; I5 |# |2 U& }$ A f
''' paxel7 U: I2 f5 z) B& b4 F& e
'''
; D9 f. ]: {0 x+ w
size = GetUrlFileSize( url, proxies )
) \4 W$ ]4 V0 q1 Q' h1 e
ranges = SpliteBlocks( size, blocks ); V W! B& ~1 a9 J* G- f4 u
\" Y+ K: _6 a! _% B0 A
threadname = [ "thread_%d" % i for i in range(0, blocks) ]
^- H2 _( {1 a6 \- w! Y- P8 E( M
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]& _; S' T' Z8 V
0 C V# j3 `! {9 P$ {, ^+ Y
tasks = []$ K- t& u8 k: H5 ^- g4 c& M
for i in range(0,blocks):
: \. O; O6 O% C: w
task = AxelPython( threadname[i], url, filename[i], ranges[i] ), I9 K) j. |. g+ }/ {
task.setDaemon( True )3 h+ p) y; n# A' U& x\" X+ W- J
task.start()$ G: M* o+ d$ Y1 h$ d( T
tasks.append( task )# U5 h! e3 ^7 @' j5 L6 M\" b! O
! Y- p- B1 z+ a) Y; _
time.sleep( 2 )4 Y+ U5 q9 |) r6 M0 S* B2 S
while islive(tasks):
' l( ^% X! Y& ?4 D8 ?# t/ X5 Y' I
downloaded = sum( [task.downloaded for task in tasks] )
$ ~8 \1 X+ l! H( U\" w+ U/ m j
process = downloaded/float(size)*100\" o$ f0 Q$ B/ G4 Y7 I' b3 o* o/ l, T% c/ h
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
4 Y8 |\" S6 l, J8 \7 U
sys.stdout.write(show)
2 X5 j4 s- T+ A7 Q4 M. h\" c9 h
sys.stdout.flush()' b; Y! H5 y2 |/ w. _& z; M: K
time.sleep( 0.5 )8 g\" t6 q! k; h8 R+ G+ p8 T; ^! t
! r. J\" q( {\" Q( x, x9 D+ j9 a9 w
filehandle = open( output, 'wb+' ) s; E* J# }+ c- g( _; L
for i in filename:# y3 [5 x. ?( U- j' Z+ |
f = open( i, 'rb' )
8 x1 O( e! h+ f' z; L) t
filehandle.write( f.read() ); H* E2 a' m Q- B, t% |$ W2 g
f.close()% k) k: p$ \8 C1 H' g) C O
try:4 ?3 Q% u$ v\" M: t8 `9 X$ f
os.remove(i)9 i# @( V, f- A6 I\" R
pass
8 U4 z( v0 T3 q+ b% @4 ^; u
except:\" X; P& S: ]6 u1 Y2 k) x6 y
pass
: [. r8 V3 s2 f: Q9 W9 }* a: o
8 [4 M$ u4 t/ W1 M
filehandle.close(): }9 Q* a8 l\" Z. B! d p
* n+ u7 V; U9 C& z W2 J/ P+ l
if __name__ == '__main__':7 Y% G& f V2 T- {8 N d
url = "http://xz1.mm667.com/xz84/images/001.jpg"5 j9 h1 O1 _: U\" l
output = '001.jpg'0 {# `+ d# a: e) T' ]. `, d) Z
paxel( url, output, blocks=4, proxies={} )