多线程下载图集

[复制链接]

字体大小: 正常放大

2744557306

1189 主题	4 听众	2934 积分

该用户从未签到

电梯直达

1^#

发表于 2024-3-31 17:18 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

#!/usr/bin/python0 T: U$ L* H\" F- D
# -*- coding: utf-8 -*-3 \; V9 P1 O; k5 _+ M; n
# filename: paxel.py
+ i4 I& y; l) i3 p: b9 C/ e$ J
'''It is a multi-thread downloading tool
5 f, l4 v: h7 t. }\" M2 A B* q$ c, C
& f! c. Z* k2 C5 C+ h
It was developed follow axel.
8 {. h; @! i; r6 ?' D/ G
Author: volans
1 r2 d( m( h. o$ t: ]+ Y' }, g
E-mail: volansw [at] gmail.com3 J7 g% P4 g7 m, `# M: p
'''- _$ u. e$ ], j# v3 y
5 J0 Y3 u% Q% M9 v2 L6 F. Z
import sys
6 U) P4 C3 Q; x4 c! M' p1 D* _
import os+ w+ }* O, w\" \) j& J! e
import time4 c( G6 Z& c) X; Y9 J# H
import urllib1 X4 e: J7 s z\" c9 Q& d
from threading import Thread: z\" H7 D# r) { u: U% c9 n
( i% ?8 O$ b, c7 g
local_proxies = {'http': 'http://131.139.58.200:8080'}! h9 I9 C- n! c4 G8 t6 g
8 I7 f6 L; |$ e; P+ F, W l
class AxelPython(Thread, urllib.FancyURLopener):
! D1 s1 z+ T W2 h3 f+ @
'''Multi-thread downloading class.2 J% y. s/ r8 b: q
. ~\" V9 W/ C# C
run() is a vitural method of Thread.
' n7 i; v/ K4 K- z$ H. G1 g# [8 l$ J
'''
% S9 M. M$ e) X* _. E4 q
def __init__(self, threadname, url, filename, ranges=0, proxies={}):
! D7 }4 b8 b1 ? e& B
Thread.__init__(self, name=threadname)
k2 N* s8 `! P& q8 n$ ^
urllib.FancyURLopener.__init__(self, proxies)! _5 x/ b- y! @* K\" X+ m2 X
self.name = threadname
+ @4 w4 n) h4 j! L' G+ R% j- ]
self.url = url6 T5 J2 j6 T6 @7 g! K `
self.filename = filename+ \6 X r( }8 c% b! _5 L
self.ranges = ranges0 P$ M3 d& Z* z, O) v8 E0 j: l2 i
self.downloaded = 01 k( [3 ]; I4 n' w2 H
$ S8 m( ?: n3 M1 j/ b
def run(self):% P; a* K\" T- L) d1 U
'''vertual function in Thread'''
5 _1 I0 H. }9 J
try:& u9 Q- D! {$ @$ K0 v8 a
self.downloaded = os.path.getsize( self.filename ) k! H9 \- \0 Q8 t- B) ?
except OSError:
9 t% Z: r4 ]$ I! ]3 \7 w: H
#print 'never downloaded'5 P1 ~4 w# M+ ?& I' L) Q- g) d& r; n: w
self.downloaded = 0% ^6 x6 F% `# J\" y+ Z
- f* o, z2 i4 Y
# rebuild start poind
' Z( P- A6 J3 k* s/ V' [5 m: z
self.startpoint = self.ranges[0] + self.downloaded
- A' `/ z8 n3 G! h
5 g+ p\" D; N2 m# _2 @6 u/ Z0 ^
# This part is completed7 y! R& v7 R6 p6 f! S- [2 {- i7 F' q
if self.startpoint >= self.ranges[1]:
$ \& {. X0 G) b4 x
print 'Part %s has been downloaded over.' % self.filename
I3 H8 q$ N. {8 W, h# v
return
2 h: q- D) s2 f) `
2 Z0 H% }, U$ P
self.oneTimeSize = 16384 #16kByte/time2 ]8 P* }8 h; T# j; c
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
3 L+ @5 E2 T/ _2 F; X# L
0 q2 }# n$ t1 q7 v; W
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))4 c2 X/ O8 u d\" Z l
# P( M/ @\" \\" n& N7 F# Z: s
self.urlhandle = self.open( self.url )
5 ^1 x% ~1 u6 O/ ?9 Q
, E3 }$ c) a! ]: ^. ~7 }
data = self.urlhandle.read( self.oneTimeSize )
. Y5 A/ j9 k& f( V1 |, y
while data: p4 T L% r8 B
filehandle = open( self.filename, 'ab+' )$ _1 ~) ]! g% u& _
filehandle.write( data )/ Q\" C+ l5 q. j) S
filehandle.close()
7 T' g' f( n) _3 p7 u% X
% h5 V8 U Z3 v$ x: \7 S
self.downloaded += len( data )
\" n- H1 }5 H5 V: J( W1 S
#print "%s" % (self.name)9 l$ I! v! H! i; a
#progress = u'\r...'. s\" r1 z1 X9 F9 Q, L! _
\" s: B) K' o) ^# @- O\" W( ~
data = self.urlhandle.read( self.oneTimeSize )
4 a9 A% I( A3 g, m$ G
4 A\" {/ e' y$ W; [ b
def GetUrlFileSize(url, proxies={}):
( N$ r\" N, L\" ]3 ^. h+ y n
urlHandler = urllib.urlopen( url, proxies=proxies ) O+ L\" S( F3 S: I6 O6 P
headers = urlHandler.info().headers
( a; ~8 H8 m4 p( `+ O! `# A1 V
length = 0* [+ _% M2 q' e8 {; I0 h
for header in headers:% ^8 U9 k2 A8 {% R2 U; i
if header.find('Length') != -1:& C% b\" U* { M J
length = header.split(':')[-1].strip()1 ]* Z6 a! n& b\" v3 y, [! r1 \
length = int(length)
# K, Q# [8 ~: v8 P' R z
return length, h1 }1 j0 d, D. x8 k
) \5 H- D5 Q3 u- x C! B. `
def SpliteBlocks(totalsize, blocknumber):! W' ?# M' T0 W
blocksize = totalsize/blocknumber
\" V4 r2 ]9 n. Z, Y% R6 u
ranges = []3 E4 |9 i6 c% E+ l* _$ G+ W
for i in range(0, blocknumber-1):
, U/ P6 R- L- P7 j\" y/ h7 K
ranges.append((i*blocksize, i*blocksize +blocksize - 1))+ C: Z, E3 P9 G- I ~' `
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))$ ?2 }8 U; V2 S9 m' B
2 U6 Q6 ~- e% j\" A; D4 ?
return ranges
. ?- |, l\" ^7 m\" M) _8 q7 a
def islive(tasks):/ ]\" w5 n/ R* }8 o
for task in tasks:
: s; D: P/ Z- T) r7 @3 U
if task.isAlive():
) w; Q( W V\" D0 J: e* j
return True( M$ n, {% d% J. m- j7 i
return False
; X3 V- g) l0 w! D
: w6 W1 L* w! R\" K' y
def paxel(url, output, blocks=6, proxies=local_proxies):
, l' f S: B) j- A3 H: o6 ^
''' paxel8 S* u7 A' ~8 D7 ~7 k\" T2 P
'''
( ]: a: w5 y6 J/ P\" z3 ^- g
size = GetUrlFileSize( url, proxies )8 S; C! M( Z) a+ o% b& K
ranges = SpliteBlocks( size, blocks )
, e- m+ S; Y$ m- q8 |/ N, u4 _
, b9 n% N' g( O: M+ H9 y
threadname = [ "thread_%d" % i for i in range(0, blocks) ]; |2 q/ |* q+ L5 Q\" Z6 t
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]2 c\" N- B) O\" X9 l) w
) G4 x& V; d3 d% l
tasks = []
, T% g$ q% U: j- z
for i in range(0,blocks):
) z4 X% [/ g7 X4 {* [
task = AxelPython( threadname[i], url, filename[i], ranges[i] )
! A7 e) o& K* M1 X
task.setDaemon( True ), V+ k3 N( K8 w3 k7 n
task.start()
8 `% X\" D8 ?; C0 ~
tasks.append( task )/ F\" W6 X( i( z8 [6 l. K
! z4 }! A: l; P' V6 ^# Y
time.sleep( 2 )' ]* U; Y& u# h0 G
while islive(tasks):. S% z* `0 z p1 f5 Q6 f6 Y
downloaded = sum( [task.downloaded for task in tasks] )\" ]5 T' H5 }* R6 `% ^* E1 ~7 l
process = downloaded/float(size)*100) h% s0 J' W; {% p4 i0 H# U* C
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
0 ?$ N; Z9 h, ?' P. B\" L9 R
sys.stdout.write(show)
, v; F$ j$ G, a8 r3 q6 O
sys.stdout.flush()
! X2 b# A; ^' X i& s\" \& r. ^
time.sleep( 0.5 )
; A+ J) b1 Y8 ^
5 P, e\" B4 j( G
filehandle = open( output, 'wb+' )' j$ z* v+ I$ Q+ Y/ m- s- j* C
for i in filename: ^, |* Q5 L. J M4 W( T7 G4 ]2 n
f = open( i, 'rb' )
+ L% g( W\" `. Z! I0 ~
filehandle.write( f.read() )
& G. F5 S- Y7 a9 E\" R! s
f.close()2 z& T& t- a: u- w+ L& H3 o' _
try:4 t5 W# N8 K\" B7 q* E8 q
os.remove(i)% A3 a' W/ m6 f\" w# E
pass8 u. M8 u7 S! q# U G6 r1 j
except:1 G4 A9 w9 g. H
pass
: ~- ~; |; [# `7 \
7 g\" B& S/ f4 ?6 L7 D
filehandle.close()
) m9 {) I\" ]! q4 `' U
5 X' _% M( w. i: b0 f8 V
if __name__ == '__main__':
- d1 r1 S: [( | L! |
url = "http://xz1.mm667.com/xz84/images/001.jpg"
+ a( p* n5 r4 Y% l# e
output = '001.jpg'
( t& g2 r$ T7 p. ~; l* [* Z
paxel( url, output, blocks=4, proxies={} )