多线程下载图集

[复制链接]

字体大小: 正常放大

2744557306

1192 主题	4 听众	2946 积分

该用户从未签到

电梯直达

1^#

发表于 2024-3-31 17:18 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

#!/usr/bin/python8 C' @7 g6 K! z' t) F. I
# -*- coding: utf-8 -*-
2 y E9 z# D V; x
# filename: paxel.py1 `2 S; l$ M) ?: h& E
7 u- t! E9 N/ w& j% |0 b
'''It is a multi-thread downloading tool
( ]# o# D9 M6 d) t9 M, D3 S
1 A6 r) ~- ] l' ?8 R
It was developed follow axel.. |; V1 ~5 Y9 l8 T; N1 T# V' d
Author: volans
9 @. Q% w: d) C3 O# f8 u
E-mail: volansw [at] gmail.com
$ H) N( [- k/ v$ @1 `: S
'''- O5 x h# U; Z, R d/ w
) A, z( s- m\" l4 [' p\" E) _1 m. ]
import sys& i2 h, ~4 c7 L
import os
\" I7 }1 F9 r0 O/ l) C
import time2 v/ Z }: L, M7 \0 Q3 |1 B
import urllib: w6 b: J( z' H+ z% B2 o
from threading import Thread! t% h9 E! s\" P6 _% W; A4 ]7 R
\" V$ M* x# M* ^, _/ g
local_proxies = {'http': 'http://131.139.58.200:8080'}
! A6 ?3 ~8 p2 i2 o
+ {2 U: e3 D1 h; {6 Z
class AxelPython(Thread, urllib.FancyURLopener):
7 V2 q% U5 D2 O6 I- |8 l
'''Multi-thread downloading class.' V+ |1 y% M$ b: f. ^! M- ?
1 p* n% ^+ Y' S6 e
run() is a vitural method of Thread.5 Q4 F. E/ w' }+ V
'''1 ^6 S! M1 t; M4 V* g1 W* ]/ C
def __init__(self, threadname, url, filename, ranges=0, proxies={}):/ e8 }0 w7 n+ {3 i9 c
Thread.__init__(self, name=threadname)
; {1 l8 J+ Q/ @
urllib.FancyURLopener.__init__(self, proxies)
\" X\" y. u3 s( D$ ` z
self.name = threadname, ~& `3 a9 B( |/ r& O
self.url = url
# ^+ |4 Q5 L }3 O
self.filename = filename# J9 `\" J# F& b5 k
self.ranges = ranges
( y9 B1 q( p+ z
self.downloaded = 0
$ y\" a+ f, E% @9 N2 f0 Q
( P! {% r' f* u- h- K' u* m6 }
def run(self):0 S; _, z4 Y/ g\" Q' i2 z
'''vertual function in Thread'''0 W% W\" R# H4 N! v% Z4 S1 O
try:+ j/ P O8 R- ^+ X% H, [) H
self.downloaded = os.path.getsize( self.filename )
- F3 z& N- N8 {3 N5 m& b( b
except OSError:
9 z4 o A' F3 v5 i0 A
#print 'never downloaded'8 l# F8 T: U9 U7 @6 d# n8 B
self.downloaded = 0
8 s\" B, T0 q. M* o2 Z2 Y
; ~5 r) U1 @\" {9 T- [
# rebuild start poind6 \# O# t* D3 d5 J3 l
self.startpoint = self.ranges[0] + self.downloaded
: H( N% A$ K0 V, H, h: i0 A
9 m, g1 ~3 V' M6 b
# This part is completed% K# y7 q/ P$ W0 w& d7 d* E; e
if self.startpoint >= self.ranges[1]:
. }\" k6 g' y* {; @ Z$ M
print 'Part %s has been downloaded over.' % self.filename
! G3 J3 I' D7 h! @$ W9 Z/ _5 \
return) p4 h( X* x8 ~3 V- _# C$ |' k2 `
, x\" e& B2 o [4 L
self.oneTimeSize = 16384 #16kByte/time
\" m2 P. B4 j/ h8 d: h7 p4 q* L3 I
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1]); C3 x; Y; M: z N7 G2 G
$ }$ N; s: p% [$ Q2 T
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
* i4 {) D; l5 H5 N F4 y/ z
0 B# v( h4 P. _1 g7 ?. n5 ?* D
self.urlhandle = self.open( self.url )
' W5 t1 Z4 C7 E2 T1 G, [
0 K1 `9 g# `* Z\" r2 @) t( R0 o
data = self.urlhandle.read( self.oneTimeSize )
~* a9 \# Z2 X/ r
while data:2 [ V. v6 ~\" G\" Z0 p5 {
filehandle = open( self.filename, 'ab+' )8 k3 X! e2 C+ V5 h# T E
filehandle.write( data )2 \: N/ D9 f3 N/ V
filehandle.close()
+ x8 C$ f k\" x+ I9 r8 K
# W+ {9 k: q v
self.downloaded += len( data )
; S+ I n) s\" O0 a% N' B
#print "%s" % (self.name)$ G, M, I* R# w3 |5 J$ f- j+ b* C
#progress = u'\r...'3 R: C0 K$ R( |+ f2 R
1 z, y7 `, s0 I; h! q4 w
data = self.urlhandle.read( self.oneTimeSize )
% }: E/ k& K6 e8 j. T' C. Y
3 @ \- r' o( z( v: i
def GetUrlFileSize(url, proxies={}): g7 ]4 S( j8 _: C0 b
urlHandler = urllib.urlopen( url, proxies=proxies )& z8 [. p' Y& J8 j
headers = urlHandler.info().headers3 n H2 z- U: V' u: J( f
length = 0' u8 J; r( ~2 X. ^7 Y& C
for header in headers:0 n% a+ g+ Z! a, T+ [ k
if header.find('Length') != -1:0 H4 T9 D, L: o* B5 g( D+ c\" J3 n
length = header.split(':')[-1].strip(): [7 P& O3 T8 n+ g+ X' V9 I
length = int(length)
; u9 ~; M9 A9 \0 w( t. T8 g. }
return length( |3 S1 g. x; C8 [
8 h- r4 e1 W5 n
def SpliteBlocks(totalsize, blocknumber):5 C1 ~+ ^' h5 F; E1 I( `' \+ }
blocksize = totalsize/blocknumber6 j! O8 J# \% l
ranges = []( X4 \: Q& i' K
for i in range(0, blocknumber-1):* Z) N* `2 G* `# `. J
ranges.append((i*blocksize, i*blocksize +blocksize - 1))
- M7 |( W! f7 X. [* X0 W
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))3 o9 h& t* X* C
4 \% E X8 H+ x! f, }# x4 M3 x
return ranges& ^( i7 _: X4 _, C8 A8 W6 L( d
def islive(tasks):4 O% z- ]; ~2 Q. s
for task in tasks:
/ I4 f) {+ K9 J
if task.isAlive():
) E, j6 P% h5 T
return True
7 D+ U. @7 o/ W6 R
return False
2 X$ L3 G; q9 | \( Z) z8 a o N
7 @+ T7 F, K f+ O* s
def paxel(url, output, blocks=6, proxies=local_proxies):
. R) E4 O; G7 w8 }
''' paxel( \- H0 B! y7 n
''': ^ |* Z L% P
size = GetUrlFileSize( url, proxies )
+ U! w\" T6 V9 E ]6 L
ranges = SpliteBlocks( size, blocks )8 t- y3 k& p, B\" ]: `
2 [& p z0 @4 K! j5 t
threadname = [ "thread_%d" % i for i in range(0, blocks) ]; {5 X' v$ W9 q) ]\" p, q
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
: n* K6 ?2 n7 h2 q
' a+ T$ N! { ~- P. L
tasks = []
+ W% Y2 { R\" h/ e- z/ `
for i in range(0,blocks):3 ^- ^: _/ @; P1 F7 ^
task = AxelPython( threadname[i], url, filename[i], ranges[i] )( h3 P: `! ?\" ]; x: T5 {* m+ C
task.setDaemon( True )
& K2 g+ ~: `- g2 G7 f/ I
task.start()& m# f0 j* E+ \ w+ _, p$ C
tasks.append( task )) A) T& k' c4 G: Q
7 M3 c1 G6 ?) p, m6 @
time.sleep( 2 )
4 z0 A! k0 {- y, c6 @
while islive(tasks):
' n1 K4 q, y; j- ]
downloaded = sum( [task.downloaded for task in tasks] )
* V# f1 u% O3 ?\" v' ]' _
process = downloaded/float(size)*100. N: o\" Y7 Z) n5 m' m
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)) f3 g( ]\" U: y. d0 Y+ U
sys.stdout.write(show); W2 ? M4 N$ R* m6 p8 p' I
sys.stdout.flush(): A0 a- N9 ?5 U$ j) `
time.sleep( 0.5 )& Y6 S* }: m* O# ^9 m O
* X$ S: a3 L q# T! G3 I
filehandle = open( output, 'wb+' )
) U6 p- c' F# D1 b1 ^, y! G
for i in filename:0 |% J) d6 y+ o# T
f = open( i, 'rb' )
' N. e( E1 }% s7 g; p8 i2 u
filehandle.write( f.read() )
1 _5 M; h G0 O2 |5 s2 ^
f.close()\" c' j: D1 A8 |
try:6 s+ H# H4 r6 N6 L$ f
os.remove(i)- J4 W0 z, n4 V8 e1 d
pass
( B- U4 u3 B0 g: x0 b7 q% K3 B
except:3 o- Z R6 n' c! I) F# f
pass
i0 W7 {; J4 Z
6 u% g1 ]# I2 e L+ O# |* [\" r
filehandle.close()' ~# p. m( U/ q6 S4 w4 W5 z9 O
; E$ o1 M9 T& f! K
if __name__ == '__main__':4 ]% W9 i- {$ P
url = "http://xz1.mm667.com/xz84/images/001.jpg"$ {8 R$ b' X% Q6 s9 o# Q
output = '001.jpg'
2 E- s' Y5 q( Z: A# i$ U
paxel( url, output, blocks=4, proxies={} )