数学建模社区-数学中国
标题:
多线程下载图集
[打印本页]
作者:
2744557306
时间:
2024-3-31 17:18
标题:
多线程下载图集
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。
#!/usr/bin/python
( r# E3 Q3 m8 v+ z$ N) k G
# -*- coding: utf-8 -*-
9 G) ]+ A% C5 t0 G, G7 u h" V+ f: |
# filename: paxel.py
# u3 ^' Q3 E% R* t8 S6 k
1 X/ Z+ y4 u$ L2 U0 ?% }
'''It is a multi-thread downloading tool
3 V# E* Z" a3 T
" o( U/ x. y& n
It was developed follow axel.
+ ~8 T& I$ g& G4 _9 j8 \& l0 |
Author: volans
# s8 A3 f/ ~! X" l0 C+ f8 N. r
E-mail: volansw [at] gmail.com
6 y, {# g1 W& w6 c0 z
'''
+ D) k7 [' E$ g# g
3 } a% t. W* b* e2 M- `1 B
import sys
. B% [1 o5 c- v' i; @
import os
6 K& ?6 [$ v$ }/ R9 o" V
import time
X8 A: C# i) V, V/ K! a5 {- z
import urllib
9 T+ n0 F/ q; h) a
from threading import Thread
q4 f) U4 D1 \! ~! e2 j
; [4 ?, }: ?- \( A' n, f; ?3 P: P
local_proxies = {'http': 'http://131.139.58.200:8080'}
/ z* M1 {$ h7 ^. m
3 ?; H7 H: a6 G7 ], u
class AxelPython(Thread, urllib.FancyURLopener):
) n: z1 S/ w+ A/ h y. v6 |2 [" }' E
'''Multi-thread downloading class.
" V8 X. U$ v) j. ~0 A
7 p( b9 s3 w6 N; K7 U9 x8 M
run() is a vitural method of Thread.
" N7 c& t2 H% p( s w
'''
7 C% z; i$ }, D. g9 P1 t" w
def __init__(self, threadname, url, filename, ranges=0, proxies={}):
9 a$ H ]# w4 c' |
Thread.__init__(self, name=threadname)
" q# S; G U) n- I+ J. E
urllib.FancyURLopener.__init__(self, proxies)
0 ]$ o W1 J! O$ q- y& v; F% v
self.name = threadname
. @* d$ y) c. [* O0 A
self.url = url
, d+ K0 q- U3 k; `8 Q* ^9 S5 X
self.filename = filename
7 _0 F" x [4 v) t
self.ranges = ranges
0 X% a' L1 H6 z2 a, r! _
self.downloaded = 0
. o- E' s. t. d& l0 V0 J8 `
! n) b. s8 m B& [
def run(self):
1 \9 ]8 x5 _$ P8 R8 y
'''vertual function in Thread'''
* R/ q$ V3 v9 y; B8 m9 V' I0 _
try:
6 i }# }! y0 k: a" y
self.downloaded = os.path.getsize( self.filename )
: J0 k! h. r% b1 W
except OSError:
0 p3 T& r9 f$ r' p% m5 H
#print 'never downloaded'
6 W3 B, @4 a* X J
self.downloaded = 0
4 L9 `) ] b; {1 z8 k z+ H( Q. |
, ~9 H/ P# F& O" E: N9 [( n/ b
# rebuild start poind
& l" @2 ?3 M. \5 k% p1 y5 ?! w# p* o
self.startpoint = self.ranges[0] + self.downloaded
4 x- j7 N. R7 c4 n
. d, B1 f% N6 N6 O
# This part is completed
/ P6 ?& |' ]" }! e% u8 P
if self.startpoint >= self.ranges[1]:
: D4 J+ i1 y" c1 L% l; f0 G+ A
print 'Part %s has been downloaded over.' % self.filename
6 ^+ t# [$ t6 X1 Z$ _
return
+ |2 K3 z' ^0 K4 y" Z) [
" m. r; y" J0 Y$ n5 w
self.oneTimeSize = 16384 #16kByte/time
; L3 r% `0 e( n2 m" }- P
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
& O. s3 T& j: e2 Y) p3 ^
: [. v. n( _! k$ L, _% m
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
7 G1 T9 \" l4 d/ Y2 A _* z! [: i
8 j8 p) G* G2 e9 {
self.urlhandle = self.open( self.url )
* U4 j3 w( N8 h. I0 y: {
: Y: x3 }8 v/ Z! Q `3 H
data = self.urlhandle.read( self.oneTimeSize )
5 A4 ?0 @2 P6 r6 F
while data:
. B% i8 B3 R& G8 x" |! t
filehandle = open( self.filename, 'ab+' )
8 e! u; q& U% `4 C- l! V# `
filehandle.write( data )
0 i; b. Z5 c( ~
filehandle.close()
7 h8 T) E( k7 u5 E0 U8 M% _. W ?; T
% T7 ]& _3 N: r& D: e
self.downloaded += len( data )
& M, e) F; S# z/ z) ^8 P
#print "%s" % (self.name)
+ F4 y% w/ y- Z! i: q
#progress = u'\r...'
+ u( M4 q, I: |( p
8 i% g4 e$ x. D1 f5 Z
data = self.urlhandle.read( self.oneTimeSize )
& X& e$ f9 S) h1 L5 f
7 d0 z! b6 c2 ^! _; o* h
def GetUrlFileSize(url, proxies={}):
& l4 H" d* X0 S% W7 {9 W1 o/ A
urlHandler = urllib.urlopen( url, proxies=proxies )
2 R, n5 Z" b. M% D" c+ R
headers = urlHandler.info().headers
8 Y" o+ E* c7 o/ M$ d$ Y
length = 0
# I( a; a4 g6 w4 V! A2 ^2 I
for header in headers:
/ N) P5 M2 m8 z5 ~; a- y# ~5 U
if header.find('Length') != -1:
9 ]; X! F3 ?. R/ a
length = header.split(':')[-1].strip()
4 p" g& Z" f' M& D
length = int(length)
& |( B$ W/ Y6 N0 t3 s a) K
return length
1 E0 \7 Y$ e/ a% c: s: [! S
2 z2 R: I4 E. n. r
def SpliteBlocks(totalsize, blocknumber):
: J5 r8 P9 \! S
blocksize = totalsize/blocknumber
" L! g, V% s7 h, H, K
ranges = []
8 @ \) h6 ?6 z$ O
for i in range(0, blocknumber-1):
- j: H$ e' r, {; z
ranges.append((i*blocksize, i*blocksize +blocksize - 1))
* Q: p3 A9 h: a- @, T
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
" e! z/ U+ V$ C0 y
6 V; O2 i5 c/ ^1 `4 m# W
return ranges
& u* ^& O, `6 w$ ~' x4 v6 o
def islive(tasks):
2 c6 M4 Z) R: m: u/ P' p7 N+ f: I- ^
for task in tasks:
. |: j* U1 x. b4 O8 b9 f
if task.isAlive():
* o1 h) f: J# c: u7 h# @, J6 y
return True
) f4 ]1 |8 B$ U0 M t
return False
: f, M. Q# x* C: M# a
$ p; T2 \0 w' p! B4 Z! Q
def paxel(url, output, blocks=6, proxies=local_proxies):
- O" y8 D) S' a, ]
''' paxel
/ i" B4 Y! J4 w) i( g
'''
6 y/ V' |% t" Y$ L5 K2 S
size = GetUrlFileSize( url, proxies )
3 q8 @* E! [6 J3 c. w
ranges = SpliteBlocks( size, blocks )
9 b/ _# G3 ]5 h7 q, c
5 J$ s. s: }0 W
threadname = [ "thread_%d" % i for i in range(0, blocks) ]
+ \1 ^4 ?4 L1 H# t3 r' p& O+ E
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
* b- N: i1 q1 G3 B2 l
# x( e6 f K0 `' L$ |+ \( A
tasks = []
7 s, z. X: T: o* z3 V
for i in range(0,blocks):
/ f4 r6 Y' x. J8 S# O9 t0 w0 c
task = AxelPython( threadname[i], url, filename[i], ranges[i] )
! M8 }' |2 @) ?4 ?5 s2 N0 q
task.setDaemon( True )
5 K% \" h/ n( |
task.start()
9 m9 f! @# P# c& C8 c3 Y, S" c
tasks.append( task )
. R+ m3 I0 o# `) d& k
* }7 L1 | ~' f5 j
time.sleep( 2 )
. ]: c& t+ v" O4 K. M
while islive(tasks):
! [, ~& q9 E- W& T" U
downloaded = sum( [task.downloaded for task in tasks] )
( d1 P; U+ {, @2 F( L1 O
process = downloaded/float(size)*100
6 ~$ u& r: a9 `9 A
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
8 S4 R5 a- P3 y3 d/ e: F
sys.stdout.write(show)
; l! L! E8 Z* M e
sys.stdout.flush()
6 i2 X4 S1 C6 e) g+ C" D
time.sleep( 0.5 )
: x/ a: l1 L4 N2 g* o% Y$ d
/ n0 c, N6 V% Y9 \
filehandle = open( output, 'wb+' )
/ Z# ?0 u0 G# U# Q7 Z
for i in filename:
2 u- K \2 Y: m+ V1 K$ U0 F) t" B
f = open( i, 'rb' )
4 i" r0 Z( S- O2 s
filehandle.write( f.read() )
S0 D: M" ]9 h+ [; R4 z
f.close()
. Y' @- n- S" i) G( v1 I2 i# E' _" n
try:
( @. _) Y5 H b0 y0 ~
os.remove(i)
1 o' M& q- D4 O0 q6 ]; h
pass
2 R6 N8 B" U' j8 t5 {
except:
p9 B7 t1 \ ^% `
pass
4 P! i/ D/ ^; {5 \
& B% L2 ^0 T, G1 `9 B2 x) V
filehandle.close()
4 s6 a9 X: P6 Z6 w6 p; S
1 k5 N" ^* _1 e6 p
if __name__ == '__main__':
4 ~! B! w; b9 T4 G* p: V$ a) P
url = "http://xz1.mm667.com/xz84/images/001.jpg"
) p+ c# p# S6 R
output = '001.jpg'
1 q& g; U0 t( X# @/ W) M0 p3 K
paxel( url, output, blocks=4, proxies={} )
复制代码
6 Y9 C+ D; ~# r8 O: e1 z2 j
欢迎光临 数学建模社区-数学中国 (http://www.madio.net/)
Powered by Discuz! X2.5