数学建模社区-数学中国
标题:
多线程下载图集
[打印本页]
作者:
2744557306
时间:
2024-3-31 17:18
标题:
多线程下载图集
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。
#!/usr/bin/python
2 h$ `0 P) A: b3 d; E' M
# -*- coding: utf-8 -*-
; [% d" S9 |: g! i/ U
# filename: paxel.py
* X. i+ V( N- J9 N: D$ s# B; g
* N: f5 Y% l* b, |% W1 D0 ]
'''It is a multi-thread downloading tool
+ w0 g v! y; j- f
/ k- Y3 ^" m* Q: u
It was developed follow axel.
C6 _8 ^8 c$ a3 q, J- L, `; F: `& l
Author: volans
+ j& q3 n" z3 D2 h
E-mail: volansw [at] gmail.com
/ s! V5 }9 a- n) @2 x& D0 b% ^
'''
: P! ]) C- r/ y
7 i m+ R" F+ N* I; r* H+ T4 o
import sys
2 N/ k5 K, V4 @2 ?
import os
+ O. h! P: j9 V5 Z/ D( u
import time
% l1 C- L6 Q; _( _8 v" s; t
import urllib
( ~# v0 y- I! e. n$ U J ?+ A6 ~
from threading import Thread
+ W) J( l4 y3 Y5 i! F2 f
i: h! v! c( g \
local_proxies = {'http': 'http://131.139.58.200:8080'}
) u2 q2 T: T, ~9 q
/ |$ H$ }: X& F/ f* G
class AxelPython(Thread, urllib.FancyURLopener):
4 `2 q9 u# I1 n# L
'''Multi-thread downloading class.
- ]4 C+ R+ o+ I( @' m& T
8 o8 D+ D! V( y, k; b/ f
run() is a vitural method of Thread.
, b5 I0 j% Z7 T7 L
'''
- d. @# S( l2 \9 \- Q
def __init__(self, threadname, url, filename, ranges=0, proxies={}):
5 D) b$ _: N1 [7 Q' Z/ K: k$ ]8 I- J
Thread.__init__(self, name=threadname)
8 l' ?( Y* p, j' `3 ~8 x+ x$ _' V8 K: a: Z
urllib.FancyURLopener.__init__(self, proxies)
' L& d6 ?' [+ Y$ [
self.name = threadname
& i/ i. f8 d6 Z# n
self.url = url
8 _4 Y T; J; V$ |8 O) A
self.filename = filename
( u* }' O$ J) Z
self.ranges = ranges
5 @; n6 G( L2 d6 l$ Y1 c y+ F4 j; f
self.downloaded = 0
: o, [" o3 Q' ~( Q$ S" @: b
: ?8 ^/ l+ k8 ~2 \9 |& u1 e
def run(self):
5 [# P* D9 [/ F+ H9 n
'''vertual function in Thread'''
( [" h# @* k; C6 K! h! h
try:
, S) N! G4 [+ Y: O9 z2 e2 G; v
self.downloaded = os.path.getsize( self.filename )
5 v. b2 F5 F* }0 w" Q
except OSError:
! f1 q' c0 J6 C# i9 n. k
#print 'never downloaded'
^- E% \0 m- W& H" W9 P; d+ a3 Q
self.downloaded = 0
/ M5 x, ^( f# H5 w
1 ?8 t& c* D0 a1 r
# rebuild start poind
5 L: e* h `3 a& {, l |" f( B
self.startpoint = self.ranges[0] + self.downloaded
8 j) X) j7 o8 k) `4 K1 h4 O
3 X1 a6 }$ e+ d {* Y
# This part is completed
7 d/ [# @& L; b; E0 v
if self.startpoint >= self.ranges[1]:
( L' @& F( t4 R$ i! k( L$ `% t
print 'Part %s has been downloaded over.' % self.filename
2 K7 L- C2 q* P
return
; f$ M9 C | O. E
% V0 f) ?$ H9 y$ \% r# v
self.oneTimeSize = 16384 #16kByte/time
$ V7 }0 B! N+ e' E; S* {
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])
! [% T) r+ K8 y
4 |, r0 E( P- n
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
4 A/ E. t' |7 U5 d% M
8 \* i* @! m% S9 V
self.urlhandle = self.open( self.url )
" P4 P/ T, ]6 ?2 x9 s8 l/ s4 D
# \$ O' ~+ J1 C! t; J& |1 W
data = self.urlhandle.read( self.oneTimeSize )
, X5 z" H, a. C s9 m
while data:
( H- o: h+ H/ b7 e; u+ W
filehandle = open( self.filename, 'ab+' )
2 P1 P8 k& E0 \( H, m" I
filehandle.write( data )
2 O; G2 t3 o5 g6 g6 [
filehandle.close()
/ M" I; s" w0 M d4 b
$ @# y# w( v: [5 U
self.downloaded += len( data )
# R2 O3 b9 S% s/ q7 Y% L
#print "%s" % (self.name)
& k2 ]1 l8 j- U
#progress = u'\r...'
+ ^- L+ S) s2 @# P5 v
1 z/ ^. n9 u0 ]; H
data = self.urlhandle.read( self.oneTimeSize )
3 o2 D, \3 W- l4 }$ x9 P5 L! o) v/ O
6 {; t q$ u! \ T/ Y
def GetUrlFileSize(url, proxies={}):
: R, o q/ J2 N+ f d
urlHandler = urllib.urlopen( url, proxies=proxies )
& y1 n* ^- \8 b# Y: F
headers = urlHandler.info().headers
1 l/ n% p/ E' f7 D" x# [
length = 0
! l! ?" N8 P: b
for header in headers:
6 W! a( ?3 q7 d' a
if header.find('Length') != -1:
& Y; R k, A, d, x7 X B
length = header.split(':')[-1].strip()
& V" S6 t |8 y/ ]1 w
length = int(length)
7 Y o2 l9 J7 D4 H& l: [$ n, `& v
return length
" G9 K. H2 F' l
: B& U3 X- R; T4 c8 R
def SpliteBlocks(totalsize, blocknumber):
9 @% q* b* `3 c1 D
blocksize = totalsize/blocknumber
; b* @( x1 D# G4 Z o8 ^
ranges = []
. n& Z3 [4 h- e" ], K k
for i in range(0, blocknumber-1):
6 L2 \# o# {& e, s" _8 Q
ranges.append((i*blocksize, i*blocksize +blocksize - 1))
# A) L1 |% U& O" o. d
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))
3 z2 `9 s5 J+ C: l
* r" c$ X" {& f7 r
return ranges
; } ] C3 m. l
def islive(tasks):
0 \( |. p l" }3 d8 r1 a
for task in tasks:
0 E) N8 Y' j" a2 K
if task.isAlive():
& q; N Y" {( {* e
return True
2 k" y" c3 t0 X B# W' X% d
return False
! K; h: C! v, {
, S! j( h: S; D. O& L
def paxel(url, output, blocks=6, proxies=local_proxies):
8 K1 H: q. Y7 }# k- X& z
''' paxel
, d7 n9 d& Y5 r& q S6 N
'''
0 i$ B- Q8 V n U$ W/ {' o6 n& C. i
size = GetUrlFileSize( url, proxies )
/ b, e8 G% A9 T M6 o. I
ranges = SpliteBlocks( size, blocks )
" |, q( o) R6 H* R0 @7 @) D" ^) ]2 s* }
; t/ t" M2 k, O' P- n- \
threadname = [ "thread_%d" % i for i in range(0, blocks) ]
6 f; D6 {- Q2 t" ~, b
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
3 |4 {6 k L- S5 q E
% N5 }5 ~- V: E) n j& g
tasks = []
; H3 K" _; M1 }$ W7 g. |; Q4 Y
for i in range(0,blocks):
+ o" j2 H& T! z% @8 ~+ R/ z
task = AxelPython( threadname[i], url, filename[i], ranges[i] )
7 j& d+ H* D& D: L% n. e! K+ ]
task.setDaemon( True )
6 N3 ?0 G* Y0 }" p5 @" i8 o
task.start()
! Y3 ^1 e' u, \+ O C" D1 j/ [* r) K/ U
tasks.append( task )
$ m$ T+ }) Z8 D' e3 y
( z3 k$ X* I2 K9 Q$ i
time.sleep( 2 )
% |. [0 `4 f% Y1 \
while islive(tasks):
$ l, v9 U6 K3 P0 Y
downloaded = sum( [task.downloaded for task in tasks] )
" _& r6 Z! S7 g1 D% M
process = downloaded/float(size)*100
9 }9 o! z) h3 V, K5 ~0 ~0 B) C4 x+ O
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
% E5 f( D' l9 N$ `8 j0 K8 L$ V
sys.stdout.write(show)
) A; a. w; W n0 b
sys.stdout.flush()
3 i5 K$ Y/ z- Q, _! I2 O0 D
time.sleep( 0.5 )
; o; V6 D$ v; u7 v3 E# R9 U
( D2 a# E9 Y9 _8 D
filehandle = open( output, 'wb+' )
* o6 g; \ s6 H( p0 L3 \/ k; J
for i in filename:
* w5 x. U& w2 Z2 ]- j
f = open( i, 'rb' )
8 o- x2 b0 N+ L# q
filehandle.write( f.read() )
. g8 w, i! o+ i% `* w" W" Z% k, x
f.close()
7 G( F1 M& P9 g6 M; N- N& H
try:
& }- o0 D7 s3 ]; A
os.remove(i)
- `" l# R9 |# `6 N, m
pass
0 B- b: t! a0 m5 K6 @) J
except:
7 v# _6 f: m* s% U( i0 ]8 N) Y9 |
pass
$ e v& w% ~5 [: h0 y' [
/ \5 R/ q: O0 Y# q" }1 f
filehandle.close()
/ @+ H0 @( X9 \" h
+ q: ?6 H4 F. s* J- N/ b
if __name__ == '__main__':
' `) g: I+ E) d+ ?
url = "http://xz1.mm667.com/xz84/images/001.jpg"
; l! S' w: E4 C( R0 Z2 Y
output = '001.jpg'
) ]& J f: }( c" M/ a
paxel( url, output, blocks=4, proxies={} )
复制代码
) |5 T) I9 A" N# y! r
欢迎光临 数学建模社区-数学中国 (http://www.madio.net/)
Powered by Discuz! X2.5