多线程下载图集

[复制链接]

字体大小: 正常放大

2744557306

1192 主题	4 听众	2946 积分

该用户从未签到

电梯直达

1^#

发表于 2024-3-31 17:18 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

网上好看的动漫图集，如果手工下载太费时了。简单分析下网页地址规律，写个多线程脚本搞定。

#!/usr/bin/python) T6 @! a0 A, F. ~% B
# -*- coding: utf-8 -*-
$ W9 Y0 c/ |8 a
# filename: paxel.py
, d4 b/ S* H- t2 H; }
' ?, k. j- X3 s. r. h
'''It is a multi-thread downloading tool
: L# W/ H$ y V$ {) {
+ z! c+ R9 z3 \: d% q* P& Q) I: Z9 h
It was developed follow axel.; w\" d\" [\" F- A
Author: volans
$ h4 r V7 q8 I9 H
E-mail: volansw [at] gmail.com5 [! a8 p\" Z2 ~\" G7 A
'''( o4 N; v- p0 {/ i\" h
) J* h, d0 L\" t t
import sys6 E ]. J: h2 a* u\" W/ N3 y P
import os
: m) ^; A: M# F) I+ _- p7 ^
import time
& @$ R# m/ x G0 v( @
import urllib
9 N3 ~. P\" y5 X4 B+ ?! d; G( `
from threading import Thread6 S( t( u\" @7 h$ j* g- v3 M
6 v# f' i) R7 w2 y- c% L
local_proxies = {'http': 'http://131.139.58.200:8080'} v8 o' w; x3 R\" J# _2 n* C8 e
0 e1 d, d\" o' R
class AxelPython(Thread, urllib.FancyURLopener):
- l9 b. o7 @\" C$ b
'''Multi-thread downloading class.0 \( z! b2 j) d: {
/ U# D0 u( N3 A3 w0 _
run() is a vitural method of Thread.6 G7 ]) E\" ~- h
'''
5 h. V5 Y& o4 z3 z
def __init__(self, threadname, url, filename, ranges=0, proxies={}):
& S: C( T% `9 H3 ]' K7 C
Thread.__init__(self, name=threadname)+ T* o7 G+ s3 @; }
urllib.FancyURLopener.__init__(self, proxies)
5 P( A) ~9 R' ~1 n2 ^( W: v) P* @+ S- Z
self.name = threadname
! \: y6 c% I3 j2 b
self.url = url
: Z' @9 o7 S; x/ e
self.filename = filename
( Y\" E' p1 b7 Z; T
self.ranges = ranges
! `6 v/ Y* F% e/ }; I1 n9 t& z
self.downloaded = 0
1 L4 k* z) n. u
) f+ O% ]* U. x
def run(self):
+ Q* |; n% M; N! x# l: ?& m+ L
'''vertual function in Thread'''
1 P# r1 Q+ W/ }% D\" ^* G) v* [
try:' P+ H1 K) d% g) h9 g9 w1 p
self.downloaded = os.path.getsize( self.filename )
0 r8 X h' R8 n4 g
except OSError:' S2 R5 p/ V% t1 j; J\" ^
#print 'never downloaded'
6 a' `4 ^ d. G: d( A& X
self.downloaded = 0: w* X) {' A' A9 d2 X% u; g
# r- @3 o ~( w c `% r
# rebuild start poind
$ i& p; R7 i4 m: v4 b
self.startpoint = self.ranges[0] + self.downloaded# a& q7 F/ d- ^ ~6 g/ D4 o$ d0 y6 L4 U& z
% Y. K; c9 A [& a, j) j
# This part is completed: f\" y' |1 q3 s\" P
if self.startpoint >= self.ranges[1]:% s$ q% I, U0 |5 Q: d# r& }$ v3 L9 r& l( r
print 'Part %s has been downloaded over.' % self.filename
* ~0 E\" P& i( z# @; h1 Z
return
$ C/ C/ o0 `$ J( h
3 R8 A: Z( W# h0 j7 W
self.oneTimeSize = 16384 #16kByte/time
' d8 c& Y- R\" y
print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])+ s1 z6 K; M7 G0 I' T
- C3 {9 o' t3 ~; x: o% y
self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
?5 |$ `) m8 k/ ?* p
, p0 |9 f2 Y+ M3 I+ ?1 J( v. |
self.urlhandle = self.open( self.url )' }7 ^3 [: I6 v
* D9 }! [* k\" p9 V( t\" ^
data = self.urlhandle.read( self.oneTimeSize )8 r1 j; z Z% Q
while data:3 h, p. y7 {3 p' o
filehandle = open( self.filename, 'ab+' )8 C! n' ~4 @# M8 q. g
filehandle.write( data )
0 D; L. b0 t\" w) u3 j
filehandle.close()
# O6 x8 n% g; w6 J
& v: p- E0 }1 J+ a( q# ?
self.downloaded += len( data )
5 P, O5 Z) \8 x! c
#print "%s" % (self.name)
6 P; T- k. u! Y* c
#progress = u'\r...'
* h' c' U: U# u& H6 o1 f7 [, S* V
8 g5 g/ v+ r% X; e5 T/ J8 n
data = self.urlhandle.read( self.oneTimeSize )
+ g+ _; k\" G8 x\" C% f& b }1 n5 M
1 o% Q# e/ L5 o$ M) R. q
def GetUrlFileSize(url, proxies={}):
1 _* [5 f+ f( [8 G6 F3 S8 q0 Y
urlHandler = urllib.urlopen( url, proxies=proxies )5 f5 R2 P8 J0 ]; k- N
headers = urlHandler.info().headers
C) e! Z) l# U9 S6 c5 T
length = 00 u+ h( g8 e, n# h1 z
for header in headers:% |! [% S, j, c5 |# r( u
if header.find('Length') != -1:
5 p7 p\" y$ L, x0 m. Z5 f$ a
length = header.split(':')[-1].strip()
& M8 D* P3 {% ^2 d3 v3 Z7 c `
length = int(length)
8 ~( Q' V2 o B8 ?\" r: H1 i
return length* z8 p\" P n) E/ l
~) g$ y- N( R) u9 c( ]
def SpliteBlocks(totalsize, blocknumber):7 Z- ~! d+ J# D4 }
blocksize = totalsize/blocknumber* n& G7 N- Y; y$ _) k1 x& j$ c r* M# Z
ranges = []7 _- m4 B4 e' | T; }
for i in range(0, blocknumber-1):( l: A7 J5 `' o8 I: m
ranges.append((i*blocksize, i*blocksize +blocksize - 1))2 w' c\" |\" T, k, M; N- z W
ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))0 i8 f* e1 S2 j. q
! z9 T- s: D: s: \2 B( v
return ranges1 q7 q\" T. k2 J7 B$ g/ X4 e6 T
def islive(tasks):
1 `, A3 t$ o* E( l, P4 F
for task in tasks:
- t/ b* t4 Q, N* s. ]4 V! p
if task.isAlive():
9 ? J9 U\" o0 e; f/ N, E- I
return True- ?3 D\" j& S+ @
return False
2 u# d& q( |' U, H' O
Q, @1 H( q8 ~# i
def paxel(url, output, blocks=6, proxies=local_proxies):+ X/ f( { g; r: H
''' paxel
7 D/ [9 r0 H y\" R$ ]
'''
7 ?$ n* D0 Z: l
size = GetUrlFileSize( url, proxies )0 l( O( ~* a\" | {3 ]3 t\" n* L
ranges = SpliteBlocks( size, blocks )
7 [# \# E% P- |
0 a5 r. S* C1 z
threadname = [ "thread_%d" % i for i in range(0, blocks) ]
. J! l/ p2 ?8 P
filename = [ "tmpfile_%d" % i for i in range(0, blocks) ], G: U A9 j& t% j
/ M* C+ @2 e2 y3 Z7 z
tasks = []
- L\" A. u6 M! v+ B; v
for i in range(0,blocks):
8 n$ G# p/ v5 z2 J, k\" s% W
task = AxelPython( threadname[i], url, filename[i], ranges[i] )
3 t1 ^/ @' E% k/ ^' n. d
task.setDaemon( True )
2 `: v- r8 I; I, H8 |
task.start()* `4 ^6 m5 Z. P4 L1 ~
tasks.append( task )& v L( j+ d. M6 E
/ U, r\" g( ]' o\" w8 R
time.sleep( 2 )
! T% e- F6 ] c7 n& ] ~+ w
while islive(tasks):6 H& L! Y( h: C; @! N# l
downloaded = sum( [task.downloaded for task in tasks] )
( M) P% O1 e! a$ l2 w
process = downloaded/float(size)*100
8 r+ I# G3 D5 \5 ^
show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)
, q$ h) \ e0 S' d
sys.stdout.write(show)
1 J* _* ]# R8 [! |
sys.stdout.flush()
K' m* v- k/ e7 i$ a. R
time.sleep( 0.5 )4 T1 |/ K* s3 E* d: h
# h6 W% ]+ y: B& Z. P1 ~; n
filehandle = open( output, 'wb+' )
5 J; p9 W7 Y0 i* s; ?$ p9 [
for i in filename:' Q\" E9 @% @1 S) m' ~- ^
f = open( i, 'rb' )1 u% H, n, @. e1 ?' s
filehandle.write( f.read() )3 s, b1 ^( I# |4 F, n& P2 \5 p
f.close()
: \: [- G' S# u3 }# M
try:
3 a! m; P$ f# I0 K% H8 l3 l
os.remove(i)
' z$ x( h& ?6 ]( E. c* q0 ~; a( c1 S% n
pass
6 F! Y9 F6 ~+ F( ?2 V+ `
except:' l, X, K1 k: ]1 C\" o( L\" ?
pass8 o7 T% Q8 h8 O& s1 h' i4 `
) i* n1 M. i K9 K( ?5 C
filehandle.close()9 o1 T; o4 a Q0 y( l% E/ L\" F
5 U& }4 U# y; C0 y& ]4 Z3 ?
if __name__ == '__main__':
7 f( D! ]( |+ M7 @8 f+ Y
url = "http://xz1.mm667.com/xz84/images/001.jpg"! U4 @* x' w |
output = '001.jpg'0 V8 J/ g9 l$ |9 J
paxel( url, output, blocks=4, proxies={} )