- 在线时间
- 481 小时
- 最后登录
- 2026-8-25
- 注册时间
- 2023-7-11
- 听众数
- 4
- 收听数
- 0
- 能力
- 0 分
- 体力
- 7859 点
- 威望
- 0 点
- 阅读权限
- 255
- 积分
- 2946
- 相册
- 0
- 日志
- 0
- 记录
- 0
- 帖子
- 1177
- 主题
- 1192
- 精华
- 0
- 分享
- 0
- 好友
- 1
该用户从未签到
 |
网上好看的动漫图集,如果手工下载太费时了。简单分析下网页地址规律,写个多线程脚本搞定。- #!/usr/bin/python8 C' @7 g6 K! z' t) F. I
- # -*- coding: utf-8 -*-
2 y E9 z# D V; x - # filename: paxel.py1 `2 S; l$ M) ?: h& E
-
7 u- t! E9 N/ w& j% |0 b - '''It is a multi-thread downloading tool
( ]# o# D9 M6 d) t9 M, D3 S -
1 A6 r) ~- ] l' ?8 R - It was developed follow axel.. |; V1 ~5 Y9 l8 T; N1 T# V' d
- Author: volans
9 @. Q% w: d) C3 O# f8 u - E-mail: volansw [at] gmail.com
$ H) N( [- k/ v$ @1 `: S - '''- O5 x h# U; Z, R d/ w
-
) A, z( s- m\" l4 [' p\" E) _1 m. ] - import sys& i2 h, ~4 c7 L
- import os
\" I7 }1 F9 r0 O/ l) C - import time2 v/ Z }: L, M7 \0 Q3 |1 B
- import urllib: w6 b: J( z' H+ z% B2 o
- from threading import Thread! t% h9 E! s\" P6 _% W; A4 ]7 R
- \" V$ M* x# M* ^, _/ g
- local_proxies = {'http': 'http://131.139.58.200:8080'}
! A6 ?3 ~8 p2 i2 o - + {2 U: e3 D1 h; {6 Z
- class AxelPython(Thread, urllib.FancyURLopener):
7 V2 q% U5 D2 O6 I- |8 l - '''Multi-thread downloading class.' V+ |1 y% M$ b: f. ^! M- ?
-
1 p* n% ^+ Y' S6 e - run() is a vitural method of Thread.5 Q4 F. E/ w' }+ V
- '''1 ^6 S! M1 t; M4 V* g1 W* ]/ C
- def __init__(self, threadname, url, filename, ranges=0, proxies={}):/ e8 }0 w7 n+ {3 i9 c
- Thread.__init__(self, name=threadname)
; {1 l8 J+ Q/ @ - urllib.FancyURLopener.__init__(self, proxies)
\" X\" y. u3 s( D$ ` z - self.name = threadname, ~& `3 a9 B( |/ r& O
- self.url = url
# ^+ |4 Q5 L }3 O - self.filename = filename# J9 `\" J# F& b5 k
- self.ranges = ranges
( y9 B1 q( p+ z - self.downloaded = 0
$ y\" a+ f, E% @9 N2 f0 Q -
( P! {% r' f* u- h- K' u* m6 } - def run(self):0 S; _, z4 Y/ g\" Q' i2 z
- '''vertual function in Thread'''0 W% W\" R# H4 N! v% Z4 S1 O
- try:+ j/ P O8 R- ^+ X% H, [) H
- self.downloaded = os.path.getsize( self.filename )
- F3 z& N- N8 {3 N5 m& b( b - except OSError:
9 z4 o A' F3 v5 i0 A - #print 'never downloaded'8 l# F8 T: U9 U7 @6 d# n8 B
- self.downloaded = 0
8 s\" B, T0 q. M* o2 Z2 Y -
; ~5 r) U1 @\" {9 T- [ - # rebuild start poind6 \# O# t* D3 d5 J3 l
- self.startpoint = self.ranges[0] + self.downloaded
: H( N% A$ K0 V, H, h: i0 A - 9 m, g1 ~3 V' M6 b
- # This part is completed% K# y7 q/ P$ W0 w& d7 d* E; e
- if self.startpoint >= self.ranges[1]:
. }\" k6 g' y* {; @ Z$ M - print 'Part %s has been downloaded over.' % self.filename
! G3 J3 I' D7 h! @$ W9 Z/ _5 \ - return) p4 h( X* x8 ~3 V- _# C$ |' k2 `
- , x\" e& B2 o [4 L
- self.oneTimeSize = 16384 #16kByte/time
\" m2 P. B4 j/ h8 d: h7 p4 q* L3 I - print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1]); C3 x; Y; M: z N7 G2 G
-
$ }$ N; s: p% [$ Q2 T - self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))
* i4 {) D; l5 H5 N F4 y/ z -
0 B# v( h4 P. _1 g7 ?. n5 ?* D - self.urlhandle = self.open( self.url )
' W5 t1 Z4 C7 E2 T1 G, [ - 0 K1 `9 g# `* Z\" r2 @) t( R0 o
- data = self.urlhandle.read( self.oneTimeSize )
~* a9 \# Z2 X/ r - while data:2 [ V. v6 ~\" G\" Z0 p5 {
- filehandle = open( self.filename, 'ab+' )8 k3 X! e2 C+ V5 h# T E
- filehandle.write( data )2 \: N/ D9 f3 N/ V
- filehandle.close()
+ x8 C$ f k\" x+ I9 r8 K - # W+ {9 k: q v
- self.downloaded += len( data )
; S+ I n) s\" O0 a% N' B - #print "%s" % (self.name)$ G, M, I* R# w3 |5 J$ f- j+ b* C
- #progress = u'\r...'3 R: C0 K$ R( |+ f2 R
-
1 z, y7 `, s0 I; h! q4 w - data = self.urlhandle.read( self.oneTimeSize )
% }: E/ k& K6 e8 j. T' C. Y - 3 @ \- r' o( z( v: i
- def GetUrlFileSize(url, proxies={}): g7 ]4 S( j8 _: C0 b
- urlHandler = urllib.urlopen( url, proxies=proxies )& z8 [. p' Y& J8 j
- headers = urlHandler.info().headers3 n H2 z- U: V' u: J( f
- length = 0' u8 J; r( ~2 X. ^7 Y& C
- for header in headers:0 n% a+ g+ Z! a, T+ [ k
- if header.find('Length') != -1:0 H4 T9 D, L: o* B5 g( D+ c\" J3 n
- length = header.split(':')[-1].strip(): [7 P& O3 T8 n+ g+ X' V9 I
- length = int(length)
; u9 ~; M9 A9 \0 w( t. T8 g. } - return length( |3 S1 g. x; C8 [
-
8 h- r4 e1 W5 n - def SpliteBlocks(totalsize, blocknumber):5 C1 ~+ ^' h5 F; E1 I( `' \+ }
- blocksize = totalsize/blocknumber6 j! O8 J# \% l
- ranges = []( X4 \: Q& i' K
- for i in range(0, blocknumber-1):* Z) N* `2 G* `# `. J
- ranges.append((i*blocksize, i*blocksize +blocksize - 1))
- M7 |( W! f7 X. [* X0 W - ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))3 o9 h& t* X* C
-
4 \% E X8 H+ x! f, }# x4 M3 x - return ranges& ^( i7 _: X4 _, C8 A8 W6 L( d
- def islive(tasks):4 O% z- ]; ~2 Q. s
- for task in tasks:
/ I4 f) {+ K9 J - if task.isAlive():
) E, j6 P% h5 T - return True
7 D+ U. @7 o/ W6 R - return False
2 X$ L3 G; q9 | \( Z) z8 a o N -
7 @+ T7 F, K f+ O* s - def paxel(url, output, blocks=6, proxies=local_proxies):
. R) E4 O; G7 w8 } - ''' paxel( \- H0 B! y7 n
- ''': ^ |* Z L% P
- size = GetUrlFileSize( url, proxies )
+ U! w\" T6 V9 E ]6 L - ranges = SpliteBlocks( size, blocks )8 t- y3 k& p, B\" ]: `
-
2 [& p z0 @4 K! j5 t - threadname = [ "thread_%d" % i for i in range(0, blocks) ]; {5 X' v$ W9 q) ]\" p, q
- filename = [ "tmpfile_%d" % i for i in range(0, blocks) ]
: n* K6 ?2 n7 h2 q - ' a+ T$ N! { ~- P. L
- tasks = []
+ W% Y2 { R\" h/ e- z/ ` - for i in range(0,blocks):3 ^- ^: _/ @; P1 F7 ^
- task = AxelPython( threadname[i], url, filename[i], ranges[i] )( h3 P: `! ?\" ]; x: T5 {* m+ C
- task.setDaemon( True )
& K2 g+ ~: `- g2 G7 f/ I - task.start()& m# f0 j* E+ \ w+ _, p$ C
- tasks.append( task )) A) T& k' c4 G: Q
- 7 M3 c1 G6 ?) p, m6 @
- time.sleep( 2 )
4 z0 A! k0 {- y, c6 @ - while islive(tasks):
' n1 K4 q, y; j- ] - downloaded = sum( [task.downloaded for task in tasks] )
* V# f1 u% O3 ?\" v' ]' _ - process = downloaded/float(size)*100. N: o\" Y7 Z) n5 m' m
- show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)) f3 g( ]\" U: y. d0 Y+ U
- sys.stdout.write(show); W2 ? M4 N$ R* m6 p8 p' I
- sys.stdout.flush(): A0 a- N9 ?5 U$ j) `
- time.sleep( 0.5 )& Y6 S* }: m* O# ^9 m O
- * X$ S: a3 L q# T! G3 I
- filehandle = open( output, 'wb+' )
) U6 p- c' F# D1 b1 ^, y! G - for i in filename:0 |% J) d6 y+ o# T
- f = open( i, 'rb' )
' N. e( E1 }% s7 g; p8 i2 u - filehandle.write( f.read() )
1 _5 M; h G0 O2 |5 s2 ^ - f.close()\" c' j: D1 A8 |
- try:6 s+ H# H4 r6 N6 L$ f
- os.remove(i)- J4 W0 z, n4 V8 e1 d
- pass
( B- U4 u3 B0 g: x0 b7 q% K3 B - except:3 o- Z R6 n' c! I) F# f
- pass
i0 W7 {; J4 Z - 6 u% g1 ]# I2 e L+ O# |* [\" r
- filehandle.close()' ~# p. m( U/ q6 S4 w4 W5 z9 O
-
; E$ o1 M9 T& f! K - if __name__ == '__main__':4 ]% W9 i- {$ P
- url = "http://xz1.mm667.com/xz84/images/001.jpg"$ {8 R$ b' X% Q6 s9 o# Q
- output = '001.jpg'
2 E- s' Y5 q( Z: A# i$ U - paxel( url, output, blocks=4, proxies={} )
复制代码 " R+ ^4 ^6 J. X ^5 l0 n8 X/ W2 G
|
zan
|