- 在线时间
- 7 小时
- 最后登录
- 2021-1-20
- 注册时间
- 2021-1-9
- 听众数
- 3
- 收听数
- 0
- 能力
- 0 分
- 体力
- 23 点
- 威望
- 0 点
- 阅读权限
- 20
- 积分
- 10
- 相册
- 0
- 日志
- 0
- 记录
- 0
- 帖子
- 7
- 主题
- 7
- 精华
- 0
- 分享
- 0
- 好友
- 0
升级   5.26% 该用户从未签到
- 自我介绍
- 我本名为我,那就是我
 |
 - #!/usr/bin/python
- 3 O# a$ s: ]: [ ?$ \\" J# -*- coding: utf-8 -*-' U( F4 W' _' s( w) i\" \! T* l
- 5 r6 Z& E: p4 d( t( u' `import sqlite3
- 7 D7 c$ g, Z% l* U8 rimport requests; p: r: I! s0 n; q0 ~+ M4 ~
- from bs4 import BeautifulSoup* A9 Z$ T. W) q, q
- from re import escape+ u2 S+ q$ o8 [1 m
- \" z1 v9 q1 z k6 R$ zif __name__ == '__main__':/ r5 `$ z$ g) ?; |
- conn = sqlite3.connect('Python.db')
- % Z2 m q4 e* X- ] c = conn.cursor()
- + D: x% ?* E( x c.execute('''CREATE TABLE IF NOT EXISTS Python (' s+ o; V8 [9 |0 g
- Url VARCHAR,- l: C7 S\" P3 [( r L\" M! u
- Title VARCHAR,$ ~\" f- @8 L* G; K
- Author VARCHAR
- - ?; k Q8 k0 n! X: ? )''')
- ) Y! t; W9 M' m7 {1 \ conn.commit()) f }5 n: s2 @3 ?/ f+ a
- ?3 x, z2 ]! K$ l4 j! C& q+ r # --------------------Split Line--------------------
- 9 d6 ^1 }) P/ p! k6 c headers = {$ W- X, A: [7 N$ h7 n9 h5 t# G: M
- "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
- V1 ?5 o- h4 @0 R: s* x3 | }
- ; r7 y% v7 ^4 `2 x
- ! k7 u+ \ x8 Q$ |4 F for i in range(1, 1046):% B+ R' ^& a; w- r# s
- url = "http://xxx/index_%s.html" % str(i)' q7 Z: f0 ]1 @' j* N2 r' @! R) ^
- req = requests.get(url=url, headers=headers)
- * u2 r2 U\" x7 d! Y req.encoding = "utf-8"
- # U1 G7 Q o9 r7 j. v7 B# V html = BeautifulSoup(req.text, "lxml")# `# ^( r' E7 h- }) h
- + H! t3 D2 F- v6 B # --------------------Split Line--------------------
- ' ^* y$ Q$ k5 h( L7 {\" I9 Z. N for div in html.find_all('div', class_='loop'):8 E7 y, i j; Y# l. H8 q5 t/ X
- content_body = div.select('h2 > a')[0]
- , y! M0 Y* {( |6 ]7 A1 d/ M' r+ A+ v content_infor = div.select('.content_infor > span:nth-child(3)')[0]
- 6 P. h; x: t K: H$ ?% M9 Y9 R) X
- $ r$ w& c; m% c) q # --------------------Split Line--------------------
- 5 y( s9 ~* w; _; |4 l cursor = c.execute(
- 6 n- @9 k# k% l0 v$ v% b "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
- 7 t! n) _# A, }- ]( _) `- l len = 0
- 5 q. `: }7 P# @) l for row in cursor:' O- w* [8 ^; i
- len = row[0]
- 0 F |3 o0 y8 \) s* q6 ~& {& a6 g! | if len > 0:5 d4 _* L# j# D: {+ ~# P
- continue2 ~1 T2 ]4 r$ j! t9 S4 V
- 5 Y8 P* W; J5 G, G
- # --------------------Split Line-------------------- R. ^1 f, A0 e
- c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (# T- w; O1 _0 ~0 T/ u' d- `
- "http://xxx" + content_body.get('href'),( J7 f7 ~- k1 G. D0 q9 ^
- escape(content_body.get('title').replace("\"", "\"\"")),
- $ j2 Z\" {( ] b% u: Q! G1 h2 i content_infor.text.replace('xxx: ', '')))1 p/ c6 p% O) Z+ l, \; M; `( X
- $ L\" \; o' D @7 f' L6 l* u1 r conn.commit()
- 7 ]\" K r0 \# A/ Q. E! |3 g: u print("第%s页" % str(i))
- - r7 [\" C; R, \: S. O4 z
- , [$ a0 t6 i; [ # --------------------Split Line--------------------* r( k2 Y, J% r
- conn.close()$ G/ Q2 F' B, `1 i8 O& a
% q0 n& G+ ~. ^. L7 f4 z
! b+ { J U( M/ x0 _5 PPython交流群:1047602540
* {/ }. H+ p; x9 c0 U) H, V- d |
zan
|