查看: 1545|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python* U, |6 x. q1 r0 Y8 j7 ^) W3 z
# -*- coding: utf-8 -*-& I: E: g+ K1 k- G\" |3 j
/ H+ h) C: f: vimport requests8 {5 m1 G5 B: A; j/ _8 S\" M
from lxml import etree4 o; i+ E# s3 i) z8 \, T
import sqlite3; _. Y% F h8 `
! H0 `. I! _2 J3 y* D# x- E
+ Q+ z\" ]# o6 \ ]def write_sql(c, text):! j( O\" Y/ q. O N
html = etree.HTML(text)
: @& C- V0 n\" _# b8 B8 J # 标题; H. x. E/ n1 V/ ~* o\" p$ y
titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')2 a K' T1 T7 Q% G
# 链接( {3 R Y5 g2 O& j) K
hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href') k! J& s' p$ Q9 V
# 日期
9 E; Q+ s9 V, Y2 y+ x ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
0 S4 f% \ W4 w& ~% P* j+ T% v; F0 X& x! f5 R' n\" N5 Q
number = 0
6 q* e. t. d; @2 G. d$ _* s; d& ` for title, href, em in zip(titles, hrefs, ems):4 m6 E9 Q2 ?% Z: P% E\" P! G
href = host + href
, v8 i K; v+ b6 B; r cursor = c.execute(2 c; j0 w/ y; s7 O) l9 J( i
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
1 N4 L- `( m- j# W7 i' q6 f8 Z( Y res = c.fetchall()
3 r& G& ?! l e: E% @ # 判断该字段是否已存在9 ~# ]3 o0 z0 P4 H& z
if res[0][0] > 0:% z* s. t5 Q\" F, ^1 v5 ]# U
continue9 A2 S* J. Z9 ]
6 l# n5 L0 W6 m c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (, k1 S0 f, n7 s5 b4 F5 e
href,\" F9 \8 P1 O: ?% g6 m5 |; I A' y
title.replace("\"", "\"\""),
1 W; A; n# B$ t4 I7 _) Y+ C6 |6 L em))
5 f& N; y4 j- {\" e# ]8 O9 \ number += 1
, Q1 D4 x5 E: e8 Q# _2 Q4 h K$ M print(title, href, em), }& q3 f/ t! }\" ^: j
' m+ O) D8 |2 K* u
conn.commit(): o5 k& D% I. ~( }5 a$ Q! P
return number > 0 ]9 k+ [2 X1 f+ z4 u) Y
6 l* G% f/ T# ^: H# ^7 E$ B; ]- \- A% m& x9 c
if __name__ == '__main__':& A4 a4 C. L- p* ?7 \. r
9 K1 N! f8 z8 {( t0 u- v conn = sqlite3.connect("Python-xxx.db")6 i, q7 c4 h. Y8 ~
c = conn.cursor()! P+ A\" T' V5 I8 q
c.execute('''CREATE TABLE IF NOT EXISTS Python (7 U) Z5 h2 w0 a/ x7 C ~! `( i6 j
Url VARCHAR,
\" Z; m; J2 Z- y8 e Title VARCHAR,
* J\" C, z5 ]1 Z1 I+ T0 _% k$ _2 K Author VARCHAR
. }+ d6 y' x; L& k( Z0 U )''')& S1 ?( y# r4 }
conn.commit()
1 X. ~\" Z; H f/ h- E% s
+ u: _\" [: q/ N host = "https://xxx"( m/ W% M\" [$ e7 B/ S8 d( B3 W5 Q( Q5 q
url = host + "/xxx"
8 N3 C+ E/ G% [% y ~* m( L req = requests.get(url)' R- q( X& G% @& I! Y5 S
req.encoding = 'utf-8'7 M' ~6 [& v; g p) _7 ]
# print(req.text)
7 S3 Z( j3 I# y/ f# j, |1 |% }' ^- i7 q! d\" Z) j
html = etree.HTML(req.text)
% C\" j3 G; K1 v+ t/ B8 K# T clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')
/ W\" M6 n, T# ~7 Q, M hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')+ P/ g\" W4 Q0 c1 T e4 N
# print(clearfix, href)5 V' V% ~# N/ j% X9 w
5 J7 ~* u5 r& J j1 e1 v9 V
for clearfix, href in zip(clearfixs, hrefs):/ W$ B- K2 ?0 x6 x4 P
print(clearfix, host + href)5 }6 ?* b: u$ T3 V8 N3 F5 A
. p/ f R. d: l: i\" v d
page = 12 M3 j' ]2 v5 ]( l6 q5 z I4 g5 o
while True:
) e- Y5 `7 @8 j) R url = host + href + "/list_%s.html" % page5 P2 `8 Q8 ?+ x
req = requests.get(url)
' P. J; }8 L$ B& l\" A8 P req.encoding = 'utf-8'
. N8 q& Y& v. E. y
0 u4 V5 N3 t5 @( \ if (not write_sql(c, req.text)):
' [8 w J( }0 R/ R\" P6 y break
& _% t% U# d% p/ l3 K
4 R% \- Q! r4 n; d print("第%s页" % page)
9 q6 U, E* d0 Y: h) {4 q8 h: p page += 1
7 Z) @! {9 X1 R; w$ M: ]9 x6 H9 e& ?# |4 F; V* {1 F, O
conn.close(); ~+ O# X1 a! J( g( w

#!/usr/bin/python* U, |6 x. q1 r0 Y8 j7 ^) W3 z 
# -*- coding: utf-8 -*-& I: E: g+ K1 k- G\" |3 j 
 
/ H+ h) C: f: vimport requests8 {5 m1 G5 B: A; j/ _8 S\" M 
from lxml import etree4 o; i+ E# s3 i) z8 \, T 
import sqlite3; _. Y% F  h8 ` 
! H0 `. I! _2 J3 y* D# x- E 
 
+ Q+ z\" ]# o6 \  ]def write_sql(c, text):! j( O\" Y/ q. O  N 
    html = etree.HTML(text) 
: @& C- V0 n\" _# b8 B8 J    # 标题; H. x. E/ n1 V/ ~* o\" p$ y 
    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')2 a  K' T1 T7 Q% G 
    # 链接( {3 R  Y5 g2 O& j) K 
    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')  k! J& s' p$ Q9 V 
    # 日期 
9 E; Q+ s9 V, Y2 y+ x    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()') 
0 S4 f% \  W4 w& ~% P* j+ T% v; F0 X& x! f5 R' n\" N5 Q 
    number = 0 
6 q* e. t. d; @2 G. d$ _* s; d& `    for title, href, em in zip(titles, hrefs, ems):4 m6 E9 Q2 ?% Z: P% E\" P! G 
        href = host + href 
, v8 i  K; v+ b6 B; r        cursor = c.execute(2 c; j0 w/ y; s7 O) l9 J( i 
            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href) 
1 N4 L- `( m- j# W7 i' q6 f8 Z( Y        res = c.fetchall() 
3 r& G& ?! l  e: E% @        # 判断该字段是否已存在9 ~# ]3 o0 z0 P4 H& z 
        if res[0][0] > 0:% z* s. t5 Q\" F, ^1 v5 ]# U 
            continue9 A2 S* J. Z9 ] 
 
6 l# n5 L0 W6 m        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (, k1 S0 f, n7 s5 b4 F5 e 
            href,\" F9 \8 P1 O: ?% g6 m5 |; I  A' y 
            title.replace("\"", "\"\""), 
1 W; A; n# B$ t4 I7 _) Y+ C6 |6 L            em)) 
5 f& N; y4 j- {\" e# ]8 O9 \        number += 1 
, Q1 D4 x5 E: e8 Q# _2 Q4 h  K$ M        print(title, href, em), }& q3 f/ t! }\" ^: j 
' m+ O) D8 |2 K* u 
    conn.commit(): o5 k& D% I. ~( }5 a$ Q! P 
    return number > 0  ]9 k+ [2 X1 f+ z4 u) Y 
 
6 l* G% f/ T# ^: H# ^7 E$ B; ]- \- A% m& x9 c 
if __name__ == '__main__':& A4 a4 C. L- p* ?7 \. r 
 
9 K1 N! f8 z8 {( t0 u- v    conn = sqlite3.connect("Python-xxx.db")6 i, q7 c4 h. Y8 ~ 
    c = conn.cursor()! P+ A\" T' V5 I8 q 
    c.execute('''CREATE TABLE IF NOT EXISTS Python (7 U) Z5 h2 w0 a/ x7 C  ~! `( i6 j 
        Url VARCHAR, 
\" Z; m; J2 Z- y8 e        Title VARCHAR, 
* J\" C, z5 ]1 Z1 I+ T0 _% k$ _2 K        Author VARCHAR 
. }+ d6 y' x; L& k( Z0 U    )''')& S1 ?( y# r4 } 
    conn.commit() 
1 X. ~\" Z; H  f/ h- E% s 
+ u: _\" [: q/ N    host = "https://xxx"( m/ W% M\" [$ e7 B/ S8 d( B3 W5 Q( Q5 q 
    url = host + "/xxx" 
8 N3 C+ E/ G% [% y  ~* m( L    req = requests.get(url)' R- q( X& G% @& I! Y5 S 
    req.encoding = 'utf-8'7 M' ~6 [& v; g  p) _7 ] 
    # print(req.text) 
7 S3 Z( j3 I# y/ f# j, |1 |% }' ^- i7 q! d\" Z) j 
    html = etree.HTML(req.text) 
% C\" j3 G; K1 v+ t/ B8 K# T    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()') 
/ W\" M6 n, T# ~7 Q, M    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')+ P/ g\" W4 Q0 c1 T  e4 N 
    # print(clearfix, href)5 V' V% ~# N/ j% X9 w 
5 J7 ~* u5 r& J  j1 e1 v9 V 
    for clearfix, href in zip(clearfixs, hrefs):/ W$ B- K2 ?0 x6 x4 P 
        print(clearfix, host + href)5 }6 ?* b: u$ T3 V8 N3 F5 A 
. p/ f  R. d: l: i\" v  d 
        page = 12 M3 j' ]2 v5 ]( l6 q5 z  I4 g5 o 
        while True: 
) e- Y5 `7 @8 j) R            url = host + href + "/list_%s.html" % page5 P2 `8 Q8 ?+ x 
            req = requests.get(url) 
' P. J; }8 L$ B& l\" A8 P            req.encoding = 'utf-8' 
. N8 q& Y& v. E. y 
0 u4 V5 N3 t5 @( \            if (not write_sql(c, req.text)): 
' [8 w  J( }0 R/ R\" P6 y                break 
& _% t% U# d% p/ l3 K 
4 R% \- Q! r4 n; d            print("第%s页" % page) 
9 q6 U, E* d0 Y: h) {4 q8 h: p            page += 1 
7 Z) @! {9 X1 R; w$ M: ]9 x6 H9 e& ?# |4 F; V* {1 F, O 
    conn.close(); ~+ O# X1 a! J( g( w

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan