查看: 1546|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
4 c$ Q( _& Z [, y# -*- coding: utf-8 -*-6 a( \9 ^# z4 [, q5 A
+ U+ Z B( V) g4 k1 |\" Eimport requests
' w0 E4 ?4 P, b& u, u, f4 mfrom lxml import etree
$ D- O4 q- w& `2 f- J$ l$ zimport sqlite3
7 }& @/ ~) F- S2 A& \+ h9 F$ L* ]. c# d
/ V$ n7 v o$ Q/ g/ X
def write_sql(c, text):& T8 h4 f4 h# O9 D4 E
html = etree.HTML(text). ^$ {! P$ ~. ?6 n, q
# 标题
2 S* k% h8 F! O0 n) e7 ? titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')
$ s% f* X! ]7 Y7 _ # 链接
8 w5 O0 H\" }1 C) K: r% H2 T hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')4 }! I2 P: l! U$ O& P
# 日期& g% P\" F$ B% `- Z5 q6 C\" u
ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
% @) G' D0 Y5 k3 ]3 h4 F& {. E3 c* n' A3 w8 q. d
number = 0
' l1 j9 E: V. d+ z5 n0 C\" y for title, href, em in zip(titles, hrefs, ems):9 {; |0 P3 o& j! q( }
href = host + href\" F. ~4 V! \2 y% {# V
cursor = c.execute(
Q1 B8 A/ ]\" S6 N1 O$ L "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
3 }* D- ~% p ~\" o\" l1 U res = c.fetchall()
' Z' Y9 s& d, {1 @ \- @ # 判断该字段是否已存在
) v0 Q\" k0 j) Q0 |( B if res[0][0] > 0:3 w4 c& r\" C$ R9 c% W
continue
7 s, w5 U# g) j, ^- V
* K' t6 g b4 L! s. h c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
, n! B7 Z2 X$ S/ R& p href,
6 K+ ~8 ~/ Q/ G/ C* Z3 l title.replace("\"", "\"\""),+ F( g/ I+ o3 K$ m$ V3 G
em))6 a8 r9 ^2 Y$ Z0 @# t+ |
number += 17 I, v3 I' Y' l
print(title, href, em)
6 e& a5 L j4 s* g\" f' C
( B2 L3 w# c\" i4 b: @0 W conn.commit()
- f2 |$ O: ~\" J. ^\" w0 u return number > 0
6 }0 L! d\" k7 q5 L$ C' B. C
: t# l: r* m6 \1 @5 W' t e: k
- I _# k# e: `5 n+ A+ |if __name__ == '__main__':
( E+ H3 X) T, `3 C% o! V8 [
- f6 l# i5 |+ r conn = sqlite3.connect("Python-xxx.db")' K: i: d% U! C6 U
c = conn.cursor()
5 a6 F- s2 O: I* Q7 h* n c.execute('''CREATE TABLE IF NOT EXISTS Python (6 ^5 O7 U3 g8 p* k$ t7 k* J
Url VARCHAR,9 F: C4 ?& X$ V. q5 u
Title VARCHAR,7 [0 Q( K& \5 b! M5 o: y\" t. R+ _+ U
Author VARCHAR
# e! z+ x4 M6 j# { )''')
8 r2 ~% z# y6 K/ o conn.commit()
4 G7 Z5 M) x) [# n& C5 w2 X. d5 A+ F' s4 f! u+ A' u3 r8 p7 T3 B
host = "https://xxx"
$ F# B+ y+ W) z. ^& {! [ url = host + "/xxx"
' r: k# S( q\" M$ w req = requests.get(url)9 d3 k3 R1 C1 u8 [! H
req.encoding = 'utf-8'
. K$ J% j5 F, t5 F5 ^5 F1 Z # print(req.text)
. H/ b& P! m, s3 o
. B& J, B( ?( j& z- K5 \7 z html = etree.HTML(req.text)
, b( R8 n1 S+ S9 f ` clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()'). |( R4 k/ |3 K2 c* |( L* }
hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')
( I6 g) l6 \: K1 a3 M # print(clearfix, href), k8 ^8 m5 W( _! B: X
& C% x* H# C1 h8 H8 m5 X: r for clearfix, href in zip(clearfixs, hrefs):3 W6 c, h6 V: E1 k/ ?9 q: L
print(clearfix, host + href)
V7 |6 N# U$ D7 Z' E
* W$ K8 T/ ~- `2 n page = 18 p. ^) B% s; j; y
while True: A H& j$ p& y& e) ?, h
url = host + href + "/list_%s.html" % page5 B4 K& }% f6 S$ w- h5 W; \5 B
req = requests.get(url)
1 ]/ C8 C- o D1 |: A$ e- G& J req.encoding = 'utf-8'
9 S# s* `# V; x4 v1 \
2 H8 d6 V6 _& Z9 [0 U) [ if (not write_sql(c, req.text)):) M2 h3 c4 P: P7 m- Q1 t' q1 i
break; C9 Y% _3 h( n
, Y$ |; v! g( {\" G2 b8 g print("第%s页" % page), l+ {& ]& Z' F/ ^0 O8 w
page += 17 y7 N7 z$ j7 ~- u9 J* q
5 ^+ C/ I6 O6 ^3 i
conn.close()
7 H: }% q0 `, e

#!/usr/bin/python 
4 c$ Q( _& Z  [, y# -*- coding: utf-8 -*-6 a( \9 ^# z4 [, q5 A 
 
+ U+ Z  B( V) g4 k1 |\" Eimport requests 
' w0 E4 ?4 P, b& u, u, f4 mfrom lxml import etree 
$ D- O4 q- w& `2 f- J$ l$ zimport sqlite3 
7 }& @/ ~) F- S2 A& \+ h9 F$ L* ]. c# d 
/ V$ n7 v  o$ Q/ g/ X 
def write_sql(c, text):& T8 h4 f4 h# O9 D4 E 
    html = etree.HTML(text). ^$ {! P$ ~. ?6 n, q 
    # 标题 
2 S* k% h8 F! O0 n) e7 ?    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()') 
$ s% f* X! ]7 Y7 _    # 链接 
8 w5 O0 H\" }1 C) K: r% H2 T    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')4 }! I2 P: l! U$ O& P 
    # 日期& g% P\" F$ B% `- Z5 q6 C\" u 
    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()') 
% @) G' D0 Y5 k3 ]3 h4 F& {. E3 c* n' A3 w8 q. d 
    number = 0 
' l1 j9 E: V. d+ z5 n0 C\" y    for title, href, em in zip(titles, hrefs, ems):9 {; |0 P3 o& j! q( } 
        href = host + href\" F. ~4 V! \2 y% {# V 
        cursor = c.execute( 
  Q1 B8 A/ ]\" S6 N1 O$ L            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href) 
3 }* D- ~% p  ~\" o\" l1 U        res = c.fetchall() 
' Z' Y9 s& d, {1 @  \- @        # 判断该字段是否已存在 
) v0 Q\" k0 j) Q0 |( B        if res[0][0] > 0:3 w4 c& r\" C$ R9 c% W 
            continue 
7 s, w5 U# g) j, ^- V 
* K' t6 g  b4 L! s. h        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
, n! B7 Z2 X$ S/ R& p            href, 
6 K+ ~8 ~/ Q/ G/ C* Z3 l            title.replace("\"", "\"\""),+ F( g/ I+ o3 K$ m$ V3 G 
            em))6 a8 r9 ^2 Y$ Z0 @# t+ | 
        number += 17 I, v3 I' Y' l 
        print(title, href, em) 
6 e& a5 L  j4 s* g\" f' C 
( B2 L3 w# c\" i4 b: @0 W    conn.commit() 
- f2 |$ O: ~\" J. ^\" w0 u    return number > 0 
6 }0 L! d\" k7 q5 L$ C' B. C 
: t# l: r* m6 \1 @5 W' t  e: k 
- I  _# k# e: `5 n+ A+ |if __name__ == '__main__': 
( E+ H3 X) T, `3 C% o! V8 [ 
- f6 l# i5 |+ r    conn = sqlite3.connect("Python-xxx.db")' K: i: d% U! C6 U 
    c = conn.cursor() 
5 a6 F- s2 O: I* Q7 h* n    c.execute('''CREATE TABLE IF NOT EXISTS Python (6 ^5 O7 U3 g8 p* k$ t7 k* J 
        Url VARCHAR,9 F: C4 ?& X$ V. q5 u 
        Title VARCHAR,7 [0 Q( K& \5 b! M5 o: y\" t. R+ _+ U 
        Author VARCHAR 
# e! z+ x4 M6 j# {    )''') 
8 r2 ~% z# y6 K/ o    conn.commit() 
4 G7 Z5 M) x) [# n& C5 w2 X. d5 A+ F' s4 f! u+ A' u3 r8 p7 T3 B 
    host = "https://xxx" 
$ F# B+ y+ W) z. ^& {! [    url = host + "/xxx" 
' r: k# S( q\" M$ w    req = requests.get(url)9 d3 k3 R1 C1 u8 [! H 
    req.encoding = 'utf-8' 
. K$ J% j5 F, t5 F5 ^5 F1 Z    # print(req.text) 
. H/ b& P! m, s3 o 
. B& J, B( ?( j& z- K5 \7 z    html = etree.HTML(req.text) 
, b( R8 n1 S+ S9 f  `    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()'). |( R4 k/ |3 K2 c* |( L* } 
    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href') 
( I6 g) l6 \: K1 a3 M    # print(clearfix, href), k8 ^8 m5 W( _! B: X 
 
& C% x* H# C1 h8 H8 m5 X: r    for clearfix, href in zip(clearfixs, hrefs):3 W6 c, h6 V: E1 k/ ?9 q: L 
        print(clearfix, host + href) 
  V7 |6 N# U$ D7 Z' E 
* W$ K8 T/ ~- `2 n        page = 18 p. ^) B% s; j; y 
        while True:  A  H& j$ p& y& e) ?, h 
            url = host + href + "/list_%s.html" % page5 B4 K& }% f6 S$ w- h5 W; \5 B 
            req = requests.get(url) 
1 ]/ C8 C- o  D1 |: A$ e- G& J            req.encoding = 'utf-8' 
9 S# s* `# V; x4 v1 \ 
2 H8 d6 V6 _& Z9 [0 U) [            if (not write_sql(c, req.text)):) M2 h3 c4 P: P7 m- Q1 t' q1 i 
                break; C9 Y% _3 h( n 
 
, Y$ |; v! g( {\" G2 b8 g            print("第%s页" % page), l+ {& ]& Z' F/ ^0 O8 w 
            page += 17 y7 N7 z$ j7 ~- u9 J* q 
5 ^+ C/ I6 O6 ^3 i 
    conn.close() 
7 H: }% q0 `, e

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan