查看: 1548|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python# [+ d/ h4 @0 N3 M
# -*- coding: utf-8 -*-8 I# `$ P& u& l j\" {3 w\" W
/ g! S) R! N' T: d) B5 d0 |import requests
8 J+ z! n) }7 `: a, [\" Nfrom lxml import etree
% t. [% h' I. P; o6 T, fimport sqlite3; P2 y; q+ F\" [/ m7 d% i/ x' c+ N
$ q4 v \\" {' N7 R2 y
/ N F) n/ }4 _$ H\" Cdef write_sql(c, text):9 W4 o9 Q\" s! K+ \5 a& h
html = etree.HTML(text)- J. F: v$ C( Z9 n* v
# 标题3 O' p& d2 B3 }\" ~; @
titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')& W; g: _+ |# `2 u: {
# 链接
7 M\" l( O3 v$ d/ y( A hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')1 R0 H$ W9 i$ {9 L# A7 C3 } \+ t
# 日期
/ M% Y0 ` ?4 G; l$ e( S7 M5 r8 H, T ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
, n8 k% i$ S2 s( x. t2 y# U
2 m# K8 `$ e: L D- d8 ^\" Z number = 0
+ I3 } n- \- Q for title, href, em in zip(titles, hrefs, ems):7 _7 q! }' C. Q2 x# J6 b
href = host + href! r7 ?: ~: ]; k, ?4 o4 P
cursor = c.execute(
$ F0 J. r8 f+ i$ Y "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)9 I\" `2 |* N1 x7 \' G3 ] V5 Y
res = c.fetchall()
& O. d! M' K8 d D. H # 判断该字段是否已存在! m% |6 s* o1 u/ E
if res[0][0] > 0:% u8 {+ F% M3 C* q( H7 z
continue
7 I- a3 `' J; D/ B+ S7 P+ x5 c2 R, ]' U. \2 g
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
* k3 H/ Y$ m( Y7 J5 l0 W href,: T. M) X, g1 y3 d D9 @& v2 x3 K* v1 I7 P
title.replace("\"", "\"\""), k7 ?* w! R r* Q5 k3 ?3 D& H
em))0 q8 @% Z# f4 c# Q/ a5 Y8 c
number += 1; d7 s. E) V- U+ D. O! ]# c. g. Q
print(title, href, em)
1 n, ^\" D- A& Q( v# t2 M, t
. d6 |3 g* s* |: L' H9 o conn.commit()# A5 {, h# n) P6 P2 y% S# N
return number > 0/ c5 y, \0 a8 V
8 B\" V- U F1 f7 N# ` F- [
( t: Z* B5 I3 i. z8 N2 Zif __name__ == '__main__':0 \; z\" y& O6 l. G
; U5 N* T3 W6 W7 k- |' ~6 ^' x conn = sqlite3.connect("Python-xxx.db")
/ N& l3 a7 V* w( [( [9 C c = conn.cursor()
% k7 w3 j. A4 y2 U1 r8 g/ x c.execute('''CREATE TABLE IF NOT EXISTS Python (- w2 E0 x6 r: x' c/ m- a# ]
Url VARCHAR,; y1 C) y. v' C$ ~+ ~# G4 B
Title VARCHAR,# h$ L# v) D* p, U1 W/ n3 T
Author VARCHAR) P( f- ~3 b) c, M l$ Q( O
)''')
, [- q; k4 S7 I' ` conn.commit()
& |6 t/ i. V/ e! G# X( u
) t& M5 m6 _2 x host = "https://xxx"
2 U9 c4 A( ~# B8 H4 u7 g# | url = host + "/xxx"
r2 v+ I) P0 [% _8 K# U' W req = requests.get(url)6 R) A6 \! y' K5 g
req.encoding = 'utf-8'
9 l9 T! H\" m; Y) Z; p0 s! _\" h8 P # print(req.text)
8 S& T1 f! \- O) y$ h3 T5 k a6 S8 L( g. k4 Z7 `
html = etree.HTML(req.text)$ l1 ^/ g7 x) z L. B! S
clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')/ \- F& h' S6 H2 E. `+ M
hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')8 h- z. U/ P$ E2 W1 X# `
# print(clearfix, href)* l, D: S9 \6 _6 m
' `. f' j7 } Y\" |% |
for clearfix, href in zip(clearfixs, hrefs): M# I2 f\" u8 u4 W# I
print(clearfix, host + href)8 k2 d3 v) \/ f( x7 R& `2 L
9 m) F6 T4 ^\" \
page = 1) ?( n5 s* o- y+ D9 [/ V& c
while True:
5 u* a\" P! |% s9 |! J url = host + href + "/list_%s.html" % page( z+ F% C& s+ x; `5 m
req = requests.get(url); x9 m0 s7 j- P* y! t
req.encoding = 'utf-8'3 n$ C5 b( c1 U% [: k: R7 V
+ O+ t4 I& M: j# M+ d
if (not write_sql(c, req.text)):\" z- w. L% F/ q4 y0 V% I
break
/ S( q6 V6 g2 c7 M6 X\" b! _
1 d; C' I0 f9 p6 P0 ~% o8 d print("第%s页" % page)
+ b8 r+ g! ]\" c$ ~7 z4 ^ page += 1$ {# c5 \. M5 q& P8 {0 F
- L5 \% L1 A2 D. S) v
conn.close()
) X' Y& ~8 k+ h\" j

#!/usr/bin/python# [+ d/ h4 @0 N3 M 
# -*- coding: utf-8 -*-8 I# `$ P& u& l  j\" {3 w\" W 
 
/ g! S) R! N' T: d) B5 d0 |import requests 
8 J+ z! n) }7 `: a, [\" Nfrom lxml import etree 
% t. [% h' I. P; o6 T, fimport sqlite3; P2 y; q+ F\" [/ m7 d% i/ x' c+ N 
$ q4 v  \\" {' N7 R2 y 
 
/ N  F) n/ }4 _$ H\" Cdef write_sql(c, text):9 W4 o9 Q\" s! K+ \5 a& h 
    html = etree.HTML(text)- J. F: v$ C( Z9 n* v 
    # 标题3 O' p& d2 B3 }\" ~; @ 
    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')& W; g: _+ |# `2 u: { 
    # 链接 
7 M\" l( O3 v$ d/ y( A    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')1 R0 H$ W9 i$ {9 L# A7 C3 }  \+ t 
    # 日期 
/ M% Y0 `  ?4 G; l$ e( S7 M5 r8 H, T    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()') 
, n8 k% i$ S2 s( x. t2 y# U 
2 m# K8 `$ e: L  D- d8 ^\" Z    number = 0 
+ I3 }  n- \- Q    for title, href, em in zip(titles, hrefs, ems):7 _7 q! }' C. Q2 x# J6 b 
        href = host + href! r7 ?: ~: ]; k, ?4 o4 P 
        cursor = c.execute( 
$ F0 J. r8 f+ i$ Y            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)9 I\" `2 |* N1 x7 \' G3 ]  V5 Y 
        res = c.fetchall() 
& O. d! M' K8 d  D. H        # 判断该字段是否已存在! m% |6 s* o1 u/ E 
        if res[0][0] > 0:% u8 {+ F% M3 C* q( H7 z 
            continue 
7 I- a3 `' J; D/ B+ S7 P+ x5 c2 R, ]' U. \2 g 
        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
* k3 H/ Y$ m( Y7 J5 l0 W            href,: T. M) X, g1 y3 d  D9 @& v2 x3 K* v1 I7 P 
            title.replace("\"", "\"\""),  k7 ?* w! R  r* Q5 k3 ?3 D& H 
            em))0 q8 @% Z# f4 c# Q/ a5 Y8 c 
        number += 1; d7 s. E) V- U+ D. O! ]# c. g. Q 
        print(title, href, em) 
1 n, ^\" D- A& Q( v# t2 M, t 
. d6 |3 g* s* |: L' H9 o    conn.commit()# A5 {, h# n) P6 P2 y% S# N 
    return number > 0/ c5 y, \0 a8 V 
8 B\" V- U  F1 f7 N# `  F- [ 
 
( t: Z* B5 I3 i. z8 N2 Zif __name__ == '__main__':0 \; z\" y& O6 l. G 
 
; U5 N* T3 W6 W7 k- |' ~6 ^' x    conn = sqlite3.connect("Python-xxx.db") 
/ N& l3 a7 V* w( [( [9 C    c = conn.cursor() 
% k7 w3 j. A4 y2 U1 r8 g/ x    c.execute('''CREATE TABLE IF NOT EXISTS Python (- w2 E0 x6 r: x' c/ m- a# ] 
        Url VARCHAR,; y1 C) y. v' C$ ~+ ~# G4 B 
        Title VARCHAR,# h$ L# v) D* p, U1 W/ n3 T 
        Author VARCHAR) P( f- ~3 b) c, M  l$ Q( O 
    )''') 
, [- q; k4 S7 I' `    conn.commit() 
& |6 t/ i. V/ e! G# X( u 
) t& M5 m6 _2 x    host = "https://xxx" 
2 U9 c4 A( ~# B8 H4 u7 g# |    url = host + "/xxx" 
  r2 v+ I) P0 [% _8 K# U' W    req = requests.get(url)6 R) A6 \! y' K5 g 
    req.encoding = 'utf-8' 
9 l9 T! H\" m; Y) Z; p0 s! _\" h8 P    # print(req.text) 
8 S& T1 f! \- O) y$ h3 T5 k  a6 S8 L( g. k4 Z7 ` 
    html = etree.HTML(req.text)$ l1 ^/ g7 x) z  L. B! S 
    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')/ \- F& h' S6 H2 E. `+ M 
    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')8 h- z. U/ P$ E2 W1 X# ` 
    # print(clearfix, href)* l, D: S9 \6 _6 m 
' `. f' j7 }  Y\" |% | 
    for clearfix, href in zip(clearfixs, hrefs):  M# I2 f\" u8 u4 W# I 
        print(clearfix, host + href)8 k2 d3 v) \/ f( x7 R& `2 L 
9 m) F6 T4 ^\" \ 
        page = 1) ?( n5 s* o- y+ D9 [/ V& c 
        while True: 
5 u* a\" P! |% s9 |! J            url = host + href + "/list_%s.html" % page( z+ F% C& s+ x; `5 m 
            req = requests.get(url); x9 m0 s7 j- P* y! t 
            req.encoding = 'utf-8'3 n$ C5 b( c1 U% [: k: R7 V 
+ O+ t4 I& M: j# M+ d 
            if (not write_sql(c, req.text)):\" z- w. L% F/ q4 y0 V% I 
                break 
/ S( q6 V6 g2 c7 M6 X\" b! _ 
1 d; C' I0 f9 p6 P0 ~% o8 d            print("第%s页" % page) 
+ b8 r+ g! ]\" c$ ~7 z4 ^            page += 1$ {# c5 \. M5 q& P8 {0 F 
- L5 \% L1 A2 D. S) v 
    conn.close() 
) X' Y& ~8 k+ h\" j

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan