查看: 1603|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python% l! k% `8 t8 d4 `! J+ r
# -*- coding: utf-8 -*-
: ]6 m. Q9 N3 k9 b( j' {
9 E% m' ^/ [$ x/ R ^import requests
& ]6 j9 S! [- V1 }& H* vfrom lxml import etree
7 P\" t2 \( `4 d: R# iimport sqlite3. b. e$ q N7 o
5 h( n- x. C4 C) q- K; r- m$ U8 c% U1 w( ]3 }& V h; u' n2 v
def write_sql(c, text):\" e# c5 `! C3 p9 i- `8 i
html = etree.HTML(text)
3 K( Q! q# h( Q& y2 ?7 f2 B\" \8 B5 I # 标题
8 t( K$ d5 ^% ]6 T titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')1 d1 C) T; ^\" Q/ ~! E: T( I
# 链接$ ~: |/ G, r# `\" L1 s! f# x
hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')7 A% u( s5 m\" p
# 日期7 x+ B6 Y+ P4 D9 k\" [- p
ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
0 X# r\" v1 u; p6 C3 H6 o+ P, a; S5 I
number = 00 l, e2 E# R3 ?2 K
for title, href, em in zip(titles, hrefs, ems):
/ \4 f6 C+ A6 A1 i. A href = host + href
. R: T3 M- f9 @; I4 b: m% u cursor = c.execute(0 {$ W: H( F# P* s! Y. N
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
& b# A0 J2 E5 x V. T- a; k4 b res = c.fetchall()( h9 I& g2 B8 f; m7 I
# 判断该字段是否已存在1 {) o$ Z6 r! H
if res[0][0] > 0:1 c$ e& ^% U. M% J
continue- a2 |% U0 n% c/ K7 `
/ F/ Y: A& J% I c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (9 `( `( R2 x( n, |\" p1 |' e( ]) s8 ^
href,2 y8 h/ m& b. M2 B
title.replace("\"", "\"\""),
# h% J+ @% B4 A; x em))1 a. Q. Y! @\" W
number += 19 R' I m5 t1 \ C# j, v( |1 H3 s0 I
print(title, href, em)( A% w1 r p. w, C! ~7 g& c
1 h7 d1 g) r) { conn.commit()
! A9 C& b- ?+ G+ R# ^( Z9 v; V4 w\" ? return number > 0
+ H8 Z4 {8 i1 V. [0 \1 J0 \) O
2 m4 L8 o7 X7 d* k$ C2 j) F% d
if __name__ == '__main__':
; I( w! D: Z\" z1 w: J6 N9 @3 P
; E\" d+ |* L& @6 {3 J& b conn = sqlite3.connect("Python-xxx.db")7 v. @+ h7 I/ r* r0 O
c = conn.cursor()
- G- t; q\" u' x& b c.execute('''CREATE TABLE IF NOT EXISTS Python (
) j# Y' g. H9 }# x8 L9 g Url VARCHAR,; K\" }- i8 y' H$ @$ R
Title VARCHAR,
% y. M7 ` y1 q1 e( M, Z/ E$ m8 j Author VARCHAR
3 G) c\" J' v$ C4 w )''')5 W, B2 D& C; H8 c, `
conn.commit()
6 P- T* L3 w }/ _3 R
6 d1 B1 L\" W b0 Y% U5 e- c host = "https://xxx"
( t4 m* c2 a# K\" j5 A' A% O4 b url = host + "/xxx", Y _\" {2 f* N1 w( ^
req = requests.get(url)) }: K o) d! S5 {, @7 M* w6 f4 y% k
req.encoding = 'utf-8'
' X# f4 f4 q+ \5 A9 X$ c: t1 a # print(req.text)
3 v\" B- ^$ o- F+ P0 b( \, L/ Y Q9 r+ }0 \
html = etree.HTML(req.text)$ Q9 s\" Q+ I, c
clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')
( Q* H7 S3 t0 y* k% ~0 g: j hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')
5 I+ y, T& C9 j1 q6 x( C% \- B: X # print(clearfix, href)
! y9 T. T+ u; Z4 Y
* c3 a$ l5 j; {$ ^6 j for clearfix, href in zip(clearfixs, hrefs):
& @5 H( l8 V5 B: g/ J) H print(clearfix, host + href)
8 C# J# I8 S+ H2 L5 v( T( P$ M( T, y4 d; d$ G8 U) t) e
page = 10 Y\" T- M Q: B* I9 a
while True:
) |3 H: `2 B1 {% _* @ E& q$ T url = host + href + "/list_%s.html" % page
/ g2 [8 U a2 S% q req = requests.get(url)
- }. g- I0 q- A2 |2 |! r\" G req.encoding = 'utf-8'
\" h) q+ n! i: Y! |
4 N# b9 l* o8 A if (not write_sql(c, req.text)):
* j! J2 j8 \6 } break$ [! K9 @4 G; _( L$ J
' k5 i/ u# ^\" a0 s2 ?6 d$ Q. z
print("第%s页" % page)6 M! `- M\" y4 c& [6 V
page += 1# O2 S! v, c/ V
K9 _4 L7 Y2 o. ^* p H2 e7 a8 I3 A
conn.close()
7 H% R- c' w: Q

#!/usr/bin/python% l! k% `8 t8 d4 `! J+ r 
# -*- coding: utf-8 -*- 
: ]6 m. Q9 N3 k9 b( j' { 
9 E% m' ^/ [$ x/ R  ^import requests 
& ]6 j9 S! [- V1 }& H* vfrom lxml import etree 
7 P\" t2 \( `4 d: R# iimport sqlite3. b. e$ q  N7 o 
 
5 h( n- x. C4 C) q- K; r- m$ U8 c% U1 w( ]3 }& V  h; u' n2 v 
def write_sql(c, text):\" e# c5 `! C3 p9 i- `8 i 
    html = etree.HTML(text) 
3 K( Q! q# h( Q& y2 ?7 f2 B\" \8 B5 I    # 标题 
8 t( K$ d5 ^% ]6 T    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')1 d1 C) T; ^\" Q/ ~! E: T( I 
    # 链接$ ~: |/ G, r# `\" L1 s! f# x 
    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')7 A% u( s5 m\" p 
    # 日期7 x+ B6 Y+ P4 D9 k\" [- p 
    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()') 
0 X# r\" v1 u; p6 C3 H6 o+ P, a; S5 I 
    number = 00 l, e2 E# R3 ?2 K 
    for title, href, em in zip(titles, hrefs, ems): 
/ \4 f6 C+ A6 A1 i. A        href = host + href 
. R: T3 M- f9 @; I4 b: m% u        cursor = c.execute(0 {$ W: H( F# P* s! Y. N 
            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href) 
& b# A0 J2 E5 x  V. T- a; k4 b        res = c.fetchall()( h9 I& g2 B8 f; m7 I 
        # 判断该字段是否已存在1 {) o$ Z6 r! H 
        if res[0][0] > 0:1 c$ e& ^% U. M% J 
            continue- a2 |% U0 n% c/ K7 ` 
 
/ F/ Y: A& J% I        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (9 `( `( R2 x( n, |\" p1 |' e( ]) s8 ^ 
            href,2 y8 h/ m& b. M2 B 
            title.replace("\"", "\"\""), 
# h% J+ @% B4 A; x            em))1 a. Q. Y! @\" W 
        number += 19 R' I  m5 t1 \  C# j, v( |1 H3 s0 I 
        print(title, href, em)( A% w1 r  p. w, C! ~7 g& c 
 
1 h7 d1 g) r) {    conn.commit() 
! A9 C& b- ?+ G+ R# ^( Z9 v; V4 w\" ?    return number > 0 
+ H8 Z4 {8 i1 V. [0 \1 J0 \) O 
2 m4 L8 o7 X7 d* k$ C2 j) F% d 
if __name__ == '__main__': 
; I( w! D: Z\" z1 w: J6 N9 @3 P 
; E\" d+ |* L& @6 {3 J& b    conn = sqlite3.connect("Python-xxx.db")7 v. @+ h7 I/ r* r0 O 
    c = conn.cursor() 
- G- t; q\" u' x& b    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
) j# Y' g. H9 }# x8 L9 g        Url VARCHAR,; K\" }- i8 y' H$ @$ R 
        Title VARCHAR, 
% y. M7 `  y1 q1 e( M, Z/ E$ m8 j        Author VARCHAR 
3 G) c\" J' v$ C4 w    )''')5 W, B2 D& C; H8 c, ` 
    conn.commit() 
6 P- T* L3 w  }/ _3 R 
6 d1 B1 L\" W  b0 Y% U5 e- c    host = "https://xxx" 
( t4 m* c2 a# K\" j5 A' A% O4 b    url = host + "/xxx", Y  _\" {2 f* N1 w( ^ 
    req = requests.get(url)) }: K  o) d! S5 {, @7 M* w6 f4 y% k 
    req.encoding = 'utf-8' 
' X# f4 f4 q+ \5 A9 X$ c: t1 a    # print(req.text) 
3 v\" B- ^$ o- F+ P0 b( \, L/ Y  Q9 r+ }0 \ 
    html = etree.HTML(req.text)$ Q9 s\" Q+ I, c 
    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()') 
( Q* H7 S3 t0 y* k% ~0 g: j    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href') 
5 I+ y, T& C9 j1 q6 x( C% \- B: X    # print(clearfix, href) 
! y9 T. T+ u; Z4 Y 
* c3 a$ l5 j; {$ ^6 j    for clearfix, href in zip(clearfixs, hrefs): 
& @5 H( l8 V5 B: g/ J) H        print(clearfix, host + href) 
8 C# J# I8 S+ H2 L5 v( T( P$ M( T, y4 d; d$ G8 U) t) e 
        page = 10 Y\" T- M  Q: B* I9 a 
        while True: 
) |3 H: `2 B1 {% _* @  E& q$ T            url = host + href + "/list_%s.html" % page 
/ g2 [8 U  a2 S% q            req = requests.get(url) 
- }. g- I0 q- A2 |2 |! r\" G            req.encoding = 'utf-8' 
\" h) q+ n! i: Y! | 
4 N# b9 l* o8 A            if (not write_sql(c, req.text)): 
* j! J2 j8 \6 }                break$ [! K9 @4 G; _( L$ J 
' k5 i/ u# ^\" a0 s2 ?6 d$ Q. z 
            print("第%s页" % page)6 M! `- M\" y4 c& [6 V 
            page += 1# O2 S! v, c/ V 
  K9 _4 L7 Y2 o. ^* p  H2 e7 a8 I3 A 
    conn.close() 
7 H% R- c' w: Q

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan