查看: 1549|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
8 R+ R( k' w5 }- q7 B! T3 s& F# -*- coding: utf-8 -*-
+ V6 l1 b# ~# N
]& C) Q) D4 @# T- m2 o. Y; himport requests8 [/ R0 Q+ W* ~* l0 c9 O8 B* H
from lxml import etree
- r: D6 Y* A9 q& \import sqlite3; k1 h2 ^9 C\" |9 K\" A. W
) s* O4 L/ a( r9 ]\" m. v p1 \8 u& Z\" T5 x# `5 j
def write_sql(c, text):. I f' M4 x5 ?\" O, j
html = etree.HTML(text)
7 s' `; J5 ]; _ ^, [; S5 [0 J # 标题
y, j( Z% ~! H3 q0 Z titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')9 S7 c+ m- S- |2 ?5 M
# 链接+ K. p. n- [# G y- Z
hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')7 R, x0 N$ ?* \% R3 m' m9 Z) y, A5 t4 J
# 日期
8 ]0 o3 h) b7 A7 v6 l& x. | ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')1 ?8 A5 }% H7 e
4 c0 i& g6 X) g* j! F m
number = 0# i+ w3 w$ m' n0 a9 C' ~
for title, href, em in zip(titles, hrefs, ems):
5 l$ @% p0 O* ` href = host + href% g6 _2 r7 m. P. _\" Y9 b
cursor = c.execute(4 g4 F\" E- I) ?2 C V+ L- M! p
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
0 a! R5 y& E; A# z\" O; Q, ^ res = c.fetchall()
* i4 p3 z( I+ U$ A1 ` # 判断该字段是否已存在: d1 w V5 I, q5 l9 {
if res[0][0] > 0:
. h2 R+ r1 W7 s4 o- K/ f continue
+ w. a7 {+ _7 n$ K( w3 f\" `$ m* `# j6 B3 I' t\" [+ K; }
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
0 J6 T* t\" q\" ~2 K$ n& s\" N6 V3 Q) p href,
. |3 b# c1 q# q- v title.replace("\"", "\"\""),
: j9 i8 z5 N6 D em))& c! ^, c! Y- c' c5 L/ f# O
number += 1/ K( Y* d5 o/ j1 M# X$ N
print(title, href, em)
; U2 J' U1 I& y3 p' ~$ m% Q1 N% |8 J0 o8 j& |% I5 I; S3 A: b
conn.commit()
8 j/ Y z8 v# o& V6 k3 a return number > 0
. S6 x4 b5 x6 G: F* i. q e: k\" `. K
: |. {% d: ^+ Y
if __name__ == '__main__':
+ f5 Q- [+ I& T% H3 |6 v- ^+ ~
, h9 }) M8 ~4 T6 ?. p conn = sqlite3.connect("Python-xxx.db")( f7 k4 I) E6 y, X* h0 Z7 S
c = conn.cursor()8 j. t\" D! l. Q: @7 e: [
c.execute('''CREATE TABLE IF NOT EXISTS Python (
0 Q) O+ D2 N8 I$ i% w6 U Url VARCHAR,: D- w) w\" A8 n\" @# t. C, \
Title VARCHAR,# ^ L( I$ X% K% C0 O% G
Author VARCHAR2 ^! k @6 S4 J4 x7 C
)''')
' r( [) z. t' f$ l+ U' ? k conn.commit()* I9 N# ^1 `: y7 s/ {
& b( b r0 H& J. D7 S host = "https://xxx"' _0 ]/ K& c5 k7 ]5 C, A
url = host + "/xxx"
2 U# H* I$ g$ m4 R* p- a p req = requests.get(url)7 [; F% c# I! F U' H: r
req.encoding = 'utf-8'
2 i8 f1 p* Z5 `6 b # print(req.text)
' z$ Z) J: i- @2 G ~. i\" l: S7 c4 x- W+ V1 n
html = etree.HTML(req.text)
' h6 ?1 F8 C6 | clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')' C: a5 I, k- s3 c# e1 T7 N& W
hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')
# N* v, C; \; e# I # print(clearfix, href)- R) u* i% T\" f2 M+ [$ c
& X2 m) P4 y3 E% z& [, G5 u+ ~ for clearfix, href in zip(clearfixs, hrefs):! A7 m* g; c0 W4 U; v
print(clearfix, host + href), B/ U* p2 r6 I
# P. |% \+ F, o5 l3 F page = 1
) n0 z\" H* t* y5 [3 a while True:
5 X: P& m! ^( |$ `+ n url = host + href + "/list_%s.html" % page
% h1 Q; P\" G/ `8 y$ I, y' R req = requests.get(url)
\" b- d6 V* ^& P9 ]8 P req.encoding = 'utf-8'! p5 h4 X* W4 q/ Q5 Q
* D\" y, L* g6 L7 L: r* v if (not write_sql(c, req.text)):
1 B0 T% @; R1 ?; X; Q' X8 \ break0 P' q\" z. L: \( Y6 r
. r. _6 [. \3 y/ H* \# V2 J8 j0 u+ Q
print("第%s页" % page)
8 T. h, E, E- U2 e page += 1
3 ~* G' t$ y% ^; V3 d. r5 F9 {: ~4 ~
conn.close()
* ?2 p1 @+ \# f. I

#!/usr/bin/python 
8 R+ R( k' w5 }- q7 B! T3 s& F# -*- coding: utf-8 -*- 
+ V6 l1 b# ~# N 
  ]& C) Q) D4 @# T- m2 o. Y; himport requests8 [/ R0 Q+ W* ~* l0 c9 O8 B* H 
from lxml import etree 
- r: D6 Y* A9 q& \import sqlite3; k1 h2 ^9 C\" |9 K\" A. W 
 
) s* O4 L/ a( r9 ]\" m. v  p1 \8 u& Z\" T5 x# `5 j 
def write_sql(c, text):. I  f' M4 x5 ?\" O, j 
    html = etree.HTML(text) 
7 s' `; J5 ]; _  ^, [; S5 [0 J    # 标题 
  y, j( Z% ~! H3 q0 Z    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')9 S7 c+ m- S- |2 ?5 M 
    # 链接+ K. p. n- [# G  y- Z 
    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')7 R, x0 N$ ?* \% R3 m' m9 Z) y, A5 t4 J 
    # 日期 
8 ]0 o3 h) b7 A7 v6 l& x. |    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')1 ?8 A5 }% H7 e 
4 c0 i& g6 X) g* j! F  m 
    number = 0# i+ w3 w$ m' n0 a9 C' ~ 
    for title, href, em in zip(titles, hrefs, ems): 
5 l$ @% p0 O* `        href = host + href% g6 _2 r7 m. P. _\" Y9 b 
        cursor = c.execute(4 g4 F\" E- I) ?2 C  V+ L- M! p 
            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href) 
0 a! R5 y& E; A# z\" O; Q, ^        res = c.fetchall() 
* i4 p3 z( I+ U$ A1 `        # 判断该字段是否已存在: d1 w  V5 I, q5 l9 { 
        if res[0][0] > 0: 
. h2 R+ r1 W7 s4 o- K/ f            continue 
+ w. a7 {+ _7 n$ K( w3 f\" `$ m* `# j6 B3 I' t\" [+ K; } 
        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
0 J6 T* t\" q\" ~2 K$ n& s\" N6 V3 Q) p            href, 
. |3 b# c1 q# q- v            title.replace("\"", "\"\""), 
: j9 i8 z5 N6 D            em))& c! ^, c! Y- c' c5 L/ f# O 
        number += 1/ K( Y* d5 o/ j1 M# X$ N 
        print(title, href, em) 
; U2 J' U1 I& y3 p' ~$ m% Q1 N% |8 J0 o8 j& |% I5 I; S3 A: b 
    conn.commit() 
8 j/ Y  z8 v# o& V6 k3 a    return number > 0 
. S6 x4 b5 x6 G: F* i. q  e: k\" `. K 
: |. {% d: ^+ Y 
if __name__ == '__main__': 
+ f5 Q- [+ I& T% H3 |6 v- ^+ ~ 
, h9 }) M8 ~4 T6 ?. p    conn = sqlite3.connect("Python-xxx.db")( f7 k4 I) E6 y, X* h0 Z7 S 
    c = conn.cursor()8 j. t\" D! l. Q: @7 e: [ 
    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
0 Q) O+ D2 N8 I$ i% w6 U        Url VARCHAR,: D- w) w\" A8 n\" @# t. C, \ 
        Title VARCHAR,# ^  L( I$ X% K% C0 O% G 
        Author VARCHAR2 ^! k  @6 S4 J4 x7 C 
    )''') 
' r( [) z. t' f$ l+ U' ?  k    conn.commit()* I9 N# ^1 `: y7 s/ { 
 
& b( b  r0 H& J. D7 S    host = "https://xxx"' _0 ]/ K& c5 k7 ]5 C, A 
    url = host + "/xxx" 
2 U# H* I$ g$ m4 R* p- a  p    req = requests.get(url)7 [; F% c# I! F  U' H: r 
    req.encoding = 'utf-8' 
2 i8 f1 p* Z5 `6 b    # print(req.text) 
' z$ Z) J: i- @2 G  ~. i\" l: S7 c4 x- W+ V1 n 
    html = etree.HTML(req.text) 
' h6 ?1 F8 C6 |    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')' C: a5 I, k- s3 c# e1 T7 N& W 
    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href') 
# N* v, C; \; e# I    # print(clearfix, href)- R) u* i% T\" f2 M+ [$ c 
 
& X2 m) P4 y3 E% z& [, G5 u+ ~    for clearfix, href in zip(clearfixs, hrefs):! A7 m* g; c0 W4 U; v 
        print(clearfix, host + href), B/ U* p2 r6 I 
 
# P. |% \+ F, o5 l3 F        page = 1 
) n0 z\" H* t* y5 [3 a        while True: 
5 X: P& m! ^( |$ `+ n            url = host + href + "/list_%s.html" % page 
% h1 Q; P\" G/ `8 y$ I, y' R            req = requests.get(url) 
\" b- d6 V* ^& P9 ]8 P            req.encoding = 'utf-8'! p5 h4 X* W4 q/ Q5 Q 
 
* D\" y, L* g6 L7 L: r* v            if (not write_sql(c, req.text)): 
1 B0 T% @; R1 ?; X; Q' X8 \                break0 P' q\" z. L: \( Y6 r 
. r. _6 [. \3 y/ H* \# V2 J8 j0 u+ Q 
            print("第%s页" % page) 
8 T. h, E, E- U2 e            page += 1 
3 ~* G' t$ y% ^; V3 d. r5 F9 {: ~4 ~ 
    conn.close() 
* ?2 p1 @+ \# f. I

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan