查看: 1526|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
; Z& _* Q; H+ W+ B1 r1 i# -*- coding: utf-8 -*-
4 q5 [\" C y- j2 Y5 F( \# t7 ^3 B1 x2 E. M) R* O0 K
import requests$ q+ i- N4 g- ~8 A0 B
from lxml import etree
0 e1 l/ P+ n\" l& M. }import sqlite38 M7 z; z! m* g
0 G3 S& W3 W' j+ Z! ?# a4 d! s; E' k1 I3 w+ g' q# R
def write_sql(c, text):
+ }3 I+ `8 y0 w$ v, [\" p& t html = etree.HTML(text)
5 U4 p1 [7 z3 |7 q # 标题. u% q- \) n R3 I# Y
titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')* @8 g: ? s, C/ D\" t9 I. L
# 链接: W! G2 p) [$ n5 C+ Y
hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')/ W. A M* ~: N, z7 N& K
# 日期
) V$ r' Q0 I4 |/ D ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')8 y$ q+ R' z5 U- w& {) p: }8 K3 M8 Q1 u
- A8 h* N5 u9 B
number = 0
5 x. X+ _; b3 E/ M# d0 p# v for title, href, em in zip(titles, hrefs, ems):
( s6 _1 g- t1 W' d href = host + href' o, e; L8 t. r) e
cursor = c.execute(
2 B' `\" W. G0 g7 e* h' r "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)- g( C+ s+ q- f( m! L
res = c.fetchall()
% \$ X# f6 F! F0 Q/ Q% m # 判断该字段是否已存在 |8 p' L\" Y j\" v( |5 G
if res[0][0] > 0:
' V0 }: @: {5 `* }3 l continue2 D& d: `, |- |8 @5 ~
* z; n$ R+ `/ c1 [- ^! C
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (3 V% w5 C\" y e @6 Q. g
href,7 E) V- l7 g( e\" x, K6 o3 E5 _
title.replace("\"", "\"\""),' I& q4 Z) s) V) M. t; _\" G; W
em))5 I& P! _9 e* J4 C: F* t9 U0 V7 K
number += 1+ L% j0 o, y# G
print(title, href, em)
; y* r% T2 T; T5 [) |: q- e# M; c* p3 Q8 L- W
conn.commit()
; o3 ?6 h( u7 h$ Q, I! s return number > 0: B A. a; o& ^; o0 a
5 E$ i4 }5 H5 j/ h6 N- f& l3 y6 P) |* ~+ \& P* z! N7 \# m
if __name__ == '__main__':
2 _9 Y% k4 |% R' t
1 A; e1 b) N6 m4 y1 R conn = sqlite3.connect("Python-xxx.db")
( j1 P+ w, Q0 m* ~5 g$ T2 E0 { c = conn.cursor()% R& J, g3 `: y- ^\" y4 L3 K/ l
c.execute('''CREATE TABLE IF NOT EXISTS Python (. W* |0 \7 |2 h\" [8 o
Url VARCHAR,+ u1 f2 b9 _9 L, n; ~% Y
Title VARCHAR, `/ W\" y\" i# F- a: \& j
Author VARCHAR; C# q7 O& u0 W, w$ E @9 m
)''')\" _+ G5 F) W( l$ r$ G\" O\" F
conn.commit()
9 z h\" ] f; c- j; w0 ]. K& N4 Z5 j\" q# |. ?& k0 C
host = "https://xxx"+ p0 L2 \0 e0 |; f
url = host + "/xxx"
7 c& u6 f( b% R2 l% V req = requests.get(url)
2 H, N: J- l( K. T5 i# v3 s req.encoding = 'utf-8'5 t1 r1 u3 n0 `
# print(req.text)
+ e* _9 C+ ]3 {2 B
& Q2 s2 t3 @& u v9 {$ ?( b html = etree.HTML(req.text)% i% ]4 O. A x* S
clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')
# b Y% d% g( l& r: x hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')
# _! O0 D2 p; M4 x1 R! V; s\" [ # print(clearfix, href)
# |/ O$ ~9 q$ x, O! f8 p9 t
6 @ P! `: n\" _; S+ g- f for clearfix, href in zip(clearfixs, hrefs):
, R+ f( p. X, K3 F: C print(clearfix, host + href)
6 z* G/ d$ J- e9 O' y\" S
5 L; N0 p3 C4 a% t' C page = 1) W7 ~* Y& O7 q& ^) p: t$ J c
while True:
2 }\" @, N n a6 d9 [ url = host + href + "/list_%s.html" % page
# q8 L- z& ]4 Y* a( D0 H req = requests.get(url)7 Z\" S4 t+ f v5 ?
req.encoding = 'utf-8'% e6 T$ J1 C\" D$ j; f1 Q9 ~4 ?
% I* r2 i6 G& ~7 j, @
if (not write_sql(c, req.text)):# B* C8 x( I! ]. E; a
break5 D8 }! r+ e/ m7 f; I
' d/ O3 w2 g! A\" t9 D4 ~5 S0 ]3 F print("第%s页" % page)
; j: K( F {9 S8 C0 f) i4 q page += 11 Q9 t- \1 T& H+ o: x5 t. S! L6 S
3 _3 X z/ H2 O2 }+ P* d
conn.close()
2 p1 j7 @% y: j: K: b: i

#!/usr/bin/python 
; Z& _* Q; H+ W+ B1 r1 i# -*- coding: utf-8 -*- 
4 q5 [\" C  y- j2 Y5 F( \# t7 ^3 B1 x2 E. M) R* O0 K 
import requests$ q+ i- N4 g- ~8 A0 B 
from lxml import etree 
0 e1 l/ P+ n\" l& M. }import sqlite38 M7 z; z! m* g 
 
0 G3 S& W3 W' j+ Z! ?# a4 d! s; E' k1 I3 w+ g' q# R 
def write_sql(c, text): 
+ }3 I+ `8 y0 w$ v, [\" p& t    html = etree.HTML(text) 
5 U4 p1 [7 z3 |7 q    # 标题. u% q- \) n  R3 I# Y 
    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')* @8 g: ?  s, C/ D\" t9 I. L 
    # 链接: W! G2 p) [$ n5 C+ Y 
    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')/ W. A  M* ~: N, z7 N& K 
    # 日期 
) V$ r' Q0 I4 |/ D    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')8 y$ q+ R' z5 U- w& {) p: }8 K3 M8 Q1 u 
- A8 h* N5 u9 B 
    number = 0 
5 x. X+ _; b3 E/ M# d0 p# v    for title, href, em in zip(titles, hrefs, ems): 
( s6 _1 g- t1 W' d        href = host + href' o, e; L8 t. r) e 
        cursor = c.execute( 
2 B' `\" W. G0 g7 e* h' r            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)- g( C+ s+ q- f( m! L 
        res = c.fetchall() 
% \$ X# f6 F! F0 Q/ Q% m        # 判断该字段是否已存在  |8 p' L\" Y  j\" v( |5 G 
        if res[0][0] > 0: 
' V0 }: @: {5 `* }3 l            continue2 D& d: `, |- |8 @5 ~ 
* z; n$ R+ `/ c1 [- ^! C 
        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (3 V% w5 C\" y  e  @6 Q. g 
            href,7 E) V- l7 g( e\" x, K6 o3 E5 _ 
            title.replace("\"", "\"\""),' I& q4 Z) s) V) M. t; _\" G; W 
            em))5 I& P! _9 e* J4 C: F* t9 U0 V7 K 
        number += 1+ L% j0 o, y# G 
        print(title, href, em) 
; y* r% T2 T; T5 [) |: q- e# M; c* p3 Q8 L- W 
    conn.commit() 
; o3 ?6 h( u7 h$ Q, I! s    return number > 0: B  A. a; o& ^; o0 a 
 
5 E$ i4 }5 H5 j/ h6 N- f& l3 y6 P) |* ~+ \& P* z! N7 \# m 
if __name__ == '__main__': 
2 _9 Y% k4 |% R' t 
1 A; e1 b) N6 m4 y1 R    conn = sqlite3.connect("Python-xxx.db") 
( j1 P+ w, Q0 m* ~5 g$ T2 E0 {    c = conn.cursor()% R& J, g3 `: y- ^\" y4 L3 K/ l 
    c.execute('''CREATE TABLE IF NOT EXISTS Python (. W* |0 \7 |2 h\" [8 o 
        Url VARCHAR,+ u1 f2 b9 _9 L, n; ~% Y 
        Title VARCHAR,  `/ W\" y\" i# F- a: \& j 
        Author VARCHAR; C# q7 O& u0 W, w$ E  @9 m 
    )''')\" _+ G5 F) W( l$ r$ G\" O\" F 
    conn.commit() 
9 z  h\" ]  f; c- j; w0 ]. K& N4 Z5 j\" q# |. ?& k0 C 
    host = "https://xxx"+ p0 L2 \0 e0 |; f 
    url = host + "/xxx" 
7 c& u6 f( b% R2 l% V    req = requests.get(url) 
2 H, N: J- l( K. T5 i# v3 s    req.encoding = 'utf-8'5 t1 r1 u3 n0 ` 
    # print(req.text) 
+ e* _9 C+ ]3 {2 B 
& Q2 s2 t3 @& u  v9 {$ ?( b    html = etree.HTML(req.text)% i% ]4 O. A  x* S 
    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()') 
# b  Y% d% g( l& r: x    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href') 
# _! O0 D2 p; M4 x1 R! V; s\" [    # print(clearfix, href) 
# |/ O$ ~9 q$ x, O! f8 p9 t 
6 @  P! `: n\" _; S+ g- f    for clearfix, href in zip(clearfixs, hrefs): 
, R+ f( p. X, K3 F: C        print(clearfix, host + href) 
6 z* G/ d$ J- e9 O' y\" S 
5 L; N0 p3 C4 a% t' C        page = 1) W7 ~* Y& O7 q& ^) p: t$ J  c 
        while True: 
2 }\" @, N  n  a6 d9 [            url = host + href + "/list_%s.html" % page 
# q8 L- z& ]4 Y* a( D0 H            req = requests.get(url)7 Z\" S4 t+ f  v5 ? 
            req.encoding = 'utf-8'% e6 T$ J1 C\" D$ j; f1 Q9 ~4 ? 
% I* r2 i6 G& ~7 j, @ 
            if (not write_sql(c, req.text)):# B* C8 x( I! ]. E; a 
                break5 D8 }! r+ e/ m7 f; I 
 
' d/ O3 w2 g! A\" t9 D4 ~5 S0 ]3 F            print("第%s页" % page) 
; j: K( F  {9 S8 C0 f) i4 q            page += 11 Q9 t- \1 T& H+ o: x5 t. S! L6 S 
3 _3 X  z/ H2 O2 }+ P* d 
    conn.close() 
2 p1 j7 @% y: j: K: b: i

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan