查看: 1524|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python. S) N9 F$ r( N# [9 v8 _' T
# -*- coding: utf-8 -*-5 h2 \, ]: B7 h, s& s% ]
0 ~2 @: l& y( f& Ximport requests
0 V5 U2 N, O# C: M; jfrom lxml import etree3 w& \ U$ \% |% }
import sqlite3: i. y# ]$ U* Y* c T1 L
S, n2 L! v$ d. m8 c
' @\" a6 F3 `: A\" V& Xdef write_sql(c, text):
4 }! s& |: @2 W# J html = etree.HTML(text)1 |) m1 u\" b+ ~/ ]* P2 ~
# 标题9 r# L1 h/ q& B0 ^9 |+ j! ~
titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')
3 o9 W9 W6 U, V: x/ u1 F) S # 链接
+ M5 P. k' d& A/ d hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')$ ]! y2 w% y9 t\" j* G
# 日期
. ~# W( I1 F$ ~/ X: D ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
% i5 ?, g' M* \
8 o( G6 i\" `+ u9 f+ |1 S+ F$ o number = 0
6 ^9 Y$ m9 n5 @% D# G2 f for title, href, em in zip(titles, hrefs, ems):
* k) q) p4 ~. x0 V9 ^ href = host + href
) O% ^9 _& ?5 `* v7 u. b cursor = c.execute(
; S3 @$ J6 {* m# Y. ?! d! g "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
+ w\" L& y\" B7 T# ] res = c.fetchall(). V- t/ {2 N7 i: ~, |
# 判断该字段是否已存在
5 l; c' K. q1 ^3 Y. H if res[0][0] > 0:+ t1 ^, X. B4 L5 s y# b
continue
. n1 O( b y r; M( r1 I) g- @5 r; h6 W
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
# ]3 u9 h# G' ~' { href,
$ A5 h- M) e3 F. g% W: J title.replace("\"", "\"\""),
8 n5 t9 A+ k1 M em))6 F5 i# h$ f& H* j; {7 W/ u
number += 1
1 z( Z! } z2 K# S print(title, href, em) o, J! g) r4 T( m0 h9 t
9 z' V* M9 h5 p conn.commit()
3 |* o. b% y/ O+ k: W8 J! I return number > 0
. Q; n5 q: l) d\" ~8 \( d8 x$ l* a' Z! k6 f\" r, p6 V
) L* X1 B9 H2 v\" `7 a8 bif __name__ == '__main__':. [. q3 ^7 Y3 \9 W
+ t3 s7 T' ~! @7 T8 Q6 t/ n conn = sqlite3.connect("Python-xxx.db")6 J0 ~5 y/ T- s/ b+ c g
c = conn.cursor()& k$ C. F& e2 z3 U) h
c.execute('''CREATE TABLE IF NOT EXISTS Python (
* C\" P* k4 f# E# B5 M2 n Url VARCHAR,; [! Z\" ^! P! E+ D! A
Title VARCHAR,
1 U5 C$ K6 j8 J6 e( J% T& S1 | Author VARCHAR! f' ] u, T# V _0 q4 j
)''')- J2 t# \- ?+ }/ \\" n, {0 H. c% j/ X
conn.commit()
& P, [1 C4 y# W5 J+ Q0 @9 G8 L0 a: ?1 m\" |
host = "https://xxx"
$ I+ I, \+ x' R5 @5 G6 d } url = host + "/xxx"( e( }5 O# b- ^. n
req = requests.get(url)8 a1 }+ w* ?: k
req.encoding = 'utf-8'2 \! X1 ^2 C6 ^* h
# print(req.text)# U. k\" J: z* L% |
6 f2 L7 P1 A& q4 y( b( X9 r! h
html = etree.HTML(req.text)
; z: F- S( i9 s, q b clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')
8 L! n8 o, m1 H7 S( `8 P, n$ f hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')8 B- |5 v. h* D/ P3 [3 f
# print(clearfix, href)% b, ^+ c* l. I1 z% e
* p1 J& {7 k0 s, s' x
for clearfix, href in zip(clearfixs, hrefs):
5 w I' q5 X- I/ i; L4 |* ~9 U+ o print(clearfix, host + href)
( a+ {8 u, g% u0 O3 B. I\" P6 z* n# b/ ]1 d7 o+ p2 d* a% P
page = 1
- w( X$ @. S5 o# d while True:+ w5 p v: S) O1 A9 u6 f2 p: {
url = host + href + "/list_%s.html" % page6 [: ^( ]6 P4 d+ A+ X# O
req = requests.get(url)$ [- M! A. z1 F. @
req.encoding = 'utf-8'
6 B. Z5 A: x) B
7 p% c) r/ @* z& O7 Y/ r if (not write_sql(c, req.text)):
6 ^/ j$ J' X- U$ m& a break
7 P6 {/ B\" E. c& ^( s
: c7 r$ |1 R# m6 A9 u print("第%s页" % page)0 O. b3 o& n3 h5 ]5 \: B% G
page += 13 R\" e* \2 i0 t
% Y) t! @- J2 N6 E3 V conn.close()9 I: k# D7 J4 L0 V/ k/ I

#!/usr/bin/python. S) N9 F$ r( N# [9 v8 _' T 
# -*- coding: utf-8 -*-5 h2 \, ]: B7 h, s& s% ] 
 
0 ~2 @: l& y( f& Ximport requests 
0 V5 U2 N, O# C: M; jfrom lxml import etree3 w& \  U$ \% |% } 
import sqlite3: i. y# ]$ U* Y* c  T1 L 
 
  S, n2 L! v$ d. m8 c 
' @\" a6 F3 `: A\" V& Xdef write_sql(c, text): 
4 }! s& |: @2 W# J    html = etree.HTML(text)1 |) m1 u\" b+ ~/ ]* P2 ~ 
    # 标题9 r# L1 h/ q& B0 ^9 |+ j! ~ 
    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()') 
3 o9 W9 W6 U, V: x/ u1 F) S    # 链接 
+ M5 P. k' d& A/ d    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')$ ]! y2 w% y9 t\" j* G 
    # 日期 
. ~# W( I1 F$ ~/ X: D    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()') 
% i5 ?, g' M* \ 
8 o( G6 i\" `+ u9 f+ |1 S+ F$ o    number = 0 
6 ^9 Y$ m9 n5 @% D# G2 f    for title, href, em in zip(titles, hrefs, ems): 
* k) q) p4 ~. x0 V9 ^        href = host + href 
) O% ^9 _& ?5 `* v7 u. b        cursor = c.execute( 
; S3 @$ J6 {* m# Y. ?! d! g            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href) 
+ w\" L& y\" B7 T# ]        res = c.fetchall(). V- t/ {2 N7 i: ~, | 
        # 判断该字段是否已存在 
5 l; c' K. q1 ^3 Y. H        if res[0][0] > 0:+ t1 ^, X. B4 L5 s  y# b 
            continue 
. n1 O( b  y  r; M( r1 I) g- @5 r; h6 W 
        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
# ]3 u9 h# G' ~' {            href, 
$ A5 h- M) e3 F. g% W: J            title.replace("\"", "\"\""), 
8 n5 t9 A+ k1 M            em))6 F5 i# h$ f& H* j; {7 W/ u 
        number += 1 
1 z( Z! }  z2 K# S        print(title, href, em)  o, J! g) r4 T( m0 h9 t 
 
9 z' V* M9 h5 p    conn.commit() 
3 |* o. b% y/ O+ k: W8 J! I    return number > 0 
. Q; n5 q: l) d\" ~8 \( d8 x$ l* a' Z! k6 f\" r, p6 V 
 
) L* X1 B9 H2 v\" `7 a8 bif __name__ == '__main__':. [. q3 ^7 Y3 \9 W 
 
+ t3 s7 T' ~! @7 T8 Q6 t/ n    conn = sqlite3.connect("Python-xxx.db")6 J0 ~5 y/ T- s/ b+ c  g 
    c = conn.cursor()& k$ C. F& e2 z3 U) h 
    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
* C\" P* k4 f# E# B5 M2 n        Url VARCHAR,; [! Z\" ^! P! E+ D! A 
        Title VARCHAR, 
1 U5 C$ K6 j8 J6 e( J% T& S1 |        Author VARCHAR! f' ]  u, T# V  _0 q4 j 
    )''')- J2 t# \- ?+ }/ \\" n, {0 H. c% j/ X 
    conn.commit() 
& P, [1 C4 y# W5 J+ Q0 @9 G8 L0 a: ?1 m\" | 
    host = "https://xxx" 
$ I+ I, \+ x' R5 @5 G6 d  }    url = host + "/xxx"( e( }5 O# b- ^. n 
    req = requests.get(url)8 a1 }+ w* ?: k 
    req.encoding = 'utf-8'2 \! X1 ^2 C6 ^* h 
    # print(req.text)# U. k\" J: z* L% | 
6 f2 L7 P1 A& q4 y( b( X9 r! h 
    html = etree.HTML(req.text) 
; z: F- S( i9 s, q  b    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()') 
8 L! n8 o, m1 H7 S( `8 P, n$ f    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')8 B- |5 v. h* D/ P3 [3 f 
    # print(clearfix, href)% b, ^+ c* l. I1 z% e 
* p1 J& {7 k0 s, s' x 
    for clearfix, href in zip(clearfixs, hrefs): 
5 w  I' q5 X- I/ i; L4 |* ~9 U+ o        print(clearfix, host + href) 
( a+ {8 u, g% u0 O3 B. I\" P6 z* n# b/ ]1 d7 o+ p2 d* a% P 
        page = 1 
- w( X$ @. S5 o# d        while True:+ w5 p  v: S) O1 A9 u6 f2 p: { 
            url = host + href + "/list_%s.html" % page6 [: ^( ]6 P4 d+ A+ X# O 
            req = requests.get(url)$ [- M! A. z1 F. @ 
            req.encoding = 'utf-8' 
6 B. Z5 A: x) B 
7 p% c) r/ @* z& O7 Y/ r            if (not write_sql(c, req.text)): 
6 ^/ j$ J' X- U$ m& a                break 
7 P6 {/ B\" E. c& ^( s 
: c7 r$ |1 R# m6 A9 u            print("第%s页" % page)0 O. b3 o& n3 h5 ]5 \: B% G 
            page += 13 R\" e* \2 i0 t 
 
% Y) t! @- J2 N6 E3 V    conn.close()9 I: k# D7 J4 L0 V/ k/ I

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan