查看: 1547|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |正序浏览

|招呼Ta 关注Ta

#!/usr/bin/python: E! `% n( s% V+ a+ U% t# P2 ?
# -*- coding: utf-8 -*-
5 q& k* M6 u h% Z\" d5 ^( h F- |3 Q' D q6 r% i7 P- j
import requests- x6 z ~# n3 K\" N
from lxml import etree( ~& \6 r, f( v, W- w5 P/ d9 L9 f) W
import sqlite3
\" ^7 M* ^. U Y3 T8 ^/ _6 ]* U% q! v
- c/ q/ U Q+ \& Q, |. ^def write_sql(c, text): N* h1 c* t5 O( V b4 A
html = etree.HTML(text)
7 n P7 B3 P) a. \6 _ # 标题
4 e) F( Y* {2 {( W0 O- Q1 _ titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')& p( d4 P) y5 x, p( w3 {
# 链接/ R/ V+ L# Z3 S\" }' a X: D, H
hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')8 T' n5 p$ q: X
# 日期
W d/ O; R% W# m+ W# S2 ~ ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
; Z4 D+ s+ F: b* E' }. h' k
1 z; A* G( p\" ^4 C! h number = 0. n2 r6 U7 {\" R- o, N
for title, href, em in zip(titles, hrefs, ems):6 e; }5 _' y3 g1 L
href = host + href& B( _9 }/ B/ q, ~0 m. s9 R
cursor = c.execute(
: J5 m. d+ D+ K; ^ "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
( E! C; y, t: R* o2 ^ res = c.fetchall()9 `, \/ D' u. O( x: t& d( D$ }\" L5 J
# 判断该字段是否已存在
5 ]9 |' O9 J3 Z. B+ ~% u: H if res[0][0] > 0:
continue
( Y- z- A4 s* o, J
$ j; x9 r; {& i+ w\" P c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (' a5 H* G) ?9 C1 l& Q
href,
\" I; j* B% n! d! e/ o+ S title.replace("\"", "\"\""), C0 w1 F7 o2 i; @$ J
em))
# g! e0 ]* s* g! V number += 1
$ w3 d6 Q% {$ Z+ G4 P) m) G6 V2 g* z print(title, href, em)
7 F2 x/ x3 U! O, A* K; d/ Q- g/ Q3 W% l
conn.commit()
6 Y% u! r: w7 i- Y5 y O return number > 08 }, n7 O8 |# x, W! B
, y! Q- _8 ]3 d8 k6 ^! _0 b3 t* L: k4 _. s: w\" x
if __name__ == '__main__':0 N# d- h j7 q6 m2 g! P, y
2 o) ^' J) r' ~3 F4 L* [' @ d conn = sqlite3.connect("Python-xxx.db")
4 y; U2 |6 _9 T& H c = conn.cursor()$ Z2 j1 ~' r! ~\" x
c.execute('''CREATE TABLE IF NOT EXISTS Python (
5 k+ Y {# F$ s8 b7 v Url VARCHAR,
5 V5 i4 L3 X: ]9 y1 b( j4 x Title VARCHAR,
) H% G/ M/ b) Z+ J r) e/ x9 o6 m9 F Author VARCHAR# S; [1 E4 q9 _3 Q- R# b
)''')0 b. K) `$ S9 f
conn.commit(); M9 z; s* ?\" h% ? h
4 j1 q% s' U- ]- C1 Y! A
host = "https://xxx"' h% Q* q- E9 v% ?5 Q! ]
url = host + "/xxx"
. j3 }0 W# K |- ^0 x+ c req = requests.get(url)9 h1 ~, r# c8 W+ v) O) ?( q
req.encoding = 'utf-8'
+ x. Q# m5 G8 C. _9 m& Y F6 U& J # print(req.text)
6 X$ K- Y. Y1 R3 `\" v' Q
3 q( I) i6 z( v, k5 C- _ html = etree.HTML(req.text)# @$ m) H7 l- z- U) Y: {4 e' k
clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')
. k2 W z6 _2 z. ?8 k% r3 j hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')# v+ T$ w$ i$ v
# print(clearfix, href)
) p) e: z# B1 I: c0 E1 Y$ C: E$ _% B% N0 f( n# ?) z
for clearfix, href in zip(clearfixs, hrefs):
: i, a. {7 |* J; v2 w7 u print(clearfix, host + href)
9 o0 }$ V; I8 l+ C2 E& z( Z* e
- q& t8 @. ^ c& J page = 1 m9 h, p6 x# D% s Z8 o; m# i$ J
while True:9 V; M' {) u& l1 v* F6 d$ J
url = host + href + "/list_%s.html" % page9 F, X* W0 x' p f) T5 A\" _
req = requests.get(url)$ A7 M$ b: ]6 y( [
req.encoding = 'utf-8': m8 ]6 y( p8 f( t! |1 G
( q6 n1 c, l0 z) p, |
if (not write_sql(c, req.text)):3 Q: H4 i- L+ e) n
break5 G ` l4 P& m\" ]
! y, s) L; h% N8 ^1 W3 q
print("第%s页" % page)' H1 W; |, o; D! N. L8 \4 t
page += 1
e* I6 S\" b# a2 ]
5 F2 H+ e8 A2 e+ n' t conn.close(): G7 p- P: G: p\" z5 V9 d4 G* a& T

#!/usr/bin/python: E! `% n( s% V+ a+ U% t# P2 ? 
# -*- coding: utf-8 -*- 
5 q& k* M6 u  h% Z\" d5 ^( h  F- |3 Q' D  q6 r% i7 P- j 
import requests- x6 z  ~# n3 K\" N 
from lxml import etree( ~& \6 r, f( v, W- w5 P/ d9 L9 f) W 
import sqlite3 
\" ^7 M* ^. U  Y3 T8 ^/ _6 ]* U% q! v 
 
- c/ q/ U  Q+ \& Q, |. ^def write_sql(c, text):  N* h1 c* t5 O( V  b4 A 
    html = etree.HTML(text) 
7 n  P7 B3 P) a. \6 _    # 标题 
4 e) F( Y* {2 {( W0 O- Q1 _    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')& p( d4 P) y5 x, p( w3 { 
    # 链接/ R/ V+ L# Z3 S\" }' a  X: D, H 
    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')8 T' n5 p$ q: X 
    # 日期 
  W  d/ O; R% W# m+ W# S2 ~    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()') 
; Z4 D+ s+ F: b* E' }. h' k 
1 z; A* G( p\" ^4 C! h    number = 0. n2 r6 U7 {\" R- o, N 
    for title, href, em in zip(titles, hrefs, ems):6 e; }5 _' y3 g1 L 
        href = host + href& B( _9 }/ B/ q, ~0 m. s9 R 
        cursor = c.execute( 
: J5 m. d+ D+ K; ^            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href) 
( E! C; y, t: R* o2 ^        res = c.fetchall()9 `, \/ D' u. O( x: t& d( D$ }\" L5 J 
        # 判断该字段是否已存在 
5 ]9 |' O9 J3 Z. B+ ~% u: H        if res[0][0] > 0: 
+ B' t: M4 g- O1 x5 Z$ w& B' Q1 J  F$ N            continue 
( Y- z- A4 s* o, J 
$ j; x9 r; {& i+ w\" P        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (' a5 H* G) ?9 C1 l& Q 
            href, 
\" I; j* B% n! d! e/ o+ S            title.replace("\"", "\"\""),  C0 w1 F7 o2 i; @$ J 
            em)) 
# g! e0 ]* s* g! V        number += 1 
$ w3 d6 Q% {$ Z+ G4 P) m) G6 V2 g* z        print(title, href, em) 
7 F2 x/ x3 U! O, A* K; d/ Q- g/ Q3 W% l 
    conn.commit() 
6 Y% u! r: w7 i- Y5 y  O    return number > 08 }, n7 O8 |# x, W! B 
 
, y! Q- _8 ]3 d8 k6 ^! _0 b3 t* L: k4 _. s: w\" x 
if __name__ == '__main__':0 N# d- h  j7 q6 m2 g! P, y 
 
2 o) ^' J) r' ~3 F4 L* [' @  d    conn = sqlite3.connect("Python-xxx.db") 
4 y; U2 |6 _9 T& H    c = conn.cursor()$ Z2 j1 ~' r! ~\" x 
    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
5 k+ Y  {# F$ s8 b7 v        Url VARCHAR, 
5 V5 i4 L3 X: ]9 y1 b( j4 x        Title VARCHAR, 
) H% G/ M/ b) Z+ J  r) e/ x9 o6 m9 F        Author VARCHAR# S; [1 E4 q9 _3 Q- R# b 
    )''')0 b. K) `$ S9 f 
    conn.commit(); M9 z; s* ?\" h% ?  h 
4 j1 q% s' U- ]- C1 Y! A 
    host = "https://xxx"' h% Q* q- E9 v% ?5 Q! ] 
    url = host + "/xxx" 
. j3 }0 W# K  |- ^0 x+ c    req = requests.get(url)9 h1 ~, r# c8 W+ v) O) ?( q 
    req.encoding = 'utf-8' 
+ x. Q# m5 G8 C. _9 m& Y  F6 U& J    # print(req.text) 
6 X$ K- Y. Y1 R3 `\" v' Q 
3 q( I) i6 z( v, k5 C- _    html = etree.HTML(req.text)# @$ m) H7 l- z- U) Y: {4 e' k 
    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()') 
. k2 W  z6 _2 z. ?8 k% r3 j    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')# v+ T$ w$ i$ v 
    # print(clearfix, href) 
) p) e: z# B1 I: c0 E1 Y$ C: E$ _% B% N0 f( n# ?) z 
    for clearfix, href in zip(clearfixs, hrefs): 
: i, a. {7 |* J; v2 w7 u        print(clearfix, host + href) 
9 o0 }$ V; I8 l+ C2 E& z( Z* e 
- q& t8 @. ^  c& J        page = 1  m9 h, p6 x# D% s  Z8 o; m# i$ J 
        while True:9 V; M' {) u& l1 v* F6 d$ J 
            url = host + href + "/list_%s.html" % page9 F, X* W0 x' p  f) T5 A\" _ 
            req = requests.get(url)$ A7 M$ b: ]6 y( [ 
            req.encoding = 'utf-8': m8 ]6 y( p8 f( t! |1 G 
( q6 n1 c, l0 z) p, | 
            if (not write_sql(c, req.text)):3 Q: H4 i- L+ e) n 
                break5 G  `  l4 P& m\" ] 
! y, s) L; h% N8 ^1 W3 q 
            print("第%s页" % page)' H1 W; |, o; D! N. L8 \4 t 
            page += 1 
  e* I6 S\" b# a2 ] 
5 F2 H+ e8 A2 e+ n' t    conn.close(): G7 p- P: G: p\" z5 V9 d4 G* a& T

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan