查看: 1609|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
9 m& U. w* p9 N/ d% G# -*- coding: utf-8 -*-
4 |, D& U. s: L\" F9 g ?8 o$ i7 w' ]; x/ D3 R& F$ ], x0 P1 w
import requests
/ s! f* w( U! Zfrom lxml import etree
' u. ?. |; X4 D1 d# B' j9 Y; Oimport sqlite3
+ G* s5 d# ]- Q! t5 ?* N3 S7 R7 Z7 N3 Q0 \& S, U! X
. K, o$ Z6 {( B
def write_sql(c, text):3 {% A3 E/ ?2 U- ?! R& l8 \
html = etree.HTML(text)
5 ] }7 K- o0 X& `: T# Z # 标题
8 B: M+ v3 A, ^2 i- a% Z\" j8 H2 G titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')$ e: @% n& g7 M% d, H
# 链接
% U* i) P9 n( J/ o! p hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')4 f\" j( b7 a! x\" V8 r/ b1 `. W- z
# 日期
: o9 F. ^$ |) Z6 \ ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
, }9 ^2 k+ t2 c; n8 E\" ^. i- W+ y% ?5 F2 b$ y3 e6 ?- l
number = 04 W6 Y* B* }3 R\" `
for title, href, em in zip(titles, hrefs, ems):\" \3 X% x, r) _6 ^8 v
href = host + href0 W8 M0 R( R. }. q
cursor = c.execute(
. n/ B+ H. U& O$ q$ }) w7 Y; Z$ V "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
2 x4 N9 Y& ?7 M7 R+ v1 Y* z: a res = c.fetchall()
/ D3 }2 |2 E* E' a8 A% l\" x # 判断该字段是否已存在- {\" f( [: T\" @\" m/ u
if res[0][0] > 0:
' p/ {+ @* l+ l( w continue
. d8 s& y' S6 \& `' S i. `
8 T4 C, T, l0 O8 ]$ F/ k, H c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
4 Y8 }0 E+ N/ J href,: |& u1 X3 K3 G4 x/ U/ |
title.replace("\"", "\"\""),
+ V& {\" d\" H% f _ em))3 y. V F& f2 Z3 U
number += 1
! Y, W# w: G\" V1 }* Q9 C5 R8 U. [5 c print(title, href, em)
/ h$ w d) r+ d+ k
) w1 a& b2 }9 k: f conn.commit()
- q+ }4 k! O1 k# ]5 {- L return number > 0
) a5 o, C. V2 K- k4 M
; m0 Y. \4 ]: D3 L- n6 |/ K s
\" A# `2 P4 z* S) X7 x1 }if __name__ == '__main__':9 A& ^0 @6 M+ ^; d
1 L! q2 t+ y M conn = sqlite3.connect("Python-xxx.db")
' { k: B- v1 H\" x* I c = conn.cursor()5 Y6 h( O+ o; V. b
c.execute('''CREATE TABLE IF NOT EXISTS Python (
$ _) t; S+ a. N: M# k& _3 x Url VARCHAR,9 d# a- Y5 m9 i, T% Z
Title VARCHAR,0 x& a; S2 z5 l3 q$ U
Author VARCHAR* K; H% U& v0 ]+ Z( }
)'''): L3 I* |; e2 w+ i& s) j$ X# @
conn.commit()+ X( q- m- b, \% I' q+ M/ j7 a- e
& B- L3 k2 d) @! a
host = "https://xxx"
url = host + "/xxx"
6 d& T\" ]. m! z% N req = requests.get(url)% D1 u% N# b5 {
req.encoding = 'utf-8'* g1 j! M( p2 K+ r+ c( O1 B
# print(req.text)% v0 L2 C4 b2 h9 _0 b/ Z& @
9 X5 A' C5 ^, Y$ W5 i( p/ e$ L
html = etree.HTML(req.text)+ d# k: t8 m9 q+ v
clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')
7 d6 H) k- K; x, B hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')0 V- ^1 [% f: l7 u Q: S
# print(clearfix, href)
4 k- _8 I) O& b/ Q0 p: ^ p\" T! t
D) r) U6 ]; E for clearfix, href in zip(clearfixs, hrefs):( u' T7 [6 i3 m. v+ k6 @
print(clearfix, host + href)
+ t3 Q5 x6 d' K1 F. u) Z1 T
' M# p6 ^+ g1 U' C: c page = 1
& Z% q b1 X: G) I4 @2 C H while True: P: N _, B4 G, O
url = host + href + "/list_%s.html" % page5 p- S3 e- y F N0 f7 \
req = requests.get(url)8 w3 p4 R' G: q J1 o, G
req.encoding = 'utf-8'2 D- O! }9 z b k% c3 Z
) g\" P; H: h+ B7 p* ^ if (not write_sql(c, req.text)):& i! K2 P' p* K4 B; [0 g4 Z) U% o( [
break
( M9 h; q; \) T+ b5 t6 R: [0 E, y1 ^1 [
print("第%s页" % page)
+ O8 ^# D$ _. `. @% c2 ` page += 1
: h) L\" z9 A$ T\" {0 o- ^1 ?- A3 H! A0 \1 m
conn.close()
! h1 E2 ?# g: e6 o0 f9 @6 X5 \

#!/usr/bin/python 
9 m& U. w* p9 N/ d% G# -*- coding: utf-8 -*- 
4 |, D& U. s: L\" F9 g  ?8 o$ i7 w' ]; x/ D3 R& F$ ], x0 P1 w 
import requests 
/ s! f* w( U! Zfrom lxml import etree 
' u. ?. |; X4 D1 d# B' j9 Y; Oimport sqlite3 
+ G* s5 d# ]- Q! t5 ?* N3 S7 R7 Z7 N3 Q0 \& S, U! X 
. K, o$ Z6 {( B 
def write_sql(c, text):3 {% A3 E/ ?2 U- ?! R& l8 \ 
    html = etree.HTML(text) 
5 ]  }7 K- o0 X& `: T# Z    # 标题 
8 B: M+ v3 A, ^2 i- a% Z\" j8 H2 G    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')$ e: @% n& g7 M% d, H 
    # 链接 
% U* i) P9 n( J/ o! p    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')4 f\" j( b7 a! x\" V8 r/ b1 `. W- z 
    # 日期 
: o9 F. ^$ |) Z6 \    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()') 
, }9 ^2 k+ t2 c; n8 E\" ^. i- W+ y% ?5 F2 b$ y3 e6 ?- l 
    number = 04 W6 Y* B* }3 R\" ` 
    for title, href, em in zip(titles, hrefs, ems):\" \3 X% x, r) _6 ^8 v 
        href = host + href0 W8 M0 R( R. }. q 
        cursor = c.execute( 
. n/ B+ H. U& O$ q$ }) w7 Y; Z$ V            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href) 
2 x4 N9 Y& ?7 M7 R+ v1 Y* z: a        res = c.fetchall() 
/ D3 }2 |2 E* E' a8 A% l\" x        # 判断该字段是否已存在- {\" f( [: T\" @\" m/ u 
        if res[0][0] > 0: 
' p/ {+ @* l+ l( w            continue 
. d8 s& y' S6 \& `' S  i. ` 
8 T4 C, T, l0 O8 ]$ F/ k, H        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
4 Y8 }0 E+ N/ J            href,: |& u1 X3 K3 G4 x/ U/ | 
            title.replace("\"", "\"\""), 
+ V& {\" d\" H% f  _            em))3 y. V  F& f2 Z3 U 
        number += 1 
! Y, W# w: G\" V1 }* Q9 C5 R8 U. [5 c        print(title, href, em) 
/ h$ w  d) r+ d+ k 
) w1 a& b2 }9 k: f    conn.commit() 
- q+ }4 k! O1 k# ]5 {- L    return number > 0 
) a5 o, C. V2 K- k4 M 
; m0 Y. \4 ]: D3 L- n6 |/ K  s 
\" A# `2 P4 z* S) X7 x1 }if __name__ == '__main__':9 A& ^0 @6 M+ ^; d 
 
1 L! q2 t+ y  M    conn = sqlite3.connect("Python-xxx.db") 
' {  k: B- v1 H\" x* I    c = conn.cursor()5 Y6 h( O+ o; V. b 
    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
$ _) t; S+ a. N: M# k& _3 x        Url VARCHAR,9 d# a- Y5 m9 i, T% Z 
        Title VARCHAR,0 x& a; S2 z5 l3 q$ U 
        Author VARCHAR* K; H% U& v0 ]+ Z( } 
    )'''): L3 I* |; e2 w+ i& s) j$ X# @ 
    conn.commit()+ X( q- m- b, \% I' q+ M/ j7 a- e 
& B- L3 k2 d) @! a 
    host = "https://xxx" 
8 P. M! O! p9 ], F* b4 x\" r  I+ P/ ~6 L    url = host + "/xxx" 
6 d& T\" ]. m! z% N    req = requests.get(url)% D1 u% N# b5 { 
    req.encoding = 'utf-8'* g1 j! M( p2 K+ r+ c( O1 B 
    # print(req.text)% v0 L2 C4 b2 h9 _0 b/ Z& @ 
9 X5 A' C5 ^, Y$ W5 i( p/ e$ L 
    html = etree.HTML(req.text)+ d# k: t8 m9 q+ v 
    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()') 
7 d6 H) k- K; x, B    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')0 V- ^1 [% f: l7 u  Q: S 
    # print(clearfix, href) 
4 k- _8 I) O& b/ Q0 p: ^  p\" T! t 
  D) r) U6 ]; E    for clearfix, href in zip(clearfixs, hrefs):( u' T7 [6 i3 m. v+ k6 @ 
        print(clearfix, host + href) 
+ t3 Q5 x6 d' K1 F. u) Z1 T 
' M# p6 ^+ g1 U' C: c        page = 1 
& Z% q  b1 X: G) I4 @2 C  H        while True:  P: N  _, B4 G, O 
            url = host + href + "/list_%s.html" % page5 p- S3 e- y  F  N0 f7 \ 
            req = requests.get(url)8 w3 p4 R' G: q  J1 o, G 
            req.encoding = 'utf-8'2 D- O! }9 z  b  k% c3 Z 
 
) g\" P; H: h+ B7 p* ^            if (not write_sql(c, req.text)):& i! K2 P' p* K4 B; [0 g4 Z) U% o( [ 
                break 
( M9 h; q; \) T+ b5 t6 R: [0 E, y1 ^1 [ 
            print("第%s页" % page) 
+ O8 ^# D$ _. `. @% c2 `            page += 1 
: h) L\" z9 A$ T\" {0 o- ^1 ?- A3 H! A0 \1 m 
    conn.close() 
! h1 E2 ?# g: e6 o0 f9 @6 X5 \

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan