查看: 1523|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python# O2 P. `/ @% G) ?
# -*- coding: utf-8 -*-
, ~9 n) E% z\" I9 w4 x7 S
% U' L! H7 m: g, t- Gimport requests
\" ^( ]* L3 L3 K: j/ W. Z0 ofrom lxml import etree
\" n, M* h3 i5 u2 S4 [import sqlite3
3 H1 |7 w+ o0 ~( v! ^\" @5 w
/ k' w# a% k; f& d5 M: j* I9 v Y; Y1 r, U
def write_sql(c, text):
( t; D+ I. v# w3 g7 H html = etree.HTML(text)8 r$ G: t8 x I1 D$ L
# 标题; z/ L' _+ E. T9 j( I- R\" [
titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()'). Y6 X) L7 M\" g' R5 u0 Z) R
# 链接
6 F- @' ~* u; x) n6 s( G$ I hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')5 y9 ?) h5 n+ |/ T& q4 x
# 日期
% F1 b2 p# s$ y9 w ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
/ j5 {: X% M# ]8 a! v9 s5 Y$ q\" i1 a) s& m& {: a! M
number = 0
' `3 j5 y6 `4 ~1 q& K8 F for title, href, em in zip(titles, hrefs, ems):
3 p! d( v5 O! _ href = host + href% x0 ?( h% m' ^6 J! }* D4 i
cursor = c.execute(
/ b9 O0 K+ L' o& a "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
: X) d2 d* h% f2 [6 p res = c.fetchall()
; \ m/ u5 f4 N' ? # 判断该字段是否已存在. ^- P V' G& q\" p3 }
if res[0][0] > 0:
$ U) b4 D: E7 K( w0 X1 E0 v: k continue
$ F( i5 O2 j: o
( n5 i6 t9 P3 k c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
+ o& ~1 a# z# d7 M1 L8 Q href,
9 A* Q# p I9 R$ g\" @/ Y$ Z title.replace("\"", "\"\""),0 T) a! C, \; }: S
em))% N3 o% w( r1 l! u, ~
number += 18 @5 B; m1 j* ^2 L8 d8 d, y
print(title, href, em)3 \( J& X9 s0 [) J ]$ O3 i3 Y
7 ]! k% J8 i! b conn.commit()0 L; J6 q2 [* G1 u# g+ t6 x$ }
return number > 0
J, j+ o! w, i$ B! F& m# f V) K& `- n/ k# ~9 j
8 M3 b3 m! t% Qif __name__ == '__main__':( R: b0 z8 D0 ^* o1 Q) n/ ~+ b5 s
( i/ v- a- h! I( e conn = sqlite3.connect("Python-xxx.db")
/ D* S8 V. M+ `: I0 `% O+ T c = conn.cursor()
/ @- N: @$ ]1 _4 Y: p c.execute('''CREATE TABLE IF NOT EXISTS Python (6 N7 f* w: M6 F% ^9 Q
Url VARCHAR,
6 G; @ ? q7 R1 L# `1 E, ? Title VARCHAR,7 E* O4 W. Z& |' E) ^$ q
Author VARCHAR
& y. i2 {5 Y0 `+ l )'''), b5 M Q% `8 }$ w
conn.commit()
* i% M* q) D. A' c
7 ~9 n\" T9 }% W& K2 G8 | host = "https://xxx"
. r3 u- i8 m, c) R url = host + "/xxx"1 v: X5 q/ m6 a! K2 Z
req = requests.get(url)% d- W( u\" U' B+ y: [$ B. g* D- \
req.encoding = 'utf-8') X) |1 q\" j+ I2 c. T! F
# print(req.text)
7 A1 u; Y( k0 V7 [( v% f9 }. R1 t+ w7 m. m+ R& A
html = etree.HTML(req.text)8 b8 x3 G8 L$ w; n8 h9 U$ \' T
clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')
0 u8 w6 J( n0 M3 ^0 Q hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')
& R6 Y7 Q5 ^4 h8 T* S. u5 t6 t0 L # print(clearfix, href)
' U( {1 n& r\" C0 f, @, @$ u7 K: h$ ]) r( \& n% e4 N: C1 ]
for clearfix, href in zip(clearfixs, hrefs):. L* S' |6 K( l9 W1 {
print(clearfix, host + href)1 C. m) J7 g1 ^. ]% s+ E0 x
% q' ~+ Z' z# {0 Y page = 1
% J! h, a3 f: A5 |! |+ C while True:0 N( q# G$ |4 Q) N+ T
url = host + href + "/list_%s.html" % page: _7 x& t- E: S# G$ i% `
req = requests.get(url)
( e' v! u$ _) \, B# E P req.encoding = 'utf-8'& x) Q8 l, Q9 X! {4 R# [5 W
+ r1 ^\" p `! T* c5 U( } T& x& G$ J
if (not write_sql(c, req.text)):* p& f( E( C, V( g' C$ e
break
% \; x7 n8 s4 ^
\" o+ ?2 w: c+ F* E( V0 i print("第%s页" % page)/ a6 V- X% e6 I2 K0 Y
page += 18 T! q: H. W' x7 m# U9 {
\" m, ^7 M9 d: m I) [+ S( M conn.close()
; l' e6 n+ Q, n& x

#!/usr/bin/python# O2 P. `/ @% G) ? 
# -*- coding: utf-8 -*- 
, ~9 n) E% z\" I9 w4 x7 S 
% U' L! H7 m: g, t- Gimport requests 
\" ^( ]* L3 L3 K: j/ W. Z0 ofrom lxml import etree 
\" n, M* h3 i5 u2 S4 [import sqlite3 
3 H1 |7 w+ o0 ~( v! ^\" @5 w 
/ k' w# a% k; f& d5 M: j* I9 v  Y; Y1 r, U 
def write_sql(c, text): 
( t; D+ I. v# w3 g7 H    html = etree.HTML(text)8 r$ G: t8 x  I1 D$ L 
    # 标题; z/ L' _+ E. T9 j( I- R\" [ 
    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()'). Y6 X) L7 M\" g' R5 u0 Z) R 
    # 链接 
6 F- @' ~* u; x) n6 s( G$ I    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')5 y9 ?) h5 n+ |/ T& q4 x 
    # 日期 
% F1 b2 p# s$ y9 w    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()') 
/ j5 {: X% M# ]8 a! v9 s5 Y$ q\" i1 a) s& m& {: a! M 
    number = 0 
' `3 j5 y6 `4 ~1 q& K8 F    for title, href, em in zip(titles, hrefs, ems): 
3 p! d( v5 O! _        href = host + href% x0 ?( h% m' ^6 J! }* D4 i 
        cursor = c.execute( 
/ b9 O0 K+ L' o& a            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href) 
: X) d2 d* h% f2 [6 p        res = c.fetchall() 
; \  m/ u5 f4 N' ?        # 判断该字段是否已存在. ^- P  V' G& q\" p3 } 
        if res[0][0] > 0: 
$ U) b4 D: E7 K( w0 X1 E0 v: k            continue 
$ F( i5 O2 j: o 
( n5 i6 t9 P3 k        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
+ o& ~1 a# z# d7 M1 L8 Q            href, 
9 A* Q# p  I9 R$ g\" @/ Y$ Z            title.replace("\"", "\"\""),0 T) a! C, \; }: S 
            em))% N3 o% w( r1 l! u, ~ 
        number += 18 @5 B; m1 j* ^2 L8 d8 d, y 
        print(title, href, em)3 \( J& X9 s0 [) J  ]$ O3 i3 Y 
 
7 ]! k% J8 i! b    conn.commit()0 L; J6 q2 [* G1 u# g+ t6 x$ } 
    return number > 0 
  J, j+ o! w, i$ B! F& m# f  V) K& `- n/ k# ~9 j 
 
8 M3 b3 m! t% Qif __name__ == '__main__':( R: b0 z8 D0 ^* o1 Q) n/ ~+ b5 s 
 
( i/ v- a- h! I( e    conn = sqlite3.connect("Python-xxx.db") 
/ D* S8 V. M+ `: I0 `% O+ T    c = conn.cursor() 
/ @- N: @$ ]1 _4 Y: p    c.execute('''CREATE TABLE IF NOT EXISTS Python (6 N7 f* w: M6 F% ^9 Q 
        Url VARCHAR, 
6 G; @  ?  q7 R1 L# `1 E, ?        Title VARCHAR,7 E* O4 W. Z& |' E) ^$ q 
        Author VARCHAR 
& y. i2 {5 Y0 `+ l    )'''), b5 M  Q% `8 }$ w 
    conn.commit() 
* i% M* q) D. A' c 
7 ~9 n\" T9 }% W& K2 G8 |    host = "https://xxx" 
. r3 u- i8 m, c) R    url = host + "/xxx"1 v: X5 q/ m6 a! K2 Z 
    req = requests.get(url)% d- W( u\" U' B+ y: [$ B. g* D- \ 
    req.encoding = 'utf-8') X) |1 q\" j+ I2 c. T! F 
    # print(req.text) 
7 A1 u; Y( k0 V7 [( v% f9 }. R1 t+ w7 m. m+ R& A 
    html = etree.HTML(req.text)8 b8 x3 G8 L$ w; n8 h9 U$ \' T 
    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()') 
0 u8 w6 J( n0 M3 ^0 Q    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href') 
& R6 Y7 Q5 ^4 h8 T* S. u5 t6 t0 L    # print(clearfix, href) 
' U( {1 n& r\" C0 f, @, @$ u7 K: h$ ]) r( \& n% e4 N: C1 ] 
    for clearfix, href in zip(clearfixs, hrefs):. L* S' |6 K( l9 W1 { 
        print(clearfix, host + href)1 C. m) J7 g1 ^. ]% s+ E0 x 
 
% q' ~+ Z' z# {0 Y        page = 1 
% J! h, a3 f: A5 |! |+ C        while True:0 N( q# G$ |4 Q) N+ T 
            url = host + href + "/list_%s.html" % page: _7 x& t- E: S# G$ i% ` 
            req = requests.get(url) 
( e' v! u$ _) \, B# E  P            req.encoding = 'utf-8'& x) Q8 l, Q9 X! {4 R# [5 W 
+ r1 ^\" p  `! T* c5 U( }  T& x& G$ J 
            if (not write_sql(c, req.text)):* p& f( E( C, V( g' C$ e 
                break 
% \; x7 n8 s4 ^ 
\" o+ ?2 w: c+ F* E( V0 i            print("第%s页" % page)/ a6 V- X% e6 I2 K0 Y 
            page += 18 T! q: H. W' x7 m# U9 { 
 
\" m, ^7 M9 d: m  I) [+ S( M    conn.close() 
; l' e6 n+ Q, n& x

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan