查看: 1550|回复: 0

Python-xpath

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-18 09:29 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python! O8 _' j; P% D2 N6 S% ~
# -*- coding: utf-8 -*-/ j& F l4 x\" t1 J' k, H
' a. ~\" }- Z: @7 l, S\" ?* mimport requests
4 u0 e A- B b6 t+ z _3 ~6 mfrom lxml import etree5 x5 g# j) w$ H
import sqlite3
# v5 y3 m* L' g) Z# Q- z- b\" `6 N2 H\" C7 m9 l, D
: P5 C( p6 a% L$ a# i% h2 W/ J& u0 |
def write_sql(c, text):
3 X* _( J' v+ l* ? html = etree.HTML(text); _* H; X\" P8 p
# 标题 P1 X0 d: s. Z: z
titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')
1 Q6 _7 u; R2 K6 t # 链接
% ?+ U( E U- ~' t9 I! e hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')
v) w! m% @, b f1 C. m. M # 日期
( h4 C& V( r, }* q ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
8 R4 t4 s; N/ g8 b\" i/ O* J1 b9 \% @; ]4 N/ V5 C
number = 04 {+ a+ g+ I% ]. [4 z4 y+ v9 a: g! t
for title, href, em in zip(titles, hrefs, ems):
\" e. F( t- o, y2 ^, ?\" K6 G href = host + href) L7 B6 C7 K$ X\" D; N* D1 T4 e# g
cursor = c.execute(
. V: `& I& l0 U, j1 D "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
( @# w& z1 P' P0 k* Y, r\" u. [ B res = c.fetchall()# q8 g0 ]$ L' G5 A) x# U0 U
# 判断该字段是否已存在
6 t% ]/ U4 H5 c* ~ if res[0][0] > 0:
% w* f+ o! P2 v4 ^& r continue
C0 P- \. @; \! W/ z+ U# o$ `2 ]; @+ t
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (6 F. O5 ~0 _' l+ A' r2 u+ \, x1 U
href,7 B% B8 h) w2 K1 I
title.replace("\"", "\"\""), d5 `/ {2 h. L\" J8 a
em))
+ Z# ~/ l1 R( i | number += 1 F' @* O+ B( D
print(title, href, em)
! L7 @- ]) [! |: A B2 t; _3 {9 p& O/ ~
conn.commit()! ]0 s4 `+ W5 U# A. L% B1 O8 d! X
return number > 06 E: d* M6 c- k
9 J, k1 y* R7 D/ a
2 f. v5 {6 {& S\" F' x( n) _! _if __name__ == '__main__':
3 N( m& e7 t. m% L\" k9 D+ T
$ B( Y: z3 L* i/ M+ | conn = sqlite3.connect("Python-xxx.db")) ^7 }( ]6 o0 O p$ ^
c = conn.cursor()
+ g! Z0 Q6 Y @0 [' d5 W c.execute('''CREATE TABLE IF NOT EXISTS Python (4 m2 X$ p6 A+ w; t( ?4 W2 u
Url VARCHAR,& [; U* U, R' L2 K. W5 C7 y
Title VARCHAR, P+ A E# ]9 O+ D6 k% }
Author VARCHAR
8 A- Y5 S8 O6 P) P! M )''')
$ k N: S0 H! J3 ]0 E3 U conn.commit(): ~ q4 ^/ I) t9 I
' i$ ~! P' [5 i
host = "https://xxx"$ r- J\" p! `0 q+ |* d7 r: B
url = host + "/xxx"& k) Z+ a* M8 ^. w
req = requests.get(url)
, ^6 N' z* z( g0 B n req.encoding = 'utf-8'
! s0 w, D' d; D I+ k. n # print(req.text)0 z! N: `& d- r V( T r
! w0 [\" F( d; l4 T3 e html = etree.HTML(req.text)) b; @+ S W% r# ^: P7 I
clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')9 H/ y6 |) {: j
hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href'). q! |& p' |% j& ?& L' h
# print(clearfix, href), c4 H1 _; S1 L2 J# M# _
\" w8 S) L, q1 }$ `4 U0 [8 c( F for clearfix, href in zip(clearfixs, hrefs):
& v7 U; e6 l5 R/ j: S. | print(clearfix, host + href); J6 F; ^1 d# }! n6 F9 ^\" i
5 Z: l6 R# p' U$ x! \0 e# k
page = 1
5 J' [, H5 P% F5 c/ x$ T% R while True:) a+ z5 r3 c- y7 N9 m
url = host + href + "/list_%s.html" % page7 S7 M, |4 X# B# n4 Z, w* h
req = requests.get(url)* S# x7 H$ r7 P) O; E% s+ W
req.encoding = 'utf-8'
( Y! P+ \3 O4 V2 C4 S\" M
7 G8 N; g3 y5 K4 @9 ]5 P. s if (not write_sql(c, req.text)):! z/ h& |' m( j' H: f' c
break
& E$ Z% Y! g6 M; J
R l% w\" M8 |. i0 a print("第%s页" % page)
: n# Z, {6 W# p: h! D9 a+ I- M0 ? page += 1( w3 A8 {( l) C& _\" t4 Y% t$ b1 X
\" h% w. ^# t: w5 Z conn.close()
& L/ v3 Y9 O5 a. j3 D; V1 ]# p

#!/usr/bin/python! O8 _' j; P% D2 N6 S% ~ 
# -*- coding: utf-8 -*-/ j& F  l4 x\" t1 J' k, H 
 
' a. ~\" }- Z: @7 l, S\" ?* mimport requests 
4 u0 e  A- B  b6 t+ z  _3 ~6 mfrom lxml import etree5 x5 g# j) w$ H 
import sqlite3 
# v5 y3 m* L' g) Z# Q- z- b\" `6 N2 H\" C7 m9 l, D 
: P5 C( p6 a% L$ a# i% h2 W/ J& u0 | 
def write_sql(c, text): 
3 X* _( J' v+ l* ?    html = etree.HTML(text); _* H; X\" P8 p 
    # 标题  P1 X0 d: s. Z: z 
    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()') 
1 Q6 _7 u; R2 K6 t    # 链接 
% ?+ U( E  U- ~' t9 I! e    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href') 
  v) w! m% @, b  f1 C. m. M    # 日期 
( h4 C& V( r, }* q    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()') 
8 R4 t4 s; N/ g8 b\" i/ O* J1 b9 \% @; ]4 N/ V5 C 
    number = 04 {+ a+ g+ I% ]. [4 z4 y+ v9 a: g! t 
    for title, href, em in zip(titles, hrefs, ems): 
\" e. F( t- o, y2 ^, ?\" K6 G        href = host + href) L7 B6 C7 K$ X\" D; N* D1 T4 e# g 
        cursor = c.execute( 
. V: `& I& l0 U, j1 D            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href) 
( @# w& z1 P' P0 k* Y, r\" u. [  B        res = c.fetchall()# q8 g0 ]$ L' G5 A) x# U0 U 
        # 判断该字段是否已存在 
6 t% ]/ U4 H5 c* ~        if res[0][0] > 0: 
% w* f+ o! P2 v4 ^& r            continue 
  C0 P- \. @; \! W/ z+ U# o$ `2 ]; @+ t 
        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (6 F. O5 ~0 _' l+ A' r2 u+ \, x1 U 
            href,7 B% B8 h) w2 K1 I 
            title.replace("\"", "\"\""),  d5 `/ {2 h. L\" J8 a 
            em)) 
+ Z# ~/ l1 R( i  |        number += 1  F' @* O+ B( D 
        print(title, href, em) 
! L7 @- ]) [! |: A  B2 t; _3 {9 p& O/ ~ 
    conn.commit()! ]0 s4 `+ W5 U# A. L% B1 O8 d! X 
    return number > 06 E: d* M6 c- k 
 
9 J, k1 y* R7 D/ a 
2 f. v5 {6 {& S\" F' x( n) _! _if __name__ == '__main__': 
3 N( m& e7 t. m% L\" k9 D+ T 
$ B( Y: z3 L* i/ M+ |    conn = sqlite3.connect("Python-xxx.db")) ^7 }( ]6 o0 O  p$ ^ 
    c = conn.cursor() 
+ g! Z0 Q6 Y  @0 [' d5 W    c.execute('''CREATE TABLE IF NOT EXISTS Python (4 m2 X$ p6 A+ w; t( ?4 W2 u 
        Url VARCHAR,& [; U* U, R' L2 K. W5 C7 y 
        Title VARCHAR,  P+ A  E# ]9 O+ D6 k% } 
        Author VARCHAR 
8 A- Y5 S8 O6 P) P! M    )''') 
$ k  N: S0 H! J3 ]0 E3 U    conn.commit(): ~  q4 ^/ I) t9 I 
' i$ ~! P' [5 i 
    host = "https://xxx"$ r- J\" p! `0 q+ |* d7 r: B 
    url = host + "/xxx"& k) Z+ a* M8 ^. w 
    req = requests.get(url) 
, ^6 N' z* z( g0 B  n    req.encoding = 'utf-8' 
! s0 w, D' d; D  I+ k. n    # print(req.text)0 z! N: `& d- r  V( T  r 
 
! w0 [\" F( d; l4 T3 e    html = etree.HTML(req.text)) b; @+ S  W% r# ^: P7 I 
    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')9 H/ y6 |) {: j 
    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href'). q! |& p' |% j& ?& L' h 
    # print(clearfix, href), c4 H1 _; S1 L2 J# M# _ 
 
\" w8 S) L, q1 }$ `4 U0 [8 c( F    for clearfix, href in zip(clearfixs, hrefs): 
& v7 U; e6 l5 R/ j: S. |        print(clearfix, host + href); J6 F; ^1 d# }! n6 F9 ^\" i 
5 Z: l6 R# p' U$ x! \0 e# k 
        page = 1 
5 J' [, H5 P% F5 c/ x$ T% R        while True:) a+ z5 r3 c- y7 N9 m 
            url = host + href + "/list_%s.html" % page7 S7 M, |4 X# B# n4 Z, w* h 
            req = requests.get(url)* S# x7 H$ r7 P) O; E% s+ W 
            req.encoding = 'utf-8' 
( Y! P+ \3 O4 V2 C4 S\" M 
7 G8 N; g3 y5 K4 @9 ]5 P. s            if (not write_sql(c, req.text)):! z/ h& |' m( j' H: f' c 
                break 
& E$ Z% Y! g6 M; J 
  R  l% w\" M8 |. i0 a            print("第%s页" % page) 
: n# Z, {6 W# p: h! D9 a+ I- M0 ?            page += 1( w3 A8 {( l) C& _\" t4 Y% t$ b1 X 
 
\" h% w. ^# t: w5 Z    conn.close() 
& L/ v3 Y9 O5 a. j3 D; V1 ]# p

xpath用着就是舒服~

转发自派生社区

Python交流群：1047602540

zan