数学建模社区-数学中国

标题: Python-xpath [打印本页]
作者: 檀俾九    时间: 2021-1-18 09:29
标题: Python-xpath
[code=python]#!/usr/bin/python! P: s& z+ g; I+ y
# -*- coding: utf-8 -*-% O1 u0 i% F: y; X
) U3 E1 j: X+ V) J+ P% O0 \( Q
import requests( O0 j* n6 x9 z; r3 d" x* s4 n+ S, Q
from lxml import etree8 F5 z2 w# u: c( |+ @$ [
import sqlite3& P: \; e) |/ Z1 @

: k$ S& J/ S4 A8 T
$ J; H- F; _$ }5 G4 Kdef write_sql(c, text):
' Q" u3 a7 _9 c# `3 {5 O- q7 B    html = etree.HTML(text)4 V+ X! I; r: i2 q0 v( K$ H
    # 标题
1 K- c+ g. B: i) E  a5 G    titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')
- Y! B8 F1 K7 m1 n- z    # 链接: `! q% n$ n  z. Q  O, J5 ]
    hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')# Z8 m" D5 p/ H: M  A
    # 日期
0 O9 q4 V  G! Q7 f3 ]9 w, v    ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')/ ?0 n" I9 ?3 }& d

" H0 u% p9 W" P( n& ~3 O0 S* u    number = 0
- B/ [+ @% Z# e! `2 E2 ]- U, y    for title, href, em in zip(titles, hrefs, ems):; y1 l9 b$ m2 Y* X& Z% S
        href = host + href3 x1 L/ ~0 R8 a
        cursor = c.execute(
8 K1 i1 v  Y2 F: r0 ~1 E8 G            "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
+ R2 Y* ?2 l0 d" N        res = c.fetchall()3 ~# {4 m$ l" }! O3 \1 t
        # 判断该字段是否已存在: I8 ^8 ]+ h& D& q/ ^! f
        if res[0][0] > 0:
* v) f% G% \7 `0 C* u  e8 |            continue: N9 ~6 l+ d6 Q- P

1 n" ]/ l) T8 G        c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
  h; P2 W, D3 ?5 r( e            href,$ j) s; x4 _9 @& b9 }, {
            title.replace("\"", "\"\""),) V4 H0 J. J2 m! \- X5 N5 D2 N
            em))
; ?- \3 k- F% x        number += 1% N- t# p8 g/ H5 Q- V
        print(title, href, em)
: P9 j8 W& _2 d4 `0 d
+ j" E" A' S( ?6 g    conn.commit()5 c3 g3 C! X4 n, ^
    return number > 0
) {& x0 w5 n- I+ |
; Z( p+ m, |: q, @# [( y& V, H) `( g7 u& M
if __name__ == '__main__':
4 y% I# u7 P& b9 H: T& i+ O- ^: U/ P# \
    conn = sqlite3.connect("Python-xxx.db")/ g$ K9 s" N& i1 X. K
    c = conn.cursor()% N- o% m! P/ N% e3 Q, w( r) T
    c.execute('''CREATE TABLE IF NOT EXISTS Python ($ T" P7 P% ^% N9 ?% }& R# g( x' [
        Url VARCHAR,
# N9 O+ ?: I- @; f, A        Title VARCHAR,
+ p4 e6 `/ M# _  ?- {        Author VARCHAR
6 ?2 ?/ i$ G- s/ q: Z1 p% \    )''')
- V" s- o; {. _    conn.commit()
$ Q0 f# L- [' ?4 s1 B2 B  b9 ~3 Z" A6 ?* v& e- l
    host = "https://xxx"
$ y" R6 d3 F) w2 z7 H4 C# k    url = host + "/xxx"' [% C1 P) U, e
    req = requests.get(url)9 x5 B( S* O+ D! y4 i  X; I
    req.encoding = 'utf-8'
6 i/ _# n4 l8 n6 f7 G    # print(req.text)
* ]) @- Z+ ]% J: ~) ~. R
, w; r: |" a9 y! p( v    html = etree.HTML(req.text)
' V* O; j. G/ V- \5 _    clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')$ z0 S' H  O/ R) ~' W8 U* [6 |
    hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')7 a( C- K- I: g) b; F4 h
    # print(clearfix, href)
- r/ G0 J& E  K- q+ k
: V& E1 S7 N4 ~$ {. I2 Y    for clearfix, href in zip(clearfixs, hrefs):4 I9 j1 b0 B9 i: A6 K0 ^
        print(clearfix, host + href)
0 P; O' Q5 c* f
: u( V- c3 d" b6 i3 q        page = 1
4 ~( Y0 n. A% s/ U- s2 V; m        while True:
) G  y6 `0 y$ J7 t& n5 r1 U            url = host + href + "/list_%s.html" % page
6 j6 G2 H4 c' k, _3 _8 Z! x7 L+ n1 N            req = requests.get(url)
/ o0 P5 u  a! c            req.encoding = 'utf-8'
" V  x1 }% V& T2 R0 F7 |8 W+ U$ \* J
            if (not write_sql(c, req.text)):
, u6 W# e7 A- I) V: Q7 c' \% Z- u& Q                break, h+ @" Z0 }) X. _5 R

( y3 O. S8 _3 j% R/ r9 `- O* I            print("第%s页" % page): [6 a2 V( g% ?2 }
            page += 1# Y$ M4 P1 e" p2 `* q1 }8 @

- T* B7 M( C- V  K    conn.close()
3 M" o5 C8 @0 U2 u[/code]
7 n: B  V6 j' {. Fxpath用着就是舒服~
! c4 f3 W! {7 c. G" p, ~! {, w5 h; h/ S+ E4 p* o
转发自派生社区
Python交流群：1047602540
1 f& y: U( L3 M1 C3 T2 |/ X& @4 l
" i& ~4 i8 @* l
9 j2 h) u1 [, `2 X5 l) @




欢迎光临 数学建模社区-数学中国 (http://www.madio.net/) Powered by Discuz! X2.5