数学建模社区-数学中国
标题:
Python-xpath
[打印本页]
作者:
檀俾九
时间:
2021-1-18 09:29
标题:
Python-xpath
[code=python]#!/usr/bin/python
+ E, {8 m/ P" m0 _' ^1 }
# -*- coding: utf-8 -*-
1 S8 F' Q7 i* a7 V0 q
) X$ K; ], M& { m9 F5 f
import requests
) S( j0 }+ ?& R. w+ j' b: R
from lxml import etree
B' J8 D; W e) V9 J
import sqlite3
$ ^9 h( r* m+ D. V- ]
7 t1 p7 T; A$ B- W8 O
! p) j- b8 ?: f1 V8 D& R
def write_sql(c, text):
3 x+ S( n3 D4 i8 K, g
html = etree.HTML(text)
& B* N7 Z9 U; p+ }) E
# 标题
G8 z8 K+ E/ r* u, C
titles = html.xpath('//ul[@class="news"]//a[@target="_blank"]/p/text()')
2 f2 ^ o; p7 U
# 链接
4 `& A. u3 V& Y% x! h1 H9 \" _- D
hrefs = html.xpath('//ul[@class="news"]//a[@target="_blank"]/@href')
. d4 S; R N6 {. P( c; `
# 日期
- C W! Y6 D2 D( |& u2 h
ems = html.xpath('//ul[@class="news"]//a[@target="_blank"]/em/text()')
# D8 g3 }) x- S6 s7 i
$ A% O& d2 g# T3 W3 W# Q$ ^
number = 0
4 w% I* \& f% K6 ]9 k# W/ f/ Q
for title, href, em in zip(titles, hrefs, ems):
1 n9 ]3 M B+ F
href = host + href
4 |. f8 T8 l# b ~" x6 C
cursor = c.execute(
1 P4 q/ H! u, ~
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % href)
1 E1 ^) n* F9 n, a8 m
res = c.fetchall()
' V2 M9 N- @! d$ _0 i# n5 g
# 判断该字段是否已存在
! v. U9 O. U& _
if res[0][0] > 0:
2 @" n7 w% s E& u( B
continue
G8 w% M$ \8 r$ u5 o
% l% h( y+ V F% H
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
* m( L% o3 [3 o) o5 N' r' M: z
href,
I2 U* b# t! W6 L1 h' c* G
title.replace("\"", "\"\""),
4 j2 R5 ]2 H4 G% y& y: P4 s* A7 l2 V
em))
( n- p' i" ]8 G) e- X* M
number += 1
6 Q+ O- _+ N, l, I& I& A5 V, |* |7 Z
print(title, href, em)
$ t6 N/ x0 O! R: X- m: Q% }
- \6 k* `' l. E7 J$ C; N$ z
conn.commit()
7 \7 C4 `! c6 N1 x" L% s6 E: a" T3 A
return number > 0
; Y% G$ [3 t: Z: d
9 I3 D! `4 i: N
. l) v9 d0 w x6 x% e1 L' F5 v) x
if __name__ == '__main__':
& F% L% j% _0 n
8 c2 G, a2 _) X- r
conn = sqlite3.connect("Python-xxx.db")
0 @5 V$ O0 |; u( ]4 ^# o/ t
c = conn.cursor()
7 S$ |$ S/ V ]3 s; _& Q
c.execute('''CREATE TABLE IF NOT EXISTS Python (
9 _: e1 C3 a6 V, M# N& [$ T# j
Url VARCHAR,
9 \ A- I5 L. Z4 n \+ G1 T
Title VARCHAR,
0 H5 {* ~; V9 N( R* Z: h* ^' K$ [
Author VARCHAR
$ F1 T v' _' M# e
)''')
; Z/ Q2 ~) \: ]) J8 V
conn.commit()
K3 f* X* w: `+ y( z: r* ^
0 ?4 p' K$ F( N& Z# e# p; q5 c
host = "https://xxx"
) [6 {: Z1 I) q: z% C2 ?$ ^0 u
url = host + "/xxx"
' a" l. Z" e+ J0 Q! i
req = requests.get(url)
4 c: }4 E" i# D( ~" \7 J
req.encoding = 'utf-8'
6 K3 g- ~0 V2 |
# print(req.text)
8 @4 @ @2 E# l
! A; I7 s( l. B" q
html = etree.HTML(req.text)
) W/ _# _: K" \9 Z' n
clearfixs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/text()')
# }. K; C, C4 c4 J+ {0 `& w( c
hrefs = html.xpath('//*[@class="nav clearfix"]//a[starts-with(@href, "/cate/")]/@href')
+ i5 V0 d- }: I# E& a1 W7 I* n; }
# print(clearfix, href)
$ o) N! d1 ]' r, V1 W; c+ J
: C1 Q" `1 B# D6 ^) ~
for clearfix, href in zip(clearfixs, hrefs):
0 k" K) X; w9 r. V4 o' F' u1 K
print(clearfix, host + href)
$ P6 F2 x D3 ~9 v
& q: `3 Y. L: c( @! Y6 A& v
page = 1
0 n/ t4 f5 S! H7 V4 e8 H
while True:
: _( F3 m5 S/ l6 f4 R
url = host + href + "/list_%s.html" % page
' I. W" N2 M1 K1 Q
req = requests.get(url)
. A! e. T& ~3 d. v' w
req.encoding = 'utf-8'
" z) d4 J2 E4 H9 Y4 a; j
3 v' W1 K; j2 ~) K
if (not write_sql(c, req.text)):
* K+ C5 `' I9 ]8 h |
break
+ u2 a0 f W) ~9 ?7 A4 c1 ]
8 q" q, f0 u0 i- u% l' X
print("第%s页" % page)
3 K3 x& N/ c: J% B4 R
page += 1
" Y9 T* t9 p7 D8 j
( e4 x4 I6 y6 [' j0 E0 \
conn.close()
% V3 c9 |: P5 z/ c
[/code]
' g e( G. ]9 Z1 g* O3 Y1 T
xpath用着就是舒服~
: K3 M4 a7 G; q: Y1 k$ W$ z
( ]& t9 A, K* B9 a. e! z
转发自
派生社区
Python交流群：1047602540
, S7 y; ^: [/ o9 e7 k7 u
. ^& Q/ p" A7 K
2 m9 }, n. o& X/ w' e6 V
欢迎光临 数学建模社区-数学中国 (http://www.madio.net/)
Powered by Discuz! X2.5