数学建模社区-数学中国
标题:
Python-网页爬虫与Sqlite3
[打印本页]
作者:
檀俾九
时间:
2021-1-14 08:44
标题:
Python-网页爬虫与Sqlite3
[code=python]#!/usr/bin/python
& L4 m. Z3 x9 Y) x
# -*- coding: utf-8 -*-
]' X5 g5 m% F% n$ ?
& H9 e' r9 D K( a; W5 \7 d
import sqlite3
/ X# g; c8 y- c8 q: X
import requests
* ~- Y- q7 W% S9 E+ [
from bs4 import BeautifulSoup
, H: f* ^- c; ?; V. @) F
from re import escape
$ p6 A7 {/ L, |6 G. k, }
2 W& o3 S5 X/ n8 A% E" Y7 t' k
if __name__ == '__main__':
& f7 X. n1 H0 ? R: C
conn = sqlite3.connect('Python.db')
% c) \7 G- _; s' x: ?
c = conn.cursor()
4 D" M5 J/ P/ a/ H& x( M
c.execute('''CREATE TABLE IF NOT EXISTS Python (
9 o0 r, R- i y% R
Url VARCHAR,
9 T7 J4 S; V$ Y- M: ?# \4 a
Title VARCHAR,
' \4 g* Q. n! f! K
Author VARCHAR
) {/ O1 d) c$ A; R5 ?
)''')
- [1 M2 c/ b* Y
conn.commit()
& w$ X, `! Y; S1 P
% F' E! T& s6 G9 O" B
# --------------------Split Line--------------------
1 b& F4 ]& C. r8 e. ^
headers = {
' g! X- m: \8 p6 z8 Z0 h a
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
, n- ~ k4 o' o, @! s; I" B4 L9 y
}
; P$ {; E; ]& x; e
9 Q7 R/ y2 ~+ G" C2 F, D; Z- D
for i in range(1, 1046):
4 C; s3 n; H: A( h
url = "http://xxx/index_%s.html" % str(i)
9 H% P; ]# v# g* E& g
req = requests.get(url=url, headers=headers)
6 R, r1 ]2 y2 s9 p2 t( V0 {
req.encoding = "utf-8"
: d; T; S; ?) h. x" R3 l
html = BeautifulSoup(req.text, "lxml")
+ E" s; W0 q" G7 G! V3 w Y$ C( i
3 l1 o$ B: m' a; c/ ~
# --------------------Split Line--------------------
d+ G/ n7 N# ~& K6 _
for div in html.find_all('div', class_='loop'):
* D: x% ?& k5 }* q
content_body = div.select('h2 > a')[0]
% C- L4 Q2 U; T
content_infor = div.select('.content_infor > span:nth-child(3)')[0]
; C! {" G3 c, W4 v4 A
) h: ~, F7 p( k/ }: y% O: w
# --------------------Split Line--------------------
7 N1 O, J8 R4 r# Q6 M( n, J3 u
cursor = c.execute(
( s0 x M" e& ]
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
" g4 x2 |- Q# E" V- P
len = 0
* {& U Z4 Z, l1 Z; q
for row in cursor:
' v7 _$ V7 @5 O1 z4 w) R9 h9 j n
len = row[0]
! ]# o( K C" O0 J/ s
if len > 0:
) d6 U4 A! T }* l$ z! v z* t, Q; O
continue
- a5 ~! G) \4 z5 C" z
: i. T) _- b! p/ O* k+ n& x/ @
# --------------------Split Line--------------------
/ ?5 c+ j! j- V0 j, z* z8 ]2 t
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
* l4 y3 |3 n- `# F* z
"http://xxx" + content_body.get('href'),
2 J4 t. O, d( u$ i5 Y
escape(content_body.get('title').replace("\"", "\"\"")),
' l. T n' U2 `! {& C# X: V
content_infor.text.replace('xxx: ', '')))
. e% o# A& o2 R$ `) _. U2 J
$ U6 O2 Q9 }5 H! q8 |) c. ~$ W, d- e
conn.commit()
0 {& L3 z6 A' F9 S! C
print("第%s页" % str(i))
2 O4 x9 g: ~2 n0 J, X7 B* }/ g+ {
! b1 G$ z1 y6 X) h
# --------------------Split Line--------------------
. p0 n' u* N2 X( a2 a
conn.close()
x+ L2 p. r; l7 ? h+ O
[/code]
4 `4 O5 O H) ?6 y: J2 [
: p2 G. |# I9 O. ?! D
转发自
派生社区
Python交流群：1047602540
% E2 L# [7 E+ q4 D
欢迎光临 数学建模社区-数学中国 (http://www.madio.net/)
Powered by Discuz! X2.5