数学建模社区-数学中国
标题:
Python-网页爬虫与Sqlite3
[打印本页]
作者:
檀俾九
时间:
2021-1-14 08:44
标题:
Python-网页爬虫与Sqlite3
[code=python]#!/usr/bin/python
9 o8 M8 x! I: {+ Q' B! L
# -*- coding: utf-8 -*-
7 L: ^- ]% p6 y3 x" C% M. @
4 ]5 z l- U$ R& H; ~% J% |
import sqlite3
* X8 ]9 _0 E; R8 p$ N) X
import requests
* p7 i8 b& H6 S
from bs4 import BeautifulSoup
8 U, x& \" q$ j/ H
from re import escape
7 O4 x- W+ ^! b& d: ?8 l
3 K6 i( f6 B0 P$ s9 w
if __name__ == '__main__':
& L: X' v4 `/ a8 c' h
conn = sqlite3.connect('Python.db')
0 K( F8 i! H8 R
c = conn.cursor()
8 C- @: B6 O9 F$ y6 v2 d$ e/ S
c.execute('''CREATE TABLE IF NOT EXISTS Python (
0 \+ o- r/ b$ {, ?9 h+ C3 x! b3 i
Url VARCHAR,
0 ], \; K3 B' \3 ^7 p3 L) Z' e
Title VARCHAR,
- E' }* i# U* C& n2 P
Author VARCHAR
4 S; T4 ^( A k# Y' p; D7 P" e" {
)''')
* `" C; L9 x+ j) t4 ~8 l$ T1 N% J
conn.commit()
* @7 `: b1 a0 E. l, R* v
6 a, M$ y8 V/ E7 V e
# --------------------Split Line--------------------
4 z O A8 E' n& i+ Y+ ?5 L4 _9 s: p
headers = {
% x6 ^( i$ t- T! ]7 K
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
) \2 f/ z# [0 S, K
}
; ?- A& o+ w! E" h S
$ P- d& N0 Z& S7 f; N0 E3 ^( C
for i in range(1, 1046):
( V1 Q+ P M6 Q$ \. ~$ P
url = "http://xxx/index_%s.html" % str(i)
3 p) r2 D7 q9 g, {
req = requests.get(url=url, headers=headers)
) V, ^( B9 H# G# F& U
req.encoding = "utf-8"
3 ]/ @1 B: l: U
html = BeautifulSoup(req.text, "lxml")
+ F. C' Y* W: F e
E- F" S4 Z1 u% I
# --------------------Split Line--------------------
a" n2 D: d( I% t6 C; T
for div in html.find_all('div', class_='loop'):
) k3 w* K# S$ V% n+ w& K9 j
content_body = div.select('h2 > a')[0]
^# y2 k4 U8 }4 n; c( a5 e
content_infor = div.select('.content_infor > span:nth-child(3)')[0]
' E o1 U. ^ N0 e' q! Y
# k8 L' }, g/ R5 C. A/ y
# --------------------Split Line--------------------
4 Q& H( \6 [( s; ]8 l9 S% {
cursor = c.execute(
, l S6 b6 h. o4 P( w2 K! k( H
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
! l/ c8 F' Q- E5 Y$ K% `9 b7 K
len = 0
) c& |, {8 X/ b h
for row in cursor:
% m2 S! O; i; \' f/ [
len = row[0]
: I: x8 a( ^6 \' @' E% m& Q, E0 v/ i1 P
if len > 0:
/ Y2 G3 a$ ^$ Q9 B' a2 U0 h: B
continue
' k8 t7 X) i4 F
5 c0 A3 X! W5 f3 W
# --------------------Split Line--------------------
. h% H8 d3 g9 m4 k: _3 j0 q
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
* v N( O1 y/ _+ B
"http://xxx" + content_body.get('href'),
`7 ^& I/ b& S7 \" z$ z& H* D
escape(content_body.get('title').replace("\"", "\"\"")),
& Z3 d8 |" t- e W! k: G8 r
content_infor.text.replace('xxx: ', '')))
! d5 h# G* H4 n1 o8 N& U
) R o6 v; W {% ]. G& _
conn.commit()
$ }) C% N5 |8 j
print("第%s页" % str(i))
0 i7 r5 @7 ^$ ?; u7 j- e3 a: }7 b
# B) u( Q- D3 U. r
# --------------------Split Line--------------------
8 U" E) o! _& ]
conn.close()
1 H n- M; Z' w, P) @( F0 r# p
[/code]
: I* D% i2 J+ v- C8 H
7 e1 h" u; l2 X
转发自
派生社区
Python交流群：1047602540
4 p5 g$ A9 M9 a
欢迎光临 数学建模社区-数学中国 (http://www.madio.net/)
Powered by Discuz! X2.5