数学建模社区-数学中国
标题:
Python-网页爬虫与Sqlite3
[打印本页]
作者:
檀俾九
时间:
2021-1-14 08:44
标题:
Python-网页爬虫与Sqlite3
[code=python]#!/usr/bin/python
; u. g0 }/ c% e! u/ N) K& J5 J% g. H
# -*- coding: utf-8 -*-
- W+ r" b7 z4 E' F
; K8 V6 i1 T4 p3 m5 E3 M
import sqlite3
% s% B6 N4 F3 e, w
import requests
* \5 D9 N5 b# C7 h, D! ?# v
from bs4 import BeautifulSoup
2 e" J$ {- U# D7 @/ F/ t4 p
from re import escape
# s7 j- \) k7 R u5 I
% W7 m' s) ~! o/ ^, e( ~
if __name__ == '__main__':
. X8 |+ ?( J5 J8 Z! R2 ^5 @9 g
conn = sqlite3.connect('Python.db')
+ I. h! v+ F5 U( U5 D+ U
c = conn.cursor()
; O, V% L) L7 p: _9 Z" K
c.execute('''CREATE TABLE IF NOT EXISTS Python (
$ V/ |9 h( W& n& o& z# @6 K
Url VARCHAR,
" `9 q( ~7 `# b! E# z0 o& L
Title VARCHAR,
- I+ f! F6 _4 B2 I
Author VARCHAR
0 \, T$ ]7 m3 \% l; i7 Z
)''')
8 E4 q0 d/ e% N
conn.commit()
7 X2 V& B) S% t6 V5 e4 v+ |
' r' b4 B: V% N) l1 A& H
# --------------------Split Line--------------------
" ]7 h+ x d Q: X2 f! Y
headers = {
6 O9 t. i. Y* a* A1 k3 {( T$ c
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
& p: Y# Y5 ^7 U$ S: r
}
, A1 n9 W( i# k4 @ m
/ a4 O& z. E1 _9 I0 \1 t
for i in range(1, 1046):
+ Y B; d- q. U1 n) S+ S8 ~9 z
url = "http://xxx/index_%s.html" % str(i)
, u+ Z# N8 j8 L7 [9 @& @* j
req = requests.get(url=url, headers=headers)
+ n9 x a4 ~' _% T9 a
req.encoding = "utf-8"
, `6 E+ |( T8 D3 a4 M
html = BeautifulSoup(req.text, "lxml")
9 M; f- p6 `3 e! z! |
7 n, O& X! j, O! w% D
# --------------------Split Line--------------------
/ L& }2 h5 g J( k" }* [0 s' P# K/ @
for div in html.find_all('div', class_='loop'):
M1 Q2 L4 c _: P" Y
content_body = div.select('h2 > a')[0]
3 \$ `! J2 I% O7 G% o- G' w5 a
content_infor = div.select('.content_infor > span:nth-child(3)')[0]
( y* }8 H4 n& b
# [2 t+ Y3 H! S+ ~) ~# h* K
# --------------------Split Line--------------------
+ ^2 d+ [; N5 P# W- f( t1 Z/ v
cursor = c.execute(
9 `. r" ~+ `, _8 Z5 O6 _9 u- l
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
9 \: Q/ A: o3 h
len = 0
0 w7 s4 {7 X6 Y1 V2 i4 ~1 U
for row in cursor:
. I( t1 }* P* J/ E5 p U! T! l4 y
len = row[0]
: ?! }5 ^; W2 ^) E# C- E
if len > 0:
. W% R! J! o" y* a% p& v
continue
* P4 q* E. F! Q# ~' z# s) E
+ H3 S/ {- ^: A$ \
# --------------------Split Line--------------------
7 q* N" ]' o d% J" X9 i
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
1 d. u( N! m% A0 x, G
"http://xxx" + content_body.get('href'),
) D% X; ^* Y8 u' l+ ?
escape(content_body.get('title').replace("\"", "\"\"")),
, ^! l2 `6 z/ S' C0 n1 n) b$ D n
content_infor.text.replace('xxx: ', '')))
; c- c" n- I0 U7 o, l$ N/ ?& o7 v
( p R: Z8 m& M# ]# t
conn.commit()
' ]' S& k" r1 @* ^, [
print("第%s页" % str(i))
3 R5 ?% _! q2 ~) R
9 f3 H/ L/ g" \
# --------------------Split Line--------------------
* j' d y8 h6 I
conn.close()
) R2 F9 \- X0 _( w5 Z1 M
[/code]
* v. |% j5 W) q0 A+ _$ N( u
- X; U! V6 q1 v" n( x
转发自
派生社区
Python交流群：1047602540
$ T% s% X# i3 J* P
欢迎光临 数学建模社区-数学中国 (http://www.madio.net/)
Powered by Discuz! X2.5