查看: 3843|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |正序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
`0 d# ?\" ?, B. v# -*- coding: utf-8 -*-5 `: D0 q7 q7 Q0 I\" q( J
! `7 G( J$ D1 t8 H
import sqlite3+ v1 z. {+ C U8 ^ `3 \
import requests1 z5 E' l5 W) }% r\" R: e& X; a
from bs4 import BeautifulSoup
- u9 V; M1 m/ \\" [- x0 @6 [from re import escape, S\" V5 Y$ g4 L5 t) z$ K
7 }5 ] H1 g( q \, R
if __name__ == '__main__':8 }+ k) m, q: t r. G/ T
conn = sqlite3.connect('Python.db')9 a$ s5 x6 i3 G\" O [7 J7 t
c = conn.cursor()
/ `$ D1 u8 ]- N& b c.execute('''CREATE TABLE IF NOT EXISTS Python (
b\" H( y0 P0 h9 |9 e% m6 p& Y0 \ Url VARCHAR,
# O\" ?( i6 a! }7 K% i( ^: a Title VARCHAR,
# i. K6 T) i. s# ^& z+ X Author VARCHAR6 h1 C6 \% o1 F; D6 W
)''')
1 h0 U2 d, D- T conn.commit()
# r4 ]\" R' j9 D7 I
- T) o) F+ c7 H2 Y, ~2 N # --------------------Split Line--------------------
% A- p/ w\" ^9 n ~ headers = {
9 Q' N# v# q% R( |0 n "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
* w. s' B/ F9 x- K+ R }
' K6 I' f# @( S( Z) D0 J
7 Z0 `7 f% J+ F) O5 u- d9 |5 O for i in range(1, 1046):
+ T6 g- d4 t5 h url = "http://xxx/index_%s.html" % str(i), I* s1 h, |# g: O
req = requests.get(url=url, headers=headers)1 U+ ^\" [& Y\" y) {% D+ K: \
req.encoding = "utf-8"7 ^' ~0 a+ t- B- B
html = BeautifulSoup(req.text, "lxml")
7 F4 u9 E( O- {, {2 r9 Z5 V
( O( [- Z- O$ n. R2 z& j5 M7 m # --------------------Split Line--------------------
* K9 ]+ ]6 ~7 N! b7 ? for div in html.find_all('div', class_='loop'):
8 P4 l4 Q7 q# u/ K$ B) M+ M content_body = div.select('h2 > a')[0]* j+ j4 T) ?6 X: P8 M# u2 a
content_infor = div.select('.content_infor > span:nth-child(3)')[0]
3 c. i+ X4 ?; Q% `6 i4 W9 n/ s* [% m6 c6 n
# --------------------Split Line--------------------
7 T- Q6 E9 S2 e& Y6 f cursor = c.execute(* {# R) O! I5 x
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
1 w8 [) Z6 }) {! j len = 0& W9 B$ ^7 @- |; G
for row in cursor:' u% ?0 ?* ]/ O, _
len = row[0]0 i: ~* j+ N) G: b( _+ u J. N
if len > 0:8 p& i) ~' ?9 T5 Q E
continue' ]0 c0 N, P, I: b% L
) R7 i! \7 t- w( X+ @( \' Y- s # --------------------Split Line--------------------3 I* i% n6 `0 _$ ~
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
: J8 t& l$ e! d1 l! w. T "http://xxx" + content_body.get('href'),
\" B% o; Q( [& d v2 Y' \' s escape(content_body.get('title').replace("\"", "\"\"")),, _1 m, ~* B+ s2 R
content_infor.text.replace('xxx: ', '')))
2 Z+ D$ h$ G+ k) }$ d: i2 \0 S\" H. }! z# N; m
conn.commit()
2 z3 L6 \% N& S print("第%s页" % str(i))3 Y! B\" ?/ D6 t# ?. U& I
' V, p) Y/ [/ K+ f# V* h2 @
# --------------------Split Line--------------------
+ ^& Q9 J3 t' y5 O( k q: l conn.close()( `; i9 g* F$ D

#!/usr/bin/python 
  `0 d# ?\" ?, B. v# -*- coding: utf-8 -*-5 `: D0 q7 q7 Q0 I\" q( J 
! `7 G( J$ D1 t8 H 
import sqlite3+ v1 z. {+ C  U8 ^  `3 \ 
import requests1 z5 E' l5 W) }% r\" R: e& X; a 
from bs4 import BeautifulSoup 
- u9 V; M1 m/ \\" [- x0 @6 [from re import escape, S\" V5 Y$ g4 L5 t) z$ K 
7 }5 ]  H1 g( q  \, R 
if __name__ == '__main__':8 }+ k) m, q: t  r. G/ T 
    conn = sqlite3.connect('Python.db')9 a$ s5 x6 i3 G\" O  [7 J7 t 
    c = conn.cursor() 
/ `$ D1 u8 ]- N& b    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
  b\" H( y0 P0 h9 |9 e% m6 p& Y0 \        Url VARCHAR, 
# O\" ?( i6 a! }7 K% i( ^: a        Title VARCHAR, 
# i. K6 T) i. s# ^& z+ X        Author VARCHAR6 h1 C6 \% o1 F; D6 W 
    )''') 
1 h0 U2 d, D- T    conn.commit() 
# r4 ]\" R' j9 D7 I 
- T) o) F+ c7 H2 Y, ~2 N    # --------------------Split Line-------------------- 
% A- p/ w\" ^9 n  ~    headers = { 
9 Q' N# v# q% R( |0 n        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36" 
* w. s' B/ F9 x- K+ R    } 
' K6 I' f# @( S( Z) D0 J 
7 Z0 `7 f% J+ F) O5 u- d9 |5 O    for i in range(1, 1046): 
+ T6 g- d4 t5 h        url = "http://xxx/index_%s.html" % str(i), I* s1 h, |# g: O 
        req = requests.get(url=url, headers=headers)1 U+ ^\" [& Y\" y) {% D+ K: \ 
        req.encoding = "utf-8"7 ^' ~0 a+ t- B- B 
        html = BeautifulSoup(req.text, "lxml") 
7 F4 u9 E( O- {, {2 r9 Z5 V 
( O( [- Z- O$ n. R2 z& j5 M7 m        # --------------------Split Line-------------------- 
* K9 ]+ ]6 ~7 N! b7 ?        for div in html.find_all('div', class_='loop'): 
8 P4 l4 Q7 q# u/ K$ B) M+ M            content_body = div.select('h2 > a')[0]* j+ j4 T) ?6 X: P8 M# u2 a 
            content_infor = div.select('.content_infor > span:nth-child(3)')[0] 
3 c. i+ X4 ?; Q% `6 i4 W9 n/ s* [% m6 c6 n 
            # --------------------Split Line-------------------- 
7 T- Q6 E9 S2 e& Y6 f            cursor = c.execute(* {# R) O! I5 x 
                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
1 w8 [) Z6 }) {! j            len = 0& W9 B$ ^7 @- |; G 
            for row in cursor:' u% ?0 ?* ]/ O, _ 
                len = row[0]0 i: ~* j+ N) G: b( _+ u  J. N 
            if len > 0:8 p& i) ~' ?9 T5 Q  E 
                continue' ]0 c0 N, P, I: b% L 
 
) R7 i! \7 t- w( X+ @( \' Y- s            # --------------------Split Line--------------------3 I* i% n6 `0 _$ ~ 
            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
: J8 t& l$ e! d1 l! w. T                "http://xxx" + content_body.get('href'), 
\" B% o; Q( [& d  v2 Y' \' s                escape(content_body.get('title').replace("\"", "\"\"")),, _1 m, ~* B+ s2 R 
                content_infor.text.replace('xxx: ', ''))) 
2 Z+ D$ h$ G+ k) }$ d: i2 \0 S\" H. }! z# N; m 
        conn.commit() 
2 z3 L6 \% N& S        print("第%s页" % str(i))3 Y! B\" ?/ D6 t# ?. U& I 
' V, p) Y/ [/ K+ f# V* h2 @ 
    # --------------------Split Line-------------------- 
+ ^& Q9 J3 t' y5 O( k  q: l    conn.close()( `; i9 g* F$ D

转发自派生社区

Python交流群：1047602540

zan