查看: 3823|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
, s B: Z$ s) S1 [7 ^; R- b# -*- coding: utf-8 -*-
, {, o\" W: g% w. N7 d/ Y4 \. T* ^; f7 J% V k( f
import sqlite3
& w3 w! e9 H8 L. K5 ?9 Z0 Simport requests
: N. u7 I# j( Ffrom bs4 import BeautifulSoup
: \4 B0 R0 Z8 b0 W4 r' m\" }* o0 a+ _from re import escape
2 i5 U& s( c! m4 ?$ q# s0 s\" i6 x8 I! X, w! L
if __name__ == '__main__':
7 Z+ U- Z6 d7 T, I conn = sqlite3.connect('Python.db')
) c$ J$ u/ p3 g( I c = conn.cursor()
* ~% C) }4 Q5 ^, a( O c.execute('''CREATE TABLE IF NOT EXISTS Python (3 P; p5 b3 T; A- j! a* X
Url VARCHAR,9 R4 Y, l) L& T\" Y. @1 A% D
Title VARCHAR,' H9 Z2 b+ [7 M( l+ h# p
Author VARCHAR
1 P! Z( t, Y7 D) G )''')* y9 K7 f8 b) Q8 f, K. q
conn.commit()8 S' _- Z& E9 F( D
6 B' a& Y5 \- `: O # --------------------Split Line--------------------
- |8 `# B1 T) M& l* S headers = {. F$ l4 \+ Q1 ?# B G
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
( c\" ^' J R+ z' O }\" W X+ \\" n( b$ S5 c% R
. p5 Y1 C& X- p( t# | for i in range(1, 1046):1 n3 L0 g, Z* S- d2 n; m) N! M
url = "http://xxx/index_%s.html" % str(i)) n) O2 d\" `; a9 K5 M1 \
req = requests.get(url=url, headers=headers)
+ x( ^& f; ]1 `2 r req.encoding = "utf-8"( @# N/ x# G. L9 x1 L6 Y
html = BeautifulSoup(req.text, "lxml")% w+ s1 m8 w$ H7 l1 S( G
. @3 C1 X: P' g3 \2 } # --------------------Split Line--------------------
+ c- x0 J3 o3 h8 d for div in html.find_all('div', class_='loop'):' R# I4 U' A. p) G
content_body = div.select('h2 > a')[0]
' {3 Q1 {' u# H1 T content_infor = div.select('.content_infor > span:nth-child(3)')[0]
% t* X2 N) ]) {, x7 m, i- z. x( D, M. Z+ X4 m- C1 t) _
# --------------------Split Line--------------------
4 j\" }7 y- z\" o4 a$ C\" b) T cursor = c.execute(
5 U/ x4 A& g: I1 i! X4 q; p7 E "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) U9 n! s3 b7 H4 Y
len = 0
0 _0 E0 N/ T( Z9 l9 @% j/ t for row in cursor:
7 T' ? n% J( B/ u len = row[0]% K/ v% {$ s6 z2 M
if len > 0:
, f* a7 z9 y- ^3 V! R continue1 @) R. i6 [\" f* ?2 O3 Z0 R8 E
* d |% r) s! @ F
# --------------------Split Line--------------------6 [\" K2 _& _\" J& x5 J1 [1 n
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
4 T9 D; O! y, D9 C "http://xxx" + content_body.get('href'),2 T, i- t: e; x& \/ ]
escape(content_body.get('title').replace("\"", "\"\"")),4 P) C+ ~\" a3 F- N; _/ {
content_infor.text.replace('xxx: ', '')))1 M\" k0 V# B5 i( [\" h9 {, Q$ L
# Y. c\" C* q# I1 x0 g6 s% Q conn.commit()
) j* A* a\" r0 v5 b6 H$ z1 j print("第%s页" % str(i))
! ~ R\" T( W0 _, J2 q' o- o$ @# S8 [
# --------------------Split Line--------------------6 r# J+ l; P; V\" d; C
conn.close()
$ j8 k\" ?( ]: O4 H5 Z& A

#!/usr/bin/python 
, s  B: Z$ s) S1 [7 ^; R- b# -*- coding: utf-8 -*- 
, {, o\" W: g% w. N7 d/ Y4 \. T* ^; f7 J% V  k( f 
import sqlite3 
& w3 w! e9 H8 L. K5 ?9 Z0 Simport requests 
: N. u7 I# j( Ffrom bs4 import BeautifulSoup 
: \4 B0 R0 Z8 b0 W4 r' m\" }* o0 a+ _from re import escape 
2 i5 U& s( c! m4 ?$ q# s0 s\" i6 x8 I! X, w! L 
if __name__ == '__main__': 
7 Z+ U- Z6 d7 T, I    conn = sqlite3.connect('Python.db') 
) c$ J$ u/ p3 g( I    c = conn.cursor() 
* ~% C) }4 Q5 ^, a( O    c.execute('''CREATE TABLE IF NOT EXISTS Python (3 P; p5 b3 T; A- j! a* X 
        Url VARCHAR,9 R4 Y, l) L& T\" Y. @1 A% D 
        Title VARCHAR,' H9 Z2 b+ [7 M( l+ h# p 
        Author VARCHAR 
1 P! Z( t, Y7 D) G    )''')* y9 K7 f8 b) Q8 f, K. q 
    conn.commit()8 S' _- Z& E9 F( D 
 
6 B' a& Y5 \- `: O    # --------------------Split Line-------------------- 
- |8 `# B1 T) M& l* S    headers = {. F$ l4 \+ Q1 ?# B  G 
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36" 
( c\" ^' J  R+ z' O    }\" W  X+ \\" n( b$ S5 c% R 
 
. p5 Y1 C& X- p( t# |    for i in range(1, 1046):1 n3 L0 g, Z* S- d2 n; m) N! M 
        url = "http://xxx/index_%s.html" % str(i)) n) O2 d\" `; a9 K5 M1 \ 
        req = requests.get(url=url, headers=headers) 
+ x( ^& f; ]1 `2 r        req.encoding = "utf-8"( @# N/ x# G. L9 x1 L6 Y 
        html = BeautifulSoup(req.text, "lxml")% w+ s1 m8 w$ H7 l1 S( G 
 
. @3 C1 X: P' g3 \2 }        # --------------------Split Line-------------------- 
+ c- x0 J3 o3 h8 d        for div in html.find_all('div', class_='loop'):' R# I4 U' A. p) G 
            content_body = div.select('h2 > a')[0] 
' {3 Q1 {' u# H1 T            content_infor = div.select('.content_infor > span:nth-child(3)')[0] 
% t* X2 N) ]) {, x7 m, i- z. x( D, M. Z+ X4 m- C1 t) _ 
            # --------------------Split Line-------------------- 
4 j\" }7 y- z\" o4 a$ C\" b) T            cursor = c.execute( 
5 U/ x4 A& g: I1 i! X4 q; p7 E                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))  U9 n! s3 b7 H4 Y 
            len = 0 
0 _0 E0 N/ T( Z9 l9 @% j/ t            for row in cursor: 
7 T' ?  n% J( B/ u                len = row[0]% K/ v% {$ s6 z2 M 
            if len > 0: 
, f* a7 z9 y- ^3 V! R                continue1 @) R. i6 [\" f* ?2 O3 Z0 R8 E 
* d  |% r) s! @  F 
            # --------------------Split Line--------------------6 [\" K2 _& _\" J& x5 J1 [1 n 
            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
4 T9 D; O! y, D9 C                "http://xxx" + content_body.get('href'),2 T, i- t: e; x& \/ ] 
                escape(content_body.get('title').replace("\"", "\"\"")),4 P) C+ ~\" a3 F- N; _/ { 
                content_infor.text.replace('xxx: ', '')))1 M\" k0 V# B5 i( [\" h9 {, Q$ L 
 
# Y. c\" C* q# I1 x0 g6 s% Q        conn.commit() 
) j* A* a\" r0 v5 b6 H$ z1 j        print("第%s页" % str(i)) 
! ~  R\" T( W0 _, J2 q' o- o$ @# S8 [ 
    # --------------------Split Line--------------------6 r# J+ l; P; V\" d; C 
    conn.close() 
$ j8 k\" ?( ]: O4 H5 Z& A

转发自派生社区

Python交流群：1047602540

zan