查看: 3819|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
( x\" F9 w: W+ C$ e9 e. v0 T. X# -*- coding: utf-8 -*-
* z# L\" f- `% ]. z/ P& y: o4 Q- B. V. I
import sqlite3* A* T7 _5 k: b\" ?
import requests' N# O8 [. _3 P3 p. {$ r
from bs4 import BeautifulSoup- m) ~8 I, f: ~0 Z; @8 d9 z
from re import escape
9 m3 c$ _4 }. Q3 w R, q
) H' J' K% t# Y7 z' z3 Jif __name__ == '__main__':% b$ c$ R1 X$ k8 M$ J\" s
conn = sqlite3.connect('Python.db')
) y5 z3 N0 y- N3 N# @ c = conn.cursor()( R+ f3 A\" h- j6 C9 K
c.execute('''CREATE TABLE IF NOT EXISTS Python (
/ G6 w% T1 I8 P3 d/ w Url VARCHAR,& r) B7 D\" Z+ S u, A
Title VARCHAR,7 t' n! x6 ?& s$ i* t2 o2 i
Author VARCHAR6 t! J; n4 T3 i6 t6 T+ A
)'''): U. v# t\" [9 K3 C
conn.commit()2 P4 c) m6 e! i: ^+ F$ ^6 l
( Q+ r U5 n% J; A/ f! \ # --------------------Split Line--------------------
/ f. a! A$ ~& f/ d4 m- H: m H headers = {
; T* g\" |( y& f9 A5 W3 b% a9 D; A "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"+ R/ R: j- l7 T: B* {
}( a# G\" Q: w! r
, H' ?, b$ B5 T# l) r
for i in range(1, 1046):
4 }% b7 j$ C$ ?; `- N url = "http://xxx/index_%s.html" % str(i)) {: s8 r4 Z: O\" v
req = requests.get(url=url, headers=headers)9 w+ h9 W8 k+ ^3 {
req.encoding = "utf-8"$ `1 O' z: s) J9 i2 i
html = BeautifulSoup(req.text, "lxml")
' S% v. N. Q. k E* _* R f X* s; d2 _2 t1 c& ~
# --------------------Split Line--------------------
! c! K, U2 ?+ _( E\" D' h7 U! [\" R: \ for div in html.find_all('div', class_='loop'):
$ H: s7 o* r, Y- C+ ~* } content_body = div.select('h2 > a')[0]
* |4 y8 x# H& L: |9 H* n content_infor = div.select('.content_infor > span:nth-child(3)')[0]: f: O* p' B7 M) n5 c; O0 D
: }# ?7 ~% W, c5 {$ } # --------------------Split Line--------------------
3 D0 S5 y- t' q8 N2 y cursor = c.execute(
8 n4 D, V$ _, }. w "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
% Q3 E: _% m\" r5 b3 r, g: i len = 0
) X! v8 a* S% B for row in cursor:, G* J3 z, ^0 N' Q- \( r% M
len = row[0]; j) g1 T6 w\" A9 j& l; `$ J6 p
if len > 0: b; P. }9 ]7 v: }- j9 }/ a/ D
continue
- F7 V& ?- O8 T3 E- \$ J, J4 q8 z5 j% S* \
# --------------------Split Line--------------------. B- Q/ G' F- E, r7 V1 \
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (! ^) c; b3 j: C6 c8 P\" n, k6 k\" Q
"http://xxx" + content_body.get('href'),* S V9 v6 C\" D: `5 N9 {
escape(content_body.get('title').replace("\"", "\"\"")),* a4 S: k6 U: Z
content_infor.text.replace('xxx: ', ''))), z# E5 S- C5 m! t9 E* r
; j% V/ k6 G8 @' @
conn.commit()
# D( x7 D8 K e$ E9 c& [2 M1 V2 X print("第%s页" % str(i))- V& O3 }. [& j3 ~$ F& Q6 J8 ], ?6 H
4 a% Y/ r% I }9 P1 `+ M5 f4 Z+ ~ # --------------------Split Line--------------------
3 U3 _( y! v; P# c conn.close()
2 e7 L$ v; c+ g. H0 z

#!/usr/bin/python 
( x\" F9 w: W+ C$ e9 e. v0 T. X# -*- coding: utf-8 -*- 
* z# L\" f- `% ]. z/ P& y: o4 Q- B. V. I 
import sqlite3* A* T7 _5 k: b\" ? 
import requests' N# O8 [. _3 P3 p. {$ r 
from bs4 import BeautifulSoup- m) ~8 I, f: ~0 Z; @8 d9 z 
from re import escape 
9 m3 c$ _4 }. Q3 w  R, q 
) H' J' K% t# Y7 z' z3 Jif __name__ == '__main__':% b$ c$ R1 X$ k8 M$ J\" s 
    conn = sqlite3.connect('Python.db') 
) y5 z3 N0 y- N3 N# @    c = conn.cursor()( R+ f3 A\" h- j6 C9 K 
    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
/ G6 w% T1 I8 P3 d/ w        Url VARCHAR,& r) B7 D\" Z+ S  u, A 
        Title VARCHAR,7 t' n! x6 ?& s$ i* t2 o2 i 
        Author VARCHAR6 t! J; n4 T3 i6 t6 T+ A 
    )'''): U. v# t\" [9 K3 C 
    conn.commit()2 P4 c) m6 e! i: ^+ F$ ^6 l 
 
( Q+ r  U5 n% J; A/ f! \    # --------------------Split Line-------------------- 
/ f. a! A$ ~& f/ d4 m- H: m  H    headers = { 
; T* g\" |( y& f9 A5 W3 b% a9 D; A        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"+ R/ R: j- l7 T: B* { 
    }( a# G\" Q: w! r 
, H' ?, b$ B5 T# l) r 
    for i in range(1, 1046): 
4 }% b7 j$ C$ ?; `- N        url = "http://xxx/index_%s.html" % str(i)) {: s8 r4 Z: O\" v 
        req = requests.get(url=url, headers=headers)9 w+ h9 W8 k+ ^3 { 
        req.encoding = "utf-8"$ `1 O' z: s) J9 i2 i 
        html = BeautifulSoup(req.text, "lxml") 
' S% v. N. Q. k  E* _* R  f  X* s; d2 _2 t1 c& ~ 
        # --------------------Split Line-------------------- 
! c! K, U2 ?+ _( E\" D' h7 U! [\" R: \        for div in html.find_all('div', class_='loop'): 
$ H: s7 o* r, Y- C+ ~* }            content_body = div.select('h2 > a')[0] 
* |4 y8 x# H& L: |9 H* n            content_infor = div.select('.content_infor > span:nth-child(3)')[0]: f: O* p' B7 M) n5 c; O0 D 
 
: }# ?7 ~% W, c5 {$ }            # --------------------Split Line-------------------- 
3 D0 S5 y- t' q8 N2 y            cursor = c.execute( 
8 n4 D, V$ _, }. w                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
% Q3 E: _% m\" r5 b3 r, g: i            len = 0 
) X! v8 a* S% B            for row in cursor:, G* J3 z, ^0 N' Q- \( r% M 
                len = row[0]; j) g1 T6 w\" A9 j& l; `$ J6 p 
            if len > 0:  b; P. }9 ]7 v: }- j9 }/ a/ D 
                continue 
- F7 V& ?- O8 T3 E- \$ J, J4 q8 z5 j% S* \ 
            # --------------------Split Line--------------------. B- Q/ G' F- E, r7 V1 \ 
            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (! ^) c; b3 j: C6 c8 P\" n, k6 k\" Q 
                "http://xxx" + content_body.get('href'),* S  V9 v6 C\" D: `5 N9 { 
                escape(content_body.get('title').replace("\"", "\"\"")),* a4 S: k6 U: Z 
                content_infor.text.replace('xxx: ', ''))), z# E5 S- C5 m! t9 E* r 
; j% V/ k6 G8 @' @ 
        conn.commit() 
# D( x7 D8 K  e$ E9 c& [2 M1 V2 X        print("第%s页" % str(i))- V& O3 }. [& j3 ~$ F& Q6 J8 ], ?6 H 
 
4 a% Y/ r% I  }9 P1 `+ M5 f4 Z+ ~    # --------------------Split Line-------------------- 
3 U3 _( y! v; P# c    conn.close() 
2 e7 L$ v; c+ g. H0 z

转发自派生社区

Python交流群：1047602540

zan