查看: 3820|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
. T4 R/ Y, X5 {# h& l# -*- coding: utf-8 -*-
* H1 J$ |. d E+ y* s2 g E. _/ v# N7 _3 R. a% A
import sqlite3
: X- c7 D0 a* @\" l0 n/ mimport requests
4 i5 t+ w6 R( _% m' A, ]from bs4 import BeautifulSoup
\" G6 ?' R! W0 D$ x2 ]- e3 N1 ]7 ]from re import escape
9 p4 Y, n5 I+ }- U
/ @* w, n6 g- E. P2 D7 }if __name__ == '__main__':4 a% Z, L }2 Y9 i- _
conn = sqlite3.connect('Python.db')
9 g7 u0 p( d. D c = conn.cursor()
3 o6 z, @7 e* ?\" t0 F c.execute('''CREATE TABLE IF NOT EXISTS Python (
S, t) V: {2 x$ A6 d. A Url VARCHAR,, }' J |0 L8 d9 J, m; ~& | p
Title VARCHAR,( r: J/ k& H: K+ b+ q
Author VARCHAR' b. |6 A6 a1 X2 X) j
)''')
* x: R# y% z) b( s conn.commit()$ x\" h0 m4 W/ f7 h7 j) t
\" X% G& T1 Q# D( Z* `7 i* Z
# --------------------Split Line--------------------! a0 f* R( f& Y/ ~. \- `: N
headers = {
! D2 d! N. z, }# N( I: O "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"% F' C6 x! p: a' R+ n B
}
( y8 D: T9 I3 O& b* j( Y7 i; g7 W0 a
for i in range(1, 1046):: t\" F& \3 y. [
url = "http://xxx/index_%s.html" % str(i)# W& N8 d& e; L6 j7 c% f
req = requests.get(url=url, headers=headers)
! n: \/ I( k, @3 |. z* b req.encoding = "utf-8"+ z3 H ]; h. C/ G6 C
html = BeautifulSoup(req.text, "lxml")$ t, d- g. F% i) r
$ L9 Q9 \4 L4 R\" @- D) P # --------------------Split Line--------------------
& a+ `2 e* ]8 j# R for div in html.find_all('div', class_='loop'):
9 ?\" B% k0 ?+ F2 h2 F5 N content_body = div.select('h2 > a')[0]# `) h1 Z. A) a
content_infor = div.select('.content_infor > span:nth-child(3)')[0]% B, d/ z1 |; ~1 L/ G! t
; j: x% K( j$ _: }1 W\" l: k # --------------------Split Line--------------------) g. y% @5 |/ X5 O$ S
cursor = c.execute(3 p! k; @1 c7 J+ J5 r
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
, C: u; ^2 t0 C8 L, m* S5 g* W len = 08 w* Y/ o/ b M& O\" W0 w# a! t9 q$ i
for row in cursor:
2 ?4 }2 i/ s5 u+ R+ i3 E' v len = row[0]
4 h |1 I7 I. r8 F9 s) { if len > 0:) R9 l O& f1 X6 \+ b! M- |, ^
continue0 a0 S6 ~0 [' k: m. \6 E5 v
, e# T& f, R, G p' |, B4 K # --------------------Split Line--------------------
3 D% ?/ g% m0 I( D2 T( A; G* } c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
% ?% r0 b9 \$ |- }' a& S1 } "http://xxx" + content_body.get('href'),, Y2 Z9 K; K: @; [\" [/ M\" L
escape(content_body.get('title').replace("\"", "\"\"")),
: l, Q\" |' ^* d+ Q2 n content_infor.text.replace('xxx: ', '')))
& R& R9 J, T5 G0 D2 C8 a- i: b- R& H
conn.commit()
1 z4 N4 H. }3 z, `& U8 F3 c& O' e print("第%s页" % str(i))
% C O% u- |& }) [. q& `
# --------------------Split Line--------------------\" H' O% q5 n5 l- O3 r; M- d
conn.close()
+ g, a/ f- }; Z/ w- L) U

#!/usr/bin/python 
. T4 R/ Y, X5 {# h& l# -*- coding: utf-8 -*- 
* H1 J$ |. d  E+ y* s2 g  E. _/ v# N7 _3 R. a% A 
import sqlite3 
: X- c7 D0 a* @\" l0 n/ mimport requests 
4 i5 t+ w6 R( _% m' A, ]from bs4 import BeautifulSoup 
\" G6 ?' R! W0 D$ x2 ]- e3 N1 ]7 ]from re import escape 
9 p4 Y, n5 I+ }- U 
/ @* w, n6 g- E. P2 D7 }if __name__ == '__main__':4 a% Z, L  }2 Y9 i- _ 
    conn = sqlite3.connect('Python.db') 
9 g7 u0 p( d. D    c = conn.cursor() 
3 o6 z, @7 e* ?\" t0 F    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
  S, t) V: {2 x$ A6 d. A        Url VARCHAR,, }' J  |0 L8 d9 J, m; ~& |  p 
        Title VARCHAR,( r: J/ k& H: K+ b+ q 
        Author VARCHAR' b. |6 A6 a1 X2 X) j 
    )''') 
* x: R# y% z) b( s    conn.commit()$ x\" h0 m4 W/ f7 h7 j) t 
\" X% G& T1 Q# D( Z* `7 i* Z 
    # --------------------Split Line--------------------! a0 f* R( f& Y/ ~. \- `: N 
    headers = { 
! D2 d! N. z, }# N( I: O        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"% F' C6 x! p: a' R+ n  B 
    } 
( y8 D: T9 I3 O& b* j( Y7 i; g7 W0 a 
    for i in range(1, 1046):: t\" F& \3 y. [ 
        url = "http://xxx/index_%s.html" % str(i)# W& N8 d& e; L6 j7 c% f 
        req = requests.get(url=url, headers=headers) 
! n: \/ I( k, @3 |. z* b        req.encoding = "utf-8"+ z3 H  ]; h. C/ G6 C 
        html = BeautifulSoup(req.text, "lxml")$ t, d- g. F% i) r 
 
$ L9 Q9 \4 L4 R\" @- D) P        # --------------------Split Line-------------------- 
& a+ `2 e* ]8 j# R        for div in html.find_all('div', class_='loop'): 
9 ?\" B% k0 ?+ F2 h2 F5 N            content_body = div.select('h2 > a')[0]# `) h1 Z. A) a 
            content_infor = div.select('.content_infor > span:nth-child(3)')[0]% B, d/ z1 |; ~1 L/ G! t 
 
; j: x% K( j$ _: }1 W\" l: k            # --------------------Split Line--------------------) g. y% @5 |/ X5 O$ S 
            cursor = c.execute(3 p! k; @1 c7 J+ J5 r 
                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
, C: u; ^2 t0 C8 L, m* S5 g* W            len = 08 w* Y/ o/ b  M& O\" W0 w# a! t9 q$ i 
            for row in cursor: 
2 ?4 }2 i/ s5 u+ R+ i3 E' v                len = row[0] 
4 h  |1 I7 I. r8 F9 s) {            if len > 0:) R9 l  O& f1 X6 \+ b! M- |, ^ 
                continue0 a0 S6 ~0 [' k: m. \6 E5 v 
 
, e# T& f, R, G  p' |, B4 K            # --------------------Split Line-------------------- 
3 D% ?/ g% m0 I( D2 T( A; G* }            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
% ?% r0 b9 \$ |- }' a& S1 }                "http://xxx" + content_body.get('href'),, Y2 Z9 K; K: @; [\" [/ M\" L 
                escape(content_body.get('title').replace("\"", "\"\"")), 
: l, Q\" |' ^* d+ Q2 n                content_infor.text.replace('xxx: ', ''))) 
& R& R9 J, T5 G0 D2 C8 a- i: b- R& H 
        conn.commit() 
1 z4 N4 H. }3 z, `& U8 F3 c& O' e        print("第%s页" % str(i)) 
% C  O% u- |& }) [. q& ` 
3 T: f: w( c% A\" F0 U2 A1 ^- e2 R$ e) f- O    # --------------------Split Line--------------------\" H' O% q5 n5 l- O3 r; M- d 
    conn.close() 
+ g, a/ f- }; Z/ w- L) U

转发自派生社区

Python交流群：1047602540

zan