查看: 3839|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
* N\" B: p1 F2 b: e# U# -*- coding: utf-8 -*-
# y\" o$ h' {+ G. ]* A8 {
' U% X5 n7 ]- b& `' iimport sqlite3
$ L+ ^, Z$ ^6 vimport requests
0 ~- w- O2 c& T\" E0 Afrom bs4 import BeautifulSoup
1 j4 r; E6 D gfrom re import escape
\" U) \; {6 ^; [( e3 n G6 t- p, @/ _4 U% u g+ {+ C9 q' V1 ?
if __name__ == '__main__':4 J5 y2 |$ e# }; x
conn = sqlite3.connect('Python.db')
7 B) X7 K0 c8 M0 c c = conn.cursor()
. O/ E2 R, h7 G8 x1 S% w# G c.execute('''CREATE TABLE IF NOT EXISTS Python () E) Y0 |% i+ H\" D/ p# y8 ]* x
Url VARCHAR,
7 O2 ~9 ^& B4 U3 n9 F Title VARCHAR,& w5 m! t\" g* J1 {
Author VARCHAR- {, t\" J8 [0 G5 S7 {/ [: h7 j
)''')
, b, D) i( z( M\" v\" U- t/ ]; E4 K conn.commit() _& [% M5 ], P5 `\" [4 V
7 ?( O- b! f+ v3 w9 Q
# --------------------Split Line--------------------
3 @; \$ ~. S( N. s) G5 a headers = {! S) X. A6 h4 j( ~' P, A) R% C1 y: _# w
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"4 I4 [6 R, V) w9 a
}
+ ~$ T' B! P5 l5 W& U
' o6 `( C- o0 @- |9 X for i in range(1, 1046):3 }7 {+ W; A2 y V) {$ d. p& d# K( j/ k
url = "http://xxx/index_%s.html" % str(i)# ?; Z+ P @+ _5 ?& y7 p
req = requests.get(url=url, headers=headers)
: ^\" W5 `# W8 Y* W3 e req.encoding = "utf-8"! N0 H( t7 m9 O( }
html = BeautifulSoup(req.text, "lxml")
6 c. l; l1 v; E {; d\" U. y) B1 E) I' M b/ H7 g
# --------------------Split Line--------------------' x1 F, c7 v& \6 `) Y& _: ^) [( W5 S
for div in html.find_all('div', class_='loop'):& d& z, {* K ^8 Q' d) D% s
content_body = div.select('h2 > a')[0]
8 _4 v% w\" W7 n( A content_infor = div.select('.content_infor > span:nth-child(3)')[0]
$ T0 l7 @- p; y* V0 P. I/ s' ~7 W: \% d3 l- d
# --------------------Split Line--------------------
8 ?5 @$ \8 D- i9 n* C2 ~- S' g2 s cursor = c.execute(
$ F6 q' c+ S+ q5 i "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))1 A6 V) i9 C+ n$ V( f* _; N( H
len = 0\" D\" |( Z/ N J! r' l9 n, d# \
for row in cursor:
7 m4 ]/ X( k& Y# I* O+ i1 u len = row[0], t8 o+ U1 `# n. r }) T
if len > 0:' H, h7 [- Q( K5 P: @9 P
continue
\" `; _0 T1 a; b& c H$ o* w% Y* D) w' M& K2 R' n, [7 P: m
# --------------------Split Line--------------------: N- [9 V& |2 w0 D* h; I7 K
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (4 j6 ?) f/ i& J\" L! h7 v! B
"http://xxx" + content_body.get('href'),) [\" D\" M3 ~: a. _3 L4 A
escape(content_body.get('title').replace("\"", "\"\"")),
3 _! z5 ~- l4 C. D. I' |4 I* q content_infor.text.replace('xxx: ', '')))
7 p( L* d. C* {. u* E: o5 z6 s. X
2 i) _& U/ @! f1 |) x0 ` conn.commit()7 f9 q; x5 H6 a5 j
print("第%s页" % str(i))
+ N2 |+ W' H( |( d: p$ n- A+ Y0 U
\" }2 z2 J9 e9 J6 [: [ # --------------------Split Line--------------------
{& n+ D& u6 r0 ?- U% x# x* r conn.close()
7 e9 n- D& @1 G, S9 O {$ u

#!/usr/bin/python 
* N\" B: p1 F2 b: e# U# -*- coding: utf-8 -*- 
# y\" o$ h' {+ G. ]* A8 { 
' U% X5 n7 ]- b& `' iimport sqlite3 
$ L+ ^, Z$ ^6 vimport requests 
0 ~- w- O2 c& T\" E0 Afrom bs4 import BeautifulSoup 
1 j4 r; E6 D  gfrom re import escape 
\" U) \; {6 ^; [( e3 n  G6 t- p, @/ _4 U% u  g+ {+ C9 q' V1 ? 
if __name__ == '__main__':4 J5 y2 |$ e# }; x 
    conn = sqlite3.connect('Python.db') 
7 B) X7 K0 c8 M0 c    c = conn.cursor() 
. O/ E2 R, h7 G8 x1 S% w# G    c.execute('''CREATE TABLE IF NOT EXISTS Python () E) Y0 |% i+ H\" D/ p# y8 ]* x 
        Url VARCHAR, 
7 O2 ~9 ^& B4 U3 n9 F        Title VARCHAR,& w5 m! t\" g* J1 { 
        Author VARCHAR- {, t\" J8 [0 G5 S7 {/ [: h7 j 
    )''') 
, b, D) i( z( M\" v\" U- t/ ]; E4 K    conn.commit()  _& [% M5 ], P5 `\" [4 V 
7 ?( O- b! f+ v3 w9 Q 
    # --------------------Split Line-------------------- 
3 @; \$ ~. S( N. s) G5 a    headers = {! S) X. A6 h4 j( ~' P, A) R% C1 y: _# w 
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"4 I4 [6 R, V) w9 a 
    } 
+ ~$ T' B! P5 l5 W& U 
' o6 `( C- o0 @- |9 X    for i in range(1, 1046):3 }7 {+ W; A2 y  V) {$ d. p& d# K( j/ k 
        url = "http://xxx/index_%s.html" % str(i)# ?; Z+ P  @+ _5 ?& y7 p 
        req = requests.get(url=url, headers=headers) 
: ^\" W5 `# W8 Y* W3 e        req.encoding = "utf-8"! N0 H( t7 m9 O( } 
        html = BeautifulSoup(req.text, "lxml") 
6 c. l; l1 v; E  {; d\" U. y) B1 E) I' M  b/ H7 g 
        # --------------------Split Line--------------------' x1 F, c7 v& \6 `) Y& _: ^) [( W5 S 
        for div in html.find_all('div', class_='loop'):& d& z, {* K  ^8 Q' d) D% s 
            content_body = div.select('h2 > a')[0] 
8 _4 v% w\" W7 n( A            content_infor = div.select('.content_infor > span:nth-child(3)')[0] 
$ T0 l7 @- p; y* V0 P. I/ s' ~7 W: \% d3 l- d 
            # --------------------Split Line-------------------- 
8 ?5 @$ \8 D- i9 n* C2 ~- S' g2 s            cursor = c.execute( 
$ F6 q' c+ S+ q5 i                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))1 A6 V) i9 C+ n$ V( f* _; N( H 
            len = 0\" D\" |( Z/ N  J! r' l9 n, d# \ 
            for row in cursor: 
7 m4 ]/ X( k& Y# I* O+ i1 u                len = row[0], t8 o+ U1 `# n. r  }) T 
            if len > 0:' H, h7 [- Q( K5 P: @9 P 
                continue 
\" `; _0 T1 a; b& c  H$ o* w% Y* D) w' M& K2 R' n, [7 P: m 
            # --------------------Split Line--------------------: N- [9 V& |2 w0 D* h; I7 K 
            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (4 j6 ?) f/ i& J\" L! h7 v! B 
                "http://xxx" + content_body.get('href'),) [\" D\" M3 ~: a. _3 L4 A 
                escape(content_body.get('title').replace("\"", "\"\"")), 
3 _! z5 ~- l4 C. D. I' |4 I* q                content_infor.text.replace('xxx: ', ''))) 
7 p( L* d. C* {. u* E: o5 z6 s. X 
2 i) _& U/ @! f1 |) x0 `        conn.commit()7 f9 q; x5 H6 a5 j 
        print("第%s页" % str(i)) 
+ N2 |+ W' H( |( d: p$ n- A+ Y0 U 
\" }2 z2 J9 e9 J6 [: [    # --------------------Split Line-------------------- 
  {& n+ D& u6 r0 ?- U% x# x* r    conn.close() 
7 e9 n- D& @1 G, S9 O  {$ u

转发自派生社区

Python交流群：1047602540

zan