查看: 3816|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python7 h9 b% [/ d% b0 R\" p
# -*- coding: utf-8 -*-5 C8 @4 B0 g2 v
/ A% j( [9 O5 l5 x$ x
import sqlite39 V& v/ t* B7 l6 e
import requests
( p5 u' C9 ?+ [from bs4 import BeautifulSoup
' y, ~1 R) ~) G5 @from re import escape
) r( c) t% ]1 p. t& N; E9 m
D |4 R- H$ h9 iif __name__ == '__main__':% T5 `2 k' V2 c | y\" I Y% y
conn = sqlite3.connect('Python.db')% E2 W; x+ X5 Y2 Y\" P& o- e& \
c = conn.cursor()
: o E* m. x\" r7 S1 o9 n6 [! L( q c.execute('''CREATE TABLE IF NOT EXISTS Python (
1 i$ D% u. d, F0 s @% F. G/ N5 ~ Url VARCHAR,8 {( M9 o7 B0 M7 d' C3 P5 T# R
Title VARCHAR,
2 [2 s7 N( C6 d9 F* B Author VARCHAR
3 F* z/ j. z7 ~\" g9 p* X& x )'''); {+ y4 v# c: n& Q
conn.commit()
8 O& D6 X. ]0 o5 Z% t5 {% M$ w+ F1 A8 ^+ e8 @2 E
# --------------------Split Line--------------------8 A3 m) E. j% L- r
headers = {, K) @. d* o\" J- h K
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"5 L6 M, K2 K: B4 u8 s
}
- h* ?$ f7 V* W) \$ p; F3 {5 p
3 }: W1 C w& @2 b2 s for i in range(1, 1046):
- }6 F7 @5 i6 V url = "http://xxx/index_%s.html" % str(i)
+ d: l; d9 |& c0 p6 v* l req = requests.get(url=url, headers=headers)
: P9 g\" p% l9 D. {* U! Y req.encoding = "utf-8"
6 a2 M. o2 B8 p html = BeautifulSoup(req.text, "lxml")0 h( w' J& d, m- }
0 z9 j+ N! p7 U; e2 [; { # --------------------Split Line--------------------
, n8 G( ~9 Z; @( `) K for div in html.find_all('div', class_='loop'):
. o/ @# T( b$ t* M$ Z. m4 [& L content_body = div.select('h2 > a')[0]+ U' w* P3 B& E6 N, G5 y b2 h
content_infor = div.select('.content_infor > span:nth-child(3)')[0]& d/ K5 i8 l6 R
' ~& W; I\" E- T2 J% R! a0 s
# --------------------Split Line--------------------
0 Y- D+ y' Y' V. z! L: ? cursor = c.execute(\" Y5 J0 }8 ?% E, \7 T0 d- _
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
# N4 B% r6 Q) f len = 04 Y9 X3 n1 X; ^
for row in cursor:0 C. F' U7 J( P7 l3 F( c/ e
len = row[0]4 @$ f% B0 m% Q
if len > 0:, U; d9 a G$ l\" q
continue: W0 l9 ]8 F: a0 J4 K- N+ o
: o& m$ A% h3 m; y- |7 X9 r9 O$ c' T
# --------------------Split Line--------------------
: a* U! `) A8 W. U3 Y3 d& i c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
9 [; r6 G: g' s6 X "http://xxx" + content_body.get('href'),
* W+ a7 Y% h# _2 [ escape(content_body.get('title').replace("\"", "\"\"")),2 d6 B5 C R) h- T
content_infor.text.replace('xxx: ', '')))
$ Z2 Z6 Y2 v. v& e, \
! a4 z0 E( G# g& p2 V$ s6 V conn.commit()$ O' I' z& b\" B: R
print("第%s页" % str(i))
8 a9 z& }# e# ]) N# t' k( s# X3 v# D' m& I1 m' P
# --------------------Split Line--------------------
8 q7 |$ c B& p* ~2 M conn.close()1 E# M: h3 ]* } U x

#!/usr/bin/python7 h9 b% [/ d% b0 R\" p 
# -*- coding: utf-8 -*-5 C8 @4 B0 g2 v 
/ A% j( [9 O5 l5 x$ x 
import sqlite39 V& v/ t* B7 l6 e 
import requests 
( p5 u' C9 ?+ [from bs4 import BeautifulSoup 
' y, ~1 R) ~) G5 @from re import escape 
) r( c) t% ]1 p. t& N; E9 m 
  D  |4 R- H$ h9 iif __name__ == '__main__':% T5 `2 k' V2 c  |  y\" I  Y% y 
    conn = sqlite3.connect('Python.db')% E2 W; x+ X5 Y2 Y\" P& o- e& \ 
    c = conn.cursor() 
: o  E* m. x\" r7 S1 o9 n6 [! L( q    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
1 i$ D% u. d, F0 s  @% F. G/ N5 ~        Url VARCHAR,8 {( M9 o7 B0 M7 d' C3 P5 T# R 
        Title VARCHAR, 
2 [2 s7 N( C6 d9 F* B        Author VARCHAR 
3 F* z/ j. z7 ~\" g9 p* X& x    )'''); {+ y4 v# c: n& Q 
    conn.commit() 
8 O& D6 X. ]0 o5 Z% t5 {% M$ w+ F1 A8 ^+ e8 @2 E 
    # --------------------Split Line--------------------8 A3 m) E. j% L- r 
    headers = {, K) @. d* o\" J- h  K 
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"5 L6 M, K2 K: B4 u8 s 
    } 
- h* ?$ f7 V* W) \$ p; F3 {5 p 
3 }: W1 C  w& @2 b2 s    for i in range(1, 1046): 
- }6 F7 @5 i6 V        url = "http://xxx/index_%s.html" % str(i) 
+ d: l; d9 |& c0 p6 v* l        req = requests.get(url=url, headers=headers) 
: P9 g\" p% l9 D. {* U! Y        req.encoding = "utf-8" 
6 a2 M. o2 B8 p        html = BeautifulSoup(req.text, "lxml")0 h( w' J& d, m- } 
 
0 z9 j+ N! p7 U; e2 [; {        # --------------------Split Line-------------------- 
, n8 G( ~9 Z; @( `) K        for div in html.find_all('div', class_='loop'): 
. o/ @# T( b$ t* M$ Z. m4 [& L            content_body = div.select('h2 > a')[0]+ U' w* P3 B& E6 N, G5 y  b2 h 
            content_infor = div.select('.content_infor > span:nth-child(3)')[0]& d/ K5 i8 l6 R 
' ~& W; I\" E- T2 J% R! a0 s 
            # --------------------Split Line-------------------- 
0 Y- D+ y' Y' V. z! L: ?            cursor = c.execute(\" Y5 J0 }8 ?% E, \7 T0 d- _ 
                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
# N4 B% r6 Q) f            len = 04 Y9 X3 n1 X; ^ 
            for row in cursor:0 C. F' U7 J( P7 l3 F( c/ e 
                len = row[0]4 @$ f% B0 m% Q 
            if len > 0:, U; d9 a  G$ l\" q 
                continue: W0 l9 ]8 F: a0 J4 K- N+ o 
: o& m$ A% h3 m; y- |7 X9 r9 O$ c' T 
            # --------------------Split Line-------------------- 
: a* U! `) A8 W. U3 Y3 d& i            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
9 [; r6 G: g' s6 X                "http://xxx" + content_body.get('href'), 
* W+ a7 Y% h# _2 [                escape(content_body.get('title').replace("\"", "\"\"")),2 d6 B5 C  R) h- T 
                content_infor.text.replace('xxx: ', ''))) 
$ Z2 Z6 Y2 v. v& e, \ 
! a4 z0 E( G# g& p2 V$ s6 V        conn.commit()$ O' I' z& b\" B: R 
        print("第%s页" % str(i)) 
8 a9 z& }# e# ]) N# t' k( s# X3 v# D' m& I1 m' P 
    # --------------------Split Line-------------------- 
8 q7 |$ c  B& p* ~2 M    conn.close()1 E# M: h3 ]* }  U  x

转发自派生社区

Python交流群：1047602540

zan