查看: 3841|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
. _! P H0 ?5 F/ O$ N4 k( |( E! `# -*- coding: utf-8 -*-
1 t G- E+ ?: U) _0 I
$ ~$ c {0 y# ~& f/ j9 J) H# kimport sqlite3! p2 s# o1 u- W$ t5 I( B
import requests
% x\" ^) \( a. R3 Q( Q6 B7 C4 ^from bs4 import BeautifulSoup
0 i0 M\" [' h8 z1 @4 z( k3 o6 Wfrom re import escape+ [4 B2 }0 G* Q! q* A7 q
- C: m; w0 h) ?( C0 i
if __name__ == '__main__':
/ V4 [6 O9 a0 g- \+ r+ C8 R conn = sqlite3.connect('Python.db')
9 X) H2 J8 c( a\" p+ F c = conn.cursor()4 W\" A# i- O+ @
c.execute('''CREATE TABLE IF NOT EXISTS Python (
6 R) A# i- ~+ Q, S Url VARCHAR,
2 r\" _, H) {; K0 D, \/ a: w Title VARCHAR,5 C- a\" H, v\" v6 H4 V; X& x
Author VARCHAR, K) n& ], w3 [8 ^\" ?# i
)''')
; ^/ `0 R+ i5 c! `2 F7 V; d( I8 w' L conn.commit()
0 M1 A( C+ u0 D. Z/ i$ B' c6 }# q* a, @% Y6 V$ C3 ~& {7 j7 y
# --------------------Split Line--------------------1 m' |* F) w. p8 e
headers = {
( o9 B\" N. J9 J4 \- Z "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36" o4 n# U2 x& H6 w2 ^( c' u
}9 L$ e G\" g7 w; |8 l
1 _# X2 A7 i& o) ]\" ^! j& |! c for i in range(1, 1046):
% t+ D$ D: U. e& q url = "http://xxx/index_%s.html" % str(i)
4 A+ [- Y8 o- ~, w7 | U: ~5 `\" U req = requests.get(url=url, headers=headers), E- \' x' C; u* W\" F
req.encoding = "utf-8"( ^8 }! i* \ Q\" X0 M
html = BeautifulSoup(req.text, "lxml")0 l\" `; o+ R5 A \3 r2 F! Y
( D8 i9 V2 c8 I4 V# r # --------------------Split Line--------------------, z: E* f7 S\" v1 B( R! B
for div in html.find_all('div', class_='loop'):, c* m+ n8 L* B3 U6 s0 V* P, V
content_body = div.select('h2 > a')[0]' `: m\" Y\" A9 L$ I
content_infor = div.select('.content_infor > span:nth-child(3)')[0]) E8 O* E5 k$ k. ^: r% Z' j/ M- @5 y
$ f, t2 Z. w6 r' ?' Z; l
# --------------------Split Line--------------------( ?3 V) l/ r3 x+ g: E! W
cursor = c.execute(
+ f c( ]( g6 d+ F, q "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))! h- Q6 ?2 C! o; R
len = 0! l9 t. b0 ]2 }\" Z* j3 Z- [2 X
for row in cursor:( n! O4 ^4 p& q {, B
len = row[0]0 i9 K5 U% \) u7 N) I
if len > 0:
' N# u3 i& p, d4 b+ U4 H continue
5 C0 T* L/ L9 u+ X! r- A9 b0 K& \: I& c) `4 C' S9 N# A
# --------------------Split Line--------------------
. }' z W5 `4 j- r. ?7 f- r' [ c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (- q8 j: ?# e6 a
"http://xxx" + content_body.get('href'),
7 [- {\" |4 D6 f6 ~- K\" f$ w escape(content_body.get('title').replace("\"", "\"\"")),
5 C- G! \( t: G& t8 ^7 \ content_infor.text.replace('xxx: ', '')))
) I- C6 D _, A. ^ T8 a: t/ z
' [' m8 p i6 Z: M conn.commit()6 P* Z/ h- ^3 h6 b
print("第%s页" % str(i))
3 L! y; Q: k1 z! q. F6 e7 ^& z4 Q
# --------------------Split Line--------------------& o) P0 o- H* g1 c2 B8 E
conn.close()8 X3 \/ y, L( h! W

#!/usr/bin/python 
. _! P  H0 ?5 F/ O$ N4 k( |( E! `# -*- coding: utf-8 -*- 
1 t  G- E+ ?: U) _0 I 
$ ~$ c  {0 y# ~& f/ j9 J) H# kimport sqlite3! p2 s# o1 u- W$ t5 I( B 
import requests 
% x\" ^) \( a. R3 Q( Q6 B7 C4 ^from bs4 import BeautifulSoup 
0 i0 M\" [' h8 z1 @4 z( k3 o6 Wfrom re import escape+ [4 B2 }0 G* Q! q* A7 q 
- C: m; w0 h) ?( C0 i 
if __name__ == '__main__': 
/ V4 [6 O9 a0 g- \+ r+ C8 R    conn = sqlite3.connect('Python.db') 
9 X) H2 J8 c( a\" p+ F    c = conn.cursor()4 W\" A# i- O+ @ 
    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
6 R) A# i- ~+ Q, S        Url VARCHAR, 
2 r\" _, H) {; K0 D, \/ a: w        Title VARCHAR,5 C- a\" H, v\" v6 H4 V; X& x 
        Author VARCHAR, K) n& ], w3 [8 ^\" ?# i 
    )''') 
; ^/ `0 R+ i5 c! `2 F7 V; d( I8 w' L    conn.commit() 
0 M1 A( C+ u0 D. Z/ i$ B' c6 }# q* a, @% Y6 V$ C3 ~& {7 j7 y 
    # --------------------Split Line--------------------1 m' |* F) w. p8 e 
    headers = { 
( o9 B\" N. J9 J4 \- Z        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"  o4 n# U2 x& H6 w2 ^( c' u 
    }9 L$ e  G\" g7 w; |8 l 
 
1 _# X2 A7 i& o) ]\" ^! j& |! c    for i in range(1, 1046): 
% t+ D$ D: U. e& q        url = "http://xxx/index_%s.html" % str(i) 
4 A+ [- Y8 o- ~, w7 |  U: ~5 `\" U        req = requests.get(url=url, headers=headers), E- \' x' C; u* W\" F 
        req.encoding = "utf-8"( ^8 }! i* \  Q\" X0 M 
        html = BeautifulSoup(req.text, "lxml")0 l\" `; o+ R5 A  \3 r2 F! Y 
 
( D8 i9 V2 c8 I4 V# r        # --------------------Split Line--------------------, z: E* f7 S\" v1 B( R! B 
        for div in html.find_all('div', class_='loop'):, c* m+ n8 L* B3 U6 s0 V* P, V 
            content_body = div.select('h2 > a')[0]' `: m\" Y\" A9 L$ I 
            content_infor = div.select('.content_infor > span:nth-child(3)')[0]) E8 O* E5 k$ k. ^: r% Z' j/ M- @5 y 
$ f, t2 Z. w6 r' ?' Z; l 
            # --------------------Split Line--------------------( ?3 V) l/ r3 x+ g: E! W 
            cursor = c.execute( 
+ f  c( ]( g6 d+ F, q                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))! h- Q6 ?2 C! o; R 
            len = 0! l9 t. b0 ]2 }\" Z* j3 Z- [2 X 
            for row in cursor:( n! O4 ^4 p& q  {, B 
                len = row[0]0 i9 K5 U% \) u7 N) I 
            if len > 0: 
' N# u3 i& p, d4 b+ U4 H                continue 
5 C0 T* L/ L9 u+ X! r- A9 b0 K& \: I& c) `4 C' S9 N# A 
            # --------------------Split Line-------------------- 
. }' z  W5 `4 j- r. ?7 f- r' [            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (- q8 j: ?# e6 a 
                "http://xxx" + content_body.get('href'), 
7 [- {\" |4 D6 f6 ~- K\" f$ w                escape(content_body.get('title').replace("\"", "\"\"")), 
5 C- G! \( t: G& t8 ^7 \                content_infor.text.replace('xxx: ', ''))) 
) I- C6 D  _, A. ^  T8 a: t/ z 
' [' m8 p  i6 Z: M        conn.commit()6 P* Z/ h- ^3 h6 b 
        print("第%s页" % str(i)) 
3 L! y; Q: k1 z! q. F6 e7 ^& z4 Q 
    # --------------------Split Line--------------------& o) P0 o- H* g1 c2 B8 E 
    conn.close()8 X3 \/ y, L( h! W

转发自派生社区

Python交流群：1047602540

zan