查看: 3844|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |正序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
3 O# a$ s: ]: [ ?$ \\" J# -*- coding: utf-8 -*-' U( F4 W' _' s( w) i\" \! T* l
5 r6 Z& E: p4 d( t( u' `import sqlite3
7 D7 c$ g, Z% l* U8 rimport requests; p: r: I! s0 n; q0 ~+ M4 ~
from bs4 import BeautifulSoup* A9 Z$ T. W) q, q
from re import escape+ u2 S+ q$ o8 [1 m
\" z1 v9 q1 z k6 R$ zif __name__ == '__main__':/ r5 `$ z$ g) ?; |
conn = sqlite3.connect('Python.db')
% Z2 m q4 e* X- ] c = conn.cursor()
+ D: x% ?* E( x c.execute('''CREATE TABLE IF NOT EXISTS Python (' s+ o; V8 [9 |0 g
Url VARCHAR,- l: C7 S\" P3 [( r L\" M! u
Title VARCHAR,$ ~\" f- @8 L* G; K
Author VARCHAR
- ?; k Q8 k0 n! X: ? )''')
) Y! t; W9 M' m7 {1 \ conn.commit()) f }5 n: s2 @3 ?/ f+ a
?3 x, z2 ]! K$ l4 j! C& q+ r # --------------------Split Line--------------------
9 d6 ^1 }) P/ p! k6 c headers = {$ W- X, A: [7 N$ h7 n9 h5 t# G: M
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
V1 ?5 o- h4 @0 R: s* x3 | }
; r7 y% v7 ^4 `2 x
! k7 u+ \ x8 Q$ |4 F for i in range(1, 1046):% B+ R' ^& a; w- r# s
url = "http://xxx/index_%s.html" % str(i)' q7 Z: f0 ]1 @' j* N2 r' @! R) ^
req = requests.get(url=url, headers=headers)
* u2 r2 U\" x7 d! Y req.encoding = "utf-8"
# U1 G7 Q o9 r7 j. v7 B# V html = BeautifulSoup(req.text, "lxml")# `# ^( r' E7 h- }) h
+ H! t3 D2 F- v6 B # --------------------Split Line--------------------
' ^* y$ Q$ k5 h( L7 {\" I9 Z. N for div in html.find_all('div', class_='loop'):8 E7 y, i j; Y# l. H8 q5 t/ X
content_body = div.select('h2 > a')[0]
, y! M0 Y* {( |6 ]7 A1 d/ M' r+ A+ v content_infor = div.select('.content_infor > span:nth-child(3)')[0]
6 P. h; x: t K: H$ ?% M9 Y9 R) X
$ r$ w& c; m% c) q # --------------------Split Line--------------------
5 y( s9 ~* w; _; |4 l cursor = c.execute(
6 n- @9 k# k% l0 v$ v% b "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
7 t! n) _# A, }- ]( _) `- l len = 0
5 q. `: }7 P# @) l for row in cursor:' O- w* [8 ^; i
len = row[0]
0 F |3 o0 y8 \) s* q6 ~& {& a6 g! | if len > 0:5 d4 _* L# j# D: {+ ~# P
continue2 ~1 T2 ]4 r$ j! t9 S4 V
5 Y8 P* W; J5 G, G
# --------------------Split Line-------------------- R. ^1 f, A0 e
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (# T- w; O1 _0 ~0 T/ u' d- `
"http://xxx" + content_body.get('href'),( J7 f7 ~- k1 G. D0 q9 ^
escape(content_body.get('title').replace("\"", "\"\"")),
$ j2 Z\" {( ] b% u: Q! G1 h2 i content_infor.text.replace('xxx: ', '')))1 p/ c6 p% O) Z+ l, \; M; `( X
$ L\" \; o' D @7 f' L6 l* u1 r conn.commit()
7 ]\" K r0 \# A/ Q. E! |3 g: u print("第%s页" % str(i))
- r7 [\" C; R, \: S. O4 z
, [$ a0 t6 i; [ # --------------------Split Line--------------------* r( k2 Y, J% r
conn.close()$ G/ Q2 F' B, `1 i8 O& a

#!/usr/bin/python 
3 O# a$ s: ]: [  ?$ \\" J# -*- coding: utf-8 -*-' U( F4 W' _' s( w) i\" \! T* l 
 
5 r6 Z& E: p4 d( t( u' `import sqlite3 
7 D7 c$ g, Z% l* U8 rimport requests; p: r: I! s0 n; q0 ~+ M4 ~ 
from bs4 import BeautifulSoup* A9 Z$ T. W) q, q 
from re import escape+ u2 S+ q$ o8 [1 m 
 
\" z1 v9 q1 z  k6 R$ zif __name__ == '__main__':/ r5 `$ z$ g) ?; | 
    conn = sqlite3.connect('Python.db') 
% Z2 m  q4 e* X- ]    c = conn.cursor() 
+ D: x% ?* E( x    c.execute('''CREATE TABLE IF NOT EXISTS Python (' s+ o; V8 [9 |0 g 
        Url VARCHAR,- l: C7 S\" P3 [( r  L\" M! u 
        Title VARCHAR,$ ~\" f- @8 L* G; K 
        Author VARCHAR 
- ?; k  Q8 k0 n! X: ?    )''') 
) Y! t; W9 M' m7 {1 \    conn.commit()) f  }5 n: s2 @3 ?/ f+ a 
 
  ?3 x, z2 ]! K$ l4 j! C& q+ r    # --------------------Split Line-------------------- 
9 d6 ^1 }) P/ p! k6 c    headers = {$ W- X, A: [7 N$ h7 n9 h5 t# G: M 
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36" 
  V1 ?5 o- h4 @0 R: s* x3 |    } 
; r7 y% v7 ^4 `2 x 
! k7 u+ \  x8 Q$ |4 F    for i in range(1, 1046):% B+ R' ^& a; w- r# s 
        url = "http://xxx/index_%s.html" % str(i)' q7 Z: f0 ]1 @' j* N2 r' @! R) ^ 
        req = requests.get(url=url, headers=headers) 
* u2 r2 U\" x7 d! Y        req.encoding = "utf-8" 
# U1 G7 Q  o9 r7 j. v7 B# V        html = BeautifulSoup(req.text, "lxml")# `# ^( r' E7 h- }) h 
 
+ H! t3 D2 F- v6 B        # --------------------Split Line-------------------- 
' ^* y$ Q$ k5 h( L7 {\" I9 Z. N        for div in html.find_all('div', class_='loop'):8 E7 y, i  j; Y# l. H8 q5 t/ X 
            content_body = div.select('h2 > a')[0] 
, y! M0 Y* {( |6 ]7 A1 d/ M' r+ A+ v            content_infor = div.select('.content_infor > span:nth-child(3)')[0] 
6 P. h; x: t  K: H$ ?% M9 Y9 R) X 
$ r$ w& c; m% c) q            # --------------------Split Line-------------------- 
5 y( s9 ~* w; _; |4 l            cursor = c.execute( 
6 n- @9 k# k% l0 v$ v% b                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
7 t! n) _# A, }- ]( _) `- l            len = 0 
5 q. `: }7 P# @) l            for row in cursor:' O- w* [8 ^; i 
                len = row[0] 
0 F  |3 o0 y8 \) s* q6 ~& {& a6 g! |            if len > 0:5 d4 _* L# j# D: {+ ~# P 
                continue2 ~1 T2 ]4 r$ j! t9 S4 V 
5 Y8 P* W; J5 G, G 
            # --------------------Split Line--------------------  R. ^1 f, A0 e 
            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (# T- w; O1 _0 ~0 T/ u' d- ` 
                "http://xxx" + content_body.get('href'),( J7 f7 ~- k1 G. D0 q9 ^ 
                escape(content_body.get('title').replace("\"", "\"\"")), 
$ j2 Z\" {( ]  b% u: Q! G1 h2 i                content_infor.text.replace('xxx: ', '')))1 p/ c6 p% O) Z+ l, \; M; `( X 
 
$ L\" \; o' D  @7 f' L6 l* u1 r        conn.commit() 
7 ]\" K  r0 \# A/ Q. E! |3 g: u        print("第%s页" % str(i)) 
- r7 [\" C; R, \: S. O4 z 
, [$ a0 t6 i; [    # --------------------Split Line--------------------* r( k2 Y, J% r 
    conn.close()$ G/ Q2 F' B, `1 i8 O& a

转发自派生社区

Python交流群：1047602540

zan