查看: 3818|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
. W n c1 J( b9 j& u) m# -*- coding: utf-8 -*-
4 C& T- @ n& s7 `3 k\" f\" U: ~. H\" h4 l: |
import sqlite3
( X1 z& O. j fimport requests# u5 H( {3 c/ M9 b+ X\" N
from bs4 import BeautifulSoup
' v7 A6 @\" B% Y8 R! d' hfrom re import escape
! S y8 Z8 X9 x2 v* O4 ?. A A! f% D& m5 I% S) i% D6 [\" l
if __name__ == '__main__':- v5 ^! Y5 G2 x& ?+ a! K& ~8 u
conn = sqlite3.connect('Python.db')
* x5 J; v3 t) u( ^ c = conn.cursor()
. Q6 {! m5 c; x8 V; E' ? c.execute('''CREATE TABLE IF NOT EXISTS Python (+ C/ K8 u/ A+ \* g
Url VARCHAR,
Title VARCHAR,
! ]2 g7 B* R& E; E- |: a/ s; q2 s Author VARCHAR
1 {3 R- t0 U/ h. j4 r$ l1 ~6 k7 b )''')( X# g0 @* B/ l& O @2 z
conn.commit()
' y S/ n6 W- y% x( q7 k' {7 C6 a/ k6 @0 e x\" B( k8 W
# --------------------Split Line--------------------
/ _! \: B. y9 S+ v, F2 L# L/ C9 F headers = {8 H% @% g$ B5 ^8 b
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
( f! Z4 u8 w8 r }
$ [9 d0 J8 H) H
* {9 r: A) j# g( l for i in range(1, 1046):
; p7 R% y1 u/ x* w url = "http://xxx/index_%s.html" % str(i)0 p, g- W$ }) {8 z) P( @\" g
req = requests.get(url=url, headers=headers)
8 E\" l1 [/ h, U req.encoding = "utf-8") z, ~) q' e! A3 n% c1 n
html = BeautifulSoup(req.text, "lxml")0 A. \! {6 t4 F- ?/ C! ~
) h3 M$ F9 N+ H) P; P0 a/ u3 V # --------------------Split Line--------------------
8 I: Q. B9 ], X7 v. y3 e for div in html.find_all('div', class_='loop'):
) p6 i7 Q5 p5 I/ x7 V) _; | content_body = div.select('h2 > a')[0]
$ h) Y. H$ a) g/ `! S7 _; }0 e2 a content_infor = div.select('.content_infor > span:nth-child(3)')[0]
4 i& l9 w) @8 s3 t. w3 F
' Q9 I3 C. S( v% o0 U% X, { # --------------------Split Line--------------------
. Q) d, M& S2 }& ^5 r- { cursor = c.execute(
( @5 j) b, n+ S\" d1 e "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
7 W2 a: a9 Z1 l# T1 [1 \0 _ len = 0- u' \: l$ K( Q/ v
for row in cursor:' \ M9 h. c# j2 B# o- ^
len = row[0]; ~' l' h1 B, {- A/ A: m3 M9 X
if len > 0:
8 a6 Y\" E6 ?4 ?# z continue
* f. }. [5 g/ g! }
$ C, T5 u3 m# {4 z* d/ @+ o) @/ ^5 ?6 K # --------------------Split Line--------------------
' D9 q' q/ X! k. }& ~7 \ c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
1 G7 k9 e* b' S- ~* c G0 W "http://xxx" + content_body.get('href'),( h# q, }: j8 n2 B2 J( b\" y6 p
escape(content_body.get('title').replace("\"", "\"\"")),) c9 H1 P' H/ Y4 G* a+ M
content_infor.text.replace('xxx: ', '')))9 o- R& o: m K- a, `
: O+ p; ~) ?% @5 s$ _ conn.commit(): W6 ^& N: v( G, Z* C
print("第%s页" % str(i))\" G. A+ _. B) P( B0 a \+ n/ }
1 e( w- N0 ]2 D. M/ F' ` # --------------------Split Line--------------------
+ L) b\" k' r. m! G% _ conn.close()( q3 ]. R/ L. B

#!/usr/bin/python 
. W  n  c1 J( b9 j& u) m# -*- coding: utf-8 -*- 
4 C& T- @  n& s7 `3 k\" f\" U: ~. H\" h4 l: | 
import sqlite3 
( X1 z& O. j  fimport requests# u5 H( {3 c/ M9 b+ X\" N 
from bs4 import BeautifulSoup 
' v7 A6 @\" B% Y8 R! d' hfrom re import escape 
! S  y8 Z8 X9 x2 v* O4 ?. A  A! f% D& m5 I% S) i% D6 [\" l 
if __name__ == '__main__':- v5 ^! Y5 G2 x& ?+ a! K& ~8 u 
    conn = sqlite3.connect('Python.db') 
* x5 J; v3 t) u( ^    c = conn.cursor() 
. Q6 {! m5 c; x8 V; E' ?    c.execute('''CREATE TABLE IF NOT EXISTS Python (+ C/ K8 u/ A+ \* g 
        Url VARCHAR, 
% W( f  K' _/ s- \4 \) U$ {$ [3 A  ]+ A3 P        Title VARCHAR, 
! ]2 g7 B* R& E; E- |: a/ s; q2 s        Author VARCHAR 
1 {3 R- t0 U/ h. j4 r$ l1 ~6 k7 b    )''')( X# g0 @* B/ l& O  @2 z 
    conn.commit() 
' y  S/ n6 W- y% x( q7 k' {7 C6 a/ k6 @0 e  x\" B( k8 W 
    # --------------------Split Line-------------------- 
/ _! \: B. y9 S+ v, F2 L# L/ C9 F    headers = {8 H% @% g$ B5 ^8 b 
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36" 
( f! Z4 u8 w8 r    } 
$ [9 d0 J8 H) H 
* {9 r: A) j# g( l    for i in range(1, 1046): 
; p7 R% y1 u/ x* w        url = "http://xxx/index_%s.html" % str(i)0 p, g- W$ }) {8 z) P( @\" g 
        req = requests.get(url=url, headers=headers) 
8 E\" l1 [/ h, U        req.encoding = "utf-8") z, ~) q' e! A3 n% c1 n 
        html = BeautifulSoup(req.text, "lxml")0 A. \! {6 t4 F- ?/ C! ~ 
 
) h3 M$ F9 N+ H) P; P0 a/ u3 V        # --------------------Split Line-------------------- 
8 I: Q. B9 ], X7 v. y3 e        for div in html.find_all('div', class_='loop'): 
) p6 i7 Q5 p5 I/ x7 V) _; |            content_body = div.select('h2 > a')[0] 
$ h) Y. H$ a) g/ `! S7 _; }0 e2 a            content_infor = div.select('.content_infor > span:nth-child(3)')[0] 
4 i& l9 w) @8 s3 t. w3 F 
' Q9 I3 C. S( v% o0 U% X, {            # --------------------Split Line-------------------- 
. Q) d, M& S2 }& ^5 r- {            cursor = c.execute( 
( @5 j) b, n+ S\" d1 e                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
7 W2 a: a9 Z1 l# T1 [1 \0 _            len = 0- u' \: l$ K( Q/ v 
            for row in cursor:' \  M9 h. c# j2 B# o- ^ 
                len = row[0]; ~' l' h1 B, {- A/ A: m3 M9 X 
            if len > 0: 
8 a6 Y\" E6 ?4 ?# z                continue 
* f. }. [5 g/ g! } 
$ C, T5 u3 m# {4 z* d/ @+ o) @/ ^5 ?6 K            # --------------------Split Line-------------------- 
' D9 q' q/ X! k. }& ~7 \            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
1 G7 k9 e* b' S- ~* c  G0 W                "http://xxx" + content_body.get('href'),( h# q, }: j8 n2 B2 J( b\" y6 p 
                escape(content_body.get('title').replace("\"", "\"\"")),) c9 H1 P' H/ Y4 G* a+ M 
                content_infor.text.replace('xxx: ', '')))9 o- R& o: m  K- a, ` 
 
: O+ p; ~) ?% @5 s$ _        conn.commit(): W6 ^& N: v( G, Z* C 
        print("第%s页" % str(i))\" G. A+ _. B) P( B0 a  \+ n/ } 
 
1 e( w- N0 ]2 D. M/ F' `    # --------------------Split Line-------------------- 
+ L) b\" k' r. m! G% _    conn.close()( q3 ]. R/ L. B

转发自派生社区

Python交流群：1047602540

zan