查看: 3840|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
7 T; E: P* z& d0 W$ E5 f9 r) Y# -*- coding: utf-8 -*-
( W4 N, D9 |+ k1 U( C
0 i6 U- ~8 z\" s' w8 C9 Kimport sqlite3, z0 y' q. S- q
import requests/ D# }4 `8 M) t: y, @$ z' J% W
from bs4 import BeautifulSoup% k' T) x( C' o5 \
from re import escape
1 M\" M, r\" U+ d) w( Z\" w2 `% O4 B3 `* w& ~! Y; p/ z
if __name__ == '__main__':: W! i% a/ h3 @
conn = sqlite3.connect('Python.db')% l7 F* @4 X\" V/ ]
c = conn.cursor(); w7 D7 @- _9 C3 u% ^+ Z7 |3 J
c.execute('''CREATE TABLE IF NOT EXISTS Python (2 U) p) b; P- l: m/ ]
Url VARCHAR,
7 P* X* X# O$ I; Z/ y+ K Title VARCHAR,
8 _$ I3 J' \6 W8 R$ K Author VARCHAR& h8 Z$ Y, d1 e3 I$ d
)''')4 `+ w Q1 I4 e- g
conn.commit()
3 N* q* \9 b1 a' ^2 @' t' z* X! n3 Q0 l: i0 P. t3 _7 |2 G- |: |% |
# --------------------Split Line--------------------
g7 K# x& Z* u: ^* D headers = {; |. E3 {: r+ m. l% U
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
2 ~8 p' _; z: z8 T; Z }8 H R; U: F4 k; Q; M
5 v8 s2 Q1 {/ d3 X$ I; J
for i in range(1, 1046):
; b' _& L) ]7 B0 x/ m url = "http://xxx/index_%s.html" % str(i). b' V3 P! y* n! L. l
req = requests.get(url=url, headers=headers)
0 a( F: q* W+ @; o\" E req.encoding = "utf-8": E- H7 X5 k2 d
html = BeautifulSoup(req.text, "lxml")
7 d, Q- ]* h9 l3 U. ^
* Q3 P0 d( K }# |, r # --------------------Split Line--------------------
3 G0 t0 h# X# s: n8 x# d5 T( B9 | for div in html.find_all('div', class_='loop'):; d; Z4 F I' t0 ^6 E
content_body = div.select('h2 > a')[0]5 \. |5 E. E( G p. B
content_infor = div.select('.content_infor > span:nth-child(3)')[0]
2 V& T6 B* ~. N- j E+ g, ?: r* ]3 {5 T# e# Q9 M2 y1 N
# --------------------Split Line--------------------' i ^8 N) s( _+ K
cursor = c.execute(
+ ^* @8 h& H% Z5 A6 d5 x l\" j "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
2 L. _* H& u+ V: \9 N9 `) ` len = 0
}( X7 X7 D( E9 g$ N/ P for row in cursor:
0 X _\" ]2 B6 } K len = row[0]
9 d5 `/ G/ z6 k/ {; [8 r. ^ if len > 0:
' o! @; B( a. F- E& c% p; E3 \ continue
% e/ p5 l; h- [; r* i
0 N# M( i6 f: r* s# Z Q$ e # --------------------Split Line--------------------( `7 I K& S5 r+ x, {6 Y2 k
c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (% B7 F8 m- E0 k
"http://xxx" + content_body.get('href'),3 m5 _/ i! S# O\" O\" }9 t
escape(content_body.get('title').replace("\"", "\"\"")),
+ y$ x# j# W2 M/ q* j! }8 v content_infor.text.replace('xxx: ', ''))). ~! ]. F3 _& E/ m9 P. O3 K
% h0 M/ a' ~5 B K4 N% i* ~9 J
conn.commit()
0 l! ]4 Q% i* s* o, | print("第%s页" % str(i))' W) t( Y1 o' `6 E3 u- M% Q* w1 r' M, _
7 |& @8 d8 I+ t
# --------------------Split Line--------------------
^, J4 [9 \2 k5 z conn.close()
7 D9 E9 k2 n; p4 R9 {

#!/usr/bin/python 
7 T; E: P* z& d0 W$ E5 f9 r) Y# -*- coding: utf-8 -*- 
( W4 N, D9 |+ k1 U( C 
0 i6 U- ~8 z\" s' w8 C9 Kimport sqlite3, z0 y' q. S- q 
import requests/ D# }4 `8 M) t: y, @$ z' J% W 
from bs4 import BeautifulSoup% k' T) x( C' o5 \ 
from re import escape 
1 M\" M, r\" U+ d) w( Z\" w2 `% O4 B3 `* w& ~! Y; p/ z 
if __name__ == '__main__':: W! i% a/ h3 @ 
    conn = sqlite3.connect('Python.db')% l7 F* @4 X\" V/ ] 
    c = conn.cursor(); w7 D7 @- _9 C3 u% ^+ Z7 |3 J 
    c.execute('''CREATE TABLE IF NOT EXISTS Python (2 U) p) b; P- l: m/ ] 
        Url VARCHAR, 
7 P* X* X# O$ I; Z/ y+ K        Title VARCHAR, 
8 _$ I3 J' \6 W8 R$ K        Author VARCHAR& h8 Z$ Y, d1 e3 I$ d 
    )''')4 `+ w  Q1 I4 e- g 
    conn.commit() 
3 N* q* \9 b1 a' ^2 @' t' z* X! n3 Q0 l: i0 P. t3 _7 |2 G- |: |% | 
    # --------------------Split Line-------------------- 
  g7 K# x& Z* u: ^* D    headers = {; |. E3 {: r+ m. l% U 
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36" 
2 ~8 p' _; z: z8 T; Z    }8 H  R; U: F4 k; Q; M 
5 v8 s2 Q1 {/ d3 X$ I; J 
    for i in range(1, 1046): 
; b' _& L) ]7 B0 x/ m        url = "http://xxx/index_%s.html" % str(i). b' V3 P! y* n! L. l 
        req = requests.get(url=url, headers=headers) 
0 a( F: q* W+ @; o\" E        req.encoding = "utf-8": E- H7 X5 k2 d 
        html = BeautifulSoup(req.text, "lxml") 
7 d, Q- ]* h9 l3 U. ^ 
* Q3 P0 d( K  }# |, r        # --------------------Split Line-------------------- 
3 G0 t0 h# X# s: n8 x# d5 T( B9 |        for div in html.find_all('div', class_='loop'):; d; Z4 F  I' t0 ^6 E 
            content_body = div.select('h2 > a')[0]5 \. |5 E. E( G  p. B 
            content_infor = div.select('.content_infor > span:nth-child(3)')[0] 
2 V& T6 B* ~. N- j  E+ g, ?: r* ]3 {5 T# e# Q9 M2 y1 N 
            # --------------------Split Line--------------------' i  ^8 N) s( _+ K 
            cursor = c.execute( 
+ ^* @8 h& H% Z5 A6 d5 x  l\" j                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
2 L. _* H& u+ V: \9 N9 `) `            len = 0 
  }( X7 X7 D( E9 g$ N/ P            for row in cursor: 
0 X  _\" ]2 B6 }  K                len = row[0] 
9 d5 `/ G/ z6 k/ {; [8 r. ^            if len > 0: 
' o! @; B( a. F- E& c% p; E3 \                continue 
% e/ p5 l; h- [; r* i 
0 N# M( i6 f: r* s# Z  Q$ e            # --------------------Split Line--------------------( `7 I  K& S5 r+ x, {6 Y2 k 
            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (% B7 F8 m- E0 k 
                "http://xxx" + content_body.get('href'),3 m5 _/ i! S# O\" O\" }9 t 
                escape(content_body.get('title').replace("\"", "\"\"")), 
+ y$ x# j# W2 M/ q* j! }8 v                content_infor.text.replace('xxx: ', ''))). ~! ]. F3 _& E/ m9 P. O3 K 
% h0 M/ a' ~5 B  K4 N% i* ~9 J 
        conn.commit() 
0 l! ]4 Q% i* s* o, |        print("第%s页" % str(i))' W) t( Y1 o' `6 E3 u- M% Q* w1 r' M, _ 
7 |& @8 d8 I+ t 
    # --------------------Split Line-------------------- 
  ^, J4 [9 \2 k5 z    conn.close() 
7 D9 E9 k2 n; p4 R9 {

转发自派生社区

Python交流群：1047602540

zan