查看: 3190|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

发表于 2021-1-14 08:44 |显示全部楼层

|招呼Ta 关注Ta

#!/usr/bin/python7 w6 l$ E\" O# W# e. [; p' L
# -*- coding: utf-8 -*-
- B2 J- K4 H( x* z7 j* Z& X3 W8 A7 T$ K) d- X% C
import sqlite3 t' U% C& K5 u, X# y7 n. J. {
import requests; n3 F1 I9 s$ ^' @% i3 {+ C$ I
from bs4 import BeautifulSoup
/ e! @) Y l$ y. W0 W8 ?( Y- gfrom re import escape4 W. O$ U7 F) e! H2 m; E. U, F4 e
& }/ }* S4 s7 g+ W' i3 Y R! rif __name__ == '__main__':
2 E. y6 Y\" _$ T7 c3 D conn = sqlite3.connect('Python.db')! C; x& S8 M2 r) V& V3 d
c = conn.cursor()7 R# I, }) g5 R0 A5 D1 F' D
c.execute('''CREATE TABLE IF NOT EXISTS Python (
9 _8 t$ ^\" o' L; c* W2 Y Url VARCHAR,) T$ N6 o$ y7 w8 k
Title VARCHAR,. T0 F) p! Z- o. K/ B6 h* Z7 F' @$ J0 ?
Author VARCHAR/ `8 f3 G& f) d+ W R0 g9 Y
)''')$ K% a8 u7 z\" z: {
conn.commit()* P' x7 p8 P5 O+ u, T7 X! O+ Q
1 w/ |% |3 B( M: D- l # --------------------Split Line--------------------
% J9 K: i+ p Y y+ _6 | headers = {
2 H4 d$ E7 j& k "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
1 c2 ~# U t% G+ i: ^ }
- q% ]- E0 K n! _0 d5 D0 o. }9 }3 m$ f# ?0 p
for i in range(1, 1046):
4 v, G3 j2 C9 @0 r' A url = "http://xxx/index_%s.html" % str(i)
: c7 n R2 |& x: L( P# X0 Z req = requests.get(url=url, headers=headers), f% K: p6 h4 U7 [3 h% g$ }
req.encoding = "utf-8"\" ~2 U9 o$ Q+ O- d: U- R% u
html = BeautifulSoup(req.text, "lxml")8 {6 G& A) x5 [+ n1 s3 ?7 o% {
( m! x2 n0 T1 l7 F4 ? # --------------------Split Line--------------------
1 r3 ^1 J4 h0 a8 x* p; p: V1 f for div in html.find_all('div', class_='loop'): Z( I, k5 D' h, _1 f
content_body = div.select('h2 > a')[0]
' f' {+ x4 J! t$ l2 d& y- X1 L content_infor = div.select('.content_infor > span:nth-child(3)')[0]
7 g$ ^7 z3 P7 f; K
/ F\" j& ^\" l! } # --------------------Split Line--------------------
: o% N& k\" I* l, B# \ cursor = c.execute(3 H: \3 M0 T* B& J- r |
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
\" T% M l) h( A, H3 b len = 0\" P7 S5 d4 v; Z, y. x2 S$ R
for row in cursor:2 l# K) K/ R7 [& E' B! P
len = row[0]
* y+ ?4 u# c H6 J3 u3 Z if len > 0: w$ x4 \0 F2 }4 r5 v* j1 N
continue# }6 M* ]# ]) C% g$ G\" Q& W! v
8 \7 a4 Z! P$ K; ?; b+ L/ C
# --------------------Split Line--------------------
3 g1 x\" p/ b0 D5 \ c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (/ `+ w7 X: C4 ~
"http://xxx" + content_body.get('href'),# a h+ n0 T9 ~+ N
escape(content_body.get('title').replace("\"", "\"\"")),
}( Z2 @# r2 B; j9 ~$ U3 M1 E/ { content_infor.text.replace('xxx: ', '')))) \4 U; G: G5 Q3 c
6 x6 x4 q( K/ K; {7 ^1 K conn.commit() J' P3 d2 a @- {9 c. G3 {
print("第%s页" % str(i))* `6 F |. I1 n7 {% ~) v) t# O
U1 [9 Q: h; o0 W6 j$ u5 s # --------------------Split Line--------------------( V. t& L$ v7 _& h3 S- V+ R
conn.close()
7 F: S9 R9 @5 U

#!/usr/bin/python7 w6 l$ E\" O# W# e. [; p' L 
# -*- coding: utf-8 -*- 
- B2 J- K4 H( x* z7 j* Z& X3 W8 A7 T$ K) d- X% C 
import sqlite3  t' U% C& K5 u, X# y7 n. J. { 
import requests; n3 F1 I9 s$ ^' @% i3 {+ C$ I 
from bs4 import BeautifulSoup 
/ e! @) Y  l$ y. W0 W8 ?( Y- gfrom re import escape4 W. O$ U7 F) e! H2 m; E. U, F4 e 
 
& }/ }* S4 s7 g+ W' i3 Y  R! rif __name__ == '__main__': 
2 E. y6 Y\" _$ T7 c3 D    conn = sqlite3.connect('Python.db')! C; x& S8 M2 r) V& V3 d 
    c = conn.cursor()7 R# I, }) g5 R0 A5 D1 F' D 
    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
9 _8 t$ ^\" o' L; c* W2 Y        Url VARCHAR,) T$ N6 o$ y7 w8 k 
        Title VARCHAR,. T0 F) p! Z- o. K/ B6 h* Z7 F' @$ J0 ? 
        Author VARCHAR/ `8 f3 G& f) d+ W  R0 g9 Y 
    )''')$ K% a8 u7 z\" z: { 
    conn.commit()* P' x7 p8 P5 O+ u, T7 X! O+ Q 
 
1 w/ |% |3 B( M: D- l    # --------------------Split Line-------------------- 
% J9 K: i+ p  Y  y+ _6 |    headers = { 
2 H4 d$ E7 j& k        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36" 
1 c2 ~# U  t% G+ i: ^    } 
- q% ]- E0 K  n! _0 d5 D0 o. }9 }3 m$ f# ?0 p 
    for i in range(1, 1046): 
4 v, G3 j2 C9 @0 r' A        url = "http://xxx/index_%s.html" % str(i) 
: c7 n  R2 |& x: L( P# X0 Z        req = requests.get(url=url, headers=headers), f% K: p6 h4 U7 [3 h% g$ } 
        req.encoding = "utf-8"\" ~2 U9 o$ Q+ O- d: U- R% u 
        html = BeautifulSoup(req.text, "lxml")8 {6 G& A) x5 [+ n1 s3 ?7 o% { 
 
( m! x2 n0 T1 l7 F4 ?        # --------------------Split Line-------------------- 
1 r3 ^1 J4 h0 a8 x* p; p: V1 f        for div in html.find_all('div', class_='loop'):  Z( I, k5 D' h, _1 f 
            content_body = div.select('h2 > a')[0] 
' f' {+ x4 J! t$ l2 d& y- X1 L            content_infor = div.select('.content_infor > span:nth-child(3)')[0] 
7 g$ ^7 z3 P7 f; K 
/ F\" j& ^\" l! }            # --------------------Split Line-------------------- 
: o% N& k\" I* l, B# \            cursor = c.execute(3 H: \3 M0 T* B& J- r  | 
                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
\" T% M  l) h( A, H3 b            len = 0\" P7 S5 d4 v; Z, y. x2 S$ R 
            for row in cursor:2 l# K) K/ R7 [& E' B! P 
                len = row[0] 
* y+ ?4 u# c  H6 J3 u3 Z            if len > 0:  w$ x4 \0 F2 }4 r5 v* j1 N 
                continue# }6 M* ]# ]) C% g$ G\" Q& W! v 
8 \7 a4 Z! P$ K; ?; b+ L/ C 
            # --------------------Split Line-------------------- 
3 g1 x\" p/ b0 D5 \            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (/ `+ w7 X: C4 ~ 
                "http://xxx" + content_body.get('href'),# a  h+ n0 T9 ~+ N 
                escape(content_body.get('title').replace("\"", "\"\"")), 
  }( Z2 @# r2 B; j9 ~$ U3 M1 E/ {                content_infor.text.replace('xxx: ', '')))) \4 U; G: G5 Q3 c 
 
6 x6 x4 q( K/ K; {7 ^1 K        conn.commit()  J' P3 d2 a  @- {9 c. G3 { 
        print("第%s页" % str(i))* `6 F  |. I1 n7 {% ~) v) t# O 
 
  U1 [9 Q: h; o0 W6 j$ u5 s    # --------------------Split Line--------------------( V. t& L$ v7 _& h3 S- V+ R 
    conn.close() 
7 F: S9 R9 @5 U

转发自派生社区

Python交流群：1047602540

zan

使用道具举报