查看: 3822|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |正序浏览

|招呼Ta 关注Ta

#!/usr/bin/python$ Q0 _: p+ x% K8 B% y& \/ l4 R& A
# -*- coding: utf-8 -*-
! G5 d) v4 | A$ R a$ c8 Z+ i$ Q8 D7 Q7 `6 ?
import sqlite31 w' {) Q. ~; _ Z
import requests1 [\" p: Q3 O0 W. @# g
from bs4 import BeautifulSoup
. p' O& y2 x$ ]5 Ufrom re import escape9 u, z9 }' N8 B3 P
( L4 n6 W6 t9 Y: [+ u
if __name__ == '__main__':4 }* m1 ]( K# s5 F
conn = sqlite3.connect('Python.db')\" @- V0 p! {$ R) c1 Q
c = conn.cursor(). c* M- ^2 C+ x: @ [
c.execute('''CREATE TABLE IF NOT EXISTS Python (\" u/ Y* u0 @* _6 `% r' ^9 C) N
Url VARCHAR,5 w& S7 e\" c8 q% ^. g. m: R! v0 O
Title VARCHAR,# h2 R4 A; i4 J+ ?% r2 s
Author VARCHAR8 \( f* a; { X$ P* ~- ^3 h
)''')
) Y* D( q5 K- |; S- C conn.commit()
, ]$ e9 H: L, f
/ j1 z) R$ c' ]5 _. R7 T\" v; n # --------------------Split Line--------------------
+ V1 O# ]9 x6 Z1 y' l J0 O- M headers = { W0 M- [' a, y1 ^& {/ J
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
! X\" J6 z! L; L) ?$ n* q; g9 |. z }% w: o' i( S6 E: L* d
; H% e9 ~ l/ `9 B/ \6 Z' Y5 P
for i in range(1, 1046):
6 f- ^/ [/ D9 }1 d/ F2 X1 q( T url = "http://xxx/index_%s.html" % str(i)
, A! W/ H# f# f9 M$ V( }1 W9 i req = requests.get(url=url, headers=headers)1 J+ c\" Q( U% _ m/ ?6 G
req.encoding = "utf-8"
1 ]: S5 I8 K2 i/ z3 ?: q\" U html = BeautifulSoup(req.text, "lxml")
% q, f# |9 O) G2 b1 w6 v& j+ x
& i( O, w/ a+ ~\" e\" f # --------------------Split Line--------------------
6 b. g# w- a# o! J F% b! B for div in html.find_all('div', class_='loop'):
; G( t6 L' k. J6 C content_body = div.select('h2 > a')[0]
' t. F5 w6 a5 w1 q\" ? content_infor = div.select('.content_infor > span:nth-child(3)')[0]0 c3 O5 i6 C) u; \ y- _- T8 j n! B
3 F1 S4 @7 h6 F
# --------------------Split Line--------------------
# Y$ {6 W5 Q# x; m5 g7 Z5 T cursor = c.execute(5 Y+ c+ D# N7 z+ A% S
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
8 {0 b0 N/ o- F/ Q' G) N9 O len = 0 Z z6 U9 y- D' T+ p
for row in cursor:; S\" H5 {+ z1 f9 d! D\" w2 O: `& Z
len = row[0]; y0 ^. I, p% V. d1 n8 d: x3 v
if len > 0:7 g6 [. F. _\" M) u* k8 P6 ]
continue
* z- F2 s1 v' ~# x4 @+ k( R! T
\" K+ ~) \3 z( N+ m7 V, q # --------------------Split Line--------------------
4 C4 [5 R# h4 S. O, ? c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (
\" q. Y( l- O4 ^$ L' e* p "http://xxx" + content_body.get('href'),
+ {2 I\" F7 J5 c0 Y S+ A escape(content_body.get('title').replace("\"", "\"\"")),9 {3 n4 {, X/ X
content_infor.text.replace('xxx: ', '')))1 Q- I! q7 {5 L. ?\" H9 Y2 d
1 P! h2 D2 m7 q1 R3 D0 x U conn.commit()& B1 s4 _* ^9 ]9 X( C& b; K6 t: T0 O2 D
print("第%s页" % str(i))- N- Z; t$ M4 Y) [
$ p% n, l8 p* i+ i- x3 q # --------------------Split Line--------------------& t! z8 t4 a, D+ t5 V
conn.close()
: [/ P `+ \2 b. g& m5 q# q

#!/usr/bin/python$ Q0 _: p+ x% K8 B% y& \/ l4 R& A 
# -*- coding: utf-8 -*- 
! G5 d) v4 |  A$ R  a$ c8 Z+ i$ Q8 D7 Q7 `6 ? 
import sqlite31 w' {) Q. ~; _  Z 
import requests1 [\" p: Q3 O0 W. @# g 
from bs4 import BeautifulSoup 
. p' O& y2 x$ ]5 Ufrom re import escape9 u, z9 }' N8 B3 P 
( L4 n6 W6 t9 Y: [+ u 
if __name__ == '__main__':4 }* m1 ]( K# s5 F 
    conn = sqlite3.connect('Python.db')\" @- V0 p! {$ R) c1 Q 
    c = conn.cursor(). c* M- ^2 C+ x: @  [ 
    c.execute('''CREATE TABLE IF NOT EXISTS Python (\" u/ Y* u0 @* _6 `% r' ^9 C) N 
        Url VARCHAR,5 w& S7 e\" c8 q% ^. g. m: R! v0 O 
        Title VARCHAR,# h2 R4 A; i4 J+ ?% r2 s 
        Author VARCHAR8 \( f* a; {  X$ P* ~- ^3 h 
    )''') 
) Y* D( q5 K- |; S- C    conn.commit() 
, ]$ e9 H: L, f 
/ j1 z) R$ c' ]5 _. R7 T\" v; n    # --------------------Split Line-------------------- 
+ V1 O# ]9 x6 Z1 y' l  J0 O- M    headers = {  W0 M- [' a, y1 ^& {/ J 
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36" 
! X\" J6 z! L; L) ?$ n* q; g9 |. z    }% w: o' i( S6 E: L* d 
; H% e9 ~  l/ `9 B/ \6 Z' Y5 P 
    for i in range(1, 1046): 
6 f- ^/ [/ D9 }1 d/ F2 X1 q( T        url = "http://xxx/index_%s.html" % str(i) 
, A! W/ H# f# f9 M$ V( }1 W9 i        req = requests.get(url=url, headers=headers)1 J+ c\" Q( U% _  m/ ?6 G 
        req.encoding = "utf-8" 
1 ]: S5 I8 K2 i/ z3 ?: q\" U        html = BeautifulSoup(req.text, "lxml") 
% q, f# |9 O) G2 b1 w6 v& j+ x 
& i( O, w/ a+ ~\" e\" f        # --------------------Split Line-------------------- 
6 b. g# w- a# o! J  F% b! B        for div in html.find_all('div', class_='loop'): 
; G( t6 L' k. J6 C            content_body = div.select('h2 > a')[0] 
' t. F5 w6 a5 w1 q\" ?            content_infor = div.select('.content_infor > span:nth-child(3)')[0]0 c3 O5 i6 C) u; \  y- _- T8 j  n! B 
3 F1 S4 @7 h6 F 
            # --------------------Split Line-------------------- 
# Y$ {6 W5 Q# x; m5 g7 Z5 T            cursor = c.execute(5 Y+ c+ D# N7 z+ A% S 
                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
8 {0 b0 N/ o- F/ Q' G) N9 O            len = 0  Z  z6 U9 y- D' T+ p 
            for row in cursor:; S\" H5 {+ z1 f9 d! D\" w2 O: `& Z 
                len = row[0]; y0 ^. I, p% V. d1 n8 d: x3 v 
            if len > 0:7 g6 [. F. _\" M) u* k8 P6 ] 
                continue 
* z- F2 s1 v' ~# x4 @+ k( R! T 
\" K+ ~) \3 z( N+ m7 V, q            # --------------------Split Line-------------------- 
4 C4 [5 R# h4 S. O, ?            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % ( 
\" q. Y( l- O4 ^$ L' e* p                "http://xxx" + content_body.get('href'), 
+ {2 I\" F7 J5 c0 Y  S+ A                escape(content_body.get('title').replace("\"", "\"\"")),9 {3 n4 {, X/ X 
                content_infor.text.replace('xxx: ', '')))1 Q- I! q7 {5 L. ?\" H9 Y2 d 
 
1 P! h2 D2 m7 q1 R3 D0 x  U        conn.commit()& B1 s4 _* ^9 ]9 X( C& b; K6 t: T0 O2 D 
        print("第%s页" % str(i))- N- Z; t$ M4 Y) [ 
 
$ p% n, l8 p* i+ i- x3 q    # --------------------Split Line--------------------& t! z8 t4 a, D+ t5 V 
    conn.close() 
: [/ P  `+ \2 b. g& m5 q# q

转发自派生社区

Python交流群：1047602540

zan