查看: 3846|回复: 0

Python-网页爬虫与Sqlite3

字体大小: 正常放大

檀俾九

7 主题	3 听众	10 积分

升级 5.26%

该用户从未签到

自我介绍: 我本名为我，那就是我

电梯直达

1^#

发表于 2021-1-14 08:44 |只看该作者 |倒序浏览

|招呼Ta 关注Ta

#!/usr/bin/python
) }) e5 ~: X. {5 n. a& ?# -*- coding: utf-8 -*-
\" C8 W4 z2 f+ |3 v( ]& D4 C6 X- H& {; A) Z6 L
import sqlite3
8 ` F0 v# v0 M# Kimport requests
a) \- g; A) |* N2 W- ufrom bs4 import BeautifulSoup4 {2 A% h- X4 o6 I0 v\" r
from re import escape
2 [1 {0 ?/ D7 p' B, q' ^7 I# A' S+ B
if __name__ == '__main__':
\" Z5 J! v8 N' V G$ l, y8 h\" { conn = sqlite3.connect('Python.db')
& r: o' w: H\" d# I3 }- K2 K c = conn.cursor()( q; h& F: Q0 y# Y% O6 q
c.execute('''CREATE TABLE IF NOT EXISTS Python (
* h4 a7 @7 `3 C\" h# w Url VARCHAR,
3 ]\" `' l\" ^+ U Title VARCHAR,% D0 P\" x; X0 Y; u8 H- P9 V! Z
Author VARCHAR
& q7 N! B* K+ T0 _$ _( M )''')
/ p$ W\" i\" V. T2 d* x$ i conn.commit()
7 J2 K4 b* ?% H
0 ~5 Z5 k+ A$ J9 _7 v # --------------------Split Line--------------------
. X3 Q4 |0 I' O: G& \; V headers = {
, B# u5 t( @7 t$ }1 b% o "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36"
* G6 I8 F' |+ f9 G, R w( v }
: Q0 f' d4 q6 |& Q) `* \4 U
+ l4 S* J1 a+ T1 y. ]& \9 `: k for i in range(1, 1046):7 N# Z1 j* B b
url = "http://xxx/index_%s.html" % str(i)& R, B& K7 M$ }6 ^! y% b3 [6 w
req = requests.get(url=url, headers=headers)
+ z+ x4 F6 h) |, i( | O req.encoding = "utf-8"% {. G5 M1 O& T
html = BeautifulSoup(req.text, "lxml")
& h4 m2 [9 \- t1 C# V) a
, y [- T j) d6 ]! c; ] # --------------------Split Line--------------------' o5 H) o) @9 r: F
for div in html.find_all('div', class_='loop'):
5 H! r' ~8 p6 h T( d5 j content_body = div.select('h2 > a')[0]' V& W2 @8 N+ \
content_infor = div.select('.content_infor > span:nth-child(3)')[0]) j) w5 M) P* _
: d6 e% h; j1 R- v* [$ s$ E
# --------------------Split Line--------------------% N9 p$ S. H7 i
cursor = c.execute(; \: _& I) C/ D4 F9 e% |1 D6 E
"SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href')))
. I5 t0 N, q! U% x0 A len = 0- G$ f; w% H! Z$ ~' F
for row in cursor:
6 \! r I\" f2 }9 r4 f3 w% F len = row[0]
2 I- `. F& E* v. @\" I0 ] if len > 0:
2 V5 u9 f' l# {* N/ C continue
( a7 n( h& H; V0 b, d' O7 w4 z |* p! v% C$ v
# --------------------Split Line--------------------
/ s/ ^3 \9 F0 F5 @ c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (7 Y1 V9 y. d( }$ C
"http://xxx" + content_body.get('href'),. ^- h6 ], M1 R$ F- M
escape(content_body.get('title').replace("\"", "\"\"")),( e& y8 [: [6 M& O+ `6 Z1 H* e
content_infor.text.replace('xxx: ', '')))
9 S\" Y) Z, B1 O1 \& X9 j- L- N: k% j\" N8 c( L
conn.commit()\" W ^# ~7 J5 W' `2 Z
print("第%s页" % str(i))0 F9 r\" _7 _; ]
! o' A& }( s& p9 i% G5 G
# --------------------Split Line--------------------
9 w8 G) Y: x5 O! k6 K' u1 J conn.close()& z. `( T* f2 J

#!/usr/bin/python 
) }) e5 ~: X. {5 n. a& ?# -*- coding: utf-8 -*- 
\" C8 W4 z2 f+ |3 v( ]& D4 C6 X- H& {; A) Z6 L 
import sqlite3 
8 `  F0 v# v0 M# Kimport requests 
  a) \- g; A) |* N2 W- ufrom bs4 import BeautifulSoup4 {2 A% h- X4 o6 I0 v\" r 
from re import escape 
2 [1 {0 ?/ D7 p' B, q' ^7 I# A' S+ B 
if __name__ == '__main__': 
\" Z5 J! v8 N' V  G$ l, y8 h\" {    conn = sqlite3.connect('Python.db') 
& r: o' w: H\" d# I3 }- K2 K    c = conn.cursor()( q; h& F: Q0 y# Y% O6 q 
    c.execute('''CREATE TABLE IF NOT EXISTS Python ( 
* h4 a7 @7 `3 C\" h# w        Url VARCHAR, 
3 ]\" `' l\" ^+ U        Title VARCHAR,% D0 P\" x; X0 Y; u8 H- P9 V! Z 
        Author VARCHAR 
& q7 N! B* K+ T0 _$ _( M    )''') 
/ p$ W\" i\" V. T2 d* x$ i    conn.commit() 
7 J2 K4 b* ?% H 
0 ~5 Z5 k+ A$ J9 _7 v    # --------------------Split Line-------------------- 
. X3 Q4 |0 I' O: G& \; V    headers = { 
, B# u5 t( @7 t$ }1 b% o        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36" 
* G6 I8 F' |+ f9 G, R  w( v    } 
: Q0 f' d4 q6 |& Q) `* \4 U 
+ l4 S* J1 a+ T1 y. ]& \9 `: k    for i in range(1, 1046):7 N# Z1 j* B  b 
        url = "http://xxx/index_%s.html" % str(i)& R, B& K7 M$ }6 ^! y% b3 [6 w 
        req = requests.get(url=url, headers=headers) 
+ z+ x4 F6 h) |, i( |  O        req.encoding = "utf-8"% {. G5 M1 O& T 
        html = BeautifulSoup(req.text, "lxml") 
& h4 m2 [9 \- t1 C# V) a 
, y  [- T  j) d6 ]! c; ]        # --------------------Split Line--------------------' o5 H) o) @9 r: F 
        for div in html.find_all('div', class_='loop'): 
5 H! r' ~8 p6 h  T( d5 j            content_body = div.select('h2 > a')[0]' V& W2 @8 N+ \ 
            content_infor = div.select('.content_infor > span:nth-child(3)')[0]) j) w5 M) P* _ 
: d6 e% h; j1 R- v* [$ s$ E 
            # --------------------Split Line--------------------% N9 p$ S. H7 i 
            cursor = c.execute(; \: _& I) C/ D4 F9 e% |1 D6 E 
                "SELECT COUNT(*) FROM Python WHERE Url = '%s'" % ("http://xxx" + content_body.get('href'))) 
. I5 t0 N, q! U% x0 A            len = 0- G$ f; w% H! Z$ ~' F 
            for row in cursor: 
6 \! r  I\" f2 }9 r4 f3 w% F                len = row[0] 
2 I- `. F& E* v. @\" I0 ]            if len > 0: 
2 V5 u9 f' l# {* N/ C                continue 
( a7 n( h& H; V0 b, d' O7 w4 z  |* p! v% C$ v 
            # --------------------Split Line-------------------- 
/ s/ ^3 \9 F0 F5 @            c.execute('INSERT INTO Python( Url, Title, Author) VALUES ( "%s", "%s", "%s")' % (7 Y1 V9 y. d( }$ C 
                "http://xxx" + content_body.get('href'),. ^- h6 ], M1 R$ F- M 
                escape(content_body.get('title').replace("\"", "\"\"")),( e& y8 [: [6 M& O+ `6 Z1 H* e 
                content_infor.text.replace('xxx: ', ''))) 
9 S\" Y) Z, B1 O1 \& X9 j- L- N: k% j\" N8 c( L 
        conn.commit()\" W  ^# ~7 J5 W' `2 Z 
        print("第%s页" % str(i))0 F9 r\" _7 _; ] 
! o' A& }( s& p9 i% G5 G 
    # --------------------Split Line-------------------- 
9 w8 G) Y: x5 O! k6 K' u1 J    conn.close()& z. `( T* f2 J

转发自派生社区

Python交流群：1047602540

zan