分享一段非常簡單的Python批量采集wordpress網(wǎng)站數(shù)據(jù)的爬蟲腳本，實(shí)現(xiàn)采集wordpress程序的網(wǎng)站的整站數(shù)據(jù)的爬蟲程序。從首頁開始，抓取href標(biāo)簽，到子頁面后還是要繼續(xù)找href標(biāo)簽，采用Python遞歸方法，直接貼代碼吧！
import re
import bs4
import urllib.request
url_home = 'https://www.zztuku.com/' #要采集的網(wǎng)站
url_pattern = url_home + '([\s\S]*)\.html' #正則表達(dá)式匹配文章頁面，此處需完善為更好的寫法
url_set = set()
url_cache = set()
url_count = 0
url_maxCount = 1000 #最大采集數(shù)量
#采集匹配文章內(nèi)容的href標(biāo)簽
def spiderURL(url, pattern):
html = urllib.request.urlopen(url).read().decode('utf8')
soup = bs4.BeautifulSoup(html, 'html.parser')
links = soup.find_all('a', href = re.compile(pattern))
for link in links:
if link['href'] not in url_cache:
url_set.add(link['href'])
return soup
#采集的過程 異常處理還需要完善，對(duì)于一些加了防采集的站，還需要處理header的，下次我們?cè)賹W(xué)習(xí)
spiderURL(url_home, url_pattern)
while len(url_set) != 0:
try:
url = url_set.pop()
url_cache.add(url)
soup = spiderURL(url, url_pattern)
page = soup.find('div', {'class':'content'})
title = page.find('h1').get_text()
autor = page.find('h4').get_text()
content = page.find('article').get_text()
print(title, autor, url)
except Exception as e:
print(url, e)
continue
else:
url_count += 1
finally:
if url_count == url_maxCount:
break
print('一共采集了： ' + str(url_count) + ' 條數(shù)據(jù)')
