【相關(guān)學(xué)習推薦：python教程】
實驗環(huán)境
1.安裝python 3.7
2.安裝requests, bs4，pymysql 模塊
實驗步驟1.安裝環(huán)境及模塊
可參考https://www.jb51.net/article/194104.htm
2.編寫代碼
# 51cto 博客頁面數(shù)據(jù)插入mysql數(shù)據(jù)庫# 導(dǎo)入模塊import reimport bs4import pymysqlimport requests# 連接數(shù)據(jù)庫賬號密碼db = pymysql.connect(host='172.171.13.229', user='root', passwd='abc123', db='test', port=3306, charset='utf8')# 獲取游標cursor = db.cursor()def open_url(url): # 連接模擬網(wǎng)頁訪問 headers = { 'user-agent': 'mozilla/5.0 (windows nt 10.0; wow64) applewebkit/537.36 (khtml, like gecko) ' 'chrome/57.0.2987.98 safari/537.36'} res = requests.get(url, headers=headers) return res# 爬取網(wǎng)頁內(nèi)容def find_text(res): soup = bs4.beautifulsoup(res.text, 'html.parser') # 博客名 titles = [] targets = soup.find_all("a", class_="tit") for each in targets: each = each.text.strip() if "置頂" in each: each = each.split(' ')[0] titles.append(each) # 閱讀量 reads = [] read1 = soup.find_all("p", class_="read fl on") read2 = soup.find_all("p", class_="read fl") for each in read1: reads.append(each.text) for each in read2: reads.append(each.text) # 評論數(shù) comment = [] targets = soup.find_all("p", class_='comment fl') for each in targets: comment.append(each.text) # 收藏 collects = [] targets = soup.find_all("p", class_='collect fl') for each in targets: collects.append(each.text) # 發(fā)布時間 dates=[] targets = soup.find_all("a", class_='time fl') for each in targets: each = each.text.split('：')[1] dates.append(each) # 插入sql 語句 sql = """insert into blog (blog_title,read_number,comment_number, collect, dates) values( '%s', '%s', '%s', '%s', '%s');""" # 替換頁面 \\\\xa0 for titles, reads, comment, collects, dates in zip(titles, reads, comment, collects, dates): reads = re.sub('\\\\s', '', reads) comment = re.sub('\\\\s', '', comment) collects = re.sub('\\\\s', '', collects) cursor.execute(sql % (titles, reads, comment, collects，dates)) db.commit() pass# 統(tǒng)計總頁數(shù)def find_depth(res): soup = bs4.beautifulsoup(res.text, 'html.parser') depth = soup.find('li', class_='next').previous_sibling.previous_sibling.text return int(depth)# 主函數(shù)def main(): host = "https://blog.51cto.com/13760351" res = open_url(host) # 打開首頁鏈接 depth = find_depth(res) # 獲取總頁數(shù) # 爬取其他頁面信息 for i in range(1, depth 1): url = host '/p' str(i) # 完整鏈接 res = open_url(url) # 打開其他鏈接 find_text(res) # 爬取數(shù)據(jù) # 關(guān)閉游標 cursor.close() # 關(guān)閉數(shù)據(jù)庫連接 db.close()if __name__ == '__main__': main()3..mysql創(chuàng)建對應(yīng)的表
create table `blog` ( `row_id` int(11) not null auto_increment comment '主鍵', `blog_title` varchar(52) default null comment '博客標題', `read_number` varchar(26) default null comment '閱讀數(shù)量', `comment_number` varchar(16) default null comment '評論數(shù)量', `collect` varchar(16) default null comment '收藏數(shù)量', `dates` varchar(16) default null comment '發(fā)布日期', primary key (`row_id`)) engine=innodb auto_increment=1 default charset=utf8;import reimport bs4import pymysqlimport requests# 連接數(shù)據(jù)庫db = pymysql.connect(host='172.171.13.229', user='root', passwd='abc123', db='test', port=3306, charset='utf8')# 獲取游標cursor = db.cursor()def open_url(url): # 連接模擬網(wǎng)頁訪問 headers = { 'user-agent': 'mozilla/5.0 (windows nt 10.0; wow64) applewebkit/537.36 (khtml, like gecko) ' 'chrome/57.0.2987.98 safari/537.36'} res = requests.get(url, headers=headers) return res# 爬取網(wǎng)頁內(nèi)容def find_text(res):