# 导入工具包import requestsfrom bs4 import BeautifulSoupimport timeimport pandas as pdimport numpy as np# 请求头headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36'}# =============================================================================# 爬取一页# =============================================================================# 爬取的网址url='/subject/34841067/comments?limit=20&status=P&sort=new_score'# 获取信息html = requests.get(url,headers=headers)# 获取内容data = html.textsoup = BeautifulSoup(data,'lxml')# 信息# 用户names = soup.select('#comments > div > ment > h3 > ment-info > a')# 评级pingjis = soup.select('#comments > div > ment > h3 > ment-info')# 日期riqis = soup.select('#comments > div > ment > h3 > ment-info > ment-time')# 内容neirongs = soup.select('#comments > div > ment > p > span')# 空listlis=[]for name,pingji,riqi,neirong in zip(names,pingjis,riqis,neirongs):pingji_re = pingji.find_all('span')lis.append([name.get_text(),pingji_re[1]['class'],pingji_re[1]['title'],riqi.get_text().strip(),neirong.get_text()])result1 = pd.DataFrame(lis,columns=['用户','评级','等级','日期','内容'])# =============================================================================# 爬取多页# =============================================================================url = ['/subject/34841067/comments?start={}&limit=20&status=P&sort=new_score'.format(i) for i in range(0,100,20)]lis2 = []for urli in url:# 获取信息html = requests.get(urli,headers=headers)# 获取内容data = html.textsoup = BeautifulSoup(data,'lxml')# 用户names = soup.select('#comments > div > ment > h3 > ment-info > a')# 评级pingjis = soup.select('#comments > div > ment > h3 > ment-info')# 日期riqis = soup.select('#comments > div > ment > h3 > ment-info > ment-time')# 内容neirongs = soup.select('#comments > div > ment > p > span')for name,pingji,riqi,neirong in zip(names,pingjis,riqis,neirongs):pingji_re = pingji.find_all('span')lis2.append([name.get_text(),pingji_re[1]['class'],pingji_re[1]['title'],riqi.get_text().strip(),neirong.get_text()])print('完成:',urli)time.sleep(np.random.randint(5,10))result2 = pd.DataFrame(lis2,columns=['用户','评级','等级','日期','内容'])
视频介绍链接: /course/detail/31518
