使用requests库，xpath库
import requestsimport timefrom lxml import etree# 去请求页面的函数def request_Header(url):headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36',}request = requests.get(url=url,headers=headers)return requestdef get_content_text(all_href):request = request_Header(all_href)etrees = etree.HTML(request.content)# 获取所有章节的内容 得到一个列表 这是由多个p标签组成的content_text = etrees.xpath('//div[@class="layui-col-md8 layui-col-sm7"]/div/div/p/text()')strs_cont = ' 'for con in content_text:strs_cont+=con+'\n'print(strs_cont)return strs_contdef main():url = '/book/sanguoyanyi.html'req = request_Header(url)# 得到网页的内容content = req.contentetrees = etree.HTML(content)# 获取所有章节内容text = etrees.xpath('//div[@class="book-mulu"]/ul/li/a')fp = open('三国演义.txt','w',encoding='utf8')# 遍历这个列表,挨个获取for a in text:# 获取标题title = a.xpath('text()')[0]print('正在下载>>>%s'%title)# 获取连接href = a.xpath('@href')[0]# 拼接urlall_href = '' + str(href)# print(all_href)# 去网页中获取内容get_content = get_content_text(all_href)fp.write(title+'\n' + str(get_content)+'\n')# time.sleep(2)print('结束下载%s' % title)fp.close()if __name__ == '__main__':main()
