帮别人写爬虫，先是单线程，太慢，改了多线程
1.单线程
import urllibimport urllib.requestimport requestsimport xlwtimport reimport stringdef set_style(name,height,bold=False):style = xlwt.XFStyle() # 初始化样式font = xlwt.Font() # 为样式创建字体font.name = name # 'Times New Roman'font.bold = boldfont.color_index = 4font.height = heightstyle.font = fontreturn styleindex=0r=0wdk=xlwt.Workbook() sheet1=wdk.add_sheet('sheet1',cell_overwrite_ok=True) #excel单元格名字row=['招聘职位','月薪','学历要求','工作经验','应聘要求','工作单位']for i in row:sheet1.write(r,index,i)index+=1r+=1import datetimestarttime = datetime.datetime.now()for pn in range(0,200,20):url = '/quanzhi?tid=4139&ie=utf8&oe=utf8&query=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98%E5%B7%A5%E7%A8%8B%E5%B8%88&city_sug=%E9%95%BF%E6%B2%99&&detailmode=close&rn=20&pn='+str(pn)#打开网页wp = urllib.request.urlopen(url)content = wp.read()hrefPatten=r"(href='.{0,200}数据挖掘工程师')"href=hrefPatten.encode("utf-8")hrefC = re.findall(href, content, re.S) #返回所有匹配正则表达式的值于列表中hre=['']*len(hrefC)for i in range (len(hrefC)):hre[i]=urllib.parse.unquote(str(hrefC[i].decode('utf-8')).replace("href='","")[:-1])#将href转化为网址for i in range (len(hre)):a=requests.get(hre[i])content1=a.content.decode('utf-8')pattern=('<span class="title line-clamp1">(.*?)</span>.*?class="salary">(.*?)</span>.*?class="xueli"></span>(.*?)</li>.*?class="minge"></span>(.*?)</li>.*?class="duty duty-box">(.*?)</p>.*?class="line-clamp2">(.*?)</p>')items=re.findall(pattern, content1, re.S)#抓取招聘信息for item in items:flag=0for _item in item:if _item=="":flag=1if flag==0:for k in range(0,6):sheet1.write(r,k,item[k])r+=1wdk.save('/home/y.xls')endtime = datetime.datetime.now()print((endtime - starttime).seconds)
时间我是80s左右
2.多线程
import urllibimport urllib.requestimport requestsimport xlwtimport reimport stringimport threadingfrom time import ctime,sleepdef set_style(name,height,bold=False):style = xlwt.XFStyle() # 初始化样式font = xlwt.Font() # 为样式创建字体font.name = name # 'Times New Roman'font.bold = boldfont.color_index = 4font.height = heightstyle.font = fontreturn styleimport datetimestarttime = datetime.datetime.now()index=0global rr=0wdk=xlwt.Workbook() sheet1=wdk.add_sheet('sheet1',cell_overwrite_ok=True) #excel单元格名字row=['招聘职位','月薪','学历要求','工作经验','应聘要求','工作单位']for i in row:sheet1.write(r,index,i)index+=1r+=1inte=[]def f(pn):url = '/quanzhi?tid=4139&ie=utf8&oe=utf8&query=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98%E5%B7%A5%E7%A8%8B%E5%B8%88&city_sug=%E9%95%BF%E6%B2%99&&detailmode=close&rn=20&pn='+str(pn)#打开网页wp = urllib.request.urlopen(url)content = wp.read()hrefPatten=r"(href='.{0,200}数据挖掘工程师')"href=hrefPatten.encode("utf-8")hrefC = re.findall(href, content, re.S) #返回所有匹配正则表达式的值于列表中hre=['']*len(hrefC)for i in range (len(hrefC)):hre[i]=urllib.parse.unquote(str(hrefC[i].decode('utf-8')).replace("href='","")[:-1])#将href转化为网址for i in range (len(hre)):a=requests.get(hre[i])content1=a.content.decode('utf-8')pattern=('<span class="title line-clamp1">(.*?)</span>.*?class="salary">(.*?)</span>.*?class="xueli"></span>(.*?)</li>.*?class="minge"></span>(.*?)</li>.*?class="duty duty-box">(.*?)</p>.*?class="line-clamp2">(.*?)</p>')items=re.findall(pattern, content1, re.S)#抓取招聘信息for item in items:flag=0for _item in item:if _item=="":flag=1if flag==0:inte.append(item)endtime = datetime.datetime.now()print((endtime - starttime).seconds)threads = []for pn in range(0,200,20):threads.append(threading.Thread(target=f,args=(pn,)))sleep(0.1)import datetimestarttime = datetime.datetime.now()if __name__ == '__main__':for t in threads:t.setDaemon(True)t.start()sleep(15)ind=1for item in inte:for k in range(0,6):sheet1.write(ind,k,item[k])ind+=1wdk.save('/home/t.xls')
15s完事
