import reimport openpyxlfrom urllib import request# 断点调试class Spider:url = 'https://www.panda.tv/cate/lol'root_pattern = '<div class="video-info">([\s\S]*?)</div>'name_pattern = '<span class="video-nickname"[\w\W]*?</i>([\s\S]*?)</span>'number_pattern = '<span class="video-number">([\s\S]*?)</span>'#获取页面def __fetch_content(self):r = request.urlopen(Spider.url)# bytes 将其转成字符串htmls = r.read()htmls = str(htmls,encoding='utf-8')return htmls#分析def __analysis(self, htmls):root_html = re.findall(Spider.root_pattern,htmls)anchors = []for html in root_html:name = re.findall(Spider.name_pattern,html)number = re.findall(Spider.number_pattern,html)anchor = {'name':name,'number':number}anchors.append(anchor)# print(anchors[0])return anchors#精炼/清洗def __refine(self, anchors):l = lambda anchor: {'name':anchor['name'][0].strip(),'number':anchor['number'][0]}return map(l,anchors)#排序def __sort(self, anchors):#py默认排序方法anchors = sorted(anchors,key=self.__sort__seed,reverse=True)return anchors#按xx排序def __sort__seed(self, anchor):r = re.findall('\d*',anchor['number'])number = float(r[0])if '万' in anchor['number']:number *= 10000return number#打印排序完的数据def __show(self, anchors):for rank in range(0,len(anchors)):print('rank ' + str(rank + 1)+ ' : ' + anchors[rank]['name']+ ' ' + anchors[rank]['number'])#导入excel表def __import_excel(self,anchors):#新建info = openpyxl.Workbook()sheet = info.activesheet.title = 'pandatv_sheet'title = ['排名','主播','人气']col = 1for t in title:sheet.cell(1,col,t)col += 1row = 2for anchor in anchors:col = 2for key,value in anchor.items():sheet.cell(row,1,row-1)sheet.cell(row,col,value)col += 1row += 1info.save('pandatv_sheet.xlsx')#入口/总控方法def go(self):htmls = self.__fetch_content()anchors = self.__analysis(htmls)anchors = list(self.__refine(anchors))anchors = self.__sort(anchors)# print(anchors)self.__import_excel(anchors)# self.__show(anchors)spider = Spider()spider.go()
