朋友说他的站挂了，想知道被收录的页面有多少是死链，于是我就想了一下流程，从Site获得收录数量当然是不精准的，不过也没有更好的地了，真实的收录只有搜索引擎数据库里面才有。。。
查询被收录页面的状态码，流程：获取收录网址 > 解析真实URL > 获取状态码
不过执行起来比较慢，不知道是BeautifulSoup还是 Location 获取真实URL地址这步慢了
#coding:utf-8import urllib2,re,requestsfrom bs4 import BeautifulSoup as bsdomain = '' #要查询的域名page_num = 10 * 10#第一个数字为要抓取的页数def gethtml(url):headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',# 'Accept-Encoding':'gzip, deflate, sdch','Accept-Language':'zh-CN,zh;q=0.8','Cache-Control':'max-age=0','Connection':'keep-alive','Cookie':'BDUSS=ng4UFVyUUpWU2hUR2R3b3hKamtpaE9ocW40LTFZcGdWeDBjbXkzdE83eDJQSE5YQVFBQUFBJCQAAAAAAAAAAAEAAADD3IYSamFjazE1NDUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHavS1d2r0tXa; ispeed_lsm=2; PSTM=1465195705; BIDUPSID=2274339847BBF9B1E97DA3ECE6469761; H_WISE_SIDS=102907_106764_106364_101556_100121_102478_102628_106368_103569_106502_106349_106665_106589_104341_106323_104000_104613_104638_106071_106599_106795; BAIDUID=D94A8DE66CF701AB5C3332B1BF883DDC:FG=1; BDSFRCVID=UEusJeC62m80hjJRoxzDhboaBeKaL6vTH6aIa6lTlb9Zx-72yRF7EG0PfOlQpYD-d1GyogKK3gOTH4jP; H_BDCLCKID_SF=fR-foIPbtKvSq5rvKbOEhPCX-fvQh4JXHD7yWCvG3455OR5Jj65Ve58JM46N2bvE3IbaWbjP5lvH8KQC3MA--fF_jxvn2PD8yj-L_KoXLqLbsq0x0-jchh_QWt8LKToxMCOMahkb5h7xOKbF056jK4JKjH0qt5cP; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a02157232777; BD_HOME=1; BD_UPN=12314353; sug=3; sugstore=1; ORIGIN=0; bdime=0; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; H_PS_645EC=a5cfUippkbo0uQPU%2F4QbUFVCqXu4W9g5gr5yrxTnJT10%2FElVEvJBbeyjWJq8QUHgepjd; BD_CK_SAM=1; BDSVRTM=323; H_PS_PSSID=1434_20317_12896_6_19860_17001_15506_11866; __bsi=16130066511508055252_00_0_I_R_326_0303_C02F_N_I_I_0',# 'Host':'','Upgrade-Insecure-Requests':'1','User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36',}req = urllib2.Request(url=url,headers=headers)html = urllib2.urlopen(req,timeout = 30).read()return htmldef status(url): #返回状态码status = requests.get(url).status_codereturn statusstatus_file = open('url_status.txt','a+')for i in range(10,page_num,10):url = '/s?wd=site%3A' + domain + '&pn=' + str(i)html = gethtml(url)soup = bs(html,"lxml")for i in soup.select('.c-showurl'):# print i.get('href')urls = i.get('href')# url_list.append(urls) header = requests.head(urls).headersheader_url = header['location'] #获取真实URLif int(status(header_url)) == 404:print status(header_url),header_url #打印状态码和真实URLstatus_file.write(str(status(header_url)) + ' ' + header_url + '\n') #获取的状态码和链接写入文件status_file.close()#获取状态码函数
借鉴的代码段
#coding: utf-8import sysimport urllibimport urllib2from BeautifulSoup import BeautifulSoupquestion_word = "吃货 程序员"url = "/s?wd=" + urllib.quote(question_word.decode(sys.stdin.encoding).encode('gbk'))htmlpage = urllib2.urlopen(url).read()soup = BeautifulSoup(htmlpage)print len(soup.findAll("table", {"class": "result"}))for result_table in soup.findAll("table", {"class": "result"}):a_click = result_table.find("a")print "-----标题----\n" + a_click.renderContents()#标题print "----链接----\n" + str(a_click.get("href"))#链接print "----描述----\n" + result_table.find("div", {"class": "c-abstract"}).renderContents()#描述print
