1. 获取代理IP
import reimport requestsproxies = []base_url = "/areaindex_35/1.html"head = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ''Chrome/81.0.4044.138 Safari/537.36','Cookie': 'Hm_lvt_1761fabf3c988e7f04bec51acd4073f4=1595389409; ''Hm_lpvt_1761fabf3c988e7f04bec51acd4073f4=1595397559 '}req1 = requests.get(url=base_url, headers=head)patten = re.findall("<tr><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td></tr>", req1.text)for index in range(1, len(patten)):proxy = patten[index]if proxy[0]+':'+proxy[1] not in proxies:proxies.append(proxy[0]+':'+proxy[1])print(proxies)print("共获取到{}个代理".format(len(proxies)))
此处使用正则表达式对网页表格数据进行提取
2. 验证代理是否可用
import telnetlibdef test_ip(ip, port):try:telnetlib.Telnet(ip, port, timeout=1)print("代理ip:{}:{}有效！".format(ip, port))except:pass
利用 Telnetlib 模块进行代理可用性测试
3. 汇总代码
import reimport telnetlibimport requestsdef test_ip(ip, port):try:telnetlib.Telnet(ip, port, timeout=1)print("代理ip:{}:{}有效！".format(ip, port))return Trueexcept:passreturn Falseproxies = []base_url = "/areaindex_35/1.html"head = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ''Chrome/81.0.4044.138 Safari/537.36','Cookie': 'Hm_lvt_1761fabf3c988e7f04bec51acd4073f4=1595389409; ''Hm_lpvt_1761fabf3c988e7f04bec51acd4073f4=1595397559 '}req1 = requests.get(url=base_url, headers=head)patten = re.findall("<tr><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td></tr>", req1.text)for index in range(1, len(patten)):proxy = patten[index]if proxy[0] + ':' + proxy[1] not in proxies:proxies.append(proxy[0] + ':' + proxy[1])# 可用代理usable_proxies = []for proxy in proxies:p = proxy.split(':')if test_ip(p[0], p[1]):usable_proxies.append(proxy)print("共获取到 {} 个代理，其中可用代理为：{} 个，详情如下:\n{}".format(len(proxies), len(usable_proxies),usable_proxies))
上述为获取代理并验证可用性的全部代码，请根据需求进行修改。
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：/xu_cxiang/article/details/112221737
