获取免费代理IP，并检验IP有效性

蔚落 2022-05-21 10:40 271阅读 0赞

爬虫程序访问网站，速度很快，很容易突破网站设置的访问次数，此情况下就会被停止访问，或者IP被封。如果此时能有一些代理IP，切换不同的代理IP去访问网站，使网站以为是从不同的机器上访问的，那么代理IP背后的自己的IP就不受影响了。就算用了代理IP也不要频繁访问网站，因为要为网站考虑一下它的压力。

1.从http://www.xicidaili.com/nn/1里获取免费代理IP。打开网页，查看源代码，分析代码结构，找到你需要的数据，用正则把 用它找出来。正则表达式是 r'<td>((\[1-9\]\\.|\[1-9\]\[0-9\]\\.|1\[0-9\]\{2\}\\.|2\[0-4\]\[0-9\]\\.|25\[0-5\]\\.)\{3\}(\[1-9\]|\[1-9\]\[0-9\]|1\[0-9\]\{2\}|2\[0-4\]\[0-9\]|25\[0-5\]))</td>\\s+<td>(\\d\{2,5\})</td>'。

2.把代理IP保存文件，留着用。但代理IP变化很快，有可能一会功夫就不能用了。所以在需要的时候抓取一下就行了。可以保存在文件里，也可以保存在数据库里。

3.检查代理IP有效性。这个操作可以放在每次抓取页面前，如果不能用就切换其他代理IP，同时把这个不能用的代理IP移除。

代码如下：分两个文件，一个获取代理IP，一个检查有效性(另外有[多进程检查][Link 1])。

# -*- coding: utf-8 -*- 
    '''
    从www.xicidaili.com获取代理IP，并保存文件
    '''
    import urllib.request as req
    import time
    import re
    import random
    
    text_html = r'd:/tmp/xici_html.txt'
    text_ips = r'd:/tmp/xici_ips.txt'
    
    class Getxi():
        def __init__(self,page):
            self.page = page
            self.url = r'http://www.xicidaili.com/nn/{}'
            
        def request_method(self,p):
            curr_time = time.time()
            sec = int(curr_time)
            micsec = int(round(curr_time*1000))
            print(sec,' == ',micsec)
            headers = {
                'Cache-Control':'max-age=0',
                'Connection': 'Keep-Alive',
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',            
                'Accept-Language': 'zh-CN,zh;q=0.8',
                'Accept-Enconding':'gzip, deflate, sdch',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
                'Host':'www.xicidili.com',
                'Referer':'http://www.xicidili.com/',
                'Pragma':'no-cache',
                'Upgrade-Insecure-Requests':1,
            }
            url_com = self.url.format(p)
            reqs = req.Request(url_com,headers=headers)
            return reqs
        def get_html(self,p):
            reqss = self.request_method(p)
            conn = req.urlopen(reqss)
            html = conn.read().decode('utf-8')
            return html
        def save_html(self,ip_html):
            with open(text_html,'a') as f:
                f.write(ip_html)
            f.close()
        def save_ips(self,ips):
            with open(text_ips,'a') as f:
                f.write(ips)
            f.close()
            
        def parse_html(self,ip_html):
            pattern = re.compile(r'<td>(([1-9]\.|[1-9][0-9]\.|1[0-9]{2}\.|2[0-4][0-9]\.|25[0-5]\.){3}([1-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))</td>\s+<td>(\d{2,5})</td>',re.S)
            tds = pattern.findall(ip_html)
            str1 = ''
            for td in tds:
                str1 += '{}:{}\n'.format(td[0].strip(),td[3].strip())
            #print(str1)
            self.save_ips(str1)
                    
        def crawler(self):
            for i in range(self.page):
                html = self.get_html(i+1)
                self.save_html(html)
                self.parse_html(html)
                time.sleep(random.randint(5,15))
                
            
    def xixi():
        page = 2
        xi = Getxi(page)
        xi.crawler()
           
    if __name__ == '__main__':
        xixi()

检查有效性：访问的网页是http://2018.ip138.com/ic.asp

# -*- coding: utf-8 -*- 
    '''
    验证代理IP的有效性
    '''
    from urllib import request
    import urllib
    import time
    import random
    import socket
    import http
    
    ips_ok_file = r'd:/tmp/xici_1_ok.txt' # 验证后，存入有效的IP
    ips_file = r'd:/tmp/xici_ips.txt' # IP列表 
    url = 'http://2018.ip138.com/ic.asp' # 检测访问ip
    User_Agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
    
    ok_ips = ''
    
    class CheckProxyIp():
        def __init__(self):
            
            pass
        def read_ips_file(self):
            with open(ips_file,'r',encoding='utf-8') as f:
                ips = f.readlines()
            f.close()
            for ip in ips:
                i = ip.strip()
                self.check_ips(i)
                time.sleep(random.randint(1,5))
    
        def check_ips(self,ip):
            global ok_ips
            proxy = {'http':ip,'https':ip}
            print(proxy)
            proxy_handler = request.ProxyHandler(proxy)
            opener = request.build_opener(proxy_handler)
            opener.addheaders = [('User-Agent',User_Agent)]
            request.install_opener(opener)
            try:
                response = request.urlopen(url,timeout=3) # 使用安装好的opener
                if(response.getcode() == 200):
                    html = response.read().decode('gbk')
                    print(len(html))
                    ok_ips += ip+'\n'
                else:
                    print('no')
            except UnicodeDecodeError as e:
                print(e)
            except urllib.error.HTTPError as e:
                print(e)
            except urllib.error.URLError as e:
                print(e)
            except socket.timeout as e:
                print(e)
            except http.client.RemoteDisconnected as e:
                print(e)
            except ConnectionResetError as e:
                print(e)
        def save_ok_ip(self):
            global ok_ips
            print('save ....')
            print(ok_ips)
            with open(ips_ok_file,'w') as f:
                f.write(ok_ips)
            f.close()
            
    def check():
        chcip = CheckProxyIp()
        chcip.read_ips_file()
        chcip.save_ok_ip()
    
    if __name__ == '__main__':
        check()

[Link 1]: https://blog.csdn.net/uvyoaa/article/details/81069033