自建代理ip池 - Python

##1. 安装ProxyPool 爬虫代理IP池
###1.1 下载安装包
	wget https://github.com/jhao104/proxy_pool/archive/2.1.0.tar.gz

###1.2 解压及安装依赖
	tar -zxvf 2.1.0.tar.gz
	cd proxy_pool-2.1.0
	pip3 install -r requirements.txt

###1.3 安装redis(可根据自己喜好，选择安装方式)
	#安装redis
	yum -y install redis
	#修改配置文件
	vim /etc/redis.conf
	requirepass 123456   #配置密码
	#启动
	systemctl start redis

##2. ProxyPool配置文件
	vim setting.py
	# 配置API服务
	HOST = "0.0.0.0"               # IP
	PORT = 5000                    # 监听端口

# 配置数据库
	DB_CONN = 'redis://:pwd@127.0.0.1:8888/0'

# 配置 ProxyFetcher
	PROXY_FETCHER = [
	    "freeProxy01",      # 这里是启用的代理抓取方法名，所有fetch方法位于fetcher/proxyFetcher.py
	    "freeProxy02",
	    # ....
	]

##3. 启动项目
	# 启动调度程序
	python proxyPool.py schedule

# 启动webApi服务
	python proxyPool.py server

##4. 程序使用代理池
	# -*- coding:utf-8 -*-
    import re
    import requests       
    import datetime
    import random
    import os
    import time
    import urllib
    
    
    def get_proxy():
        return requests.get("http://192.168.31.60:5010/get/").json()
    
    def delete_proxy(proxy):
        requests.get("http://192.168.31.60:5010/delete/?proxy={}".format(proxy))
    
    uaList = ['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (   KHTML, like Gecko) Chrome/80.0.3987.100 Safari/537.36']
    
    for i in range(0,100):
        TIME=random.randint(1,3)
        url = "http://www.bao-server.club"
        header={'User-Agent': random.choice(uaList)}
    
    
        proxy = get_proxy().get("proxy")
        print(proxy)
        httpproxy_handler = urllib.request.ProxyHandler({"http" : proxy})
        opener = urllib.request.build_opener(httpproxy_handler)
        request = urllib.request.Request(url,headers=header)
        try:
            response = opener.open(request)
        except Exception:
            delete_proxy(proxy)
    
        strhtml = response.read().decode('utf-8')
    
        time.sleep(TIME)