Feat/iplocate source and fresh proxy serving

.python-version

@@ -0,0 +1 @@
+proxy_pool

api/proxyApi.py

@@ -16,6 +16,7 @@
 """
 __author__ = 'JHao'
 
+import logging
 import platform
 from werkzeug.wrappers import Response
 from flask import Flask, jsonify, request
@@ -25,6 +26,10 @@
 from handler.proxyHandler import ProxyHandler
 from handler.configHandler import ConfigHandler
 
+# Suppress requestor IP logging from werkzeug logger to ensure zero-log requestor privacy
+log = logging.getLogger('werkzeug')
+log.setLevel(logging.ERROR)
+
 app = Flask(__name__)
 conf = ConfigHandler()
 proxy_handler = ProxyHandler()
@@ -42,31 +47,80 @@ def force_type(cls, response, environ=None):
 app.response_class = JsonResponse
 
 api_list = [
-    {"url": "/get", "params": "type: ''https'|''", "desc": "get a proxy"},
-    {"url": "/pop", "params": "", "desc": "get and delete a proxy"},
-    {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unable proxy"},
-    {"url": "/all", "params": "type: ''https'|''", "desc": "get all proxy from proxy pool"},
-    {"url": "/count", "params": "", "desc": "return proxy count"}
-    # 'refresh': 'refresh proxy pool',
+    {"url": "/get", "params": "type: 'https'|'', residential: 'true'|'false'", "desc": "get a proxy"},
+    {"url": "/pop", "params": "type: 'https'|'', residential: 'true'|'false'", "desc": "get and delete a proxy"},
+    {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unusable proxy"},
+    {"url": "/all", "params": "type: 'https'|'', residential: 'true'|'false', num: 'int'", "desc": "get all proxies from proxy pool"},
+    {"url": "/count", "params": "", "desc": "return proxy count statistics"}
 ]
 
 
+@app.before_request
+def verify_token():
+    auth_token = conf.authToken
+    if not auth_token:
+        return None
+
+    auth_header = request.headers.get("Authorization", "")
+    req_token = None
+    if auth_header.startswith("Bearer "):
+        req_token = auth_header[7:].strip()
+    elif auth_header.startswith("Token "):
+        req_token = auth_header[6:].strip()
+    elif auth_header:
+        req_token = auth_header.strip()
+
+    if not req_token:
+        req_token = (
+            request.headers.get("X-API-Token")
+            or request.headers.get("X-Auth-Token")
+            or request.headers.get("Api-Key")
+            or request.args.get("token")
+            or request.args.get("api_key")
+        )
+
+    if req_token != auth_token:
+        return jsonify({"code": 401, "src": "Unauthorized: invalid or missing token"}), 401
+
+
+def _parse_params():
+    req_type = request.args.get("type", "").lower()
+    https = (req_type == 'https')
+    res_arg = request.args.get("residential", "").lower() or request.args.get("is_residential", "").lower()
+    if res_arg in ["true", "1", "yes"] or req_type == "residential":
+        residential = True
+    elif res_arg in ["false", "0", "no"]:
+        residential = False
+    else:
+        residential = None
+
+    num_arg = request.args.get("num") or request.args.get("count")
+    num = None
+    if num_arg is not None:
+        try:
+            num = int(num_arg)
+        except ValueError:
+            num = None
+
+    return https, residential, num
+
+
 @app.route('/')
 def index():
     return {'url': api_list}
 
 
 @app.route('/get/')
 def get():
-    https = request.args.get("type", "").lower() == 'https'
-    proxy = proxy_handler.get(https)
+    https, residential, _ = _parse_params()
+    proxy = proxy_handler.get(https=https, residential=residential)
     return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
 
 
 @app.route('/pop/')
 def pop():
-    https = request.args.get("type", "").lower() == 'https'
-    proxy = proxy_handler.pop(https)
+    https, residential, _ = _parse_params()
+    proxy = proxy_handler.pop(https=https, residential=residential)
     return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
 
 
@@ -78,8 +132,8 @@ def refresh():
 
 @app.route('/all/')
 def getAll():
-    https = request.args.get("type", "").lower() == 'https'
-    proxies = proxy_handler.getAll(https)
+    https, residential, num = _parse_params()
+    proxies = proxy_handler.getAll(https=https, residential=residential, num=num)
     return jsonify([_.to_dict for _ in proxies])
 
 
@@ -95,12 +149,21 @@ def getCount():
     proxies = proxy_handler.getAll()
     http_type_dict = {}
     source_dict = {}
+    residential_count = 0
     for proxy in proxies:
         http_type = 'https' if proxy.https else 'http'
         http_type_dict[http_type] = http_type_dict.get(http_type, 0) + 1
+        if proxy.is_residential:
+            residential_count += 1
         for source in proxy.source.split('/'):
-            source_dict[source] = source_dict.get(source, 0) + 1
-    return {"http_type": http_type_dict, "source": source_dict, "count": len(proxies)}
+            if source:
+                source_dict[source] = source_dict.get(source, 0) + 1
+    return {
+        "http_type": http_type_dict,
+        "residential": residential_count,
+        "source": source_dict,
+        "count": len(proxies)
+    }
 
 
 def runFlask():
@@ -127,12 +190,16 @@ def load(self):
 
         _options = {
             'bind': '%s:%s' % (conf.serverHost, conf.serverPort),
-            'workers': 4,
-            'accesslog': '-',  # log to stdout
-            'access_log_format': '%(h)s %(l)s %(t)s "%(r)s" %(s)s "%(a)s"'
+            'workers': conf.gunicornWorkers,
+            'threads': conf.gunicornThreads,
+            'accesslog': None,  # Disable accesslog to enforce requestor privacy (zero logs)
+            'max_requests': 1000,
+            'max_requests_jitter': 50,
+            'timeout': 60
         }
         StandaloneApplication(app, _options).run()
 
 
 if __name__ == '__main__':
     runFlask()
+

db/dbClient.py

@@ -86,8 +86,8 @@ def __initDbClient(self):
                                                                                      password=self.db_pwd,
                                                                                      db=self.db_name)
 
-    def get(self, https, **kwargs):
-        return self.client.get(https, **kwargs)
+    def get(self, https=False, residential=None, **kwargs):
+        return self.client.get(https=https, residential=residential, **kwargs)
 
     def put(self, key, **kwargs):
         return self.client.put(key, **kwargs)
@@ -101,11 +101,12 @@ def delete(self, key, **kwargs):
     def exists(self, key, **kwargs):
         return self.client.exists(key, **kwargs)
 
-    def pop(self, https, **kwargs):
-        return self.client.pop(https, **kwargs)
+    def pop(self, https=False, residential=None, **kwargs):
+        return self.client.pop(https=https, residential=residential, **kwargs)
+
+    def getAll(self, https=False, residential=None, num=None, raw=False):
+        return self.client.getAll(https=https, residential=residential, num=num, raw=raw)
 
-    def getAll(self, https):
-        return self.client.getAll(https)
 
     def clear(self):
         return self.client.clear()

db/redisClient.py

@@ -17,6 +17,8 @@
 from redis.exceptions import TimeoutError, ConnectionError, ResponseError
 from redis.connection import BlockingConnectionPool
 from handler.logHandler import LogHandler
+from handler.configHandler import ConfigHandler
+from datetime import datetime
 from random import choice
 from redis import Redis
 import json
@@ -48,19 +50,72 @@ def __init__(self, **kwargs):
                                                                    protocol=2,
                                                                    **kwargs))
 
-    def get(self, https):
-        """
-        返回一个代理
+    def _filter_proxy(self, proxy_str, https=False, residential=None):
+        try:
+            data = json.loads(proxy_str)
+            if https and not data.get("https"):
+                return False
+            if residential is True and not (data.get("is_residential") or data.get("residential")):
+                return False
+            if residential is False and (data.get("is_residential") or data.get("residential")):
+                return False
+            return True
+        except Exception:
+            return False
+
+    @staticmethod
+    def _is_working(data):
+        """ 最近一次校验是否通过 """
+        status = data.get("last_status")
+        return status is True or str(status).lower() == "true"
+
+    @staticmethod
+    def _check_age(data):
+        """ 距上次校验的秒数; 无法解析时返回 None """
+        last_time = data.get("last_time") or ""
+        try:
+            checked = datetime.strptime(last_time, "%Y-%m-%d %H:%M:%S")
+        except (ValueError, TypeError):
+            return None
+        return (datetime.now() - checked).total_seconds()
+
+    def _live_proxies(self, https=False, residential=None):
+        """
+        返回「最近校验通过」的代理列表, 按校验时间从新到旧排序。
+        优先返回 PROXY_FRESH_SECONDS 秒内校验通过的代理;
+        若无满足时效要求的, 退回到所有校验通过的代理, 避免空池。
+        """
+        max_age = ConfigHandler().proxyFreshSeconds
+        working = []
+        for proxy_str in self.__conn.hvals(self.name):
+            if not self._filter_proxy(proxy_str, https=https, residential=residential):
+                continue
+            try:
+                data = json.loads(proxy_str)
+            except Exception:
+                continue
+            if not self._is_working(data):
+                continue
+            working.append((self._check_age(data), proxy_str))
+
+        def _sort_key(item):
+            age = item[0]
+            return age if age is not None else float("inf")
+
+        working.sort(key=_sort_key)
+        if max_age and max_age > 0:
+            fresh = [p for age, p in working if age is not None and age <= max_age]
+            if fresh:
+                return fresh
+        return [p for _, p in working]
+
+    def get(self, https=False, residential=None):
+        """
+        返回一个「最近校验通过」的代理
         :return:
         """
-        if https:
-            items = self.__conn.hvals(self.name)
-            proxies = list(filter(lambda x: json.loads(x).get("https"), items))
-            return choice(proxies) if proxies else None
-        else:
-            proxies = self.__conn.hkeys(self.name)
-            proxy = choice(proxies) if proxies else None
-            return self.__conn.hget(self.name, proxy) if proxy else None
+        proxies = self._live_proxies(https=https, residential=residential)
+        return choice(proxies) if proxies else None
 
     def put(self, proxy_obj):
         """
@@ -71,12 +126,12 @@ def put(self, proxy_obj):
         data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
         return data
 
-    def pop(self, https):
+    def pop(self, https=False, residential=None):
         """
         弹出一个代理
         :return: dict {proxy: value}
         """
-        proxy = self.get(https)
+        proxy = self.get(https=https, residential=residential)
         if proxy:
             self.__conn.hdel(self.name, json.loads(proxy).get("proxy", ""))
         return proxy if proxy else None
@@ -105,16 +160,21 @@ def update(self, proxy_obj):
         """
         return self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
 
-    def getAll(self, https):
+    def getAll(self, https=False, residential=None, num=None, raw=False):
         """
-        字典形式返回所有代理, 使用changeTable指定hash name
+        字典形式返回代理列表, 使用changeTable指定hash name。
+        raw=False(默认): 只返回「最近校验通过」的代理, 校验时间从新到旧排序(供 /get /all 使用);
+        raw=True: 返回池中全部代理(供调度器复检使用)。
         :return:
         """
-        items = self.__conn.hvals(self.name)
-        if https:
-            return list(filter(lambda x: json.loads(x).get("https"), items))
+        if raw:
+            items = self.__conn.hvals(self.name)
+            proxies = [x for x in items if self._filter_proxy(x, https=https, residential=residential)]
         else:
-            return items
+            proxies = self._live_proxies(https=https, residential=residential)
+        if num is not None and isinstance(num, int) and num >= 0:
+            proxies = proxies[:num]
+        return proxies
 
     def clear(self):
         """
@@ -128,8 +188,19 @@ def getCount(self):
         返回代理数量
         :return:
         """
-        proxies = self.getAll(https=False)
-        return {'total': len(proxies), 'https': len(list(filter(lambda x: json.loads(x).get("https"), proxies)))}
+        proxies = self.__conn.hvals(self.name)
+        parsed = []
+        for x in proxies:
+            try:
+                parsed.append(json.loads(x))
+            except Exception:
+                pass
+        return {
+            'total': len(parsed),
+            'https': len([p for p in parsed if p.get("https")]),
+            'residential': len([p for p in parsed if p.get("is_residential") or p.get("residential")])
+        }
+
 
     def changeTable(self, name):
         """