diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..8a945f94a --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +proxy_pool diff --git a/README.md b/README.md index 640c1e157..d8ae00999 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,10 @@ +# ProxyPool - Production-Ready Crawler Proxy IP Pool -ProxyPool 爬虫代理IP池 -======= -[![Tests](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml/badge.svg)](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml) -[![codecov](https://codecov.io/gh/jhao104/proxy_pool/graph/badge.svg?token=8WHGkrQA6E)](https://codecov.io/gh/jhao104/proxy_pool) -[![](https://img.shields.io/badge/Powered%20by-@j_hao104-green.svg)](http://www.spiderpy.cn/blog/) -[![Packagist](https://img.shields.io/packagist/l/doctrine/orm.svg)](https://github.com/jhao104/proxy_pool/blob/master/LICENSE) -[![GitHub contributors](https://img.shields.io/github/contributors/jhao104/proxy_pool.svg)](https://github.com/jhao104/proxy_pool/graphs/contributors) -[![](https://img.shields.io/badge/language-Python-green.svg)](https://github.com/jhao104/proxy_pool) +[![Tests](https://github.com/anand870/proxy_pool/actions/workflows/test.yml/badge.svg)](https://github.com/anand870/proxy_pool/actions/workflows/test.yml) +[![Python Version](https://img.shields.io/badge/Python-3.8--3.12-blue.svg)](https://docs.python.org/3/) +[![License](https://img.shields.io/packagist/l/doctrine/orm.svg)](LICENSE) +``` ______ ______ _ | ___ \_ | ___ \ | | | |_/ / \__ __ __ _ __ _ | |_/ /___ ___ | | @@ -16,221 +13,304 @@ ProxyPool 爬虫代理IP池 \_| |_| \___/ /_/\_\ \__ |\_| \___/ \___/ \_____\ __ / / /___ / +``` -### ProxyPool - -爬虫代理IP池项目,主要功能为定时采集网上发布的免费代理验证入库,定时验证入库的代理保证代理的可用性,提供API和CLI两种使用方式。同时你也可以扩展代理源以增加代理池IP的质量和数量。 - -* 文档: [document](https://jhao104.github.io/proxy_pool/) - -* 支持版本: -[![](https://img.shields.io/badge/Python-3.8-blue.svg)](https://docs.python.org/3.8/) -[![](https://img.shields.io/badge/Python-3.9-blue.svg)](https://docs.python.org/3.9/) -[![](https://img.shields.io/badge/Python-3.10-blue.svg)](https://docs.python.org/3.10/) -[![](https://img.shields.io/badge/Python-3.11-blue.svg)](https://docs.python.org/3.11/) - -* 测试地址: http://demo.spiderpy.cn (勿压谢谢) +An automated, high-performance proxy pool for web scrapers and crawlers. It automatically fetches free public proxies, validates their availability on a scheduled basis, persists healthy proxies in Redis/SSDB, and exposes them via a lightweight RESTful API server. -* 付费代理推荐: [亮数据 Bright Data](https://get.brightdata.com/github_jh)(前身 Luminati).全球代理与网络抓取行业头部领导者。覆盖 195+ 国家的 1.5亿+ 真人住宅IP,亲测成功率极高,轻松突破反爬封锁。需要高质量代理IP的可以注册后联系中文客服。[申请免费试用](https://get.brightdata.com/github_jh) (PS:用不明白的同学可以参考这个[使用教程](https://www.cnblogs.com/jhao/p/15611785.html))。 +--- -   想自建爬虫?接入 [Bright Data MCP Server](https://get.brightdata.com/cd3yy5),让 Claude、Cursor、Windsurf 等 AI 助手直接实时抓取网页——自动破解验证码、绕过地区限制。[Scraper Studio](https://get.brightdata.com/cd3yy5) 支持 AI 一键生成或 JS 代码定制,全托管基础设施运行,无需自购代理、无需搭服务器,分钟级上线。所有产品底层均由同一套顶级代理网络驱动。 +## Key Features -   API 产品现享7折 + 免费试用额度,注册后可联系中文客服快速上手。(用不明白的同学可参考使用教程,或注册后直接使用互动 AI 智能助手) -👉 [https://get.brightdata.com/cd3yy5](https://get.brightdata.com/cd3yy5) +- 🚀 **Production Ready for Cloud Free Tiers**: Optimized memory footprint (<300MB) for seamless deployment on **Google Cloud Compute Engine `e2-micro`** and **Oracle Cloud Always Free VM** instances. +- 🏡 **Residential IP Filtering**: Filter and fetch only residential proxies on-demand (`?residential=true` or `?type=residential`). +- 🔐 **Token-Based Authentication System**: Protect your API endpoints with configurable token authentication (`Authorization: Bearer `, `X-API-Token`, or `?token=`). +- 🛡️ **Zero-Log Requestor Privacy**: Strict privacy protection — no client IP addresses, headers, or requestor details are ever logged or retained in log files. +- ⚡ **Automated Scheduling & Validation**: Built-in APScheduler constantly checks proxy latency, availability, and failure thresholds. +- 🔌 **Pluggable Architecture**: Modular proxy fetchers with automatic directory discovery and runtime hot-reloading. +--- -### 运行项目 +## Quick Start & Local Setup -##### 下载代码: +### Prerequisites +- Python 3.8+ +- Redis Server (local or remote) -* git clone +### Installation ```bash -git clone https://github.com/jhao104/proxy_pool.git -``` - -* releases +# Clone the repository +git clone https://github.com/anand870/proxy_pool.git +cd proxy_pool -```bash -https://github.com/jhao104/proxy_pool/releases 下载对应zip文件 +# Install dependencies +pip install -r requirements.txt ``` -##### 安装依赖: +### Running Locally ```bash -pip install -r requirements.txt -``` +# Start the proxy scheduler (fetcher & validator daemon) +python proxyPool.py schedule -##### 更新配置: +# Start the Web API server +python proxyPool.py server +``` +--- -```python -# setting.py 为项目配置文件 +## Production Deployment (Google Cloud `e2-micro`) -# 配置API服务 +### Method 1: Deploy to Remote GCP VM from Local Machine (One-Command) -HOST = "0.0.0.0" # IP -PORT = 5000 # 监听端口 +Run `remote_deploy.sh` directly from your local terminal. It will connect via SSH to your running GCP `e2-micro` VM, install `git` & dependencies, configure GCP firewall rules for port `5010`, pull code, and start the service: +```bash +# Basic usage (Target instance name: my-gcp-micro-instance, Zone: us-central1-a) +./remote_deploy.sh -# 配置数据库 +# Deploy with custom instance name, zone, and secret AUTH_TOKEN: +INSTANCE_NAME="my-proxy-vm" ZONE="us-central1-a" AUTH_TOKEN="my_secret_token_123" ./remote_deploy.sh +``` -DB_CONN = 'redis://:pwd@127.0.0.1:8888/0' +### Method 2: Deploy Directly on Host Machine (Systemd) +On your GCP Compute Engine VM host instance, run the local deployment script: -# 配置代理源(可选) -# 默认自动扫描 fetcher/sources/ 目录下所有 enabled=True 的代理源 -# 如需禁用某些代理源,在黑名单中添加其 name 即可 -# PROXY_FETCHER_EXCLUDE = ["freevpnnode"] +```bash +chmod +x deploy.sh proxy_pool.sh +./deploy.sh ``` - -#### 启动项目: +This script will: +1. Create a Python virtual environment (`venv`) and install dependencies. +2. Generate `.env` configuration template. +3. Install and register `proxy_pool.service` under `systemd`. +4. Start the production service via Gunicorn. ```bash -# 如果已经具备运行条件, 可用通过proxyPool.py启动。 -# 程序分为: schedule 调度程序 和 server Api服务 - -# 启动调度程序 -python proxyPool.py schedule - -# 启动webApi服务 -python proxyPool.py server - +# Manage via systemd +sudo systemctl start proxy_pool +sudo systemctl status proxy_pool +sudo systemctl stop proxy_pool +sudo systemctl restart proxy_pool ``` -### Docker Image +### Checking Server Status & Logs + +To verify if the server is running, check process logs, or inspect port binding: + +- **Stream Live Logs**: + ```bash + sudo journalctl -u proxy_pool.service -f + ``` +- **View Recent Log History**: + ```bash + sudo journalctl -u proxy_pool.service -n 50 --no-pager + ``` +- **Check Service Health & Status**: + ```bash + sudo systemctl status proxy_pool + ``` +- **Verify Port 5010 Binding**: + ```bash + sudo lsof -i :5010 + ``` + +### Method 3: Containerized Deployment (Docker & Docker Compose) ```bash -docker pull jhao104/proxy_pool +# Set your token in environment (optional) +export AUTH_TOKEN="my_secret_token_123" -docker run --env DB_CONN=redis://:password@ip:port/0 -p 5010:5010 jhao104/proxy_pool:latest -``` -### docker-compose - -项目目录下运行: -``` bash +# Build and launch with memory limits docker-compose up -d ``` -### 使用 - -* Api +--- -启动web服务后, 默认配置下会开启 http://127.0.0.1:5010 的api接口服务: +## Token Authentication -| api | method | Description | params| -| ----| ---- | ---- | ----| -| / | GET | api介绍 | None | -| /get | GET | 随机获取一个代理| 可选参数: `?type=https` 过滤支持https的代理| -| /pop | GET | 获取并删除一个代理| 可选参数: `?type=https` 过滤支持https的代理| -| /all | GET | 获取所有代理 |可选参数: `?type=https` 过滤支持https的代理| -| /count | GET | 查看代理数量 |None| -| /delete | GET | 删除代理 |`?proxy=host:ip`| +Set `AUTH_TOKEN` in your environment or `.env` file to enable token protection: +```ini +AUTH_TOKEN=my_secret_token_123 +``` -* 爬虫使用 - -  如果要在爬虫代码中使用的话, 可以将此api封装成函数直接使用,例如: +When enabled, API requests must include the token via any of the following 3 formats: + +1. **HTTP Authorization Header** *(Recommended)*: + ```bash + curl -H "Authorization: Bearer my_secret_token_123" "http://:5010/get/" + ``` +2. **Custom Header**: + ```bash + curl -H "X-API-Token: my_secret_token_123" "http://:5010/get/" + ``` +3. **URL Query Parameter**: + ```bash + curl "http://:5010/get/?token=my_secret_token_123" + ``` + +*If the token is missing or invalid, the API responds with `HTTP 401 Unauthorized`.* + +--- + +## Residential IP Filtering & Count Parameter + +Filter and fetch proxies marked as residential or datacenter IPs using `?residential=true|false`, or limit returned proxies using `?num=N`: + +- **Fetch a random residential proxy**: + ```bash + curl "http://127.0.0.1:5010/get/?residential=true" + ``` +- **Fetch a random non-residential (datacenter) proxy**: + ```bash + curl "http://127.0.0.1:5010/get/?residential=false" + ``` +- **Pop and delete a residential proxy**: + ```bash + curl "http://127.0.0.1:5010/pop/?residential=true" + ``` +- **List residential proxies with quantity limit**: + ```bash + curl "http://127.0.0.1:5010/all/?residential=true&num=5" + ``` +- **List non-residential proxies**: + ```bash + curl "http://127.0.0.1:5010/all/?residential=false" + ``` +- **Proxy count statistics (including residential count)**: + ```bash + curl "http://127.0.0.1:5010/count/" + # Response: {"http_type": {"http": 10, "https": 5}, "residential": 6, "source": {...}, "count": 15} + ``` + +--- + +## Zero-Log Requestor Privacy + +To ensure complete requestor anonymity: +- **Werkzeug Logger**: Suppressed HTTP access logging of client IP addresses. +- **Gunicorn Server**: Configured with `accesslog = None` in production, ensuring client IP (`%(h)s`) and User-Agent (`%(a)s`) are never recorded or written to disk. + +--- + +## RESTful API Reference + +| Endpoint | Method | Description | Parameters | +|----------|--------|-------------|------------| +| `/` | GET | List available API routes | None | +| `/get` | GET | Get a random proxy | `type=https`, `residential=true|false` | +| `/pop` | GET | Get and delete a proxy | `type=https`, `residential=true|false` | +| `/all` | GET | Get proxies from pool | `type=https`, `residential=true|false`, `num=N` | +| `/count` | GET | Get proxy count & statistics | None | +| `/delete` | GET | Delete an invalid proxy | `proxy=host:port` | + +### Usage Example in Python (Scraper Integration) ```python import requests -def get_proxy(): - return requests.get("http://127.0.0.1:5010/get/").json() +API_URL = "http://127.0.0.1:5010" +HEADERS = {"Authorization": "Bearer my_secret_token_123"} -def delete_proxy(proxy): - requests.get("http://127.0.0.1:5010/delete/?proxy={}".format(proxy)) +def get_residential_proxy(): + response = requests.get(f"{API_URL}/get/", params={"residential": "true"}, headers=HEADERS) + return response.json().get("proxy") -# your spider code +def delete_proxy(proxy): + requests.get(f"{API_URL}/delete/", params={"proxy": proxy}, headers=HEADERS) -def getHtml(): - # .... +def fetch_target_page(url): retry_count = 5 - proxy = get_proxy().get("proxy") while retry_count > 0: + proxy = get_residential_proxy() + if not proxy: + break try: - html = requests.get('http://www.example.com', proxies={"http": "http://{}".format(proxy)}) - # 使用代理访问 - return html + resp = requests.get(url, proxies={"http": f"http://{proxy}", "https": f"http://{proxy}"}, timeout=10) + if resp.status_code == 200: + return resp.text except Exception: retry_count -= 1 - # 删除代理池中代理 - delete_proxy(proxy) + delete_proxy(proxy) return None ``` -### 扩展代理 - -  项目默认包含几个免费的代理获取源,但是免费的毕竟质量有限,所以如果直接运行可能拿到的代理质量不理想。所以,提供了代理获取的扩展方法。 +--- -  添加一个新的代理源方法如下: +## Extending Proxy Sources -* 1、在 `fetcher/sources/` 目录下新建 `.py` 文件,继承 `BaseFetcher` 基类,声明 `name`/`url`/`enabled` 属性,实现 `fetch()` 方法以生成器(yield)形式返回`host:port`格式的代理,例如: +Create a new `.py` file in `fetcher/sources/` inheriting from `BaseFetcher`: ```python from fetcher.baseFetcher import BaseFetcher from util.webRequest import WebRequest -class MyProxyFetcher(BaseFetcher): - """我的代理源""" - - name = "myproxy" +class CustomFetcher(BaseFetcher): + name = "custom_fetcher" url = "https://www.example.com/" enabled = True + is_residential = True # Tag proxies from this source as residential def fetch(self): - r = WebRequest().get("https://www.example.com/api/proxies") - for item in r.json: - yield item["ip"] + ":" + item["port"] + response = WebRequest().get("https://www.example.com/api/proxies") + for item in response.json: + yield f"{item['ip']}:{item['port']}" ``` -* 2、添加好后,`schedule` 进程下次抓取时会自动扫描 `fetcher/sources/` 目录并启用新代理源,无需修改配置。 - -  可用 `python proxyPool.py fetcher` 命令查看当前启用的代理源列表。 +The scheduler automatically scans `fetcher/sources/` on the next iteration and enables new fetchers without restarting the application. -  如需临时禁用某个代理源,在 [setting.py](setting.py) 的 `PROXY_FETCHER_EXCLUDE` 黑名单中添加其 `name` 即可。 +--- -### 免费代理源 +## Supported Public Proxy Sources - 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): - - | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | - |--------------| ---- |------|-----|---------------------------------------------------|---------------------------------------------------------| - | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | - | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | - | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | - | 小幻代理 | ✔ | ★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | - | 89代理 | ✔ | ★★ | ** | [地址](https://www.89ip.cn) | [`ip89.py`](/fetcher/sources/ip89.py) | - | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | - | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | - | 66代理 | ✔ | ★★ | * | [地址](https://www.66daili.com) | [`daili66.py`](/fetcher/sources/daili66.py) | - | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | - | FreeVPNNode | ✔ | ★★ | * | [地址](https://cn.freevpnnode.com) | [`freevpnnode.py`](/fetcher/sources/freevpnnode.py) | - | Geonode | ✔ | ★★ | ** | [地址](https://geonode.com) | [`geonode.py`](/fetcher/sources/geonode.py) | - | RoundProxies | ✔ | ★ | * | [地址](https://roundproxies.com/free-proxy-list) | [`roundproxies.py`](/fetcher/sources/roundproxies.py) | +| Source Name | Enabled | Residential Tag | File Path | +|-------------|---------|-----------------|-----------| +| Geonode | ✔ | Yes | [`geonode.py`](fetcher/sources/geonode.py) | +| RoundProxies | ✔ | Yes | [`roundproxies.py`](fetcher/sources/roundproxies.py) | +| Proxifly | ✔ | No | [`proxifly.py`](fetcher/sources/proxifly.py) | +| Kuaidaili | ✔ | No | [`kuaidaili.py`](fetcher/sources/kuaidaili.py) | +| Kxdaili | ✔ | No | [`kxdaili.py`](fetcher/sources/kxdaili.py) | +| IP3366 | ✔ | No | [`ip3366.py`](fetcher/sources/ip3366.py) | +| Ihuan | ✔ | No | [`ihuan.py`](fetcher/sources/ihuan.py) | +| IP89 | ✔ | No | [`ip89.py`](fetcher/sources/ip89.py) | +| DocIP | ✔ | No | [`docip.py`](fetcher/sources/docip.py) | +| GoodIPs | ✔ | No | [`goodips.py`](fetcher/sources/goodips.py) | +| Daili66 | ✔ | No | [`daili66.py`](fetcher/sources/daili66.py) | +| FreeVPNNode | ✔ | No | [`freevpnnode.py`](fetcher/sources/freevpnnode.py) | - - 如果还有其他好的免费代理网站, 可以在提交在[issues](https://github.com/jhao104/proxy_pool/issues/71), 下次更新时会考虑在项目中支持。 +--- -### 问题反馈 +## Configuration Reference (`setting.py` / `.env`) -  任何问题欢迎在[Issues](https://github.com/jhao104/proxy_pool/issues) 中反馈,同时也可以到我的[博客](http://www.spiderpy.cn/blog/message)中留言。 +| Variable | Default | Description | +|----------|---------|-------------| +| `HOST` | `0.0.0.0` | API server binding address | +| `PORT` | `5010` | API server port | +| `AUTH_TOKEN` | `""` | API token authentication (empty = disabled) | +| `DB_CONN` | `redis://:pwd@127.0.0.1:6379/0` | DB connection string (Redis or SSDB) | +| `TABLE_NAME` | `use_proxy` | Database table / hash key name | +| `GUNICORN_WORKERS` | `2` | Gunicorn worker processes (optimized for 1GB RAM) | +| `GUNICORN_THREADS` | `2` | Threads per Gunicorn worker | +| `VERIFY_TIMEOUT` | `10` | Proxy validation timeout in seconds | +| `POOL_SIZE_MIN` | `20` | Minimum proxy threshold before triggering re-fetch | +| `TIMEZONE` | `Asia/Shanghai` | Scheduler timezone | -  你的反馈会让此项目变得更加完美。 +--- -### 贡献代码 +## Running Tests -  本项目仅作为基本的通用的代理池架构,不接收特有功能(当然,不限于特别好的idea)。 - -  本项目依然不够完善,如果发现bug或有新的功能添加,请在[Issues](https://github.com/jhao104/proxy_pool/issues)中提交bug(或新功能)描述,我会尽力改进,使她更加完美。 - -  这里感谢以下contributor的无私奉献: - -  [@kangnwh](https://github.com/kangnwh) | [@bobobo80](https://github.com/bobobo80) | [@halleywj](https://github.com/halleywj) | [@newlyedward](https://github.com/newlyedward) | [@wang-ye](https://github.com/wang-ye) | [@gladmo](https://github.com/gladmo) | [@bernieyangmh](https://github.com/bernieyangmh) | [@PythonYXY](https://github.com/PythonYXY) | [@zuijiawoniu](https://github.com/zuijiawoniu) | [@netAir](https://github.com/netAir) | [@scil](https://github.com/scil) | [@tangrela](https://github.com/tangrela) | [@highroom](https://github.com/highroom) | [@luocaodan](https://github.com/luocaodan) | [@vc5](https://github.com/vc5) | [@1again](https://github.com/1again) | [@obaiyan](https://github.com/obaiyan) | [@zsbh](https://github.com/zsbh) | [@jiannanya](https://github.com/jiannanya) | [@Jerry12228](https://github.com/Jerry12228) | [@zeyudada](https://github.com/zeyudada) +```bash +# Run unit & API test suite +pytest +# View coverage report +pytest --cov=. --cov-report=term-missing +``` -### Release Notes +--- - [changelog](https://jhao104.github.io/proxy_pool/changelog/) +## License -Featured|HelloGitHub +Distributed under the MIT License. See [LICENSE](LICENSE) for details. diff --git a/api/proxyApi.py b/api/proxyApi.py index bd2de57e2..5e7f2cdac 100644 --- a/api/proxyApi.py +++ b/api/proxyApi.py @@ -16,6 +16,7 @@ """ __author__ = 'JHao' +import logging import platform from werkzeug.wrappers import Response from flask import Flask, jsonify, request @@ -25,6 +26,10 @@ from handler.proxyHandler import ProxyHandler from handler.configHandler import ConfigHandler +# Suppress requestor IP logging from werkzeug logger to ensure zero-log requestor privacy +log = logging.getLogger('werkzeug') +log.setLevel(logging.ERROR) + app = Flask(__name__) conf = ConfigHandler() proxy_handler = ProxyHandler() @@ -42,15 +47,64 @@ def force_type(cls, response, environ=None): app.response_class = JsonResponse api_list = [ - {"url": "/get", "params": "type: ''https'|''", "desc": "get a proxy"}, - {"url": "/pop", "params": "", "desc": "get and delete a proxy"}, - {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unable proxy"}, - {"url": "/all", "params": "type: ''https'|''", "desc": "get all proxy from proxy pool"}, - {"url": "/count", "params": "", "desc": "return proxy count"} - # 'refresh': 'refresh proxy pool', + {"url": "/get", "params": "type: 'https'|'', residential: 'true'|'false'", "desc": "get a proxy"}, + {"url": "/pop", "params": "type: 'https'|'', residential: 'true'|'false'", "desc": "get and delete a proxy"}, + {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unusable proxy"}, + {"url": "/all", "params": "type: 'https'|'', residential: 'true'|'false', num: 'int'", "desc": "get all proxies from proxy pool"}, + {"url": "/count", "params": "", "desc": "return proxy count statistics"} ] +@app.before_request +def verify_token(): + auth_token = conf.authToken + if not auth_token: + return None + + auth_header = request.headers.get("Authorization", "") + req_token = None + if auth_header.startswith("Bearer "): + req_token = auth_header[7:].strip() + elif auth_header.startswith("Token "): + req_token = auth_header[6:].strip() + elif auth_header: + req_token = auth_header.strip() + + if not req_token: + req_token = ( + request.headers.get("X-API-Token") + or request.headers.get("X-Auth-Token") + or request.headers.get("Api-Key") + or request.args.get("token") + or request.args.get("api_key") + ) + + if req_token != auth_token: + return jsonify({"code": 401, "src": "Unauthorized: invalid or missing token"}), 401 + + +def _parse_params(): + req_type = request.args.get("type", "").lower() + https = (req_type == 'https') + res_arg = request.args.get("residential", "").lower() or request.args.get("is_residential", "").lower() + if res_arg in ["true", "1", "yes"] or req_type == "residential": + residential = True + elif res_arg in ["false", "0", "no"]: + residential = False + else: + residential = None + + num_arg = request.args.get("num") or request.args.get("count") + num = None + if num_arg is not None: + try: + num = int(num_arg) + except ValueError: + num = None + + return https, residential, num + + @app.route('/') def index(): return {'url': api_list} @@ -58,15 +112,15 @@ def index(): @app.route('/get/') def get(): - https = request.args.get("type", "").lower() == 'https' - proxy = proxy_handler.get(https) + https, residential, _ = _parse_params() + proxy = proxy_handler.get(https=https, residential=residential) return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} @app.route('/pop/') def pop(): - https = request.args.get("type", "").lower() == 'https' - proxy = proxy_handler.pop(https) + https, residential, _ = _parse_params() + proxy = proxy_handler.pop(https=https, residential=residential) return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} @@ -78,8 +132,8 @@ def refresh(): @app.route('/all/') def getAll(): - https = request.args.get("type", "").lower() == 'https' - proxies = proxy_handler.getAll(https) + https, residential, num = _parse_params() + proxies = proxy_handler.getAll(https=https, residential=residential, num=num) return jsonify([_.to_dict for _ in proxies]) @@ -95,12 +149,21 @@ def getCount(): proxies = proxy_handler.getAll() http_type_dict = {} source_dict = {} + residential_count = 0 for proxy in proxies: http_type = 'https' if proxy.https else 'http' http_type_dict[http_type] = http_type_dict.get(http_type, 0) + 1 + if proxy.is_residential: + residential_count += 1 for source in proxy.source.split('/'): - source_dict[source] = source_dict.get(source, 0) + 1 - return {"http_type": http_type_dict, "source": source_dict, "count": len(proxies)} + if source: + source_dict[source] = source_dict.get(source, 0) + 1 + return { + "http_type": http_type_dict, + "residential": residential_count, + "source": source_dict, + "count": len(proxies) + } def runFlask(): @@ -127,12 +190,16 @@ def load(self): _options = { 'bind': '%s:%s' % (conf.serverHost, conf.serverPort), - 'workers': 4, - 'accesslog': '-', # log to stdout - 'access_log_format': '%(h)s %(l)s %(t)s "%(r)s" %(s)s "%(a)s"' + 'workers': conf.gunicornWorkers, + 'threads': conf.gunicornThreads, + 'accesslog': None, # Disable accesslog to enforce requestor privacy (zero logs) + 'max_requests': 1000, + 'max_requests_jitter': 50, + 'timeout': 60 } StandaloneApplication(app, _options).run() if __name__ == '__main__': runFlask() + diff --git a/db/dbClient.py b/db/dbClient.py index 4d9554b18..4356c6c3a 100644 --- a/db/dbClient.py +++ b/db/dbClient.py @@ -86,8 +86,8 @@ def __initDbClient(self): password=self.db_pwd, db=self.db_name) - def get(self, https, **kwargs): - return self.client.get(https, **kwargs) + def get(self, https=False, residential=None, **kwargs): + return self.client.get(https=https, residential=residential, **kwargs) def put(self, key, **kwargs): return self.client.put(key, **kwargs) @@ -101,11 +101,12 @@ def delete(self, key, **kwargs): def exists(self, key, **kwargs): return self.client.exists(key, **kwargs) - def pop(self, https, **kwargs): - return self.client.pop(https, **kwargs) + def pop(self, https=False, residential=None, **kwargs): + return self.client.pop(https=https, residential=residential, **kwargs) + + def getAll(self, https=False, residential=None, num=None, raw=False): + return self.client.getAll(https=https, residential=residential, num=num, raw=raw) - def getAll(self, https): - return self.client.getAll(https) def clear(self): return self.client.clear() diff --git a/db/redisClient.py b/db/redisClient.py index 5f17e4c5a..aea9d993c 100644 --- a/db/redisClient.py +++ b/db/redisClient.py @@ -17,6 +17,8 @@ from redis.exceptions import TimeoutError, ConnectionError, ResponseError from redis.connection import BlockingConnectionPool from handler.logHandler import LogHandler +from handler.configHandler import ConfigHandler +from datetime import datetime from random import choice from redis import Redis import json @@ -48,19 +50,72 @@ def __init__(self, **kwargs): protocol=2, **kwargs)) - def get(self, https): - """ - 返回一个代理 + def _filter_proxy(self, proxy_str, https=False, residential=None): + try: + data = json.loads(proxy_str) + if https and not data.get("https"): + return False + if residential is True and not (data.get("is_residential") or data.get("residential")): + return False + if residential is False and (data.get("is_residential") or data.get("residential")): + return False + return True + except Exception: + return False + + @staticmethod + def _is_working(data): + """ 最近一次校验是否通过 """ + status = data.get("last_status") + return status is True or str(status).lower() == "true" + + @staticmethod + def _check_age(data): + """ 距上次校验的秒数; 无法解析时返回 None """ + last_time = data.get("last_time") or "" + try: + checked = datetime.strptime(last_time, "%Y-%m-%d %H:%M:%S") + except (ValueError, TypeError): + return None + return (datetime.now() - checked).total_seconds() + + def _live_proxies(self, https=False, residential=None): + """ + 返回「最近校验通过」的代理列表, 按校验时间从新到旧排序。 + 优先返回 PROXY_FRESH_SECONDS 秒内校验通过的代理; + 若无满足时效要求的, 退回到所有校验通过的代理, 避免空池。 + """ + max_age = ConfigHandler().proxyFreshSeconds + working = [] + for proxy_str in self.__conn.hvals(self.name): + if not self._filter_proxy(proxy_str, https=https, residential=residential): + continue + try: + data = json.loads(proxy_str) + except Exception: + continue + if not self._is_working(data): + continue + working.append((self._check_age(data), proxy_str)) + + def _sort_key(item): + age = item[0] + return age if age is not None else float("inf") + + working.sort(key=_sort_key) + if max_age and max_age > 0: + fresh = [p for age, p in working if age is not None and age <= max_age] + if fresh: + return fresh + return [p for _, p in working] + + def get(self, https=False, residential=None): + """ + 返回一个「最近校验通过」的代理 :return: """ - if https: - items = self.__conn.hvals(self.name) - proxies = list(filter(lambda x: json.loads(x).get("https"), items)) - return choice(proxies) if proxies else None - else: - proxies = self.__conn.hkeys(self.name) - proxy = choice(proxies) if proxies else None - return self.__conn.hget(self.name, proxy) if proxy else None + proxies = self._live_proxies(https=https, residential=residential) + return choice(proxies) if proxies else None def put(self, proxy_obj): """ @@ -71,12 +126,12 @@ def put(self, proxy_obj): data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json) return data - def pop(self, https): + def pop(self, https=False, residential=None): """ 弹出一个代理 :return: dict {proxy: value} """ - proxy = self.get(https) + proxy = self.get(https=https, residential=residential) if proxy: self.__conn.hdel(self.name, json.loads(proxy).get("proxy", "")) return proxy if proxy else None @@ -105,16 +160,21 @@ def update(self, proxy_obj): """ return self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json) - def getAll(self, https): + def getAll(self, https=False, residential=None, num=None, raw=False): """ - 字典形式返回所有代理, 使用changeTable指定hash name + 字典形式返回代理列表, 使用changeTable指定hash name。 + raw=False(默认): 只返回「最近校验通过」的代理, 校验时间从新到旧排序(供 /get /all 使用); + raw=True: 返回池中全部代理(供调度器复检使用)。 :return: """ - items = self.__conn.hvals(self.name) - if https: - return list(filter(lambda x: json.loads(x).get("https"), items)) + if raw: + items = self.__conn.hvals(self.name) + proxies = [x for x in items if self._filter_proxy(x, https=https, residential=residential)] else: - return items + proxies = self._live_proxies(https=https, residential=residential) + if num is not None and isinstance(num, int) and num >= 0: + proxies = proxies[:num] + return proxies def clear(self): """ @@ -128,8 +188,19 @@ def getCount(self): 返回代理数量 :return: """ - proxies = self.getAll(https=False) - return {'total': len(proxies), 'https': len(list(filter(lambda x: json.loads(x).get("https"), proxies)))} + proxies = self.__conn.hvals(self.name) + parsed = [] + for x in proxies: + try: + parsed.append(json.loads(x)) + except Exception: + pass + return { + 'total': len(parsed), + 'https': len([p for p in parsed if p.get("https")]), + 'residential': len([p for p in parsed if p.get("is_residential") or p.get("residential")]) + } + def changeTable(self, name): """ diff --git a/db/ssdbClient.py b/db/ssdbClient.py index 559539905..82930475c 100644 --- a/db/ssdbClient.py +++ b/db/ssdbClient.py @@ -19,6 +19,8 @@ from redis.exceptions import TimeoutError, ConnectionError, ResponseError from redis.connection import BlockingConnectionPool from handler.logHandler import LogHandler +from handler.configHandler import ConfigHandler +from datetime import datetime from random import choice from redis import Redis import json @@ -48,19 +50,72 @@ def __init__(self, **kwargs): protocol=2, **kwargs)) - def get(self, https): + def _filter_proxy(self, proxy_str, https=False, residential=None): + try: + data = json.loads(proxy_str) + if https and not data.get("https"): + return False + if residential is True and not (data.get("is_residential") or data.get("residential")): + return False + if residential is False and (data.get("is_residential") or data.get("residential")): + return False + return True + except Exception: + return False + + @staticmethod + def _is_working(data): + """ 最近一次校验是否通过 """ + status = data.get("last_status") + return status is True or str(status).lower() == "true" + + @staticmethod + def _check_age(data): + """ 距上次校验的秒数; 无法解析时返回 None """ + last_time = data.get("last_time") or "" + try: + checked = datetime.strptime(last_time, "%Y-%m-%d %H:%M:%S") + except (ValueError, TypeError): + return None + return (datetime.now() - checked).total_seconds() + + def _live_proxies(self, https=False, residential=None): + """ + 返回「最近校验通过」的代理列表, 按校验时间从新到旧排序。 + 优先返回 PROXY_FRESH_SECONDS 秒内校验通过的代理; + 若无满足时效要求的, 退回到所有校验通过的代理, 避免空池。 + """ + max_age = ConfigHandler().proxyFreshSeconds + working = [] + for proxy_str in self.__conn.hgetall(self.name).values(): + if not self._filter_proxy(proxy_str, https=https, residential=residential): + continue + try: + data = json.loads(proxy_str) + except Exception: + continue + if not self._is_working(data): + continue + working.append((self._check_age(data), proxy_str)) + + def _sort_key(item): + age = item[0] + return age if age is not None else float("inf") + + working.sort(key=_sort_key) + if max_age and max_age > 0: + fresh = [p for age, p in working if age is not None and age <= max_age] + if fresh: + return fresh + return [p for _, p in working] + + def get(self, https=False, residential=None): """ - 从hash中随机返回一个代理 + 从hash中随机返回一个「最近校验通过」的代理 :return: """ - if https: - items_dict = self.__conn.hgetall(self.name) - proxies = list(filter(lambda x: json.loads(x).get("https"), items_dict.values())) - return choice(proxies) if proxies else None - else: - proxies = self.__conn.hkeys(self.name) - proxy = choice(proxies) if proxies else None - return self.__conn.hget(self.name, proxy) if proxy else None + proxies = self._live_proxies(https=https, residential=residential) + return choice(proxies) if proxies else None def put(self, proxy_obj): """ @@ -71,12 +126,12 @@ def put(self, proxy_obj): result = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json) return result - def pop(self, https): + def pop(self, https=False, residential=None): """ 顺序弹出一个代理 :return: proxy """ - proxy = self.get(https) + proxy = self.get(https=https, residential=residential) if proxy: self.__conn.hdel(self.name, json.loads(proxy).get("proxy", "")) return proxy if proxy else None @@ -105,16 +160,21 @@ def update(self, proxy_obj): """ self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json) - def getAll(self, https): + def getAll(self, https=False, residential=None, num=None, raw=False): """ - 字典形式返回所有代理, 使用changeTable指定hash name + 字典形式返回代理列表, 使用changeTable指定hash name。 + raw=False(默认): 只返回「最近校验通过」的代理, 校验时间从新到旧排序(供 /get /all 使用); + raw=True: 返回池中全部代理(供调度器复检使用)。 :return: """ - item_dict = self.__conn.hgetall(self.name) - if https: - return list(filter(lambda x: json.loads(x).get("https"), item_dict.values())) + if raw: + item_dict = self.__conn.hgetall(self.name) + proxies = [x for x in item_dict.values() if self._filter_proxy(x, https=https, residential=residential)] else: - return item_dict.values() + proxies = self._live_proxies(https=https, residential=residential) + if num is not None and isinstance(num, int) and num >= 0: + proxies = proxies[:num] + return proxies def clear(self): """ @@ -128,8 +188,19 @@ def getCount(self): 返回代理数量 :return: """ - proxies = self.getAll(https=False) - return {'total': len(proxies), 'https': len(list(filter(lambda x: json.loads(x).get("https"), proxies)))} + item_dict = self.__conn.hgetall(self.name) + parsed = [] + for x in item_dict.values(): + try: + parsed.append(json.loads(x)) + except Exception: + pass + return { + 'total': len(parsed), + 'https': len([p for p in parsed if p.get("https")]), + 'residential': len([p for p in parsed if p.get("is_residential") or p.get("residential")]) + } + def changeTable(self, name): """ diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 000000000..385cb722b --- /dev/null +++ b/deploy.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# ============================================================================== +# ProxyPool Host Machine Production Deployment Script +# Optimized for Google Cloud Compute Engine e2-micro Always Free Tier +# ============================================================================== +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +log_info "Starting ProxyPool Host Deployment Setup for GCP e2-micro..." + +# 1. Check Python installation +PYTHON_CMD="" +if command -v python3 &>/dev/null; then + PYTHON_CMD="python3" +elif command -v python &>/dev/null; then + PYTHON_CMD="python" +else + log_error "Python 3 is required but not installed." + exit 1 +fi + +PY_VER=$($PYTHON_CMD -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') +log_info "Detected Python version: $PY_VER ($PYTHON_CMD)" + +# 2. Setup Virtual Environment +VENV_DIR="$SCRIPT_DIR/venv" +if [ ! -d "$VENV_DIR" ]; then + log_info "Creating virtual environment in $VENV_DIR..." + $PYTHON_CMD -m venv "$VENV_DIR" +fi + +VENV_PYTHON="$VENV_DIR/bin/python" +VENV_PIP="$VENV_DIR/bin/pip" + +log_info "Upgrading pip and installing requirements..." +$VENV_PIP install --upgrade pip -q +$VENV_PIP install -r requirements.txt -q + +# Ensure log directory exists with write permissions +mkdir -p "$SCRIPT_DIR/log" +chmod 777 "$SCRIPT_DIR/log" 2>/dev/null || true + + +# 3. Setup Default .env Configuration +ENV_FILE="$SCRIPT_DIR/.env" +if [ ! -f "$ENV_FILE" ]; then + log_info "Creating default .env configuration file..." + cat < "$ENV_FILE" +# Server Configuration +HOST=0.0.0.0 +PORT=5010 + +# API Token Authentication (Set non-empty token string to enable protection) +AUTH_TOKEN= + +# Database Connection (Default local Redis) +DB_CONN=redis://@127.0.0.1:6379/0 + +# Production Gunicorn Performance Tuning for GCP e2-micro Free Tier (1GB RAM) +GUNICORN_WORKERS=2 +GUNICORN_THREADS=2 +LOG_LEVEL=INFO + +# Proxy Fetching & Validation +POOL_SIZE_MIN=20 +VERIFY_TIMEOUT=10 +MAX_FAIL_COUNT=0 +TIMEZONE=Asia/Shanghai +EOF + log_info "Generated default .env configuration at $ENV_FILE" +fi + +# 4. Check Redis Service +if command -v redis-cli &>/dev/null; then + if redis-cli ping &>/dev/null; then + log_info "Local Redis server is running." + else + log_warn "Local Redis server is installed but not currently responding to ping." + fi +else + log_warn "redis-cli not found in PATH. Ensure Redis server is reachable via DB_CONN in .env" +fi + +# 5. Setup Systemd Service (if sudo/systemd available) +SERVICE_DEST="/etc/systemd/system/proxy_pool.service" +if [ -d "/etc/systemd/system" ] && command -v systemctl &>/dev/null; then + if [ "$EUID" -eq 0 ] || command -v sudo &>/dev/null; then + log_info "Configuring systemd service proxy_pool.service..." + + # Prepare service file with actual path + TEMP_SERVICE=$(mktemp) + cat < "$TEMP_SERVICE" +[Unit] +Description=ProxyPool Production Service (GCP Compute Engine e2-micro Free Tier) +After=network.target redis-server.service redis.service + +[Service] +Type=simple +WorkingDirectory=$SCRIPT_DIR +Environment="PATH=$VENV_DIR/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin" +EnvironmentFile=-$SCRIPT_DIR/.env +ExecStart=$SCRIPT_DIR/proxy_pool.sh start --fg +ExecStop=$SCRIPT_DIR/proxy_pool.sh stop +Restart=always +RestartSec=5 +LimitNOFILE=65536 +MemoryMax=350M + +[Install] +WantedBy=multi-user.target +EOF + if [ "$EUID" -eq 0 ]; then + cp "$TEMP_SERVICE" "$SERVICE_DEST" + systemctl daemon-reload + else + sudo cp "$TEMP_SERVICE" "$SERVICE_DEST" + sudo systemctl daemon-reload + fi + rm -f "$TEMP_SERVICE" + log_info "Installed systemd service to $SERVICE_DEST" + log_info "To start service via systemd: sudo systemctl start proxy_pool" + log_info "To enable auto-start on boot: sudo systemctl enable proxy_pool" + fi +fi + +log_info "Deployment setup complete for GCP Compute Engine e2-micro!" +log_info "" +log_info "Usage:" +log_info " Start in background: ./proxy_pool.sh start" +log_info " Check status: ./proxy_pool.sh status" +log_info " Stop service: ./proxy_pool.sh stop" +log_info " Systemd start: sudo systemctl start proxy_pool" diff --git a/docker-compose.yml b/docker-compose.yml index 9d1a10ba4..14a67e317 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,30 @@ -version: '2' +version: '3.8' + services: proxy_pool: build: . container_name: proxy_pool + restart: always ports: - "5010:5010" - links: + depends_on: - proxy_redis environment: DB_CONN: "redis://@proxy_redis:6379/0" + AUTH_TOKEN: "${AUTH_TOKEN:-}" + GUNICORN_WORKERS: "2" + GUNICORN_THREADS: "2" + deploy: + resources: + limits: + memory: 300M + proxy_redis: - image: "redis" - container_name: proxy_redis \ No newline at end of file + image: "redis:alpine" + container_name: proxy_redis + restart: always + command: redis-server --save "" --appendonly no + deploy: + resources: + limits: + memory: 150M \ No newline at end of file diff --git a/docs/api.md b/docs/api.md index 875d378c8..e26032495 100644 --- a/docs/api.md +++ b/docs/api.md @@ -7,12 +7,24 @@ | 接口 | 方法 | 说明 | 参数 | |------|------|------|------| | `/` | GET | 返回 API 列表 | 无 | -| `/get` | GET | 随机返回一个代理 | 可选:`?type=https` 过滤 HTTPS 代理 | -| `/pop` | GET | 返回并删除一个代理 | 可选:`?type=https` 过滤 HTTPS 代理 | -| `/all` | GET | 返回所有代理 | 可选:`?type=https` 过滤 HTTPS 代理 | -| `/count` | GET | 返回代理数量统计 | 无 | +| `/get` | GET | 随机返回一个代理 | 可选:`?type=https` 过滤 HTTPS 代理;`?residential=true|false` 过滤住宅 IP / 非住宅 IP 代理 | +| `/pop` | GET | 返回并删除一个代理 | 可选:`?type=https` 过滤 HTTPS 代理;`?residential=true|false` 过滤住宅 IP / 非住宅 IP 代理 | +| `/all` | GET | 返回代理列表 | 可选:`?type=https` 过滤 HTTPS 代理;`?residential=true|false` 过滤住宅 IP / 非住宅 IP 代理;`?num=N` 指定返回代理数量 | +| `/count` | GET | 返回代理数量统计(含 residential 住宅 IP 统计) | 无 | | `/delete` | GET | 删除指定代理 | `?proxy=host:port` | +## 认证与隐私 (Token Authentication & Zero-Log Privacy) + +### Token 身份验证 +当在环境变量或 `setting.py` 中配置 `AUTH_TOKEN` 时,所有 API 接口均需携带 Token。支持以下任意一种方式: +- Header 请求头: `Authorization: Bearer ` +- Header 请求头: `X-API-Token: ` 或 `Api-Key: ` +- URL 查询参数: `?token=` 或 `?api_key=` + +### 零日志隐私保障 (Zero-Log Privacy) +为保护请求者隐私,生产模式下已禁用请求者 IP (Client IP) 和 User-Agent 的访问日志记录,绝不留存或打印任何请求者身份特征信息。 + + ## 调用示例 ### 在爬虫中使用 diff --git a/fetcher/baseFetcher.py b/fetcher/baseFetcher.py index f22f31e25..ae721ec04 100644 --- a/fetcher/baseFetcher.py +++ b/fetcher/baseFetcher.py @@ -24,6 +24,8 @@ class BaseFetcher(object): # ---- 子类可覆盖 ---- enabled = True # 是否启用,设为 False 可禁用该源 + is_residential = False # 是否为住宅代理源 + def fetch(self): """爬取代理,yield "host:port" 字符串""" diff --git a/fetcher/sources/docip.py b/fetcher/sources/docip.py index 15cbeea33..482266daa 100644 --- a/fetcher/sources/docip.py +++ b/fetcher/sources/docip.py @@ -25,6 +25,8 @@ class DocipFetcher(BaseFetcher): name = "docip" url = "https://www.docip.net/" + enabled = False # 站点已下线 (connection refused), 2026/09/03 禁用 + def fetch(self): r = WebRequest().get("https://www.docip.net/data/free.json", timeout=10) try: diff --git a/fetcher/sources/geonode.py b/fetcher/sources/geonode.py index 8e3171047..bf730eb4c 100644 --- a/fetcher/sources/geonode.py +++ b/fetcher/sources/geonode.py @@ -24,6 +24,10 @@ class GeonodeFetcher(BaseFetcher): name = "geonode" url = "https://geonode.com/" + is_residential = True + + enabled = False # API 持续返回空数据, 2026/09/03 禁用 + def fetch(self): url = ("https://proxylist.geonode.com/api/proxy-list?" diff --git a/fetcher/sources/goodips.py b/fetcher/sources/goodips.py index fd1ba2a10..19fa37c91 100644 --- a/fetcher/sources/goodips.py +++ b/fetcher/sources/goodips.py @@ -22,6 +22,8 @@ class GoodipsFetcher(BaseFetcher): name = "goodips" url = "https://www.goodips.com/" + enabled = False # 反爬拦截, tree 为 None 导致解析失败, 2026/09/03 禁用 + def fetch(self): url = "https://www.goodips.com/" tree = WebRequest().get(url, verify=False).tree diff --git a/fetcher/sources/iplocate.py b/fetcher/sources/iplocate.py new file mode 100644 index 000000000..f28616bd8 --- /dev/null +++ b/fetcher/sources/iplocate.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name: iplocate.py + Description : IPLocate free-proxy-list 代理源 + Author : JHao + date: 2026/09/03 +------------------------------------------------- + Change Activity: + 2026/09/03: 新增 iplocate 源 +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + + +class IPLocateFetcher(BaseFetcher): + """IPLocate free-proxy-list https://github.com/iplocate/free-proxy-list""" + + name = "iplocate" + url = "https://github.com/iplocate/free-proxy-list" + + enabled = True + + def fetch(self): + # 该列表每行形如 "http://ip:port" / "socks5://ip:port" + # 代理池内部统一按 http 代理处理, 故仅采集 http/https 行, socks 行跳过 + raw_url = ("https://github.com/iplocate/free-proxy-list/" + "raw/refs/heads/main/all-proxies.txt") + try: + r = WebRequest().get(raw_url, timeout=10) + except Exception as e: + logger.error("ProxyFetch - iplocate: %s" % e) + return + proxies = [] + for line in r.text.splitlines(): + line = line.strip() + if not line or "://" not in line: + continue + scheme, addr = line.split("://", 1) + if scheme.lower() not in ("http", "https"): + continue + proxies.extend(self.parseProxiesFromText(addr)) + for proxy in self.yieldUniqueProxies(proxies): + yield proxy + + +if __name__ == '__main__': + for proxy in IPLocateFetcher().fetch(): + print(proxy) diff --git a/fetcher/sources/kuaidaili.py b/fetcher/sources/kuaidaili.py index 3489d7ecf..4684324d5 100644 --- a/fetcher/sources/kuaidaili.py +++ b/fetcher/sources/kuaidaili.py @@ -24,6 +24,8 @@ class KuaidailiFetcher(BaseFetcher): name = "kuaidaili" url = "https://www.kuaidaili.com" + enabled = False # 反爬拦截, 返回非 HTML 导致解析失败, 2026/09/03 禁用 + def fetch(self, page_count=1): url_pattern = [ 'https://www.kuaidaili.com/free/inha/{}/', diff --git a/fetcher/sources/roundproxies.py b/fetcher/sources/roundproxies.py index ba1d0af36..7f4cca9ce 100644 --- a/fetcher/sources/roundproxies.py +++ b/fetcher/sources/roundproxies.py @@ -25,6 +25,8 @@ class RoundProxiesFetcher(BaseFetcher): url = "https://roundproxies.com/free-proxy-list" enabled = True + is_residential = True + def fetch(self): page_size = 50 diff --git a/fetcher/sources/zdaye.py b/fetcher/sources/zdaye.py index 8eeda8b79..c5358db5d 100644 --- a/fetcher/sources/zdaye.py +++ b/fetcher/sources/zdaye.py @@ -25,6 +25,8 @@ class ZdayeFetcher(BaseFetcher): name = "zdaye" url = "https://www.zdaye.com/dayProxy.html" + enabled = False # 页面结构变更导致解析 IndexError, 2026/09/03 禁用 + def fetch(self): start_url = "https://www.zdaye.com/free/" html_tree = WebRequest().get(start_url, verify=False).tree diff --git a/handler/configHandler.py b/handler/configHandler.py index 37a923493..fe05f3a98 100644 --- a/handler/configHandler.py +++ b/handler/configHandler.py @@ -32,6 +32,22 @@ def serverHost(self): def serverPort(self): return os.environ.get("PORT", setting.PORT) + @LazyProperty + def authToken(self): + return os.getenv("AUTH_TOKEN", getattr(setting, "AUTH_TOKEN", "")) + + @LazyProperty + def gunicornWorkers(self): + return int(os.getenv("GUNICORN_WORKERS", getattr(setting, "GUNICORN_WORKERS", 2))) + + @LazyProperty + def gunicornThreads(self): + return int(os.getenv("GUNICORN_THREADS", getattr(setting, "GUNICORN_THREADS", 2))) + + @LazyProperty + def logLevel(self): + return os.getenv("LOG_LEVEL", getattr(setting, "LOG_LEVEL", "INFO")) + @LazyProperty def dbConn(self): return os.getenv("DB_CONN", setting.DB_CONN) @@ -73,6 +89,10 @@ def maxFailCount(self): def poolSizeMin(self): return int(os.getenv("POOL_SIZE_MIN", setting.POOL_SIZE_MIN)) + @LazyProperty + def proxyFreshSeconds(self): + return int(os.getenv("PROXY_FRESH_SECONDS", getattr(setting, "PROXY_FRESH_SECONDS", 900))) + @LazyProperty def proxyRegion(self): return bool(os.getenv("PROXY_REGION", setting.PROXY_REGION)) diff --git a/handler/logHandler.py b/handler/logHandler.py index 45cd1201d..873924d91 100644 --- a/handler/logHandler.py +++ b/handler/logHandler.py @@ -62,19 +62,23 @@ def __setFileHandler__(self, level=None): :param level: :return: """ - file_name = os.path.join(LOG_PATH, '{name}.log'.format(name=self.name)) - # 设置日志回滚, 保存在log目录, 一天保存一个文件, 保留15天 - file_handler = TimedRotatingFileHandler(filename=file_name, when='D', interval=1, backupCount=15) - file_handler.suffix = '%Y%m%d.log' - if not level: - file_handler.setLevel(self.level) - else: - file_handler.setLevel(level) - formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s') + try: + file_name = os.path.join(LOG_PATH, '{name}.log'.format(name=self.name)) + # 设置日志回滚, 保存在log目录, 一天保存一个文件, 保留15天 + file_handler = TimedRotatingFileHandler(filename=file_name, when='D', interval=1, backupCount=15) + file_handler.suffix = '%Y%m%d.log' + if not level: + file_handler.setLevel(self.level) + else: + file_handler.setLevel(level) + formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s') + + file_handler.setFormatter(formatter) + self.file_handler = file_handler + self.addHandler(file_handler) + except (PermissionError, OSError): + pass - file_handler.setFormatter(formatter) - self.file_handler = file_handler - self.addHandler(file_handler) def __setStreamHandler__(self, level=None): """ diff --git a/handler/proxyHandler.py b/handler/proxyHandler.py index 32e215e5d..878ac07e4 100644 --- a/handler/proxyHandler.py +++ b/handler/proxyHandler.py @@ -26,22 +26,23 @@ def __init__(self): self.db = DbClient(self.conf.dbConn) self.db.changeTable(self.conf.tableName) - def get(self, https=False): + def get(self, https=False, residential=None): """ return a proxy Args: https: True/False + residential: True/False/None Returns: """ - proxy = self.db.get(https) + proxy = self.db.get(https=https, residential=residential) return Proxy.createFromJson(proxy) if proxy else None - def pop(self, https): + def pop(self, https=False, residential=None): """ return and delete a useful proxy :return: """ - proxy = self.db.pop(https) + proxy = self.db.pop(https=https, residential=residential) if proxy: return Proxy.createFromJson(proxy) return None @@ -61,14 +62,17 @@ def delete(self, proxy): """ return self.db.delete(proxy.proxy) - def getAll(self, https=False): + def getAll(self, https=False, residential=None, num=None, raw=False): """ get all proxy from pool as Proxy list + raw=False: 仅返回最近校验通过的代理(供 API 使用); + raw=True: 返回池中全部代理(供调度器复检使用)。 :return: """ - proxies = self.db.getAll(https) + proxies = self.db.getAll(https=https, residential=residential, num=num, raw=raw) return [Proxy.createFromJson(_) for _ in proxies] + def exists(self, proxy): """ check proxy exists diff --git a/helper/fetch.py b/helper/fetch.py index 62970685a..077891f32 100644 --- a/helper/fetch.py +++ b/helper/fetch.py @@ -101,21 +101,37 @@ def __init__(self, fetcher_class, proxy_dict): def run(self): fetcher_name = self.fetcher_class.name + fetcher_instance = self.fetcher_class() + fetcher_is_res = getattr(self.fetcher_class, 'is_residential', False) or getattr(fetcher_instance, 'is_residential', False) self.log.info("ProxyFetch - {func}: start".format(func=fetcher_name)) try: - for proxy in self.fetcher_class().fetch(): - self.log.info('ProxyFetch - %s: %s ok' % (fetcher_name, proxy.ljust(23))) - proxy = proxy.strip() - if proxy in self.proxy_dict: - self.proxy_dict[proxy].add_source(fetcher_name) + for item in fetcher_instance.fetch(): + res_flag = fetcher_is_res + if isinstance(item, Proxy): + proxy_str = item.proxy + res_flag = item.is_residential or fetcher_is_res + elif isinstance(item, tuple) and len(item) >= 2: + proxy_str = item[0] + if isinstance(item[1], dict): + res_flag = item[1].get("is_residential", fetcher_is_res) else: - self.proxy_dict[proxy] = Proxy( - proxy, source=fetcher_name) + proxy_str = str(item) + + proxy_str = proxy_str.strip() + self.log.info('ProxyFetch - %s: %s ok' % (fetcher_name, proxy_str.ljust(23))) + if proxy_str in self.proxy_dict: + self.proxy_dict[proxy_str].add_source(fetcher_name) + if res_flag: + self.proxy_dict[proxy_str].is_residential = True + else: + self.proxy_dict[proxy_str] = Proxy( + proxy_str, source=fetcher_name, is_residential=res_flag) except Exception as e: self.log.error("ProxyFetch - {func}: error".format(func=fetcher_name)) self.log.error(str(e)) + class Fetcher(object): name = "fetcher" diff --git a/helper/proxy.py b/helper/proxy.py index 396a84239..0ce82aeca 100644 --- a/helper/proxy.py +++ b/helper/proxy.py @@ -18,16 +18,17 @@ class Proxy(object): def __init__(self, proxy, fail_count=0, region="", anonymous="", - source="", check_count=0, last_status="", last_time="", https=False): + source="", check_count=0, last_status="", last_time="", https=False, is_residential=False): self._proxy = proxy self._fail_count = fail_count self._region = region self._anonymous = anonymous - self._source = source.split('/') + self._source = source.split('/') if isinstance(source, str) else source self._check_count = check_count self._last_status = last_status self._last_time = last_time self._https = https + self._is_residential = is_residential @classmethod def createFromJson(cls, proxy_json): @@ -40,7 +41,8 @@ def createFromJson(cls, proxy_json): check_count=_dict.get("check_count", 0), last_status=_dict.get("last_status", ""), last_time=_dict.get("last_time", ""), - https=_dict.get("https", False) + https=_dict.get("https", False), + is_residential=_dict.get("is_residential", _dict.get("residential", False)) ) @property @@ -88,6 +90,11 @@ def https(self): """ 是否支持https """ return self._https + @property + def is_residential(self): + """ 是否为住宅IP """ + return self._is_residential + @property def to_dict(self): """ 属性字典 """ @@ -99,7 +106,8 @@ def to_dict(self): "source": self.source, "check_count": self.check_count, "last_status": self.last_status, - "last_time": self.last_time} + "last_time": self.last_time, + "is_residential": self.is_residential} @property def to_json(self): @@ -126,6 +134,10 @@ def last_time(self, value): def https(self, value): self._https = value + @is_residential.setter + def is_residential(self, value): + self._is_residential = bool(value) + @region.setter def region(self, value): self._region = value @@ -134,3 +146,4 @@ def add_source(self, source_str): if source_str: self._source.append(source_str) self._source = list(set(self._source)) + diff --git a/helper/scheduler.py b/helper/scheduler.py index ef0431881..4f5fd7a10 100644 --- a/helper/scheduler.py +++ b/helper/scheduler.py @@ -39,7 +39,7 @@ def __runProxyCheck(): proxy_queue = Queue() if proxy_handler.db.getCount().get("total", 0) < proxy_handler.conf.poolSizeMin: __runProxyFetch() - for proxy in proxy_handler.getAll(): + for proxy in proxy_handler.getAll(raw=True): proxy_queue.put(proxy) Checker("use", proxy_queue) diff --git a/provision_gcp_instance.sh b/provision_gcp_instance.sh new file mode 100755 index 000000000..2c1fa43e5 --- /dev/null +++ b/provision_gcp_instance.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Exit immediately if a command exits with a non-zero status +set -e + +# ============================================================================== +# Configuration Variables +# Note: Google Cloud uses 'e2-micro' as its micro instance type (equivalent to AWS ec2 micro). +# ============================================================================== +INSTANCE_NAME="${INSTANCE_NAME:-my-gcp-micro-instance}" +ZONE="${ZONE:-us-central1-a}" +MACHINE_TYPE="${MACHINE_TYPE:-e2-micro}" +IMAGE_FAMILY="${IMAGE_FAMILY:-ubuntu-2204-lts}" +IMAGE_PROJECT="${IMAGE_PROJECT:-ubuntu-os-cloud}" + +echo "============================================================" +echo " Provisioning Google Cloud Instance" +echo " Instance Name: ${INSTANCE_NAME}" +echo " Machine Type : ${MACHINE_TYPE}" +echo " Zone : ${ZONE}" +echo " Image : ${IMAGE_FAMILY} (${IMAGE_PROJECT})" +echo "============================================================" + +# Define startup script to update apt and install git upon VM boot +STARTUP_SCRIPT=$(cat << 'EOF' +#!/bin/bash +export DEBIAN_FRONTEND=noninteractive +apt-get update -y +apt-get install -y git +echo "Git installation completed at $(date)" >> /var/log/startup-script-git.log +git --version >> /var/log/startup-script-git.log +EOF +) + +# Provision the Compute Engine VM instance +gcloud compute instances create "${INSTANCE_NAME}" \ + --zone="${ZONE}" \ + --machine-type="${MACHINE_TYPE}" \ + --image-family="${IMAGE_FAMILY}" \ + --image-project="${IMAGE_PROJECT}" \ + --metadata=startup-script="${STARTUP_SCRIPT}" \ + --scopes=cloud-platform + +echo "" +echo " Successfully created instance '${INSTANCE_NAME}'." +echo " Git is being installed automatically via the startup script." +echo "" +echo "To SSH into your instance:" +echo " gcloud compute ssh ${INSTANCE_NAME} --zone=${ZONE}" +echo "" +echo "To check git version inside the VM:" +echo " gcloud compute ssh ${INSTANCE_NAME} --zone=${ZONE} --command='git --version'" diff --git a/proxyPool.py b/proxyPool.py index 9b18b7845..5f35e2ff7 100644 --- a/proxyPool.py +++ b/proxyPool.py @@ -12,6 +12,9 @@ """ __author__ = 'JHao' +from dotenv import load_dotenv +load_dotenv() + import click from helper.launcher import startServer, startScheduler from setting import BANNER, VERSION diff --git a/proxy_pool.service b/proxy_pool.service new file mode 100644 index 000000000..5b74376fd --- /dev/null +++ b/proxy_pool.service @@ -0,0 +1,18 @@ +[Unit] +Description=ProxyPool Production Service (GCP Compute Engine e2-micro Free Tier) +After=network.target redis-server.service redis.service + +[Service] +Type=simple +WorkingDirectory=/opt/proxy_pool +Environment="PATH=/opt/proxy_pool/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin" +EnvironmentFile=-/opt/proxy_pool/.env +ExecStart=/opt/proxy_pool/venv/bin/bash proxy_pool.sh start --fg +ExecStop=/opt/proxy_pool/venv/bin/bash proxy_pool.sh stop +Restart=always +RestartSec=5 +LimitNOFILE=65536 +MemoryMax=350M + +[Install] +WantedBy=multi-user.target diff --git a/proxy_pool.sh b/proxy_pool.sh old mode 100644 new mode 100755 diff --git a/remote_deploy.sh b/remote_deploy.sh new file mode 100755 index 000000000..33b5417b2 --- /dev/null +++ b/remote_deploy.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# ============================================================================== +# Remote Deployment Script for GCP Compute Engine e2-micro VM Instance +# Run this script locally to deploy proxy_pool onto a running GCP e2-micro VM. +# ============================================================================== +set -e + +# Configuration (Can be passed via arguments or environment variables) +INSTANCE_NAME="${1:-${INSTANCE_NAME:-my-gcp-micro-instance}}" +ZONE="${2:-${ZONE:-us-central1-a}}" +GIT_REPO_URL="${3:-${GIT_REPO_URL:-https://github.com/anand870/proxy_pool.git}}" +TARGET_DIR="${TARGET_DIR:-/opt/proxy_pool}" +AUTH_TOKEN="${AUTH_TOKEN:-}" +PORT="${PORT:-5010}" + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +log_info "=================================================================" +log_info " Deploying ProxyPool to Remote GCP Compute Engine Instance" +log_info " Instance Name: ${INSTANCE_NAME}" +log_info " Zone: ${ZONE}" +log_info " Git Repo: ${GIT_REPO_URL}" +log_info " Target Dir: ${TARGET_DIR}" +log_info " Port: ${PORT}" +log_info "=================================================================" + +# Check gcloud CLI +if ! command -v gcloud &>/dev/null; then + log_error "'gcloud' CLI tool is required to SSH into Google Cloud instances." + log_error "Please install Google Cloud SDK: https://cloud.google.com/sdk/docs/install" + exit 1 +fi + +log_info "[1/5] Opening GCP Firewall port ${PORT} to public ingress..." +# Create firewall rule allowing port 5010 for instances tagged with proxy-pool +gcloud compute firewall-rules create "allow-proxy-pool-${PORT}" \ + --allow="tcp:${PORT}" \ + --source-ranges="0.0.0.0/0" \ + --target-tags="proxy-pool" \ + --description="Allow public ingress traffic on port ${PORT} for ProxyPool" &>/dev/null || log_warn "Firewall rule 'allow-proxy-pool-${PORT}' already exists or updated." + + +# Add network tag proxy-pool to the VM instance +gcloud compute instances add-tags "${INSTANCE_NAME}" \ + --zone="${ZONE}" \ + --tags="proxy-pool" &>/dev/null || log_warn "Network tag 'proxy-pool' already present on ${INSTANCE_NAME}." + +log_info "[2/5] Installing git, python3, venv, and redis on remote GCP VM..." +gcloud compute ssh "${INSTANCE_NAME}" --zone="${ZONE}" --command=" + sudo apt-get update -y && \ + sudo apt-get install -y git python3 python3-venv redis-server curl +" + +log_info "[3/5] Cloning or pulling project repository on remote GCP VM..." +gcloud compute ssh "${INSTANCE_NAME}" --zone="${ZONE}" --command=" + sudo mkdir -p ${TARGET_DIR} && \ + sudo chown -R \$USER:\$USER ${TARGET_DIR} && \ + if [ -d '${TARGET_DIR}/.git' ]; then + echo 'Updating existing repository...' && \ + cd ${TARGET_DIR} && git fetch --all && (git reset --hard origin/master || git pull) + else + echo 'Cloning repository...' && \ + git clone ${GIT_REPO_URL} ${TARGET_DIR} + fi +" + +log_info "[4/5] Running deploy.sh on remote GCP VM..." +gcloud compute ssh "${INSTANCE_NAME}" --zone="${ZONE}" --command=" + cd ${TARGET_DIR} && \ + chmod +x deploy.sh proxy_pool.sh && \ + ./deploy.sh +" + +if [ -n "${AUTH_TOKEN}" ]; then + log_info "Setting AUTH_TOKEN in remote .env configuration..." + gcloud compute ssh "${INSTANCE_NAME}" --zone="${ZONE}" --command=" + sed -i 's/^AUTH_TOKEN=.*/AUTH_TOKEN=${AUTH_TOKEN}/' ${TARGET_DIR}/.env + " +fi + +log_info "[5/5] Starting ProxyPool production service on remote GCP VM..." +gcloud compute ssh "${INSTANCE_NAME}" --zone="${ZONE}" --command=" + if command -v systemctl &>/dev/null && [ -f /etc/systemd/system/proxy_pool.service ]; then + sudo systemctl restart proxy_pool && \ + sudo systemctl status proxy_pool --no-pager + else + cd ${TARGET_DIR} && ./proxy_pool.sh restart + fi +" + +EXTERNAL_IP=$(gcloud compute instances describe "${INSTANCE_NAME}" --zone="${ZONE}" --format='get(networkInterfaces[0].accessConfigs[0].natIP)' 2>/dev/null || echo "") + +log_info "" +log_info "=================================================================" +log_info " Remote Deployment Completed Successfully!" +log_info " ProxyPool Public URL: http://${EXTERNAL_IP}:${PORT}" +if [ -n "${AUTH_TOKEN}" ]; then + log_info " Token Auth Enabled: Header 'Authorization: Bearer ${AUTH_TOKEN}' or '?token=${AUTH_TOKEN}'" +else + log_info " Token Auth: Disabled (Pass AUTH_TOKEN='your_token' to enable)" +fi +log_info "=================================================================" diff --git a/requirements.txt b/requirements.txt index c658ae713..fc3558d42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ requests==2.31.0 -gunicorn==19.9.0 -lxml==4.9.2 +gunicorn>=21.2.0 +lxml>=5.0.0 redis>=4.2.0 APScheduler==3.10.0;python_version>="3.10" APScheduler==3.2.0;python_version<"3.10" click==8.0.1 Flask==2.1.1 werkzeug>=2.0,<2.2 +python-dotenv>=1.0.0 diff --git a/setting.py b/setting.py index 868f32bb7..73e348727 100644 --- a/setting.py +++ b/setting.py @@ -32,12 +32,24 @@ PORT = 5010 +# API token authentication system (Empty string disabled, set string to enable) +AUTH_TOKEN = "" + +# Gunicorn production options (Optimized for GCP Compute Engine e2-micro Free Tier) + +GUNICORN_WORKERS = 2 +GUNICORN_THREADS = 2 +LOG_LEVEL = "INFO" + + +import os + # ############### database config ################### # db connection uri # example: # Redis: redis://:password@ip:port/db # Ssdb: ssdb://:password@ip:port -DB_CONN = 'redis://:pwdstring@127.0.0.1:6379/0' +DB_CONN = os.getenv('DB_CONN', 'redis://127.0.0.1:6379/0') # proxy table name TABLE_NAME = 'use_proxy' @@ -66,6 +78,10 @@ # proxyCheck时代理数量少于POOL_SIZE_MIN触发抓取 POOL_SIZE_MIN = 20 +# /get 与 /all 接口只返回最近 PROXY_FRESH_SECONDS 秒内校验通过的代理, +# 避免返回距上次校验过久(可能已失效)的代理。设为 0 关闭时间过滤。 +PROXY_FRESH_SECONDS = 900 + # ############# proxy attributes ################# # 是否启用代理地域属性 PROXY_REGION = True diff --git a/source_health.py b/source_health.py new file mode 100644 index 000000000..b7843b40f --- /dev/null +++ b/source_health.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name: source_health.py + Description : 逐个运行 fetcher/sources/ 下的代理源,实时校验其产出的代理, + 统计每个源「抓到多少 / 可用多少 / 可用率」以及哪些源已失效。 + 不依赖数据库,不修改任何业务代码。 + Author : analysis tool +------------------------------------------------- + 用法: + python source_health.py # 跑全部源 + python source_health.py scdn ip89 # 只跑指定源 + python source_health.py --no-validate # 只抓取不校验(快) +------------------------------------------------- +""" +import os +import sys +import time +import json +from concurrent.futures import ThreadPoolExecutor + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +os.chdir(os.path.dirname(os.path.abspath(__file__))) + +from helper.fetch import _discover_fetchers # noqa: E402 +from helper.proxy import Proxy # noqa: E402 +from helper.check import DoValidator # noqa: E402 + +MAX_WORKERS = 50 + + +def collect(only=None): + """运行每个源的 fetch(),返回 {source_name: set(proxy_str)} 与耗时/错误信息""" + src_map = {} + meta = {} + for fc in _discover_fetchers([]): + if only and fc.name not in only: + continue + got = set() + err = "" + t0 = time.time() + try: + for item in fc().fetch(): + s = item.proxy if hasattr(item, "proxy") else ( + item[0] if isinstance(item, tuple) else str(item)) + got.add(s.strip()) + except Exception as e: # noqa: BLE001 + err = repr(e) + src_map[fc.name] = got + meta[fc.name] = (round(time.time() - t0, 1), err) + flag = "OK " if got else ("ERR " if err else "EMPTY") + print("fetch %-14s %s %4d %5.1fs %s" + % (fc.name, flag, len(got), meta[fc.name][0], err[:70])) + return src_map, meta + + +def validate(proxies): + """并发校验,返回 {proxy_str: (http_ok, https_ok)}""" + def _check(p): + pr = Proxy(p) + try: + pr = DoValidator.validator(pr, "use") + return p, bool(pr.last_status), bool(pr.https) + except Exception: # noqa: BLE001 + return p, False, False + + out = {} + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex: + for p, ok, https in ex.map(_check, list(proxies)): + out[p] = (ok, https) + return out + + +def main(): + args = [a for a in sys.argv[1:] if not a.startswith("-")] + do_validate = "--no-validate" not in sys.argv + only = set(args) or None + + src_map, meta = collect(only) + + all_proxies = {} + for name, ps in src_map.items(): + for p in ps: + all_proxies.setdefault(p, []).append(name) + print("\nunique proxies: %d" % len(all_proxies)) + + res = {} + if do_validate and all_proxies: + t0 = time.time() + res = validate(all_proxies.keys()) + print("validation took %.0fs\n" % (time.time() - t0)) + + print("%-14s %7s %7s %6s %8s %s" + % ("source", "fetched", "working", "rate", "https_ok", "note")) + print("-" * 70) + rows = [] + for name, ps in src_map.items(): + tot = len(ps) + work = sum(1 for p in ps if res.get(p, (False, False))[0]) + https = sum(1 for p in ps if res.get(p, (False, False))[1]) + rate = (work / tot * 100) if tot else 0.0 + note = meta[name][1][:40] if meta[name][1] else ( + "" if tot else "no proxies returned") + rows.append((name, tot, work, rate, https, note)) + + for name, tot, work, rate, https, note in sorted(rows, key=lambda r: -r[2]): + print("%-14s %7d %7d %5.0f%% %8d %s" + % (name, tot, work, rate, https, note)) + + if res: + tot_all = len(all_proxies) + work_all = sum(1 for v in res.values() if v[0]) + print("-" * 70) + print("%-14s %7d %7d %5.0f%%" + % ("TOTAL (uniq)", tot_all, work_all, + work_all / tot_all * 100 if tot_all else 0)) + + stamp = time.strftime("%Y%m%d_%H%M%S") + out_path = os.path.join("log", "source_health_%s.json" % stamp) + try: + json.dump( + {"generated": stamp, + "fetched": {k: sorted(v) for k, v in src_map.items()}, + "meta": meta, + "validation": res}, + open(out_path, "w"), indent=2) + print("\nraw result -> %s" % out_path) + except Exception as e: # noqa: BLE001 + print("could not write json: %s" % e) + + +if __name__ == "__main__": + main() diff --git a/tests/api/test_proxy_api.py b/tests/api/test_proxy_api.py index 012a11e88..165d2f90f 100644 --- a/tests/api/test_proxy_api.py +++ b/tests/api/test_proxy_api.py @@ -63,13 +63,41 @@ def test_get_https_filter(self, client, mocks): assert resp.status_code == 200 data = resp.get_json() assert data["https"] is True - mocks["get"].assert_called_with(True) + mocks["get"].assert_called_with(https=True, residential=None) def test_get_http_filter(self, client, mocks): mocks["get"].return_value = None client.get("/get/") - mocks["get"].assert_called_with(False) + mocks["get"].assert_called_with(https=False, residential=None) + + def test_get_residential_filter(self, client, mocks): + proxy = Proxy("1.1.1.1:8080", source="test", is_residential=True) + mocks["get"].return_value = proxy + + resp = client.get("/get/?residential=true") + assert resp.status_code == 200 + data = resp.get_json() + assert data["is_residential"] is True + mocks["get"].assert_called_with(https=False, residential=True) + + def test_get_residential_false_filter(self, client, mocks): + proxy = Proxy("1.1.1.1:8080", source="test", is_residential=False) + mocks["get"].return_value = proxy + + resp = client.get("/get/?residential=false") + assert resp.status_code == 200 + data = resp.get_json() + assert data["is_residential"] is False + mocks["get"].assert_called_with(https=False, residential=False) + + def test_get_type_residential(self, client, mocks): + proxy = Proxy("1.1.1.1:8080", source="test", is_residential=True) + mocks["get"].return_value = proxy + + resp = client.get("/get/?type=residential") + assert resp.status_code == 200 + mocks["get"].assert_called_with(https=False, residential=True) class TestPop: @@ -82,6 +110,23 @@ def test_pop_returns_proxy(self, client, mocks): assert resp.status_code == 200 data = resp.get_json() assert data["proxy"] == "1.2.3.4:8080" + mocks["pop"].assert_called_with(https=False, residential=None) + + def test_pop_residential_filter(self, client, mocks): + proxy = Proxy("1.2.3.4:8080", source="test", is_residential=True) + mocks["pop"].return_value = proxy + + resp = client.get("/pop/?residential=1") + assert resp.status_code == 200 + mocks["pop"].assert_called_with(https=False, residential=True) + + def test_pop_residential_false_filter(self, client, mocks): + proxy = Proxy("1.2.3.4:8080", source="test", is_residential=False) + mocks["pop"].return_value = proxy + + resp = client.get("/pop/?residential=false") + assert resp.status_code == 200 + mocks["pop"].assert_called_with(https=False, residential=False) def test_pop_no_proxy(self, client, mocks): mocks["pop"].return_value = None @@ -107,6 +152,38 @@ def test_all_returns_list(self, client, mocks): assert data[0]["proxy"] == "1.2.3.4:8080" assert data[1]["proxy"] == "5.6.7.8:443" + def test_all_residential_filter(self, client, mocks): + proxies = [Proxy("1.1.1.1:8080", source="test", is_residential=True)] + mocks["getAll"].return_value = proxies + + resp = client.get("/all/?residential=true") + assert resp.status_code == 200 + mocks["getAll"].assert_called_with(https=False, residential=True, num=None) + + def test_all_residential_false_filter(self, client, mocks): + proxies = [Proxy("1.1.1.1:8080", source="test", is_residential=False)] + mocks["getAll"].return_value = proxies + + resp = client.get("/all/?residential=false") + assert resp.status_code == 200 + mocks["getAll"].assert_called_with(https=False, residential=False, num=None) + + def test_all_num_filter(self, client, mocks): + proxies = [Proxy("1.1.1.1:8080", source="test")] + mocks["getAll"].return_value = proxies + + resp = client.get("/all/?num=1") + assert resp.status_code == 200 + mocks["getAll"].assert_called_with(https=False, residential=None, num=1) + + def test_all_residential_false_and_num_filter(self, client, mocks): + proxies = [Proxy("1.1.1.1:8080", source="test", is_residential=False)] + mocks["getAll"].return_value = proxies + + resp = client.get("/all/?residential=false&num=1") + assert resp.status_code == 200 + mocks["getAll"].assert_called_with(https=False, residential=False, num=1) + def test_all_empty(self, client, mocks): mocks["getAll"].return_value = [] @@ -132,8 +209,8 @@ class TestCount: def test_count_returns_stats(self, client, mocks): proxies = [ - Proxy("1.2.3.4:8080", source="freeProxy01", https=False), - Proxy("5.6.7.8:443", source="freeProxy02", https=True), + Proxy("1.2.3.4:8080", source="freeProxy01", https=False, is_residential=False), + Proxy("5.6.7.8:443", source="freeProxy02", https=True, is_residential=True), ] mocks["getAll"].return_value = proxies @@ -141,6 +218,7 @@ def test_count_returns_stats(self, client, mocks): assert resp.status_code == 200 data = resp.get_json() assert data["count"] == 2 + assert data["residential"] == 1 assert data["http_type"]["http"] == 1 assert data["http_type"]["https"] == 1 assert data["source"]["freeProxy01"] == 1 @@ -152,10 +230,44 @@ def test_count_empty(self, client, mocks): resp = client.get("/count/") data = resp.get_json() assert data["count"] == 0 + assert data["residential"] == 0 assert data["http_type"] == {} assert data["source"] == {} +class TestTokenAuth: + + def test_token_auth_missing(self, client): + with patch("api.proxyApi.conf.authToken", "secret-token"): + resp = client.get("/get/") + assert resp.status_code == 401 + data = resp.get_json() + assert data["code"] == 401 + + def test_token_auth_invalid(self, client): + with patch("api.proxyApi.conf.authToken", "secret-token"): + resp = client.get("/get/", headers={"Authorization": "Bearer wrong-token"}) + assert resp.status_code == 401 + + def test_token_auth_valid_bearer_header(self, client, mocks): + mocks["get"].return_value = Proxy("1.2.3.4:8080") + with patch("api.proxyApi.conf.authToken", "secret-token"): + resp = client.get("/get/", headers={"Authorization": "Bearer secret-token"}) + assert resp.status_code == 200 + + def test_token_auth_valid_x_api_token(self, client, mocks): + mocks["get"].return_value = Proxy("1.2.3.4:8080") + with patch("api.proxyApi.conf.authToken", "secret-token"): + resp = client.get("/get/", headers={"X-API-Token": "secret-token"}) + assert resp.status_code == 200 + + def test_token_auth_valid_query_param(self, client, mocks): + mocks["get"].return_value = Proxy("1.2.3.4:8080") + with patch("api.proxyApi.conf.authToken", "secret-token"): + resp = client.get("/get/?token=secret-token") + assert resp.status_code == 200 + + class TestRefresh: def test_refresh_returns_success(self, client): diff --git a/tests/conftest.py b/tests/conftest.py index 4ec30bda6..3fea0fb8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -85,7 +85,8 @@ def app(): with patch.object(proxy_handler, "get") as mock_get, \ patch.object(proxy_handler, "pop") as mock_pop, \ patch.object(proxy_handler, "getAll") as mock_getAll, \ - patch.object(proxy_handler, "delete") as mock_delete: + patch.object(proxy_handler, "delete") as mock_delete, \ + patch("api.proxyApi.conf.authToken", None): flask_app._test_mocks = { "get": mock_get, "pop": mock_pop, diff --git a/tests/integration/test_redis_client.py b/tests/integration/test_redis_client.py index 9c048ca0f..4fc664c0b 100644 --- a/tests/integration/test_redis_client.py +++ b/tests/integration/test_redis_client.py @@ -113,6 +113,31 @@ def test_get_all_https_filter(self, redis_client): https_proxies = redis_client.getAll(https=True) assert len(https_proxies) == 1 + def test_get_all_residential_filter(self, redis_client): + res_proxy = Proxy("1.1.1.1:8080", source="test", is_residential=True) + non_res_proxy = Proxy("2.2.2.2:8080", source="test", is_residential=False) + redis_client.put(res_proxy) + redis_client.put(non_res_proxy) + + res_only = redis_client.getAll(residential=True) + assert len(res_only) == 1 + assert json.loads(res_only[0])["proxy"] == "1.1.1.1:8080" + + non_res_only = redis_client.getAll(residential=False) + assert len(non_res_only) == 1 + assert json.loads(non_res_only[0])["proxy"] == "2.2.2.2:8080" + + all_proxies = redis_client.getAll(residential=None) + assert len(all_proxies) == 2 + + def test_get_all_num_filter(self, redis_client): + redis_client.put(_make_proxy("1.2.3.4:8080")) + redis_client.put(_make_proxy("5.6.7.8:8080")) + redis_client.put(_make_proxy("9.9.9.9:8080")) + + limited = redis_client.getAll(num=2) + assert len(limited) == 2 + class TestRedisGetCount: diff --git a/tests/integration/test_ssdb_client.py b/tests/integration/test_ssdb_client.py index ab046ef7f..bc9366838 100644 --- a/tests/integration/test_ssdb_client.py +++ b/tests/integration/test_ssdb_client.py @@ -112,6 +112,31 @@ def test_get_all_https_filter(self, ssdb_client): https_proxies = list(ssdb_client.getAll(https=True)) assert len(https_proxies) == 1 + def test_get_all_residential_filter(self, ssdb_client): + res_proxy = Proxy("1.1.1.1:8080", source="test", is_residential=True) + non_res_proxy = Proxy("2.2.2.2:8080", source="test", is_residential=False) + ssdb_client.put(res_proxy) + ssdb_client.put(non_res_proxy) + + res_only = ssdb_client.getAll(residential=True) + assert len(res_only) == 1 + assert json.loads(res_only[0])["proxy"] == "1.1.1.1:8080" + + non_res_only = ssdb_client.getAll(residential=False) + assert len(non_res_only) == 1 + assert json.loads(non_res_only[0])["proxy"] == "2.2.2.2:8080" + + all_proxies = ssdb_client.getAll(residential=None) + assert len(all_proxies) == 2 + + def test_get_all_num_filter(self, ssdb_client): + ssdb_client.put(_make_proxy("1.2.3.4:8080")) + ssdb_client.put(_make_proxy("5.6.7.8:8080")) + ssdb_client.put(_make_proxy("9.9.9.9:8080")) + + limited = ssdb_client.getAll(num=2) + assert len(limited) == 2 + class TestSsdbGetCount: diff --git a/tests/unit/test_db_client.py b/tests/unit/test_db_client.py index 49ae429c4..293f17214 100644 --- a/tests/unit/test_db_client.py +++ b/tests/unit/test_db_client.py @@ -98,7 +98,7 @@ def test_get(self): db = self._make_client() db.client.get.return_value = '{"proxy": "1.2.3.4:8080"}' result = db.get(True) - db.client.get.assert_called_once_with(True) + db.client.get.assert_called_once_with(https=True, residential=None) assert result == '{"proxy": "1.2.3.4:8080"}' def test_put(self): @@ -127,16 +127,17 @@ def test_pop(self): db = self._make_client() db.client.pop.return_value = '{"proxy": "1.2.3.4:8080"}' result = db.pop(True) - db.client.pop.assert_called_once_with(True) + db.client.pop.assert_called_once_with(https=True, residential=None) assert result == '{"proxy": "1.2.3.4:8080"}' def test_getAll(self): db = self._make_client() db.client.getAll.return_value = [] result = db.getAll(False) - db.client.getAll.assert_called_once_with(False) + db.client.getAll.assert_called_once_with(https=False, residential=None, num=None, raw=False) assert result == [] + def test_clear(self): db = self._make_client() db.clear() diff --git a/tests/unit/test_proxy.py b/tests/unit/test_proxy.py index d8f1bb680..2543daa18 100644 --- a/tests/unit/test_proxy.py +++ b/tests/unit/test_proxy.py @@ -31,6 +31,7 @@ def test_default_values(self): assert p.last_status == "" assert p.last_time == "" assert p.https is False + assert p.is_residential is False def test_custom_values(self): p = Proxy( @@ -43,6 +44,7 @@ def test_custom_values(self): last_status=True, last_time="2024-01-01 00:00:00", https=True, + is_residential=True, ) assert p.proxy == "5.6.7.8:443" assert p.fail_count == 3 @@ -53,6 +55,8 @@ def test_custom_values(self): assert p.last_status is True assert p.last_time == "2024-01-01 00:00:00" assert p.https is True + assert p.is_residential is True + def test_source_with_slash(self): """source 含 / 时应被拆分为列表,读回时用 / 连接""" @@ -67,7 +71,8 @@ def test_to_dict_keys(self): p = Proxy("1.2.3.4:8080") d = p.to_dict expected_keys = {"proxy", "https", "fail_count", "region", "anonymous", - "source", "check_count", "last_status", "last_time"} + "source", "check_count", "last_status", "last_time", "is_residential"} + assert set(d.keys()) == expected_keys def test_to_dict_values(self): diff --git a/tests/unit/test_proxy_handler.py b/tests/unit/test_proxy_handler.py index 441b89a75..f49d3922e 100644 --- a/tests/unit/test_proxy_handler.py +++ b/tests/unit/test_proxy_handler.py @@ -48,7 +48,7 @@ def test_get_returns_proxy(self): assert result is not None assert result.proxy == "1.2.3.4:8080" assert result.https is False - handler._mock_db.get.assert_called_once_with(False) + handler._mock_db.get.assert_called_once_with(https=False, residential=None) def test_get_returns_none_when_empty(self): """DbClient 返回 None -> None""" @@ -69,7 +69,7 @@ def test_get_https_forwarded(self): assert result is not None assert result.https is True - handler._mock_db.get.assert_called_once_with(True) + handler._mock_db.get.assert_called_once_with(https=True, residential=None) class TestProxyHandlerPop: @@ -85,7 +85,7 @@ def test_pop_returns_proxy(self): assert result is not None assert result.proxy == "1.2.3.4:8080" - handler._mock_db.pop.assert_called_once_with(False) + handler._mock_db.pop.assert_called_once_with(https=False, residential=None) def test_pop_returns_none_when_empty(self): """pop 无数据时返回 None""" @@ -138,7 +138,8 @@ def test_getAll_returns_proxy_list(self): assert len(result) == 2 assert result[0].proxy == "1.2.3.4:8080" assert result[1].proxy == "5.6.7.8:443" - handler._mock_db.getAll.assert_called_once_with(False) + handler._mock_db.getAll.assert_called_once_with(https=False, residential=None, num=None, raw=False) + def test_getAll_empty_returns_empty_list(self): """getAll 无数据返回空列表"""