422 lines
16 KiB
Python
422 lines
16 KiB
Python
"""
|
|
FastRDP-NG: Proxy Manager Module.
|
|
Fetches free SOCKS5/HTTP proxy lists, tests them, and provides rotating proxy
|
|
support for all outbound connections. Integrates with scanner + spray modules.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import List, Optional, Tuple, Callable
|
|
|
|
import aiohttp
|
|
from aiohttp_socks import open_connection as socks_open_connection
|
|
from aiohttp_socks import ProxyType, ProxyConnectionError
|
|
|
|
logger = logging.getLogger("FastRDP-NG")
|
|
|
|
# ── Proxy sources (free proxy lists from GitHub) ─────────────────────────
|
|
PROXY_SOURCES = [
|
|
# Proxifly format: JSON array of {ip, port, protocol, ...}
|
|
"https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies.json",
|
|
# TheSpeedX format: ip:port per line (HTTP)
|
|
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt",
|
|
# TheSpeedX format: ip:port per line (SOCKS4)
|
|
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt",
|
|
# TheSpeedX format: ip:port per line (SOCKS5)
|
|
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt",
|
|
# ProxyScrape format: ip:port per line
|
|
"https://api.proxyscrape.com/v2/?request=displayproxies&protocol=socks5&timeout=10000&country=all",
|
|
"https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=10000&country=all",
|
|
# Monosans format: ip:port per line
|
|
"https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/all.txt",
|
|
]
|
|
|
|
# Test URL for proxy verification (fast, reliable endpoint)
|
|
PROXY_TEST_URL = "http://httpbin.org/ip"
|
|
PROXY_TEST_TIMEOUT = 8.0
|
|
|
|
# Default credentials for proxy testing (most free proxies don't use auth)
|
|
NO_AUTH = ("", "")
|
|
|
|
# ── How often to refresh from sources (seconds) ──────────────────────────
|
|
REFRESH_INTERVAL = 300 # 5 minutes
|
|
|
|
|
|
@dataclass
|
|
class ProxyEntry:
|
|
"""A single proxy entry with status tracking."""
|
|
host: str
|
|
port: int
|
|
protocol: str # "socks5", "socks4", "http"
|
|
alive: bool = False
|
|
latency: float = 0.0 # ms
|
|
last_tested: float = 0.0
|
|
failures: int = 0
|
|
username: str = ""
|
|
password: str = ""
|
|
|
|
@property
|
|
def proxy_type(self) -> ProxyType:
|
|
if self.protocol == "socks4":
|
|
return ProxyType.SOCKS4
|
|
elif self.protocol == "socks5":
|
|
return ProxyType.SOCKS5
|
|
else:
|
|
return ProxyType.HTTP
|
|
|
|
@property
|
|
def url(self) -> str:
|
|
"""Return proxy URL string for aiohttp_socks."""
|
|
if self.username:
|
|
return f"{self.protocol}://{self.username}:{self.password}@{self.host}:{self.port}"
|
|
return f"{self.protocol}://{self.host}:{self.port}"
|
|
|
|
def __str__(self):
|
|
status = "\u2713" if self.alive else "\u2717"
|
|
return f"{status} {self.protocol}://{self.host}:{self.port} ({self.latency:.0f}ms)"
|
|
|
|
|
|
class ProxyManager:
|
|
"""
|
|
Manages a dynamic list of proxies with fetching, testing, rotation.
|
|
Thread-safe for use across asyncio event loops.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.proxies: List[ProxyEntry] = []
|
|
self._working: List[ProxyEntry] = []
|
|
self._rotation_index = 0
|
|
self._lock = asyncio.Lock()
|
|
|
|
# State
|
|
self.enabled = False
|
|
self.auto_rotate = True
|
|
self.testing = False
|
|
self.fetching = False
|
|
self.last_refresh = 0.0
|
|
|
|
# Stats
|
|
self.total_fetched = 0
|
|
self.working_count = 0
|
|
self.failed_count = 0
|
|
|
|
# Session for HTTP requests
|
|
self._session: Optional[aiohttp.ClientSession] = None
|
|
|
|
async def _get_session(self) -> aiohttp.ClientSession:
|
|
if self._session is None or self._session.closed:
|
|
self._session = aiohttp.ClientSession(
|
|
timeout=aiohttp.ClientTimeout(total=15)
|
|
)
|
|
return self._session
|
|
|
|
async def close(self):
|
|
if self._session and not self._session.closed:
|
|
await self._session.close()
|
|
|
|
# ── FETCHING ──────────────────────────────────────────────────────────
|
|
|
|
async def fetch_from_sources(
|
|
self,
|
|
progress_callback: Optional[Callable[[str], None]] = None
|
|
) -> int:
|
|
"""
|
|
Fetch proxy lists from all configured sources.
|
|
Returns total number of unique proxies collected.
|
|
"""
|
|
async with self._lock:
|
|
if self.fetching:
|
|
return len(self.proxies)
|
|
self.fetching = True
|
|
|
|
try:
|
|
session = await self._get_session()
|
|
all_proxies: List[Tuple[str, int, str]] = [] # (host, port, protocol)
|
|
seen = set()
|
|
|
|
for source_url in PROXY_SOURCES:
|
|
try:
|
|
if progress_callback:
|
|
progress_callback(f"Fetching: {source_url.split('/')[-1][:40]}...")
|
|
|
|
async with session.get(source_url, timeout=10) as resp:
|
|
if resp.status != 200:
|
|
continue
|
|
text = await resp.text()
|
|
|
|
# Parse based on source URL pattern
|
|
if "proxifly" in source_url.lower() or source_url.endswith(".json"):
|
|
# JSON format: [{"ip":"...","port":...,"protocol":"..."}]
|
|
parsed = self._parse_proxifly_json(text)
|
|
else:
|
|
# Plain text: ip:port per line
|
|
parsed = self._parse_plain_text(text)
|
|
|
|
for host, port, protocol in parsed:
|
|
key = (host, port, protocol)
|
|
if key not in seen:
|
|
seen.add(key)
|
|
all_proxies.append((host, port, protocol))
|
|
|
|
if progress_callback:
|
|
progress_callback(f" -> Got {len(parsed)} proxies from {source_url.split('/')[-1][:30]}")
|
|
|
|
except (asyncio.TimeoutError, aiohttp.ClientError, Exception) as e:
|
|
logger.debug(f"Proxy source failed: {source_url[:60]} - {e}")
|
|
continue
|
|
|
|
# Deduplicate and create entries
|
|
new_proxies = []
|
|
for host, port, protocol in all_proxies:
|
|
# Skip duplicates with existing entries
|
|
exists = any(
|
|
p.host == host and p.port == port and p.protocol == protocol
|
|
for p in self.proxies
|
|
)
|
|
if not exists:
|
|
new_proxies.append(ProxyEntry(
|
|
host=host, port=port, protocol=protocol
|
|
))
|
|
|
|
self.proxies.extend(new_proxies)
|
|
self.total_fetched = len(self.proxies)
|
|
|
|
if progress_callback:
|
|
progress_callback(f"\U0001f4e1 Total: {len(self.proxies)} unique proxies collected")
|
|
|
|
self.last_refresh = time.time()
|
|
return len(self.proxies)
|
|
|
|
finally:
|
|
async with self._lock:
|
|
self.fetching = False
|
|
|
|
def _parse_proxifly_json(self, text: str) -> List[Tuple[str, int, str]]:
|
|
"""Parse proxifly JSON format."""
|
|
import json
|
|
result = []
|
|
try:
|
|
data = json.loads(text)
|
|
if isinstance(data, list):
|
|
for entry in data:
|
|
ip = entry.get("ip", "") or entry.get("host", "")
|
|
port = entry.get("port", 0)
|
|
protocol = entry.get("protocol", "socks5").lower()
|
|
if ip and port:
|
|
# Map protocol names
|
|
if protocol in ("socks5", "socks4", "http", "https"):
|
|
if protocol == "https":
|
|
protocol = "http"
|
|
result.append((ip, int(port), protocol))
|
|
except (json.JSONDecodeError, ValueError):
|
|
pass
|
|
return result
|
|
|
|
def _parse_plain_text(self, text: str) -> List[Tuple[str, int, str]]:
|
|
"""Parse 'ip:port' per line format."""
|
|
result = []
|
|
# Determine protocol from context (will be overwritten by caller)
|
|
# We try to autodetect or default to socks5
|
|
for line in text.splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or line.startswith("//"):
|
|
continue
|
|
# Try ip:port format
|
|
if ":" in line and not line.startswith("["):
|
|
parts = line.split(":")
|
|
if len(parts) == 2:
|
|
ip, port_str = parts
|
|
try:
|
|
port = int(port_str)
|
|
if 1 <= port <= 65535:
|
|
# Default to socks5 (source determines actual type)
|
|
result.append((ip, port, "socks5"))
|
|
except ValueError:
|
|
pass
|
|
return result
|
|
|
|
# ── TESTING ───────────────────────────────────────────────────────────
|
|
|
|
async def test_proxy(
|
|
self,
|
|
proxy: ProxyEntry,
|
|
test_url: str = PROXY_TEST_URL,
|
|
timeout: float = PROXY_TEST_TIMEOUT
|
|
) -> bool:
|
|
"""
|
|
Test a single proxy by making an HTTP request through it.
|
|
Returns True if proxy is working.
|
|
"""
|
|
start = time.time()
|
|
try:
|
|
# Use aiohttp with ProxyConnector for testing
|
|
from aiohttp_socks import ProxyConnector
|
|
|
|
connector = ProxyConnector(
|
|
proxy_type=proxy.proxy_type,
|
|
proxy_host=proxy.host,
|
|
proxy_port=proxy.port,
|
|
username=proxy.username or None,
|
|
password=proxy.password or None,
|
|
rdns=True
|
|
)
|
|
|
|
async with aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=timeout)) as session:
|
|
async with session.get(test_url, timeout=timeout) as resp:
|
|
if resp.status == 200:
|
|
proxy.alive = True
|
|
proxy.latency = (time.time() - start) * 1000 # ms
|
|
proxy.last_tested = time.time()
|
|
proxy.failures = 0
|
|
return True
|
|
else:
|
|
proxy.alive = False
|
|
proxy.failures += 1
|
|
return False
|
|
|
|
except (ProxyConnectionError, asyncio.TimeoutError,
|
|
aiohttp.ClientError, OSError, ConnectionError):
|
|
proxy.alive = False
|
|
proxy.failures += 1
|
|
return False
|
|
finally:
|
|
proxy.last_tested = time.time()
|
|
|
|
async def test_all(
|
|
self,
|
|
max_concurrent: int = 100,
|
|
progress_callback: Optional[Callable[[int, int], None]] = None
|
|
) -> int:
|
|
"""
|
|
Test all untested/failed proxies in the pool.
|
|
Returns number of working proxies found.
|
|
"""
|
|
async with self._lock:
|
|
if self.testing:
|
|
return self.working_count
|
|
self.testing = True
|
|
|
|
try:
|
|
# Test proxies that haven't been tested or have failures
|
|
to_test = [
|
|
p for p in self.proxies
|
|
if not p.alive or p.failures > 0
|
|
]
|
|
|
|
if not to_test:
|
|
# Test a random sample if all are already alive
|
|
to_test = random.sample(self.proxies, min(50, len(self.proxies)))
|
|
|
|
total = len(to_test)
|
|
tested = 0
|
|
working = 0
|
|
|
|
sem = asyncio.Semaphore(max_concurrent)
|
|
|
|
async def test_one(proxy: ProxyEntry):
|
|
nonlocal tested, working
|
|
async with sem:
|
|
ok = await self.test_proxy(proxy)
|
|
tested += 1
|
|
if ok:
|
|
working += 1
|
|
if progress_callback and tested % 10 == 0:
|
|
progress_callback(tested, total)
|
|
|
|
tasks = [asyncio.create_task(test_one(p)) for p in to_test]
|
|
if tasks:
|
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
# Update working list
|
|
self._working = [p for p in self.proxies if p.alive]
|
|
self.working_count = len(self._working)
|
|
self.failed_count = len(self.proxies) - self.working_count
|
|
|
|
return self.working_count
|
|
|
|
finally:
|
|
async with self._lock:
|
|
self.testing = False
|
|
|
|
# ── ROTATION ──────────────────────────────────────────────────────────
|
|
|
|
async def get_proxy(self) -> Optional[ProxyEntry]:
|
|
"""
|
|
Get the next working proxy (round-robin if auto_rotate is enabled).
|
|
Returns None if no working proxies available.
|
|
"""
|
|
async with self._lock:
|
|
if not self._working:
|
|
return None
|
|
|
|
if self.auto_rotate and len(self._working) > 1:
|
|
proxy = self._working[self._rotation_index % len(self._working)]
|
|
self._rotation_index += 1
|
|
else:
|
|
proxy = self._working[self._rotation_index % len(self._working)]
|
|
|
|
return proxy
|
|
|
|
def mark_bad(self, proxy: ProxyEntry):
|
|
"""Mark a proxy as failed (connection error)."""
|
|
proxy.failures += 1
|
|
if proxy.failures >= 3:
|
|
proxy.alive = False
|
|
if proxy in self._working:
|
|
self._working.remove(proxy)
|
|
self.working_count = len(self._working)
|
|
self.failed_count = len(self.proxies) - self.working_count
|
|
|
|
# ── CONNECTION WRAPPER ────────────────────────────────────────────────
|
|
|
|
async def open_connection(self, host: str, port: int, **kwargs):
|
|
"""
|
|
Open a TCP connection through the current proxy.
|
|
If proxy is disabled or no working proxies available, falls back to
|
|
direct `asyncio.open_connection`.
|
|
|
|
Returns (reader, writer) - same as asyncio.open_connection.
|
|
"""
|
|
if not self.enabled:
|
|
return await asyncio.open_connection(host, port, **kwargs)
|
|
|
|
proxy = await self.get_proxy()
|
|
if proxy is None:
|
|
# No working proxy — fallback to direct
|
|
logger.debug("No working proxy available, falling back to direct connection")
|
|
return await asyncio.open_connection(host, port, **kwargs)
|
|
|
|
try:
|
|
return await asyncio.wait_for(
|
|
socks_open_connection(
|
|
proxy_url=proxy.url,
|
|
host=host,
|
|
port=port,
|
|
**kwargs
|
|
),
|
|
timeout=kwargs.get("timeout", 10.0) if "timeout" in kwargs else 10.0
|
|
)
|
|
except (ProxyConnectionError, asyncio.TimeoutError, OSError, ConnectionError) as e:
|
|
self.mark_bad(proxy)
|
|
# Retry with next proxy
|
|
logger.debug(f"Proxy {proxy} failed: {e}, trying next...")
|
|
return await self.open_connection(host, port, **kwargs)
|
|
|
|
# ── STATS ─────────────────────────────────────────────────────────────
|
|
|
|
def get_stats(self) -> dict:
|
|
"""Return proxy pool statistics."""
|
|
return {
|
|
"enabled": self.enabled,
|
|
"total": len(self.proxies),
|
|
"working": self.working_count,
|
|
"failed": self.failed_count,
|
|
"fetching": self.fetching,
|
|
"testing": self.testing,
|
|
"auto_rotate": self.auto_rotate,
|
|
"last_refresh": self.last_refresh,
|
|
}
|