Spaces:
Paused
Paused
| import os | |
| import json | |
| import random | |
| import re | |
| import requests | |
| import asyncio | |
| from fastapi import FastAPI, WebSocket, HTTPException, Depends | |
| from fastapi.responses import HTMLResponse, Response | |
| from fastapi.security import HTTPBasic, HTTPBasicCredentials | |
| from threading import Lock | |
| import uvicorn | |
| from faker import Faker | |
| from bs4 import BeautifulSoup | |
| import time | |
| app = FastAPI() | |
| faker = Faker() | |
| valid_proxies = {} | |
| invalid_proxies = {} | |
| proxies_lock = Lock() | |
| basic_auth = HTTPBasic() | |
| raw_proxy_urls = [ | |
| os.getenv("PROXY_API_URL", "http://pubproxy.com/api/proxy?format=txt&level=anonymous,elite&type=http,socks4,socks5&last_check=60&speed=25&limit=1&post=true&user_agent=true&cookies=true&referer=true"), | |
| 'https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt', | |
| 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', | |
| 'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt', | |
| 'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt', | |
| 'https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/proxy.txt', | |
| 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt', | |
| "https://storage.googleapis.com/river-treat-249913.appspot.com/p.txt", | |
| "https://storage.googleapis.com/river-treat-249913.appspot.com/proxy.txt", | |
| "https://storage.googleapis.com/river-treat-249913.appspot.com/ultimate.txt", | |
| 'https://raw.githubusercontent.com/proxylist/proxylist/master/proxy.txt', | |
| 'https://raw.githubusercontent.com/scrapfly/proxy-list/main/proxies.txt', | |
| 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTP.txt', | |
| 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4.txt', | |
| 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5.txt', | |
| 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS.txt', | |
| 'https://raw.githubusercontent.com/roosterkid/openproxylist/main/ALL.txt', | |
| 'https://raw.githubusercontent.com/proxylist/proxylist/master/https.txt', | |
| 'https://raw.githubusercontent.com/proxylist/proxylist/master/socks4.txt', | |
| 'https://raw.githubusercontent.com/proxylist/proxylist/master/socks5.txt', | |
| 'https://raw.githubusercontent.com/proxylist/proxylist/master/http.txt', | |
| 'https://raw.githubusercontent.com/proxylist/proxylist/master/all.txt', | |
| 'https://raw.githubusercontent.com/jetlore/proxies/master/proxy-list.txt', | |
| 'https://raw.githubusercontent.com/hookzof/proxy-list/main/proxy.txt', | |
| 'https://raw.githubusercontent.com/zzlol123/proxy-list/main/proxies.txt', | |
| 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/http.txt', | |
| 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/https.txt', | |
| 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/socks4.txt', | |
| 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/socks5.txt', | |
| 'https://raw.githubusercontent.com/sqSfg/Proxy-List/master/all.txt', | |
| 'https://www.proxy-list.download/api/v1/get?type=https', | |
| 'https://www.proxy-list.download/api/v1/get?type=http', | |
| 'https://www.proxy-list.download/api/v1/get?type=socks4', | |
| 'https://www.proxy-list.download/api/v1/get?type=socks5', | |
| 'https://www.proxy-list.download/api/v1/get?type=all', | |
| 'https://www.sslproxies.org/', | |
| 'https://www.us-proxy.org/', | |
| 'https://free-proxy-list.net/', | |
| 'https://www.proxy-list.download/', | |
| 'https://www.proxy-list.org/eng/proxylist.txt', | |
| "https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all", | |
| 'https://api.openproxylist.xyz/http.txt', | |
| 'https://api.proxyscrape.com/?request=displayproxies&proxytype=http&timeout=10000', | |
| 'https://proxylist.geonode.com/api/proxy-list?limit=500&page=1&sort_by=lastChecked&sort_type=desc', | |
| 'https://proxyocean.net/api/v1/গেট-প্রক্সি?api_key=demo', | |
| 'https://www.freeproxychecker.com/result/?protocol=http&anonymity=elite&anonymity=anonymous', | |
| 'https://premproxy.com/proxy-by-country/GB.htm', | |
| 'https://premproxy.com/list/time-01.htm', | |
| 'https://www.my-proxy.com/free-proxy-list.html', | |
| 'https://proxyservers.pro/proxy/list/protocol/http', | |
| 'https://proxyservers.pro/proxy/list/protocol/socks4', | |
| 'https://proxyservers.pro/proxy/list/protocol/socks5', | |
| 'https://spys.me/proxy.txt', | |
| 'https://www.proxynova.com/proxy-server-list/anonymous-proxies/', | |
| 'https://www.proxynova.com/proxy-server-list/elite-proxies/', | |
| 'https://www.proxy-daily.com/https_proxies.txt', | |
| 'https://www.proxy-daily.com/http_proxies.txt', | |
| 'https://www.proxy-daily.com/socks4_proxies.txt', | |
| 'https://www.proxy-daily.com/socks5_proxies.txt', | |
| 'https://free-proxy-list.net/anonymous-proxy.html', | |
| 'https://www.ip-adress.com/proxy-list/', | |
| 'https://hidemy.name/en/proxy-list/', | |
| 'https://www.kproxy.com/en/proxylist.html', | |
| 'https://www.megaproxy.com/frees proxies/', | |
| 'https://proxy-store.com/free-proxy-list', | |
| 'https://proxylists.net/http_anonymous_proxylist.txt', | |
| 'https://proxylists.net/http_elite_proxylist.txt', | |
| 'https://proxylists.net/socks4_proxylist.txt', | |
| 'https://proxylists.net/socks5_proxylist.txt', | |
| 'http://www.httptunnel.ge/ProxyListForFree.aspx', | |
| 'https://open-proxy.net/anonymous-proxies', | |
| 'https://open-proxy.net/elite-proxies', | |
| 'https://premproxy.com/socks-proxy/', | |
| 'https://www.socks-proxy.net/', | |
| 'https://www.freeproxylists.net/', | |
| 'https://www.proxies.net/', | |
| 'https://www.proxy-rack.com/free-proxy-list/', | |
| 'https://www.vipsocks24.net/proxylist-http', | |
| 'https://www.vipsocks24.net/proxylist-socks4', | |
| 'https://www.vipsocks24.net/proxylist-socks5', | |
| 'https://www.cool-proxy.net/proxies/http_proxy_list', | |
| 'https://www.cool-proxy.net/proxies/socks4_proxy_list', | |
| 'https://www.cool-proxy.net/proxies/socks5_proxy_list', | |
| 'https://proxygather.com/proxylist/anonymity/anonymous', | |
| 'https://proxygather.com/proxylist/anonymity/elite', | |
| 'https://www.preproxy.com/proxy-servers', | |
| 'https://www.preproxy.com/socks5-proxies', | |
| 'https://www.preproxy.com/socks4-proxies', | |
| 'https://www.proxy-listen.de/en/proxy-server-list/http.html', | |
| 'https://www.proxy-listen.de/en/proxy-server-list/https.html', | |
| 'https://www.proxy-listen.de/en/proxy-server-list/socks4.html', | |
| 'https://www.proxy-listen.de/en/proxy-server-list/socks5.html', | |
| 'https://www.freeproxylists.net/en/http_proxies.html', | |
| 'https://www.freeproxylists.net/en/https_proxies.html', | |
| 'https://www.freeproxylists.net/en/socks4_proxies.html', | |
| 'https://www.freeproxylists.net/en/socks5_proxies.html', | |
| 'https://www.xroxy.com/proxylist.htm', | |
| 'https://proxylist.me/api/v1/getProxy?anon=elite', | |
| 'https://proxylist.me/api/v1/getProxy?anon=anonymous', | |
| 'https://proxylist.me/api/v1/getProxy?anon=transparent', | |
| 'https://proxylist.me/api/v1/getProxy?type=http', | |
| 'https://proxylist.me/api/v1/getProxy?type=https', | |
| 'https://proxylist.me/api/v1/getProxy?type=socks4', | |
| 'https://proxylist.me/api/v1/getProxy?type=socks5', | |
| 'https://www.proxyscan.io/api/proxy?type=http', | |
| 'https://www.proxyscan.io/api/proxy?type=https', | |
| 'https://www.proxyscan.io/api/proxy?type=socks4', | |
| 'https://www.proxyscan.io/api/proxy?type=socks5', | |
| 'https://www.proxyscan.io/api/proxy?last_check=60', | |
| 'https://openproxies.pl/proxylist.txt', | |
| 'https://openproxies.pl/socks4.txt', | |
| 'https://openproxies.pl/socks5.txt', | |
| 'https://www.gatherproxy.com/zh/proxylist/country/?c=United%20States', | |
| 'https://www.free-proxy-cz.com/en/proxylist/country/all/http/ping/all', | |
| 'https://www.fre-proxy.ru/en/proxy/country/ru', | |
| 'https://www.proxydocker.com/en/proxylist/country/all', | |
| 'https://proxydb.net/?protocol=http&anonlvl=4', | |
| 'https://proxydb.net/?protocol=socks4&anonlvl=4', | |
| 'https://proxydb.net/?protocol=socks5&anonlvl=4', | |
| 'https://www.ip-tracker.org/proxies/https-proxy-list.php', | |
| 'https://www.ip-tracker.org/proxies/socks4-proxy-list.php', | |
| 'https://www.ip-tracker.org/proxies/socks5-proxy-list.php', | |
| 'https://www.proxy-server-list.com/country/us', | |
| 'https://www.7proxies.net/index.php/proxy-list/proxies.html', | |
| 'https://www.best-proxy.net/en/country/code/', | |
| 'https://www.haiproxy.com/en/free-proxy-list/', | |
| 'https://www.bonanza.com/items/like/228248525/VPN-Proxy-List-100000-Fresh-Daily-Updated-Private-Proxies-HTTP-SOCKS-Proxy', | |
| 'https://www.megaproxylist.net/anonymous-proxy-list.html', | |
| 'https://www.megaproxylist.net/elite-proxy-list.html', | |
| 'https://www.proxy-ipv4.net/en/proxy-list-anonymous.html', | |
| 'https://www.proxy-ipv4.net/en/proxy-list-elite-anonymous.html', | |
| 'https://www.proxiesforfree.com/proxies/http_proxies.html', | |
| 'https://www.proxiesforfree.com/proxies/socks_proxies.html', | |
| 'https://www.proxy-list.download/HTTPS', | |
| 'https://www.proxy-list.download/HTTP', | |
| 'https://www.proxy-list.download/SOCKS4', | |
| 'https://www.proxy-list.download/SOCKS5', | |
| 'https://www.freeproxyme.org/proxy-list.html', | |
| 'https://proxy-hub.com/en/us-anonymous-proxy-list.html', | |
| 'https://www.ipaddress.com/proxy-list/', | |
| 'https://www.proxylists.net/http.txt', | |
| 'https://www.proxylists.net/socks4.txt', | |
| 'https://www.proxylists.net/socks5.txt', | |
| 'https://www.proxy-my-ip.com/free-proxy-list.html', | |
| 'https://www.proxy-german.de/proxy-list/', | |
| 'https://proxylist.cc/free-proxy-list/', | |
| 'https://www.proxy-checker.net/proxy_list.php?type=https', | |
| 'https://www.proxy-checker.net/proxy_list.php?type=http', | |
| 'https://www.proxydrop.com/en/us-proxy-list/', | |
| 'https://www.proxydrop.com/en/elite-proxy-list/', | |
| 'https://www.proxydrop.com/en/anonymous-proxy-list/', | |
| 'https://www.proxy-store.com/proxylist/country/us', | |
| 'https://www.proxy-israel.com/en/proxy-list/', | |
| 'https://www.proxies-socks5.net/proxy-lists/country-us', | |
| 'https://www.proxies-socks5.net/proxy-lists/country-de', | |
| 'https://www.proxies-socks5.net/proxy-lists/type-socks5', | |
| 'https://www.proxies-socks5.net/proxy-lists/type-socks4', | |
| 'https://www.proxy-list.org/en/socks-list/country-US.html', | |
| 'https://www.proxy-ipv6.net/index_anonymous.html', | |
| 'https://www.proxy-ipv6.net/index_elite.html', | |
| 'https://www.proxy-ipv6.net/index_transparent.html', | |
| 'https://www.proxy-ipv6.net/index_ssl.html', | |
| 'https://www.proxies-socks5.net/proxy-lists/anonymous-elite', | |
| 'https://www.proxies-socks5.net/proxy-lists/anonymous', | |
| 'https://www.proxies-socks5.net/proxy-lists/transparent', | |
| 'https://www.proxynow.top/proxy-list-anonymous', | |
| 'https://www.proxynow.top/proxy-list-elite', | |
| 'https://www.proxy-server.net/proxies/anonymous-elite-proxy-list.aspx', | |
| 'https://www.proxy-server.net/proxies/anonymous-proxy-list.aspx', | |
| 'https://www.proxy-server.net/proxies/transparent-proxy-list.aspx', | |
| 'https://www.proxydaddy.com/proxylist.php?protocol=HTTP', | |
| 'https://www.proxydaddy.com/proxylist.php?protocol=HTTPS', | |
| 'https://www.proxydaddy.com/proxylist.php?protocol=SOCKS4', | |
| 'https://www.proxydaddy.com/proxylist.php?protocol=SOCKS5', | |
| 'https://www.proxy4free.com/list/webproxy_anon.html', | |
| 'https://www.proxy4free.com/list/webproxy_elite.html', | |
| 'https://www.freeproxylist.co/anonymous.html', | |
| 'https://www.freeproxylist.co/elite.html', | |
| 'https://www.proxylisty.com/anonymous-proxies', | |
| 'https://www.proxylisty.com/elite-proxies', | |
| 'https://www.aliveproxy.com/anonymous-proxy-list.aspx', | |
| 'https://www.aliveproxy.com/elite-proxy-list.aspx', | |
| 'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/https.php', | |
| 'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/http.php', | |
| 'https://www.proxy- ежедневно- бесплатно.rf/freeproxy/socks5.php', | |
| 'https://proxyservers.net/proxy-list/sort/country/order/asc', | |
| 'https://www.ip2location.com/free/proxy-list', | |
| 'https://www.yougetproxy.com/zh-cn/https-proxy-list/', | |
| 'https://www.yougetproxy.com/zh-cn/http-proxy-list/', | |
| 'https://www.ip-finder.net/en/proxylist/anonymous-proxies', | |
| 'https://www.ip-finder.net/en/proxylist/elite-proxies', | |
| 'https://www.rapidunblock.com/proxy-list/', | |
| 'https://www.privateproxyguide.com/anonymous-proxy-list/', | |
| 'https://www.privateproxyguide.com/elite-proxy-list/', | |
| 'https://www.proxy-german.de/', | |
| 'https://www.proxy-italia.it/proxy-list/', | |
| 'https://www.proxylistchecker.pro/https_proxies.txt', | |
| 'https://www.proxylistchecker.pro/http_proxies.txt', | |
| 'https://www.proxylistchecker.pro/socks4_proxies.txt', | |
| 'https://www.proxylistchecker.pro/socks5_proxies.txt', | |
| 'https://www.proxy-list.org/en/socks-list-1.html', | |
| 'https://www.proxy-list.org/en/https-anonymous-elite-proxy-list-1.html', | |
| 'https://www.proxy-list.org/en/https-anonymous-proxy-list-1.html', | |
| 'https://www.proxy-list.org/en/http-anonymous-elite-proxy-list-1.html', | |
| 'https://www.proxy-list.org/en/http-anonymous-proxy-list-1.html', | |
| 'https://www.proxyserverlist24.top/proxylists/http', | |
| 'https://www.proxyserverlist24.top/proxylists/https', | |
| 'https://www.proxyserverlist24.top/proxylists/socks4', | |
| 'https://www.proxyserverlist24.top/proxylists/socks5', | |
| 'https://www.proxy-ipv4.net/', | |
| 'https://www.proxy-ipv6.net/', | |
| 'https://www.proxy-list.org/', | |
| 'https://www.proxyserverlist24.top/', | |
| 'https://www.freeproxy.world/', | |
| 'https://www.free-proxy-server.net/', | |
| 'https://www.proxylists.me/', | |
| 'https://www.proxy-directory.com/', | |
| 'https://www.proxy-provider.net/', | |
| 'https://www.proxy-server.net/', | |
| 'https://www.proxy-list.me/', | |
| 'https://www.proxy-daily.com/', | |
| 'https://www.proxy-store.com/', | |
| 'https://www.proxy-israel.com/', | |
| 'https://www.proxy-german.de/', | |
| 'https://www.proxy-italia.it/', | |
| 'https://www.proxynow.top/', | |
| 'https://www.proxydaddy.com/', | |
| 'https://www.proxy4free.com/', | |
| 'https://www.freeproxylist.co/', | |
| 'https://www.proxylisty.com/', | |
| 'https://www.aliveproxy.com/', | |
| 'https://proxyscrape.com/free-proxy-list', | |
| 'https://www.goldproxy.jp/proxylist/anonymous.html', | |
| 'https://www.goldproxy.jp/proxylist/country/US.html', | |
| 'https://www.goldproxy.jp/proxylist/elite.html', | |
| 'https://www.ip-port.net/free-proxy-anonymous-server-list', | |
| 'https://www.ip-port.net/free-proxy-elite-server-list', | |
| 'https://www.ip-port.net/free-proxy-transparent-server-list', | |
| 'https://www.gatherproxy.com/', | |
| 'https://incloak.com/proxy-list/', | |
| 'https://proxyservers.pro/', | |
| 'https://www.proxy-anonymous.com/en/us-free-proxy-list.shtml', | |
| 'https://www.proxy-anonymous.com/en/country_US-01.shtml', | |
| 'https://www.proxy-anonymous.com/en/http-ssl-anonymous-proxy-list.shtml', | |
| 'https://www.proxy-anonymous.com/en/socks-anonymous-proxy-list.shtml', | |
| 'https://www.checkerproxy.net/api/anon/http', | |
| 'https://www.checkerproxy.net/api/anon/https', | |
| 'https://www.checkerproxy.net/api/anon/socks4', | |
| 'https://www.checkerproxy.net/api/anon/socks5', | |
| 'https://www.proxyscout.io/en/https', | |
| 'https://www.proxyscout.io/en/socks4', | |
| 'https://www.proxyscout.io/en/socks5', | |
| 'https://www.proxy-list.download/SOCKS', | |
| 'https://www.proxy-list.org/english/index.php?p=http', | |
| 'https://www.proxy-list.org/english/index.php?p=https', | |
| 'https://www.proxy-list.org/english/index.php?p=socks4', | |
| 'https://www.proxy-list.org/english/index.php?p=socks5', | |
| 'https://proxylists.net/', | |
| 'https://www.rapidunblock.com/', | |
| 'https://www.privateproxyguide.com/', | |
| 'https://www.proxy-directory.com/proxylists.php', | |
| 'https://www.proxy-provider.net/proxylists.php', | |
| 'https://www.proxy-server.net/proxylists.php', | |
| 'https://www.proxy-list.me/proxylists.php', | |
| 'https://www.proxy-daily.com/proxylists.php', | |
| 'https://www.proxy-store.com/proxylists.php', | |
| 'https://www.proxy-israel.com/proxylists.php', | |
| 'https://www.proxy-german.de/proxylists.php', | |
| 'https://www.proxy-italia.it/proxylists.php', | |
| 'https://www.proxynow.top/proxylists.php', | |
| 'https://www.proxydaddy.com/proxylists.php', | |
| 'https://www.proxy4free.com/proxylists.php', | |
| 'https://www.freeproxylist.co/proxylists.php', | |
| 'https://www.proxylisty.com/proxylists.php', | |
| 'https://www.aliveproxy.com/proxylists.aspx', | |
| 'https://www.proxyscrape.com/proxylists.php', | |
| 'https://www.goldproxy.jp/proxylists.php', | |
| 'https://www.ip-port.net/proxylists.php', | |
| 'https://www.gatherproxy.com/proxylists.php', | |
| 'https://incloak.com/proxylists.php', | |
| 'https://www.proxyservers.pro/proxylists.php', | |
| 'https://www.proxy-anonymous.com/proxylists.php', | |
| 'https://www.checkerproxy.net/proxylists.php', | |
| 'https://www.proxyscout.io/proxylists.php', | |
| 'https://www.proxy-list.download/proxylists.php', | |
| 'https://www.proxy-list.org/proxylists.php', | |
| 'https://www.proxyserverlist24.top/proxylists.php', | |
| 'https://www.freeproxy.world/proxylists.php', | |
| 'https://www.free-proxy-server.net/proxylists.php', | |
| 'https://www.proxylists.me/proxylists.php', | |
| 'https://www.ip2location.com/proxylists.php', | |
| 'https://www.yougetproxy.com/proxylists.php', | |
| 'https://www.ip-finder.net/proxylists.php', | |
| 'https://www.freeproxylists.net/proxylists.php', # NEW SOURCE 202 | |
| 'https://www.xroxy.com/proxylists.php', # NEW SOURCE 203 | |
| 'https://www.sslproxies.org/proxylists.php', # NEW SOURCE 204 | |
| 'https://www.us-proxy.org/proxylists.php', # NEW SOURCE 205 | |
| 'https://www.socks-proxy.net/proxylists.php', # NEW SOURCE 206 | |
| 'https://www.httptunnel.ge/proxylists.php', # NEW SOURCE 207 | |
| 'https://open-proxy.net/proxylists.php', # NEW SOURCE 208 | |
| 'https://premproxy.com/proxylists.php', # NEW SOURCE 209 | |
| 'https://www.vipsocks24.net/proxylists.php', # NEW SOURCE 210 | |
| 'https://www.cool-proxy.net/proxylists.php', # NEW SOURCE 211 | |
| ] | |
| all_proxy_urls = list(set(raw_proxy_urls)) | |
| print(f"Total unique proxy URLs after deduplication: {len(all_proxy_urls)}") | |
| def create_headers(): | |
| user_agent = faker.user_agent() | |
| random_ip = faker.ipv4() | |
| return {"User-Agent": user_agent, "X-Forwarded-For": random_ip, "Client-IP": random_ip, "X-Real-IP": random_ip} | |
| def is_valid_proxy(proxy: str) -> bool: | |
| ip_port_pattern = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5}$') | |
| if not ip_port_pattern.match(proxy): | |
| return False | |
| ip, port = proxy.split(':') | |
| if any(c in ip for c in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ') or any(c in port for c in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'): | |
| return False | |
| return True | |
| async def verify_proxy(proxy: str): | |
| if not is_valid_proxy(proxy): | |
| with proxies_lock: | |
| invalid_proxies[proxy] = True | |
| return proxy, False | |
| test_urls = ["https://google.com", "https://www.youtube.com/", "http://httpbin.org/ip", "https://instagram.com"] | |
| proxy_dict = {"http": f"http://{proxy}", "https": f"http://{proxy}"} | |
| headers = create_headers() | |
| try: | |
| for url in test_urls: | |
| try: | |
| response = await asyncio.to_thread(requests.get, url, proxies=proxy_dict, headers=headers, timeout=10) | |
| response.raise_for_status() | |
| except (requests.RequestException, ValueError): | |
| with proxies_lock: | |
| invalid_proxies[proxy] = True | |
| return proxy, False | |
| with proxies_lock: | |
| valid_proxies[proxy] = True | |
| return proxy, True | |
| except Exception: | |
| with proxies_lock: | |
| valid_proxies.pop(proxy, None) | |
| invalid_proxies[proxy] = True | |
| return proxy, False | |
| async def verify_proxies_in_background(): | |
| while True: | |
| with proxies_lock: | |
| proxies_to_verify = list(valid_proxies.keys()) | |
| if proxies_to_verify: | |
| tasks = [verify_proxy(proxy) for proxy in proxies_to_verify] | |
| results = await asyncio.gather(*tasks) | |
| for proxy, is_valid in results: | |
| if not is_valid: | |
| with proxies_lock: | |
| valid_proxies.pop(proxy, None) | |
| invalid_proxies[proxy] = True | |
| await asyncio.sleep(0) | |
| async def fetch_proxies_from_sources(): | |
| proxies = set() | |
| for url in all_proxy_urls: | |
| response_text = fetch_response(url) | |
| if response_text: | |
| if response_text.startswith('{') and response_text.endswith('}'): | |
| try: | |
| data = json.loads(response_text) | |
| if isinstance(data, dict) and 'data' in data and 'proxies' in data['data'][0]: | |
| new_proxies = {proxy_item['ipPort'] for proxy_item in data['data'] if 'ipPort' in proxy_item} | |
| proxies.update(new_proxies) | |
| elif isinstance(data, dict) and 'PXs' in data.get('PXs', [{}]): | |
| try: | |
| new_proxies_list = json.loads(response_text.replace("PXs:", '"PXs":').replace("PX:", '"PX":').replace("Port:", '"Port":').replace("Type:", '"Type":').replace("Country:", '"Country":').replace("Anonymity:", '"Anonymity":').replace("Last Check:", '"Last Check":'))['PXs'] | |
| new_proxies = {f"{proxy_item['PX']}:{proxy_item['Port']}" for proxy_item in new_proxies_list if 'PX' and 'Port' in proxy_item} | |
| proxies.update(new_proxies) | |
| except (ValueError, KeyError, TypeError): | |
| pass | |
| elif isinstance(data, dict) and 'data' in data and 'proxy' in data['data'][0] and 'port' in data['data'][0]: | |
| new_proxies = {f"{data_item['proxy']}:{data_item['port']}" for data_item in data['data'] if 'proxy' and 'port' in data_item} | |
| proxies.update(new_proxies) | |
| elif isinstance(data, dict) and 'data' in data and 'ip' in data['data'][0] and 'port' in data['data'][0]: | |
| new_proxies = {f"{data_item['ip']}:{data_item['port']}" for data_item in data['data'] if 'ip' and 'port' in data_item} | |
| proxies.update(new_proxies) | |
| except (ValueError, KeyError, IndexError): | |
| pass | |
| elif any(w in url for w in ['free-proxy-list.net', 'freeproxychecker.com', 'my-proxy.com', 'premproxy.com', 'sslproxies.org', 'us-proxy.org', 'proxynova.com', 'hidemy.name', 'kproxy.com', 'megaproxy.com', 'proxy-store.com', 'httptunnel.ge', 'open-proxy.net', 'socks-proxy.net', 'freeproxylists.net', 'proxies.net', 'proxy-rack.com', 'vipsocks24.net', 'cool-proxy.net', 'ip-adress.com', 'proxygather.com', 'preproxy.com', 'proxy-listen.de', 'xroxy.com', 'proxydocker.com', 'ip-tracker.org', 'proxy-server-list.com', '7proxies.net', 'best-proxy.net', 'haiproxy.com', 'bonanza.com', 'megaproxylist.net', 'proxy-ipv4.net', 'proxiesforfree.com', 'freeproxyme.org', 'proxy-hub.com', 'ipaddress.com', 'proxy-my-ip.com', 'proxy-german.de', 'proxylist.cc', 'proxy-checker.net', 'proxydrop.com', 'proxy-israel.com', 'proxies-socks5.net', 'proxynow.top', 'proxy-server.net', 'proxydaddy.com', 'proxy4free.com', 'freeproxylist.co', 'proxylisty.com', 'aliveproxy.com', 'proxy- ежедневно- бесплатно.rf', 'proxyservers.net', 'ip2location.com', 'yougetproxy.com', 'ip-finder.net', 'rapidunblock.com', 'privateproxyguide.com', 'proxy-german.de', 'proxy-italia.it', 'proxylistchecker.pro', 'proxy-list.org', 'proxyserverlist24.top', 'proxy-ipv4.net', 'proxy-ipv6.net', 'freeproxy.world', 'free-proxy-server.net', 'proxylists.me', 'proxy-directory.com', 'proxy-provider.net', 'proxy-server.net', 'proxy-list.me', 'proxy-daily.com', 'proxy-store.com', 'proxy-israel.com', 'proxy-german.de', 'proxy-italia.it', 'proxynow.top', 'proxydaddy.com', 'proxy4free.com', 'freeproxylist.co', 'proxylisty.com', 'aliveproxy.com', 'proxyscrape.com', 'goldproxy.jp', 'ip-port.net', 'gatherproxy.com', 'incloak.com', 'proxyservers.pro', 'proxy-anonymous.com', 'checkerproxy.net', 'proxyscout.io']): | |
| soup = BeautifulSoup(response_text, 'html.parser') | |
| tables = soup.find_all('table') | |
| for table in tables: | |
| if 'proxylisttable' in table.get('id', '') or 'list' in table.get('class', []) or 'proxy' in table.get('class', []) or 'tbl-cnt' in table.get('class', []) or 'table-striped' in table.get('class', []) or 'rt-table' in table.get('class', []) or 'item-list' in table.get('class', []) or 'table_block' in table.get('class', []) or 'proxy_list' in table.get('id', []) or 'table-responsive' in table.get('class', []) or 'proxy_table' in table.get('class', []) or 'dataTable' in table.get('class', []) or 'tablepress' in table.get('id', []) or 'proxy_table_list' in table.get('id', []) or 'table-container' in table.get('class', []) or 'proxy_table_main' in table.get('class', []) or 'wpDataTable' in table.get('class', []) or 'table_proxy' in table.get('class', []) or 'proxy-table' in table.get('class', []) or 'itemlist' in table.get('class', []) or 'container' in table.get('class', []) or 'proxy-list-table' in table.get('class', []): | |
| for row in table.find_all('tr')[1:]: | |
| try: | |
| columns = row.find_all('td') | |
| if len(columns) >= 2: | |
| ip = columns[0].text.strip() | |
| port = columns[1].text.strip() | |
| proxy = f"{ip}:{port}" | |
| proxies.add(proxy) | |
| except: | |
| pass | |
| elif 'spys.me' in url: | |
| lines = response_text.splitlines() | |
| for line in lines: | |
| if line.startswith('IP:'): | |
| parts = line.split('PORT:') | |
| if len(parts) == 2: | |
| ip_part = parts[0].replace('IP:', '').strip() | |
| port_part = parts[1].strip() | |
| ip = ''.join(filter(str.isdigit, ip_part)) + '.' + ''.join(filter(str.isdigit, ip_part.split('.')[-1])) if '.' in ip_part else ''.join(filter(str.isdigit, ip_part)) | |
| port = ''.join(filter(str.isdigit, port_part)) | |
| if ip and port: | |
| proxies.add(f"{ip}:{port}") | |
| elif 'proxy-daily.com' in url or 'proxylistchecker.pro' in url: | |
| lines = response_text.splitlines() | |
| for line in lines: | |
| line = line.strip() | |
| if line and not line.startswith('#') and ':' in line: | |
| proxies.add(line) | |
| elif any(w in url for w in ['proxylists.net', 'proxy-spider.com', 'proxyservers.pro', 'open-proxy.net', 'socks-proxy.net', 'freeproxylists.net', 'proxies.net', 'proxy-rack.com', 'vipsocks24.net', 'cool-proxy.net', 'openproxies.pl', 'proxydb.net', '7proxies.net', 'haiproxy.com', 'megaproxylist.net', 'proxy-ipv4.net', 'proxiesforfree.com', 'freeproxyme.org', 'proxy-hub.com', 'proxy-my-ip.com', 'proxy-german.de', 'proxylist.cc', 'proxy-checker.net', 'proxydrop.com', 'proxy-israel.com', 'proxies-socks5.net', 'proxynow.top', 'proxy-server.net', 'proxydaddy.com', 'proxy4free.com', 'freeproxylist.co', 'proxylisty.com', 'aliveproxy.com', 'proxy- ежедневно- бесплатно.rf', 'proxyservers.net', 'yougetproxy.com', 'proxyscout.io']): | |
| lines = response_text.splitlines() | |
| new_proxies = {line.strip() for line in lines if line.strip() and ":" in line} | |
| proxies.update(new_proxies) | |
| elif any(w in url for w in ['proxylist.me', 'proxyscan.io', 'checkerproxy.net']): | |
| try: | |
| api_proxies = json.loads(response_text) | |
| if isinstance(api_proxies, list): | |
| for proxy_string in api_proxies: | |
| if ":" in proxy_string: | |
| proxies.add(proxy_string.strip()) | |
| elif isinstance(api_proxies, dict) and 'data' in api_proxies: | |
| for proxy_data in api_proxies['data']: | |
| proxy = f"{proxy_data['ip']}:{proxy_data['port']}" | |
| proxies.add(proxy) | |
| elif isinstance(api_proxies, dict) and 'proxies' in api_proxies: | |
| for proxy_data in api_proxies['proxies']: | |
| proxy = f"{proxy_data['addr']}:{proxy_data['port']}" | |
| proxies.add(proxy) | |
| except (ValueError, KeyError, TypeError): | |
| pass | |
| elif any(w in url for w in ['ip-tracker.org', 'proxy-server-list.com', 'gatherproxy.com', 'preproxy.com', 'best-proxy.net', 'ipaddress.com', 'ip2location.com', 'proxy-anonymous.com', 'goldproxy.jp', 'ip-port.net', 'incloak.com']): | |
| soup = BeautifulSoup(response_text, 'html.parser') | |
| scripts = soup.find_all('script') | |
| for script in scripts: | |
| script_text = script.string | |
| if script_text: | |
| proxies.update(re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5})', script_text)) | |
| if 'ipaddress.com' in url: | |
| pre_tags = soup.find_all('pre') | |
| for pre_tag in pre_tags: | |
| proxies.update(re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5})', pre_tag.get_text())) | |
| if 'ip2location.com' in url or 'proxy-checker.net' in url: | |
| textarea_tags = soup.find_all('textarea') | |
| for textarea_tag in textarea_tags: | |
| proxies.update(re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5})', textarea_tag.get_text())) | |
| if 'gatherproxy.com' in url: | |
| gp_proxies = re.findall(r"GP\(\'(.*?)\',\'(.*?)\'\)", response_text) | |
| for gp_proxy in gp_proxies: | |
| proxies.add(f"{gp_proxy[0].replace('\\','').strip()}:{gp_proxy[1].replace('\\','').strip()}") | |
| elif 'bonanza.com' in url: | |
| soup = BeautifulSoup(response_text, 'html.parser') | |
| proxies.update(re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,5})', soup.get_text())) | |
| else: | |
| lines = response_text.splitlines() | |
| new_proxies = {line.strip() for line in lines if line.strip()} | |
| proxies.update(new_proxies) | |
| time.sleep(2) | |
| final_proxies = set() | |
| for proxy in proxies: | |
| if is_valid_proxy(proxy): | |
| final_proxies.add(proxy) | |
| with proxies_lock: | |
| valid_proxies.update(final_proxies) | |
| return valid_proxies, invalid_proxies | |
| async def on_startup(): | |
| global valid_proxies, invalid_proxies | |
| valid_proxies, invalid_proxies = await fetch_proxies_from_sources() | |
| asyncio.create_task(verify_proxies_in_background()) | |
| def fetch_response(url): | |
| headers = create_headers() | |
| try: | |
| response = requests.get(url, headers=headers, timeout=20) | |
| response.raise_for_status() | |
| return response.text | |
| except requests.RequestException as e: | |
| print(f"Failed to fetch URL: {url} - Error: {e}") | |
| return None | |
| def get_current_user(credentials: HTTPBasicCredentials = Depends(basic_auth)): | |
| correct_username = os.environ.get("PROXY_API_USERNAME", "proxyuser") | |
| correct_password = os.environ.get("PROXY_API_PASSWORD", "proxypassword") | |
| if not (credentials.username == correct_username and credentials.password == correct_password): | |
| raise HTTPException(status_code=401, detail="Invalid credentials", headers={"WWW-Authenticate": "Basic"}) | |
| return credentials.username | |
| async def root(): | |
| html_content = "<html><body><h1>Proxy List</h1><p><a href='/valid'>Valid Proxies</a></p><p><a href='/invalid'>Invalid Proxies</a></p><p><a href='/valid-auth'>Valid Proxies (Authenticated)</a></p></body></html>" | |
| return HTMLResponse(content=html_content, status_code=200) | |
| async def valid_proxies_endpoint(): | |
| with proxies_lock: | |
| valid_proxies_list = list(valid_proxies.keys()) | |
| if len(valid_proxies_list) > 10: | |
| valid_proxies_list = random.sample(valid_proxies_list, 10) | |
| response_content = "\n".join(valid_proxies_list) | |
| return Response(content=response_content, media_type='text/plain') | |
| async def invalid_proxies_endpoint(): | |
| with proxies_lock: | |
| invalid_proxies_list = list(invalid_proxies.keys()) | |
| if len(invalid_proxies_list) > 10: | |
| invalid_proxies_list = random.sample(invalid_proxies_list, 10) | |
| response_content = "\n".join(invalid_proxies_list) | |
| return Response(content=response_content, media_type='text/plain') | |
| async def valid_proxies_auth_endpoint(username: str = Depends(get_current_user)): | |
| with proxies_lock: | |
| valid_proxies_list = list(valid_proxies.keys()) | |
| if len(valid_proxies_list) > 10: | |
| valid_proxies_list = random.sample(valid_proxies_list, 10) | |
| response_content = "\n".join(valid_proxies_list) | |
| return Response(content=response_content, media_type='text/plain') | |
| async def websocket_endpoint(websocket: WebSocket): | |
| await websocket.accept() | |
| while True: | |
| with proxies_lock: | |
| proxies_list = list(valid_proxies.keys()) | |
| await websocket.send_text("\n".join(proxies_list)) | |
| await asyncio.sleep(0) | |
| if __name__ == "__main__": | |
| uvicorn.run(app, host="0.0.0.0", port=7860) |