changed proxy pool

docker-compose.yml

@@ -0,0 +1,18 @@
+version: '3.8'
+
+services:
+  freeplay2api:
+    build: .
+    ports:
+      - "7860:7860"
+    environment:
+      - API_KEY=sk-123456
+    volumes:
+      - ./data:/app/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7860/v1/models"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s

freeplay2api.py

@@ -159,6 +159,22 @@ class FreeplayClient:
 
     def register(self) -> Optional[Dict]:
         logging.info("REGISTER FUNCTION STARTED")
+        
+        # Wait for proxy pool to have at least one proxy before starting registration
+        if self.proxy_pool:
+            logging.info("Waiting for proxy pool to initialize...")
+            wait_time = 0
+            while self.proxy_pool.get_count() == 0 and wait_time < 60:  # Wait up to 60 seconds
+                time.sleep(1)
+                wait_time += 1
+                if wait_time % 10 == 0:  # Log every 10 seconds
+                    logging.info(f"Still waiting for proxies... ({wait_time}s elapsed)")
+            
+            if self.proxy_pool.get_count() > 0:
+                logging.info(f"Proxy pool ready with {self.proxy_pool.get_count()} proxies")
+            else:
+                logging.warning("Proxy pool initialization timed out, proceeding anyway")
+        
         url = "https://app.freeplay.ai/app_data/auth/signup"
         headers = {
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
@@ -201,24 +217,14 @@ class FreeplayClient:
                     continue
                 
                 logging.info(f"ABOUT TO MAKE REQUEST WITH PROXY {proxy_info['full'] if proxy_info else 'None'}")
-                try:
-                    response = requests.post(
-                        url,
-                        data=json.dumps(payload),
-                        headers=headers,
-                        proxies=proxies,
-                        timeout=20,
-                    )
-                    logging.info(f"REQUEST COMPLETED WITH STATUS {response.status_code}")
-                except requests.exceptions.ProxyError as proxy_err:
-                    logging.warning(f"CAUGHT PROXY ERROR: {proxy_err}")
-                    if self.proxy_pool and proxy_info:
-                        self.proxy_pool.remove_proxy(proxy_info['ip'], proxy_info['port'])
-                    logging.info(f"CONTINUING AFTER INNER PROXY ERROR TO ATTEMPT {attempt + 1}")
-                    continue
-                except Exception as inner_err:
-                    logging.error(f"CAUGHT OTHER ERROR IN REQUEST: {inner_err}")
-                    raise  # Re-raise to be caught by outer handlers
+                response = requests.post(
+                    url,
+                    data=json.dumps(payload),
+                    headers=headers,
+                    proxies=proxies,
+                    timeout=20,
+                )
+                logging.info(f"REQUEST COMPLETED WITH STATUS {response.status_code}")
                 
                 if response.status_code == 200:
                     data = response.json()
@@ -237,10 +243,18 @@ class FreeplayClient:
                 logging.warning(f"Registration attempt {attempt}/50 failed with status {response.status_code}: {response.text}")
                 logging.info(f"CONTINUING TO NEXT ATTEMPT {attempt + 1}")
 
+            except requests.exceptions.ProxyError as e:
+                logging.warning(f"CAUGHT REQUESTS.EXCEPTIONS.PROXYERROR: {e}")
+                if self.proxy_pool and proxy_info:
+                    logging.info(f"ABOUT TO REMOVE PROXY {proxy_info['ip']}:{proxy_info['port']}")
+                    self.proxy_pool.remove_proxy(proxy_info['ip'], proxy_info['port'])
+                    logging.info(f"PROXY REMOVAL COMPLETED")
+                logging.info(f"CONTINUING AFTER PROXY ERROR TO ATTEMPT {attempt + 1}")
+                continue
+            
             except Exception as e:
-                logging.error(f"An unexpected error occurred during registration on attempt {attempt}/50: {e}. Retrying...")
+                logging.error(f"CAUGHT GENERIC EXCEPTION: {type(e).__name__}: {e}")
                 logging.info(f"CONTINUING AFTER UNEXPECTED ERROR TO ATTEMPT {attempt + 1}")
-                # Explicitly continue to next iteration
                 continue
             
 
@@ -421,17 +435,16 @@ def initialize_app():
         default_config = {
             "HOST": "0.0.0.0",
             "PORT": 7860,
-            "ACCOUNTS_FILE": "accounts.json",
+            "ACCOUNTS_FILE": "data/accounts.json",
             "LOW_BALANCE_THRESHOLD": 2.0,
             "ACTIVE_KEY_THRESHOLD": 5,
             "CHECK_INTERVAL_SECONDS": 5,
             "REGISTRATION_CONCURRENCY": 1,
             "USE_PROXY_POOL": True,
             "PROXY_POOL_CONFIG": {
-                "target_count": 20,
-                "min_threshold": 5,
-                "check_interval": 30,
-                "proxy_protocol": "https"
+                "target_count": 10,
+                "min_threshold": 3,
+                "check_interval": 30
             }
         }
         try:
@@ -453,7 +466,7 @@ def initialize_app():
             proxy_pool_config = config.get("PROXY_POOL_CONFIG", {})
             logging.info(f"Proxy pool config being used: {proxy_pool_config}")
             proxy_pool = ProxyPool(proxy_pool_config)
-            # The initialize method will now run in the background
+            # Initialize proxy pool asynchronously to avoid blocking app startup
             threading.Thread(target=proxy_pool.initialize, daemon=True).start()
         else:
             logging.info("Proxy pool is disabled in config.")

proxy_pool.py

@@ -3,19 +3,17 @@ import threading
 import time
 import random
 from datetime import datetime, timedelta
+from urllib.parse import urlparse
 
 class ProxyPool:
     def __init__(self, options={}):
-        self.target_count = options.get('target_count', 20)
-        self.batch_size = options.get('batch_size', 20)
+        self.target_count = options.get('target_count', 10)  # Reduced since we get one at a time
         self.test_timeout = options.get('test_timeout', 5)
         self.request_timeout = options.get('request_timeout', 5)
         self.target_url = options.get('target_url', 'https://app.freeplay.ai/')
-        self.concurrent_requests = options.get('concurrent_requests', 10)
-        self.min_threshold = options.get('min_threshold', 5)
+        self.min_threshold = options.get('min_threshold', 3)  # Reduced threshold
         self.check_interval = options.get('check_interval', 30)
-        self.proxy_protocol = options.get('proxy_protocol', 'http')
-        self.max_refill_attempts = options.get('max_refill_attempts', 20)
+        self.max_refill_attempts = options.get('max_refill_attempts', 10)
         self.retry_delay = options.get('retry_delay', 1)
 
         self.available_proxies = []
@@ -45,7 +43,10 @@ class ProxyPool:
         with self.lock:
             if len(self.available_proxies) <= self.min_threshold and not self.is_refilling:
                 print(f"Available proxies ({len(self.available_proxies)}) below threshold ({self.min_threshold}), starting refill")
-                self.refill_proxies()
+                # Start refill in a separate thread to avoid blocking
+                refill_thread = threading.Thread(target=self.refill_proxies)
+                refill_thread.daemon = True
+                refill_thread.start()
         if self.is_initialized:
             self.check_timer = threading.Timer(self.check_interval, self.check_and_refill)
             self.check_timer.start()
@@ -62,33 +63,26 @@ class ProxyPool:
                 attempts += 1
                 print(f"Refill attempt #{attempts}, current available: {len(self.available_proxies)}/{self.target_count}")
                 
-                remaining_needed = self.target_count - len(self.available_proxies)
-                batch_size_needed = max(self.batch_size, remaining_needed * 2)
-                
-                proxies = self.get_proxies_from_provider(batch_size_needed)
-                if not proxies:
-                    print(f"No proxies received, retrying in {self.retry_delay} seconds...")
+                proxy_url = self.get_proxy_from_provider()
+                if not proxy_url:
+                    print(f"No proxy received, retrying in {self.retry_delay} seconds...")
                     time.sleep(self.retry_delay)
                     continue
 
-                new_proxies = self.filter_existing_proxies(proxies)
-                if not new_proxies:
-                    print("All fetched proxies already exist, getting new ones...")
+                if self.is_proxy_duplicate(proxy_url):
+                    print(f"Proxy {proxy_url} already exists, getting new one...")
                     continue
 
-                threads = []
-                for proxy in new_proxies:
-                    thread = threading.Thread(target=self.test_and_add_proxy, args=(proxy,))
-                    threads.append(thread)
-                    thread.start()
+                if self.test_and_add_proxy(proxy_url):
+                    print(f"Successfully added proxy, current available: {len(self.available_proxies)}/{self.target_count}")
+                else:
+                    print(f"Proxy {proxy_url} failed test, trying next...")
                 
-                for thread in threads:
-                    thread.join()
-
                 if len(self.available_proxies) >= self.target_count:
                     break
-                if len(self.available_proxies) < self.target_count:
-                    time.sleep(self.retry_delay)
+                    
+                time.sleep(0.5)  # Small delay between requests
+                
         except Exception as e:
             print(f"Error during proxy refill: {e}")
         finally:
@@ -98,45 +92,65 @@ class ProxyPool:
             else:
                 print(f"Max refill attempts reached, current available: {len(self.available_proxies)}/{self.target_count}")
 
-    def test_and_add_proxy(self, proxy):
-        if self.test_proxy(proxy):
-            with self.lock:
-                if not any(p['full'] == f"{self.proxy_protocol}://{proxy}" for p in self.available_proxies):
-                    ip, port = proxy.split(':')
-                    proxy_obj = {
-                        'ip': ip,
-                        'port': port,
-                        'protocol': self.proxy_protocol,
-                        'full': f"{self.proxy_protocol}://{proxy}",
-                        'added_at': datetime.now().isoformat()
-                    }
-                    self.available_proxies.append(proxy_obj)
-                    print(f"Successfully added proxy: {proxy_obj['full']}, current available: {len(self.available_proxies)}/{self.target_count}")
-
-    def filter_existing_proxies(self, proxies):
-        existing = {f"{p['ip']}:{p['port']}" for p in self.available_proxies}
-        return [p for p in proxies if p not in existing]
-
-    def get_proxies_from_provider(self, count=None):
+    def get_proxy_from_provider(self):
         try:
-            request_count = count or self.batch_size
-            url = f"https://proxy.scdn.io/api/get_proxy.php?protocol={self.proxy_protocol}&count={request_count}"
-            print(f"Getting proxies from: {url}")
+            url = "https://proxy.doudouzi.me/random/us?host=us.proxy302.com"
+            print(f"Getting proxy from: {url}")
             response = requests.get(url, timeout=10)
             if response.status_code == 200:
-                data = response.json()
-                if data.get('code') == 200:
-                    print(f"Successfully got {data['data']['count']} proxies")
-                    return data['data']['proxies']
-            print(f"Failed to get proxies. Status: {response.status_code}, Response: {response.text}")
-            return []
+                proxy_url = response.text.strip()
+                print(f"Successfully got proxy: {proxy_url}")
+                return proxy_url
+            print(f"Failed to get proxy. Status: {response.status_code}, Response: {response.text}")
+            return None
         except Exception as e:
-            print(f"Error getting proxies: {e}")
-            return []
+            print(f"Error getting proxy: {e}")
+            return None
+
+    def parse_proxy_url(self, proxy_url):
+        """Parse proxy URL like http://username:password@host:port"""
+        try:
+            parsed = urlparse(proxy_url)
+            return {
+                'protocol': parsed.scheme,
+                'username': parsed.username,
+                'password': parsed.password,
+                'host': parsed.hostname,
+                'port': str(parsed.port),
+                'full': proxy_url
+            }
+        except Exception as e:
+            print(f"Error parsing proxy URL {proxy_url}: {e}")
+            return None
+
+    def is_proxy_duplicate(self, proxy_url):
+        """Check if proxy already exists in the pool"""
+        # Note: This method assumes the caller already holds self.lock
+        return any(p['full'] == proxy_url for p in self.available_proxies)
+
+    def test_and_add_proxy(self, proxy_url):
+        if self.test_proxy(proxy_url):
+            with self.lock:
+                if not self.is_proxy_duplicate(proxy_url):
+                    parsed = self.parse_proxy_url(proxy_url)
+                    if parsed:
+                        proxy_obj = {
+                            'ip': parsed['host'],
+                            'port': parsed['port'],
+                            'protocol': parsed['protocol'],
+                            'username': parsed['username'],
+                            'password': parsed['password'],
+                            'full': parsed['full'],
+                            'added_at': datetime.now().isoformat()
+                        }
+                        self.available_proxies.append(proxy_obj)
+                        print(f"Successfully added proxy: {parsed['host']}:{parsed['port']}")
+                        return True
+        return False
 
     def test_proxy(self, proxy_url):
         try:
-            proxies = {self.proxy_protocol: f"{self.proxy_protocol}://{proxy_url}"}
+            proxies = {'http': proxy_url, 'https': proxy_url}
             headers = {
                 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
             }
@@ -145,7 +159,7 @@ class ProxyPool:
             if is_valid:
                 print(f"Proxy {proxy_url} successfully tested, status: {response.status_code}")
             else:
-                print(f"Proxy {proxy_url} failed test, status: {response.status_code}, headers: {response.headers}")
+                print(f"Proxy {proxy_url} failed test, status: {response.status_code}")
             return is_valid
         except Exception as e:
             print(f"Proxy {proxy_url} request error: {e}")
@@ -172,7 +186,15 @@ class ProxyPool:
                 print(f"Removed proxy {ip}:{port}, current available: {len(self.available_proxies)}")
             else:
                 print(f"Could not find proxy to remove {ip}:{port}")
-            self.check_and_refill()
+            
+            # Check if we need to refill asynchronously to avoid blocking the caller
+            if len(self.available_proxies) <= self.min_threshold and not self.is_refilling:
+                print(f"Available proxies ({len(self.available_proxies)}) below threshold ({self.min_threshold}), starting async refill")
+                # Start refill in a separate thread to avoid blocking
+                refill_thread = threading.Thread(target=self.refill_proxies)
+                refill_thread.daemon = True
+                refill_thread.start()
+            
             return removed
 
     def get_all_proxies(self):