Spaces:
Running
Running
Isaac Quarenta commited on
Commit ·
efbacd8
1
Parent(s): 5e3306f
Fix: pooler fallback agresivo (mais regiões + session pooler), fix SQLite %s→?, fix finetuning queries, pooler retry em todas tentativas
Browse files- modules/database_pg.py +33 -18
- modules/finetuning_pipeline.py +4 -7
modules/database_pg.py
CHANGED
|
@@ -120,7 +120,12 @@ class DatabasePG:
|
|
| 120 |
# ================================================================
|
| 121 |
# CONEXÃO
|
| 122 |
# ================================================================
|
| 123 |
-
_SUPAVISOR_REGIONS = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
|
| 125 |
def _is_ipv6_only(self, host: str) -> bool:
|
| 126 |
"""Check if hostname resolves only to IPv6 (no A record)."""
|
|
@@ -164,17 +169,18 @@ class DatabasePG:
|
|
| 164 |
# Supavisor user format: postgres.<ref>
|
| 165 |
pooler_user = f"postgres.{ref}"
|
| 166 |
|
| 167 |
-
# Try each region
|
| 168 |
for region in self._SUPAVISOR_REGIONS:
|
| 169 |
pooler_host = f"aws-0-{region}.pooler.supabase.com"
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
|
|
|
| 178 |
|
| 179 |
logger.warning("All Supavisor pooler regions unreachable via IPv4")
|
| 180 |
return None
|
|
@@ -227,10 +233,13 @@ class DatabasePG:
|
|
| 227 |
import re
|
| 228 |
match = re.match(r'postgresql://([^:]+):([^@]+)@([^:/]+)(?::(\d+))?/(.+)', dsn)
|
| 229 |
if match:
|
| 230 |
-
user, password, host, port,
|
| 231 |
-
|
| 232 |
port = port or '5432'
|
| 233 |
-
|
|
|
|
|
|
|
|
|
|
| 234 |
if 'connect_timeout' not in dsn:
|
| 235 |
sep = '&' if '?' in dsn else '?'
|
| 236 |
dsn = f"{dsn}{sep}connect_timeout=10"
|
|
@@ -259,19 +268,25 @@ class DatabasePG:
|
|
| 259 |
except psycopg2.OperationalError as e:
|
| 260 |
# Try Supavisor pooler fallback if direct connection fails
|
| 261 |
pooler_dsn = self._conn_params.get('pooler_dsn')
|
| 262 |
-
if pooler_dsn
|
| 263 |
-
logger.warning(f"Direct connection failed ({
|
| 264 |
try:
|
| 265 |
conn = self._connect_with_dsn(pooler_dsn, keepalive_opts)
|
| 266 |
conn.autocommit = False
|
| 267 |
-
# Update the primary DSN so future retries use pooler directly
|
| 268 |
self._conn_params['dsn'] = pooler_dsn
|
| 269 |
-
|
|
|
|
| 270 |
logger.success("Connected via Supavisor pooler")
|
| 271 |
return conn
|
| 272 |
except Exception as pooler_err:
|
| 273 |
logger.error(f"Supavisor pooler also failed: {pooler_err}")
|
| 274 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 275 |
if 'SSL connection has been closed' in str(e) and attempt < self.max_retries - 1:
|
| 276 |
time.sleep(self.retry_delay * (2 ** attempt))
|
| 277 |
continue
|
|
|
|
| 120 |
# ================================================================
|
| 121 |
# CONEXÃO
|
| 122 |
# ================================================================
|
| 123 |
+
_SUPAVISOR_REGIONS = [
|
| 124 |
+
'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2',
|
| 125 |
+
'eu-west-1', 'eu-west-2', 'eu-west-3', 'eu-central-1',
|
| 126 |
+
'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'ap-northeast-2',
|
| 127 |
+
'sa-east-1', 'ca-central-1', 'me-south-1', 'af-south-1'
|
| 128 |
+
]
|
| 129 |
|
| 130 |
def _is_ipv6_only(self, host: str) -> bool:
|
| 131 |
"""Check if hostname resolves only to IPv6 (no A record)."""
|
|
|
|
| 169 |
# Supavisor user format: postgres.<ref>
|
| 170 |
pooler_user = f"postgres.{ref}"
|
| 171 |
|
| 172 |
+
# Try each region: first transaction pooler (6543), then session pooler (5432)
|
| 173 |
for region in self._SUPAVISOR_REGIONS:
|
| 174 |
pooler_host = f"aws-0-{region}.pooler.supabase.com"
|
| 175 |
+
for pool_port, pool_name in [(6543, 'transaction'), (5432, 'session')]:
|
| 176 |
+
try:
|
| 177 |
+
_orig_getaddrinfo(pooler_host, pool_port, socket.AF_INET, socket.SOCK_STREAM)
|
| 178 |
+
pooler_dsn = f"postgresql://{pooler_user}:{password}@{pooler_host}:{pool_port}/postgres?sslmode=require"
|
| 179 |
+
logger.info(f"Supavisor {pool_name} pooler found: {pooler_host} ({region}:{pool_port})")
|
| 180 |
+
return pooler_dsn
|
| 181 |
+
except (socket.gaierror, OSError):
|
| 182 |
+
logger.debug(f"Supavisor {pool_name} pooler {region}:{pool_port} not reachable via IPv4, trying next...")
|
| 183 |
+
continue
|
| 184 |
|
| 185 |
logger.warning("All Supavisor pooler regions unreachable via IPv4")
|
| 186 |
return None
|
|
|
|
| 233 |
import re
|
| 234 |
match = re.match(r'postgresql://([^:]+):([^@]+)@([^:/]+)(?::(\d+))?/(.+)', dsn)
|
| 235 |
if match:
|
| 236 |
+
user, password, host, port, dbname_full = match.groups()
|
| 237 |
+
dbname = dbname_full.split('?')[0]
|
| 238 |
port = port or '5432'
|
| 239 |
+
ipv4_host = self._resolve_host_to_ipv4(host)
|
| 240 |
+
if ipv4_host != host:
|
| 241 |
+
dsn = f"postgresql://{user}:{password}@{ipv4_host}:{port}/{dbname_full}"
|
| 242 |
+
logger.debug(f"Replaced hostname with IPv4 in DSN: {host} -> {ipv4_host}")
|
| 243 |
if 'connect_timeout' not in dsn:
|
| 244 |
sep = '&' if '?' in dsn else '?'
|
| 245 |
dsn = f"{dsn}{sep}connect_timeout=10"
|
|
|
|
| 268 |
except psycopg2.OperationalError as e:
|
| 269 |
# Try Supavisor pooler fallback if direct connection fails
|
| 270 |
pooler_dsn = self._conn_params.get('pooler_dsn')
|
| 271 |
+
if pooler_dsn:
|
| 272 |
+
logger.warning(f"Direct connection failed (attempt {attempt + 1}), trying Supavisor pooler...")
|
| 273 |
try:
|
| 274 |
conn = self._connect_with_dsn(pooler_dsn, keepalive_opts)
|
| 275 |
conn.autocommit = False
|
|
|
|
| 276 |
self._conn_params['dsn'] = pooler_dsn
|
| 277 |
+
if 'pooler_dsn' in self._conn_params:
|
| 278 |
+
del self._conn_params['pooler_dsn']
|
| 279 |
logger.success("Connected via Supavisor pooler")
|
| 280 |
return conn
|
| 281 |
except Exception as pooler_err:
|
| 282 |
logger.error(f"Supavisor pooler also failed: {pooler_err}")
|
| 283 |
+
# If pooler fails, rebuild pooler DSN with next region
|
| 284 |
+
if attempt < self.max_retries - 1:
|
| 285 |
+
new_pooler_dsn = self._build_supavisor_dsn(params.get('dsn', ''))
|
| 286 |
+
if new_pooler_dsn:
|
| 287 |
+
self._conn_params['pooler_dsn'] = new_pooler_dsn
|
| 288 |
+
logger.info("Retrying with next Supavisor region...")
|
| 289 |
+
# Reconnect one-shot: SSL closed mid-handshake (Render idle drop).
|
| 290 |
if 'SSL connection has been closed' in str(e) and attempt < self.max_retries - 1:
|
| 291 |
time.sleep(self.retry_delay * (2 ** attempt))
|
| 292 |
continue
|
modules/finetuning_pipeline.py
CHANGED
|
@@ -552,18 +552,15 @@ class FinetuningPipeline:
|
|
| 552 |
tone_level, hostility_score, emotion_label, quality_score,
|
| 553 |
embedding_input, embedding_output, similarity_score)
|
| 554 |
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
| 555 |
-
RETURNING id;
|
| 556 |
"""), (user_id, conversation_id, input_message, expected_response,
|
| 557 |
tone_level, hostility_score, emotion_label, auto_quality,
|
| 558 |
input_emb_bytes.tobytes(), output_emb_bytes.tobytes(), similarity))
|
| 559 |
|
| 560 |
-
|
| 561 |
-
if
|
| 562 |
-
self.logger.error(f"❌ [FINETUNING]
|
| 563 |
return -1
|
| 564 |
|
| 565 |
-
# RealDictCursor retorna dict, não tupla - acessa por chave
|
| 566 |
-
example_id = result['id'] if isinstance(result, dict) else result[0]
|
| 567 |
self.logger.info(f"✅ [FINETUNING] Exemplo #{example_id} armazenado | Emotion={emotion_label} | Similarity={similarity:.3f}")
|
| 568 |
return example_id
|
| 569 |
except Exception as cur_err:
|
|
@@ -733,9 +730,9 @@ class FinetuningPipeline:
|
|
| 733 |
cur.execute(self.db._prepare_query("""
|
| 734 |
INSERT INTO training_cycles (cycle_number, cycle_type, status)
|
| 735 |
VALUES (%s, %s, 'started')
|
| 736 |
-
RETURNING id;
|
| 737 |
"""), (current_cycle, cycle_type))
|
| 738 |
|
|
|
|
| 739 |
self.logger.info(f"🚀 [CYCLE {current_cycle}] Tipo={cycle_type} | Session={session_id}")
|
| 740 |
return session_id
|
| 741 |
except Exception as e:
|
|
|
|
| 552 |
tone_level, hostility_score, emotion_label, quality_score,
|
| 553 |
embedding_input, embedding_output, similarity_score)
|
| 554 |
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
|
|
| 555 |
"""), (user_id, conversation_id, input_message, expected_response,
|
| 556 |
tone_level, hostility_score, emotion_label, auto_quality,
|
| 557 |
input_emb_bytes.tobytes(), output_emb_bytes.tobytes(), similarity))
|
| 558 |
|
| 559 |
+
example_id = cur.lastrowid
|
| 560 |
+
if not example_id:
|
| 561 |
+
self.logger.error(f"❌ [FINETUNING] lastrowid não retornou ID")
|
| 562 |
return -1
|
| 563 |
|
|
|
|
|
|
|
| 564 |
self.logger.info(f"✅ [FINETUNING] Exemplo #{example_id} armazenado | Emotion={emotion_label} | Similarity={similarity:.3f}")
|
| 565 |
return example_id
|
| 566 |
except Exception as cur_err:
|
|
|
|
| 730 |
cur.execute(self.db._prepare_query("""
|
| 731 |
INSERT INTO training_cycles (cycle_number, cycle_type, status)
|
| 732 |
VALUES (%s, %s, 'started')
|
|
|
|
| 733 |
"""), (current_cycle, cycle_type))
|
| 734 |
|
| 735 |
+
session_id = f"cycle_{current_cycle}_{int(time.time())}"
|
| 736 |
self.logger.info(f"🚀 [CYCLE {current_cycle}] Tipo={cycle_type} | Session={session_id}")
|
| 737 |
return session_id
|
| 738 |
except Exception as e:
|