Todas as partidas

Dockerfile

@@ -1,34 +1,36 @@
+# Usa uma imagem base mais recente e suportada
 FROM python:3.9-slim-bookworm
 
+# Define o diretório de trabalho
 WORKDIR /app
 
-# Install git + LFS for data cloning
-RUN apt-get update && apt-get install -y git git-lfs && git-lfs install && rm -rf /var/lib/apt/lists/*
-
-# Clone data files from the space repo
+# --- ETAPA 1: INSTALAR O GIT E CLONAR O SEU DATASET COMPLETO ---
+RUN apt-get update && apt-get install -y git git-lfs && git-lfs install
 RUN git clone https://huggingface.co/spaces/Finish-him/prometheus-embedding-generator ./dados && cd dados && git lfs pull
 
-# Python deps
+# --- ETAPA 2: CONFIGURAR O AMBIENTE PYTHON E PRÉ-AQUECER O CACHE ---
 COPY requirements.txt .
 RUN pip install --no-cache-dir --upgrade -r requirements.txt
 
-# Cache dir for models
+# Define o caminho da cache
 ENV HF_HOME=/app/cache/huggingface
 ENV SENTENCE_TRANSFORMERS_HOME=/app/cache/torch
-RUN mkdir -p $HF_HOME $SENTENCE_TRANSFORMERS_HOME
+RUN mkdir -p $HF_HOME && mkdir -p $SENTENCE_TRANSFORMERS_HOME
 
-# Pre-download model
-RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('intfloat/multilingual-e5-large', cache_folder='/app/cache/torch')"
+# Pré-aquece o cache com o modelo e5-large
+RUN python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer('intfloat/multilingual-e5-large', cache_folder=os.environ.get('SENTENCE_TRANSFORMERS_HOME', '/app/cache/torch'))"
 
-# Create output dirs with proper permissions
-RUN mkdir -p /app/dados_extraidos /app/output && \
-    chown -R 1000:1000 /app/dados_extraidos /app/output /app/cache
+# --- ETAPA 3: PREPARAR DIRETÓRIOS E COPIAR SCRIPT ---
 
-# Copy the actual app file (was train.py, now app.py)
-COPY app.py .
+# *** A CORREÇÃO ESTÁ AQUI ***
+# Cria os diretórios necessários e dá permissão ao usuário padrão (1000)
+RUN mkdir -p /app/dados_extraidos && \
+    mkdir -p /app/output && \
+    chown -R 1000:1000 /app/dados_extraidos /app/output /app/cache
 
-ENV GRADIO_SERVER_NAME="0.0.0.0"
-ENV GRADIO_SERVER_PORT="7860"
-EXPOSE 7860
+# Copia o script de treinamento
+COPY train.py .
 
-CMD ["python", "app.py"]
+# --- ETAPA 4: EXECUTAR O SCRIPT ---
+# Executa o script e, após o término, mantém o contêiner a correr.
+CMD ["sh", "-c", "python train.py && sleep infinity"]

requirements.txt

@@ -1,5 +1,4 @@
 sentence-transformers
 numpy
 torch
-tqdm
-gradio==4.44.0
+tqdm

app.py → train.py

@@ -1,27 +1,24 @@
-# --- 1. IMPORTS ---
 import os
 import glob
 import json
 import csv
 import numpy as np
+from tqdm.auto import tqdm
 from sentence_transformers import SentenceTransformer
 import zipfile
 import xml.etree.ElementTree as ET
-import gradio as gr
-import shutil
 
-# --- 2. CONFIGURAÇÕES ---
-DATA_DIR = "dados"
-EXTRACT_DIR = os.path.join(DATA_DIR, "dados_extraidos")
-OUTPUT_FILENAME = "meus_embeddings_e5_large.npy"
+DATA_DIR = "/app/dados"
+EXTRACT_DIR = "/app/dados_extraidos"
 
-# --- 3. FUNÇÕES DE PROCESSAMENTO ---
 def setup_data():
     os.makedirs(EXTRACT_DIR, exist_ok=True)
-    zip_files = glob.glob(os.path.join(DATA_DIR, "*.zip"))
+    zip_files = glob.glob(DATA_DIR + "/**/*.zip", recursive=True)
     if not zip_files:
+        print("Nenhum arquivo .zip encontrado, usando o diretório de dados principal.")
         return DATA_DIR
     for zip_path in zip_files:
+        print(f"Descompactando {zip_path}...")
         with zipfile.ZipFile(zip_path, 'r') as zf:
             zf.extractall(EXTRACT_DIR)
     return EXTRACT_DIR
@@ -46,87 +43,75 @@ def serialize_item_to_text(item_dict):
         return str(item_dict)
     for key, value in item_dict.items():
         if isinstance(value, dict):
-            parts.append(f"{key} ({serialize_item_to_text(value)})")
+            nested_text = serialize_item_to_text(value)
+            parts.append(f"{key} ({nested_text})")
         elif isinstance(value, list):
-            list_str = ", ".join([serialize_item_to_text(i) for i in value])
+            list_str = ', '.join([serialize_item_to_text(i) for i in value])
             parts.append(f"{key}: [{list_str}]")
         else:
             parts.append(f"{key}: {value}")
     return ", ".join(parts)
 
-# --- 4. PIPELINE PRINCIPAL ---
-def run_full_process():
-    yield "Iniciando... Descompactando arquivos..."
+def main():
     process_dir = setup_data()
-
     csv.field_size_limit(10_000_000)
-    all_files = (
-        glob.glob(os.path.join(process_dir, "**/*.json"), recursive=True)
-        + glob.glob(os.path.join(process_dir, "**/*.csv"), recursive=True)
-        + glob.glob(os.path.join(process_dir, "**/*.xml"), recursive=True)
-    )
-    yield f"Encontrados {len(all_files)} arquivos para processar."
+
+    all_files = glob.glob(process_dir + "/**/*.json", recursive=True) + \
+                glob.glob(process_dir + "/**/*.csv", recursive=True) + \
+                glob.glob(process_dir + "/**/*.xml", recursive=True)
+    print(f"\n🔎 Encontrados {len(all_files)} arquivos (JSON, CSV, XML) para processar.")
+
+    if not all_files:
+        return
 
     documents = []
-    for idx, filepath in enumerate(all_files):
+    for filepath in tqdm(all_files, desc="Processando arquivos"):
         try:
-            yield f"Processando arquivo {idx + 1}/{len(all_files)}: {os.path.basename(filepath)}"
-            if filepath.endswith(".json"):
-                with open(filepath, "r", encoding="utf-8") as f:
+            if filepath.endswith('.json'):
+                with open(filepath, 'r', encoding='utf-8') as f:
                     data = json.load(f)
                     if isinstance(data, list):
-                        for item in data:
-                            documents.append(serialize_item_to_text(item))
+                        for item in data: documents.append(serialize_item_to_text(item))
                     else:
                         documents.append(serialize_item_to_text(data))
-            elif filepath.endswith(".csv"):
-                with open(filepath, "r", encoding="utf-8") as f:
+            elif filepath.endswith('.csv'):
+                with open(filepath, 'r', encoding='utf-8') as f:
                     reader = csv.DictReader(f)
-                    for row in reader:
-                        documents.append(serialize_item_to_text(row))
-            elif filepath.endswith(".xml"):
+                    for row in reader: documents.append(serialize_item_to_text(row))
+            elif filepath.endswith('.xml'):
                 tree = ET.parse(filepath)
                 root = tree.getroot()
                 xml_dict = {root.tag: xml_to_dict(root)}
                 documents.append(serialize_item_to_text(xml_dict))
         except Exception as e:
-            yield f"Erro ao processar {os.path.basename(filepath)}: {e}"
+            print(f"⚠️ Erro ao processar o arquivo {filepath}: {e}")
 
-    yield f"Processamento concluido! {len(documents)} documentos criados."
+    print(f"\nProcessamento de arquivos concluído! {len(documents)} documentos foram criados.")
     if not documents:
-        yield "Nenhum documento encontrado. Processo encerrado."
         return
 
-    # --- ETAPA 2: GERAÇÃO DE EMBEDDINGS ---
-    yield "Carregando modelo intfloat/multilingual-e5-large..."
-    cache_path = "./model_cache"
-    os.makedirs(cache_path, exist_ok=True)
+    cache_path = os.environ.get('SENTENCE_TRANSFORMERS_HOME', '/app/cache/torch')
+    
+    print("Carregando modelo de alta performance: intfloat/multilingual-e5-large")
     model = SentenceTransformer(
-        "intfloat/multilingual-e5-large", cache_folder=cache_path
+        'intfloat/multilingual-e5-large',
+        cache_folder=cache_path
     )
+    
+    batch_size = 128
+    output_filename = '/app/output/meus_embeddings_e5_large.npy'
+    
+    if os.path.exists(output_filename):
+        os.remove(output_filename)
 
-    yield f"Gerando embeddings para {len(documents)} documentos..."
-    batch_size = 32
-    all_embeddings = []
-    for i in range(0, len(documents), batch_size):
-        batch = documents[i : i + batch_size]
-        embeddings = model.encode(batch, show_progress_bar=False)
-        all_embeddings.append(embeddings)
-        yield f"Batch {i // batch_size + 1}/{(len(documents) - 1) // batch_size + 1} concluido."
-
-    final_embeddings = np.vstack(all_embeddings)
-    np.save(OUTPUT_FILENAME, final_embeddings)
-    yield f"Embeddings salvos em {OUTPUT_FILENAME}! Shape: {final_embeddings.shape}"
-    yield f"Processo completo! {final_embeddings.shape[0]} embeddings de dimensao {final_embeddings.shape[1]}."
-
-# --- 5. INTERFACE GRADIO ---
-with gr.Blocks(title="Prometheus Embedding Generator") as demo:
-    gr.Markdown("# Prometheus Embedding Generator")
-    gr.Markdown("Gera embeddings a partir dos dados do repositorio usando multilingual-e5-large.")
-
-    run_btn = gr.Button("Iniciar Processamento", variant="primary")
-    output = gr.Textbox(label="Progresso", lines=15, interactive=False)
+    print(f"🚀 Iniciando geração de embeddings (lotes de {batch_size}).")
+    for i in tqdm(range(0, len(documents), batch_size), desc="Gerando Embeddings"):
+        batch = documents[i:i + batch_size]
+        batch_embeddings = model.encode(batch, show_progress_bar=False)
+        with open(output_filename, 'ab') as f_out:
+            np.save(f_out, batch_embeddings)
 
-    run_btn.click(fn=run_full_process, outputs=output)
+    print(f"✅ Processo finalizado! Embeddings salvos em '{output_filename}'.")
 
-demo.launch()
+if __name__ == "__main__":
+        main()