Upload 7 files

aimodel.py

@@ -6,9 +6,9 @@ import uuid
 import re
 from fastai.vision.all import *
 from pathlib import Path
-import torch.nn as nn
 import random
 import shutil
+import aiutils
 
 MEMORY_SIZE = 50
 SUCCESS_MEMORY_RATIO = 0.5
@@ -49,6 +49,7 @@ def get_memory_samples(n_samples=None):
         return memory_samples
 
 def prepare_dataset():
+    aiutils.prepare_dirs()
     dataset_dir = Path(utils.DATASET_DIR)
     dataset_dir.mkdir(parents=True, exist_ok=True)
     for item in dataset_dir.iterdir():
@@ -98,22 +99,7 @@ def train_model():
     num_classes = len(dls.vocab)
 
     # Criar um novo Learner com o número correto de classes
-    learn = vision_learner(dls, resnet18, metrics=accuracy, pretrained=True)
-
-    if os.path.exists(utils.MODEL_PATH):
-        try:
-            # Carregar o modelo completo (incluindo a camada de classificação antiga)
-            learn.load(utils.MODEL_PATH.replace(".pkl", ""))
-            logging.info("Modelo existente carregado.")
-
-            # Substituir a camada de classificação (a última camada linear)
-            # Precisamos acessar o modelo subjacente (PyTorch) para fazer isso
-            num_ftrs = learn.model.fc.in_features
-            learn.model.fc = nn.Linear(num_ftrs, num_classes).to(learn.model.fc.weight.device)
-            logging.info("Camada de classificação substituída.")
-
-        except Exception as e:
-            logging.error(f"Erro ao carregar o modelo existente: {e}. Treinando do zero.")
+    learn = aiutils.get_learner()
 
     learn.fine_tune(5)
     learn.export(utils.MODEL_PATH)

aiutils.py

@@ -0,0 +1,75 @@
+from fastai.vision.all import *
+from pathlib import Path
+import utils
+import os
+import logging
+import random
+import json
+
+MEMORY_SIZE = 50
+SUCCESS_MEMORY_RATIO = 0.5
+
+def load_success_fail_data(path):
+    datas = []
+    if os.path.exists(path):
+        try:
+            with open(path, "r") as f:
+                datas = json.load(f)
+        except json.JSONDecodeError:
+            logging.error(f"JSONDecodeError in {path}")
+    return datas
+
+def get_memory_samples(n_samples=None):
+    success_data = load_success_fail_data(utils.SUCCESS_PATH)
+    fail_data = load_success_fail_data(utils.FAIL_PATH)
+
+    memory_samples = []
+    num_success = int((n_samples if n_samples else MEMORY_SIZE) * SUCCESS_MEMORY_RATIO)
+    num_fail = (n_samples if n_samples else MEMORY_SIZE) - num_success
+
+    if success_data:
+        sampled_success = random.sample(success_data, min(num_success, len(success_data)))
+        for item in sampled_success:
+            memory_samples.append({'url': item['url'], 'type': item['predicted']})
+
+    if fail_data:
+        sampled_fail = random.sample(fail_data, min(num_fail, len(fail_data)))
+        for item in sampled_fail:
+            memory_samples.append({'url': item['url'], 'type': item['correct']})
+
+    if n_samples is not None and len(memory_samples) > n_samples:
+        return random.sample(memory_samples, n_samples)
+    elif n_samples is None and len(memory_samples) > MEMORY_SIZE:
+        return random.sample(memory_samples, MEMORY_SIZE)
+    else:
+        return memory_samples
+
+def prepare_dirs():
+    base_dir = utils.BASE_DIR
+    os.makedirs(base_dir, exist_ok=True)  # Creates the base directory if it doesn't exist
+
+    campaign_types = ["DEMAND", "OFFER", "PROZIS", "NO_CAMPAIGN"]
+
+    for campaign_type in campaign_types:
+        campaign_folder = os.path.join(base_dir, campaign_type.lower())
+        os.makedirs(campaign_folder, exist_ok=True)
+        print(f"Folder created: {campaign_folder}")
+
+def get_learner():
+    learn = vision_learner(dls, resnet18, metrics=accuracy, pretrained=True)
+
+    if os.path.exists(utils.MODEL_PATH):
+        try:
+            # Carregar o modelo completo (incluindo a camada de classificação antiga)
+            learn.load(utils.MODEL_PATH.replace(".pkl", ""))
+            logging.info("Modelo existente carregado.")
+
+            # Substituir a camada de classificação (a última camada linear)
+            # Precisamos acessar o modelo subjacente (PyTorch) para fazer isso
+            num_ftrs = learn.model.fc.in_features
+            learn.model.fc = nn.Linear(num_ftrs, num_classes).to(learn.model.fc.weight.device)
+            logging.info("Camada de classificação substituída.")
+
+        except Exception as e:
+            logging.error(f"Erro ao carregar o modelo existente: {e}. Treinando do zero.")
+            

client.py

@@ -9,7 +9,7 @@ API_URL = "https://marioprzbasto-campaign-detector.hf.space"
 
 st.title("Classificador de Campanhas Prozis")
 
-opcao = st.selectbox("Escolha a ação:", ["Success", "Fail", "Upload Model", "Classificar", "Feedback", "Bulk Classification", "URLs"])
+opcao = st.selectbox("Escolha a ação:", ["Success", "Fail", "Upload Model", "Classificar", "Feedback", "Bulk Classification", "URLs", "Download da Pasta /tmp"])
 
 def load_image_from_url(url):
     try:
@@ -38,7 +38,7 @@ if opcao == "Classificar":
 
     if st.button("Classificar"):
         if image_url:
-            response = requests.post(f"{API_URL}/predict", data={"url": image_url})
+            response = requests.post(f"{API_URL}/predict", data={"id": str(uuid.uuid4()), "url": image_url})
             if response.status_code == 200:
                 st.success(response.json())
             else:
@@ -181,6 +181,8 @@ elif opcao == "Success":
                             with col1:
                                 st.image(img, caption="Imagem de sucesso", use_container_width=True)
                             with col2:
+                                if 'id' in item:
+                                    st.write(f"**ID:** {item['id']}")
                                 st.write(f"**URL:** {item['url']}")
                                 if 'predicted' in item:
                                     st.write(f"**Previsto:** {item['predicted']}")
@@ -206,6 +208,8 @@ elif opcao == "Fail":
                             with col1:
                                 st.image(img, caption="Imagem de falha", use_container_width=True)
                             with col2:
+                                if 'id' in item:
+                                    st.write(f"**ID:** {item['id']}")
                                 st.write(f"**URL:** {item['url']}")
                                 if 'predicted' in item:
                                     st.write(f"**Previsto:** {item['predicted']}")
@@ -232,4 +236,21 @@ elif opcao == "Upload Model":
                 except requests.exceptions.RequestException as e:
                     st.error(f"Erro ao enviar o modelo: {e}")
                     if response is not None:
-                        st.error(f"Detalhes do erro: {response.text}")
+                        st.error(f"Detalhes do erro: {response.text}")
+
+elif opcao == "Download da Pasta /tmp":
+        st.subheader("Download da Pasta /tmp como ZIP")
+        if st.button("Gerar e Baixar ZIP"):
+            try:
+                response = requests.get(f"{API_URL}/download_selected_tmp", stream=True)
+                response.raise_for_status()
+                st.download_button(
+                    label="Clique para Baixar /tmp.zip",
+                    data=response.content,
+                    file_name="tmp_contents.zip",
+                    mime="application/zip",
+                )
+            except requests.exceptions.RequestException as e:
+                st.error(f"Erro ao solicitar o download: {e}")
+                if response is not None:
+                    st.error(f"Detalhes do erro: {response.status_code} - {response.text}")

utils.py

@@ -10,7 +10,7 @@ PREDICT_DIR = "/tmp/predict"
 MODEL_PATH = "/tmp/model.pkl"
 FAIL_PATH = "/tmp/fails.json"
 SUCCESS_PATH = "/tmp/success.json"
-ZIP_PATH = "/tmp"
+ZIP_PATH = "/tmp/my_app_temp/download.zip"
 MAX_JSON_SIZE = 100
 
 def truncate_json(file_path):