Deployment: Synchronized system with new model architecture and streaming notebook v3.0

ConfereAI_FastTrain_Colab_v3.ipynb

@@ -0,0 +1,198 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "header"
+      },
+      "source": [
+        "# 🚀 ConfereAI - Fast Training v3.0 (Streaming Edition)\n",
+        "Este notebook permite treinar o motor neural do ConfereAI utilizando a GPU do Colab e datasets do Hugging Face sem download.\n",
+        "\n",
+        "**Novidade:** Suporte a Streaming para datasets gigantes (ex: BRSpeech-DF 243GB).\n",
+        "\n",
+        "**Instruções:**\n",
+        "1. Selecione **T4 GPU** em `Ambiente de Execução`.\n",
+        "2. Escolha se quer fazer upload de um ZIP ou usar um dataset remoto.\n",
+        "3. Execute as células."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "setup"
+      },
+      "outputs": [],
+      "source": [
+        "# @title 1. Instalar Dependências\n",
+        "!pip install -q transformers[torch] datasets librosa soundfile huggingface_hub accelerate"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "config"
+      },
+      "outputs": [],
+      "source": [
+        "# @title 2. Configurações do Hugging Face\n",
+        "HF_TOKEN = \"\" # @param {type:\"string\"}\n",
+        "REPO_ID = \"TEDDyx86/confereai-wav2vec2\" # @param {type:\"string\"}\n",
+        "\n",
+        "from huggingface_hub import HfApi, login\n",
+        "if HF_TOKEN:\n",
+        "    login(token=HF_TOKEN)\n",
+        "else:\n",
+        "    print(\"❌ Por favor, insira o seu HF_TOKEN!\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "upload"
+      },
+      "outputs": [],
+      "source": [
+        "# @title 3. Carregamento do Dataset\n",
+        "DATASET_SOURCE = \"Hugging Face Hub\" # @param [\"Upload ZIP\", \"Hugging Face Hub\"]\n",
+        "REMOTE_DATASET = \"AKCIT-Deepfake/BRSpeech-DF\" # @param {type:\"string\"}\n",
+        "\n",
+        "import os\n",
+        "import shutil\n",
+        "import zipfile\n",
+        "from google.colab import files\n",
+        "from datasets import load_dataset\n",
+        "\n",
+        "DATASET_DIR = \"dataset_training\"\n",
+        "RAW_DATASET = None\n",
+        "\n",
+        "if DATASET_SOURCE == \"Upload ZIP\":\n",
+        "    uploaded = files.upload()\n",
+        "    if uploaded:\n",
+        "        dataset_zip = list(uploaded.keys())[0]\n",
+        "        if os.path.exists(DATASET_DIR): shutil.rmtree(DATASET_DIR)\n",
+        "        os.makedirs(DATASET_DIR)\n",
+        "        with zipfile.ZipFile(dataset_zip, 'r') as zip_ref:\n",
+        "            zip_ref.extractall(DATASET_DIR)\n",
+        "        print(f\"✅ Dataset local extraído em: {DATASET_DIR}\")\n",
+        "else:\n",
+        "    print(f\"🌐 Conectando a {REMOTE_DATASET} via Streaming...\")\n",
+        "    RAW_DATASET = load_dataset(REMOTE_DATASET, streaming=True)\n",
+        "    print(f\"✅ Pronto para treinar com {REMOTE_DATASET}\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "training"
+      },
+      "outputs": [],
+      "source": [
+        "# @title 4. Executar Treinamento (Fine-Tuning)\n",
+        "import torch\n",
+        "from torch.utils.data import Dataset, IterableDataset\n",
+        "from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2ForSequenceClassification, Trainer, TrainingArguments\n",
+        "import librosa\n",
+        "\n",
+        "BASE_MODEL = \"HyperMoon/wav2vec2-base-960h-finetuned-deepfake\"\n",
+        "OUTPUT_DIR = \"local_finetuned_model\"\n",
+        "processor = Wav2Vec2FeatureExtractor.from_pretrained(BASE_MODEL)\n",
+        "\n",
+        "class LocalDeepfakeDataset(Dataset):\n",
+        "    def __init__(self, root_dir, processor):\n",
+        "        self.files = []\n",
+        "        self.processor = processor\n",
+        "        for label, folder in enumerate(['real', 'fake']):\n",
+        "            path = os.path.join(root_dir, folder)\n",
+        "            if os.path.exists(path):\n",
+        "                for f in os.listdir(path):\n",
+        "                    if f.endswith(('.wav', '.mp3', '.flac')):\n",
+        "                        self.files.append({\"path\": os.path.join(path, f), \"label\": label})\n",
+        "\n",
+        "    def __len__(self): return len(self.files)\n",
+        "    def __getitem__(self, idx):\n",
+        "        item = self.files[idx]\n",
+        "        speech, _ = librosa.load(item[\"path\"], sr=16000)\n",
+        "        inputs = self.processor(speech, sampling_rate=16000, return_tensors=\"pt\", padding=\"max_length\", max_length=160000, truncation=True)\n",
+        "        return {\"input_values\": inputs.input_values[0], \"labels\": torch.tensor(item[\"label\"], dtype=torch.long)}\n",
+        "\n",
+        "class StreamingHFDataset(IterableDataset):\n",
+        "    def __init__(self, hf_dataset, processor):\n",
+        "        self.hf_dataset = hf_dataset\n",
+        "        self.processor = processor\n",
+        "    def __iter__(self):\n",
+        "        for example in self.hf_dataset['train']:\n",
+        "            audio = example['audio']\n",
+        "            label = example['label']\n",
+        "            inputs = self.processor(audio['array'], sampling_rate=16000, return_tensors=\"pt\", padding=\"max_length\", max_length=160000, truncation=True)\n",
+        "            yield {\"input_values\": inputs.input_values[0], \"labels\": torch.tensor(label, dtype=torch.long)}\n",
+        "\n",
+        "if DATASET_SOURCE == \"Upload ZIP\":\n",
+        "    train_data = LocalDeepfakeDataset(DATASET_DIR, processor)\n",
+        "else:\n",
+        "    train_data = StreamingHFDataset(RAW_DATASET, processor)\n",
+        "\n",
+        "model = Wav2Vec2ForSequenceClassification.from_pretrained(BASE_MODEL, num_labels=2, ignore_mismatched_sizes=True)\n",
+        "if hasattr(model, 'freeze_feature_extractor'): model.freeze_feature_extractor()\n",
+        "\n",
+        "training_args = TrainingArguments(\n",
+        "    output_dir=\"./results\",\n",
+        "    num_train_epochs=3,\n",
+        "    per_device_train_batch_size=4,\n",
+        "    gradient_accumulation_steps=4,\n",
+        "    learning_rate=3e-5,\n",
+        "    logging_steps=10,\n",
+        "    max_steps=1000 if DATASET_SOURCE != \"Upload ZIP\" else -1, \n",
+        "    report_to=\"none\"\n",
+        ")\n",
+        "\n",
+        "trainer = Trainer(model=model, args=training_args, train_dataset=train_data)\n",
+        "print(\"🚀 Iniciando treinamento...\")\n",
+        "trainer.train()\n",
+        "\n",
+        "model.save_pretrained(OUTPUT_DIR)\n",
+        "processor.save_pretrained(OUTPUT_DIR)\n",
+        "print(f\"✅ Modelo salvo em {OUTPUT_DIR}\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "push"
+      },
+      "outputs": [],
+      "source": [
+        "# @title 5. Sincronizar com Hugging Face (Model Repo)\n",
+        "api = HfApi()\n",
+        "api.upload_folder(\n",
+        "    folder_path=OUTPUT_DIR,\n",
+        "    repo_id=REPO_ID,\n",
+        "    repo_type=\"model\",\n",
+        "    token=HF_TOKEN,\n",
+        "    commit_message=\"🤖 Auto-Update: Cérebro aprimorado com dataset BR\"\n",
+        ")\n",
+        "print(f\"✨ Sucesso! O novo cérebro está disponível em: https://huggingface.co/{REPO_ID}\")"
+      ]
+    }
+  ]
+}

dashboard/index.html

@@ -277,7 +277,7 @@
 
             <div class="footer-links">
 
-                <a href="https://huggingface.co/HyperMoon/wav2vec2-base-960h-finetuned-deepfake" target="_blank">Modelo IA</a>
+                <a href="https://huggingface.co/TEDDyx86/confereai-wav2vec2" target="_blank">Cérebro Pessoal v1.0</a>
 
             </div>
 

execution/inference_wav2vec.py

@@ -10,7 +10,8 @@ import os
 LOCAL_MODEL_DIR = "./local_finetuned_model"
 # Prioridade: 1. Pasta Local (Upload direto) | 2. Repo Customizado (Variável de Ambiente) | 3. Modelo Base
 CUSTOM_MODEL_REPO = os.environ.get("CUSTOM_MODEL_REPO", "TEDDyx86/confereai-wav2vec2")
-BASE_MODEL = "HyperMoon/wav2vec2-base-960h-finetuned-deepfake"
+# Modelo especializado treinado pelo usuário (Cérebro Pessoal)
+BASE_MODEL = "TEDDyx86/confereai-wav2vec2"
 
 # Singleton para carregar o modelo e processador apenas uma vez
 _feature_extractor = None