add all files

.gitignore

@@ -0,0 +1,152 @@
+### Database ###
+*.sqlite3
+
+### bin ###
+*.bin
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Environments
+*.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+### venv ###
+pyvenv.cfg
+pip-selfcheck.json
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

.gradio/flagged/dataset1.csv

@@ -0,0 +1,12 @@
+Upload image for emotion analysis,Vision Emotion Analysis Results,timestamp
+.gradio\flagged\Upload image for emotion analysis\4e8dadac1774d97fcc36\surprised.jpeg,"Predicted Emotion: SURPRISE
+
+Probabilities:
+angry: 3.97%
+disgust: 0.45%
+fear: 10.07%
+happy: 1.09%
+sad: 0.99%
+surprise: 82.87%
+neutral: 0.55%
+",2025-01-15 02:28:39.690721

app.py

@@ -0,0 +1,167 @@
+import gradio as gr
+import numpy as np
+from PIL import Image
+import sys
+import os
+from text.test import predict_sentiment, SentimentRequest
+from speech.test import classify_emotion as classify_speech_emotion
+from vision.test import classify_emotion as classify_vision_emotion
+import librosa
+import torch
+import asyncio
+
+os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
+
+def process_text(text, api_key):
+    try:
+        if not api_key.strip():
+            return "Please provide an OpenAI API key"
+            
+        # Set the API key as environment variable
+        os.environ["OPENAI_API_KEY"] = api_key.strip()
+        
+        # Create request object
+        request = SentimentRequest(text=text)
+        
+        # Get prediction
+        result = asyncio.run(predict_sentiment(request))
+        
+        return f"Predicted Sentiment: {result['sentiment'].upper()}"
+    except Exception as e:
+        return f"Error processing text: {str(e)}"
+
+def process_speech(audio_path):
+    try:
+        # Load audio file
+        waveform, sr = librosa.load(audio_path, sr=None)
+        waveform = waveform.astype(np.float32)
+        
+        # Get prediction
+        predicted_emotion, emotion_probs = classify_speech_emotion(waveform, sr)
+        
+        # Format the output
+        output_text = f"Predicted Emotion: {predicted_emotion.upper()}\n\nProbabilities:\n"
+        for emotion, prob in emotion_probs.items():
+            output_text += f"{emotion}: {prob:.2%}\n"
+        
+        return output_text
+    except Exception as e:
+        return f"Error processing audio: {str(e)}"
+
+def process_image(image):
+    try:
+        if image is None:
+            return "Please provide an image"
+            
+        # Convert to PIL Image if needed
+        if not isinstance(image, Image.Image):
+            image = Image.fromarray(image)
+            
+        # Get prediction
+        prob_dict, most_likely_emotion = classify_vision_emotion(image)
+        
+        # Format the output
+        output_text = f"Predicted Emotion: {most_likely_emotion.upper()}\n\nProbabilities:\n"
+        for emotion, prob in prob_dict.items():
+            output_text += f"{emotion}: {prob:.2%}\n"
+        
+        return output_text
+    except Exception as e:
+        return f"Error processing image: {str(e)}"
+
+def create_interface():
+    # Text analysis
+    with gr.Row():
+        api_key_input = gr.Textbox(
+            label="OpenAI API Key",
+            placeholder="Enter your OpenAI API key...",
+            type="password"
+        )
+    text_input = gr.Textbox(
+        label="Enter text for sentiment analysis",
+        placeholder="Type your text here...",
+        lines=3
+    )
+    text_output = gr.Textbox(label="Text Sentiment Analysis Results")
+    text_interface = gr.Interface(
+        fn=process_text,
+        inputs=[text_input, api_key_input],
+        outputs=text_output,
+        title="Text Sentiment Analysis",
+        description="Analyze the sentiment of text input as positive or negative.",
+        theme=gr.themes.Soft(),
+        css="footer {display: none !important;}"
+    )
+
+    # Speech analysis
+    speech_input = gr.Audio(
+        label="Upload audio for emotion analysis",
+        type="filepath"
+    )
+    speech_output = gr.Textbox(label="Speech Emotion Analysis Results")
+    speech_interface = gr.Interface(
+        fn=process_speech,
+        inputs=speech_input,
+        outputs=speech_output,
+        title="Speech Emotion Analysis",
+        description="Analyze the emotion in speech audio.",
+        theme=gr.themes.Soft(),
+        css="footer {display: none !important;}"
+    )
+
+    # Vision analysis
+    vision_input = gr.Image(
+        label="Upload image for emotion analysis",
+        type="pil"
+    )
+    vision_output = gr.Textbox(label="Vision Emotion Analysis Results")
+    vision_interface = gr.Interface(
+        fn=process_image,
+        inputs=vision_input,
+        outputs=vision_output,
+        title="Vision Emotion Analysis",
+        description="Analyze the emotion in facial expressions.",
+        theme=gr.themes.Soft(),
+        css="footer {display: none !important;}"
+    )
+
+    demo = gr.TabbedInterface(
+        [text_interface, speech_interface, vision_interface],
+        ["Text Analysis", "Speech Analysis", "Vision Analysis"],
+        title="Multimodal Sentiment Analysis",
+        theme=gr.themes.Soft(),
+        css="""
+            .gradio-container {
+                max-width: 900px !important;
+                margin: auto;
+                padding: 20px;
+                border-radius: 10px;
+                box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+            }
+            .tabs {
+                margin-bottom: 20px;
+                border-bottom: 2px solid #eee;
+            }
+            .tab-button {
+                padding: 10px 20px;
+                margin-right: 10px;
+                border-radius: 5px 5px 0 0;
+                background-color: #f5f5f5;
+            }
+            .tab-button.selected {
+                background-color: #2196F3;
+                color: white;
+            }
+            footer {display: none !important;}
+        """
+    )
+    
+    return demo
+
+if __name__ == "__main__":
+    try:
+        demo = create_interface()
+        demo.launch(share=True)
+    except Exception as e:
+        print(f"Error launching application: {str(e)}")
+        sys.exit(1)

speech/.gitignore

@@ -0,0 +1,154 @@
+best_ser_whisper.pkl
+
+### Database ###
+*.sqlite3
+
+### bin ###
+*.bin
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Environments
+*.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+### venv ###
+pyvenv.cfg
+pip-selfcheck.json
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+

speech/README.md

@@ -0,0 +1,115 @@
+### API Documentation: Speech Emotion Recognition (SER)
+
+This API allows you to upload an audio file and get the predicted emotion along with the probabilities for all emotion classes using a fine-tuned Speech Emotion Recognition (SER) model (openai/whisper-small). I saved some audio files for testing [here](./dataset_my/pilot_test). To create the api on localhost, run [test.py](./test.py) using ```python test.py``` command.
+
+---
+
+### Base URL
+```
+http://127.0.0.1:8000
+```
+
+---
+
+### Endpoints
+
+#### 1. **Classify Emotion**
+This endpoint accepts an audio file and returns the predicted emotion and probabilities for all emotion classes.
+
+- **Endpoint**: `POST /classify-emotion/`
+- **Request Body**:
+  - **Key**: `audio_file`
+  - **Value**: The audio file to analyze (e.g., `.wav`).
+- **Response**:
+  - **Success (200 OK)**:
+    ```json
+    {
+      "predicted_emotion": "happy",
+      "emotion_probabilities": {
+        "neutral": 0.1,
+        "happy": 0.8,
+        "sad": 0.05,
+        "angry": 0.03,
+        "fearful": 0.01,
+        "disgusted": 0.005,
+        "surprised": 0.005
+      }
+    }
+    ```
+  - **Error (500 Internal Server Error)**:
+    ```json
+    {
+      "detail": "Error message describing the issue."
+    }
+    ```
+
+---
+
+### Special Instructions for Use with Postman
+
+#### Step 1: Set Up the Request
+1. Open Postman.
+2. Set the request type to `POST`.
+3. Enter the URL:
+   ```
+   http://127.0.0.1:8000/classify-emotion/
+   ```
+
+#### Step 2: Configure the Body
+1. Go to the **Body** tab.
+2. Select **form-data**.
+
+#### Step 3: Add the File
+1. In the **Key** field, enter `audio_file` (this must match the parameter name in the API).
+2. Hover over the **Value** field, and click the dropdown that says **Text**. Change it to **File**.
+3. Click **Choose File** and select your audio file (e.g., `test_audio.wav`).
+
+#### Step 4: Send the Request
+1. Click the **Send** button to submit the request.
+
+---
+
+### Example Postman Configuration
+
+#### **Headers**:
+- Postman automatically sets the `Content-Type` to `multipart/form-data` when you use the **form-data** option. You don’t need to manually add this header.
+
+#### **Body**:
+| Key         | Value              
+|-------------|--------------------
+| `audio_file`| `test_audio.wav`   
+
+---
+
+### Example Workflow in Postman
+
+1. **Set Up the Request**:
+   - Method: `POST`
+   - URL: `http://127.0.0.1:8000/classify-emotion/`
+   - Body: `form-data`
+   - Key: `audio_file`
+   - Value: `test_audio.wav` (select as **File**)
+
+2. **Send the Request**:
+   - Click **Send**.
+
+3. **Check the Response**:
+   - If successful, you’ll see the predicted emotion and probabilities:
+     ```json
+     {
+       "predicted_emotion": "happy",
+       "emotion_probabilities": {
+         "neutral": 0.1,
+         "happy": 0.8,
+         "sad": 0.05,
+         "angry": 0.03,
+         "fearful": 0.01,
+         "disgusted": 0.005,
+         "surprised": 0.005
+       }
+     }
+     ```
+   - If there’s an error, check the error message and debug accordingly.
+
+---
+

speech/gradio_demo.py

@@ -0,0 +1,69 @@
+import gradio as gr
+import numpy as np
+import librosa
+import torch
+from .train import label_mapping, device, MAX_DURATION, processor
+
+label_to_name = {v: k for k, v in label_mapping.items()}
+BEST_MODEL = torch.load("best_ser_whisper.pkl", map_location=torch.device('cpu'))
+# BEST_MODEL = best_model
+
+def classify_emotion(audio_input):
+    if audio_input is None:
+        return "No audio input detected. Please check your microphone or upload an audio file.", {}
+
+    # Handle both microphone and uploaded file inputs
+    if isinstance(audio_input, tuple):  # Microphone input
+        sr, waveform = audio_input
+        waveform = waveform.astype(np.float32)
+    elif isinstance(audio_input, str):  # File upload input
+        try:
+            waveform, sr = librosa.load(audio_input, sr=None)
+            waveform = waveform.astype(np.float32)
+        except Exception as e:
+            return f"Error loading audio file: {e}", {}
+    else:
+        return "Invalid audio input.", {}
+
+    # Resample if necessary
+    if sr != 16000:
+        waveform = librosa.resample(waveform, orig_sr=sr, target_sr=16000)
+        sr = 16000
+
+    # Trim or pad audio to max_duration
+    max_samples = int(MAX_DURATION * sr)
+    if len(waveform) > max_samples:
+        waveform = waveform[:max_samples]
+    elif len(waveform) < max_samples:
+        waveform = np.pad(waveform, (0, max_samples - len(waveform)), "constant")
+
+    processed_inputs = processor(
+        waveform, sampling_rate=sr, return_tensors="pt", return_attention_mask=True
+    )
+    inputs = processed_inputs.input_features.to(device)
+
+    with torch.no_grad():
+        logits = BEST_MODEL(inputs).logits
+    probabilities = torch.softmax(logits, dim=1).squeeze().cpu().numpy()
+
+    emotion_probs = {}
+    for i, prob in enumerate(probabilities):
+        emotion_probs[label_to_name[i + 1]] = prob
+
+    # Get the most likely class
+    predicted_label_index = torch.argmax(logits, dim=1).item()
+    predicted_label_name = label_to_name[predicted_label_index + 1]
+
+    return predicted_label_name, emotion_probs
+
+iface = gr.Interface(
+    fn=classify_emotion,
+    inputs=gr.Audio(sources=["microphone", "upload"], type="numpy"),
+    outputs=[
+        gr.Label(label="Predicted Emotion"),
+        gr.Label(num_top_classes=len(label_mapping), label="Probabilities"),
+    ],
+    live=True,
+)
+
+iface.launch(share=True)

speech/test.py

@@ -0,0 +1,81 @@
+from fastapi import FastAPI, File, UploadFile, HTTPException
+import numpy as np
+import librosa
+import torch
+from .train import label_mapping, device, MAX_DURATION, processor
+import io
+import uvicorn
+import warnings
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+app = FastAPI()
+
+# Load the best model
+BEST_MODEL = torch.load("F:\\sociofi_my\\projects\\sentiment_analysis\\speech\\best_ser_whisper.pkl", map_location=torch.device('cpu'), weights_only=False)
+BEST_MODEL.eval()
+
+# Reverse label mapping
+label_to_name = {v: k for k, v in label_mapping.items()}
+
+# Function to classify emotion
+def classify_emotion(audio_input: np.ndarray, sr: int):
+    # Resample if necessary
+    if sr != 16000:
+        audio_input = librosa.resample(audio_input, orig_sr=sr, target_sr=16000)
+        sr = 16000
+
+    # Trim or pad audio to max_duration
+    max_samples = int(MAX_DURATION * sr)
+    if len(audio_input) > max_samples:
+        audio_input = audio_input[:max_samples]
+    elif len(audio_input) < max_samples:
+        audio_input = np.pad(audio_input, (0, max_samples - len(audio_input)), "constant")
+
+    # Preprocess the audio
+    processed_inputs = processor(
+        audio_input, sampling_rate=sr, return_tensors="pt", return_attention_mask=True
+    )
+    inputs = processed_inputs.input_features.to(device)
+
+    # Perform inference
+    with torch.no_grad():
+        logits = BEST_MODEL(inputs).logits
+    probabilities = torch.softmax(logits, dim=1).squeeze().cpu().numpy()
+
+    # Map probabilities to emotion labels
+    emotion_probs = {}
+    for i, prob in enumerate(probabilities):
+        emotion_probs[label_to_name[i + 1]] = float(prob)  # Convert to float for JSON serialization
+
+    # Get the most likely class
+    predicted_label_index = torch.argmax(logits, dim=1).item()
+    predicted_label_name = label_to_name[predicted_label_index + 1]
+
+    return predicted_label_name, emotion_probs
+
+# Endpoint to classify emotion
+@app.post("/classify-emotion/")
+async def classify_emotion_endpoint(audio_file: UploadFile = File(...)):
+    try:
+        # Read the uploaded audio file
+        audio_bytes = await audio_file.read()
+        audio_file = io.BytesIO(audio_bytes)
+        
+        # Load the audio file using librosa
+        waveform, sr = librosa.load(audio_file, sr=None)
+        waveform = waveform.astype(np.float32)
+        
+        # Classify the emotion
+        predicted_emotion, emotion_probs = classify_emotion(waveform, sr)
+        
+        return {
+            "predicted_emotion": predicted_emotion,
+            "emotion_probabilities": emotion_probs
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error processing audio file: {str(e)}")
+
+# Run the FastAPI app
+if __name__ == '__main__':
+    uvicorn.run(app, host="0.0.0.0", port=8000)
+    

speech/train.py

@@ -0,0 +1,257 @@
+import numpy as np
+import pandas as pd
+import os
+from transformers import WhisperProcessor, WhisperForAudioClassification
+import torch
+import librosa
+import numpy as np
+from torch.utils.data import Dataset, DataLoader
+import os
+from torch.optim import AdamW
+from torch.optim.lr_scheduler import ReduceLROnPlateau
+from tqdm import tqdm
+from sklearn.metrics import accuracy_score, f1_score
+import pickle
+import logging
+logging.getLogger("transformers.modeling_utils").setLevel(logging.ERROR)
+
+label_mapping = {
+    'neutral': 1,
+    'calm': 2,
+    'happy': 3,
+    'sad': 4,
+    'angry': 5,
+    'fearful': 6,
+    'disgust': 7,
+    'surprised': 8
+}
+
+MAX_DURATION = 10
+
+
+model_id = "openai/whisper-small"
+processor = WhisperProcessor.from_pretrained(model_id) 
+model = WhisperForAudioClassification.from_pretrained(model_id,
+    num_labels=8, # Number of emotion classes (01 = neutral, 02 = calm, 03 = happy, 04 = sad, 05 = angry, 06 = fearful, 07 = disgust, 08 = surprised)
+    ignore_mismatched_sizes=True
+)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model.to(device)
+
+class SpeechEmotionDataset(Dataset):
+    def __init__(self, data_dirs, processor, label_mapping, max_duration=MAX_DURATION):
+        self.data_dirs = data_dirs
+        self.processor = processor
+        self.label_mapping = label_mapping
+        self.max_duration = max_duration  # in seconds
+        self.file_paths = []
+        self.labels = []
+
+        for data_dir in self.data_dirs:
+            for filename in os.listdir(data_dir):
+                if filename.endswith(".wav"):
+                    self.file_paths.append(os.path.join(data_dir, filename))
+                    parts = filename.split('-') # split the filename into parts using '-' as the delimeter
+                    emotion_code = int(parts[2]) # extract the emotion code from the third position
+                    self.labels.append(self.map_ravdess_emotion_to_label(emotion_code))
+
+
+    def __len__(self):
+        return len(self.file_paths)
+
+    def __getitem__(self, idx):
+        file_path = self.file_paths[idx]
+        label = self.labels[idx]
+        try:
+            audio, sr = librosa.load(file_path, sr=16000)  # Whisper expects 16kHz audio
+        except Exception as e:
+            print(f"Error loading {file_path}: {e}")
+            return None  # Skip this item on error
+        
+        
+        # Trim or pad audio to max_duration
+        max_samples = int(self.max_duration * sr)
+        if len(audio) > max_samples:
+            audio = audio[:max_samples]
+        elif len(audio) < max_samples:
+             audio = np.pad(audio, (0, max_samples - len(audio)), 'constant')
+
+        processed_inputs = self.processor(
+            audio, sampling_rate=sr, return_tensors="pt", return_attention_mask=True
+        )
+        
+        inputs = processed_inputs.input_features
+        attention_mask = processed_inputs.attention_mask
+        
+        return inputs.squeeze(0).to(device), \
+                attention_mask.squeeze(0).to(device), \
+                torch.tensor(label).to(device)
+    # num_mel_channels = 80 (generally 80 for Whisper)
+    # mel_length = 3000 (ensured with the padding logic in processor)
+    # Then:
+    # Before squeeze(0):
+    # input_features shape: [1, 80, 3000]
+    # attention_mask shape: [1, 3000]
+    # After squeeze(0):
+    # input_features shape: [80, 3000]
+    # attention_mask shape: [3000]
+    # When the DataLoader forms a batch of size 8:
+    # input_features batch shape: [8, 80, 3000]
+    # attention_mask batch shape: [8, 3000]
+    
+    def map_ravdess_emotion_to_label(self, emotion_code):
+            """Maps RAVDESS emotion codes to integer labels."""
+            if emotion_code == 1:
+                return self.label_mapping['neutral']
+            elif emotion_code == 2:
+                return self.label_mapping['calm']
+            elif emotion_code == 3:
+                 return self.label_mapping['happy']
+            elif emotion_code == 4:
+                return self.label_mapping['sad']
+            elif emotion_code == 5:
+                 return self.label_mapping['angry']
+            elif emotion_code == 6:
+                return self.label_mapping['fearful']
+            elif emotion_code == 7:
+                return self.label_mapping['disgust']
+            elif emotion_code == 8:
+                 return self.label_mapping['surprised']
+            else:
+                raise ValueError(f"Invalid emotion code: {emotion_code}")
+
+def calculate_metrics(model, data_loader):
+    model.eval()
+    predictions = []
+    true_labels = []
+    with torch.no_grad():
+        for inputs, _, labels in tqdm(data_loader):
+            outputs = model(inputs).logits
+            # outputs shape: [batch_size, num_labels]
+            predicted_labels = torch.argmax(outputs, dim=1)
+            # predicted_labels shape: [batch_size]
+            predictions.extend(predicted_labels.cpu().numpy())
+            true_labels.extend(labels.cpu().numpy())
+
+    accuracy = accuracy_score(true_labels, predictions)
+    macro_f1 = f1_score(true_labels, predictions, average="macro")
+    return accuracy, macro_f1
+
+if __name__ == "__main__":
+    train_data_dirs = [
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_01",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_02",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_03",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_04",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_05",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_06",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_07",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_08",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_09",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_10",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_11",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_12",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_13",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_14"
+    ]
+    val_data_dirs = ["dataset_my\\Audio_Song_Actors_01-24\\Actor_15",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_16",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_17",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_19",]
+    test_data_dirs = ["dataset_my\\Audio_Song_Actors_01-24\\Actor_20",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_21",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_22",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_23",
+        "dataset_my\\Audio_Song_Actors_01-24\\Actor_24"]
+
+    train_dataset = SpeechEmotionDataset(train_data_dirs, processor, label_mapping, MAX_DURATION)
+    val_dataset = SpeechEmotionDataset(val_data_dirs, processor, label_mapping, MAX_DURATION)
+    test_dataset = SpeechEmotionDataset(test_data_dirs, processor, label_mapping, MAX_DURATION)
+
+    # filter out any None values that were returned due to error
+    train_dataset.file_paths = [
+        f for i, f in enumerate(train_dataset.file_paths) if train_dataset[i] is not None
+    ]
+    train_dataset.labels = [
+        l for i, l in enumerate(train_dataset.labels) if train_dataset[i] is not None
+    ]
+    val_dataset.file_paths = [
+        f for i, f in enumerate(val_dataset.file_paths) if val_dataset[i] is not None
+    ]
+    val_dataset.labels = [
+        l for i, l in enumerate(val_dataset.labels) if val_dataset[i] is not None
+    ]
+    test_dataset.file_paths = [
+        f for i, f in enumerate(test_dataset.file_paths) if test_dataset[i] is not None
+    ]
+    test_dataset.labels = [
+        l for i, l in enumerate(test_dataset.labels) if test_dataset[i] is not None
+    ]
+
+    batch_size = 8
+    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
+    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
+    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
+
+    print(f"train loader (size = {len(train_loader)}) ::")
+    for data, _, labels in train_loader:
+        print(f'Data batch shape: {data.shape}')
+        print(f'Labels batch shape: {labels.shape}')
+        break
+
+    print(f"\nval loader (size = {len(val_loader)}) ::")
+    for data, _, labels in val_loader:
+        print(f'Data batch shape: {data.shape}')
+        print(f'Labels batch shape: {labels.shape}')
+        break
+
+    print(f"\ntest loader (size = {len(test_loader)}) ::")
+    for data, _, labels in test_loader:
+        print(f'Data batch shape: {data.shape}')
+        print(f'Labels batch shape: {labels.shape}')
+        break
+    
+    optimizer = AdamW(model.parameters(), lr=1e-4)
+    scheduler = ReduceLROnPlateau(
+        optimizer, mode="max", factor=0.1, patience=3, verbose=True
+    )
+    loss_fn = torch.nn.CrossEntropyLoss()
+    num_epochs = 10
+    best_val_f1 = 0.0
+    
+    for epoch in range(num_epochs):
+        # training
+        model.train()
+        total_loss = 0
+        for batch in tqdm(train_loader, desc=f"Epoch {epoch+1}/{num_epochs} [Train]"):
+            inputs, _, labels = batch
+            # inputs shape: [batch_size, num_mel_channels, mel_length]
+            # attention_mask shape: [batch_size, mel_length]
+            # labels shape: [batch_size]
+            optimizer.zero_grad()
+            outputs = model(inputs).logits
+            # outputs shape: [batch_size, num_labels]
+            loss = loss_fn(outputs, labels)
+            loss.backward()
+            optimizer.step()
+            total_loss += loss.item()
+
+        train_loss = total_loss / len(train_loader)
+        train_accuracy, train_macro_f1 = calculate_metrics(model, train_loader)
+
+        # validation
+        val_accuracy, val_macro_f1 = calculate_metrics(model, val_loader)
+
+        # update learning rate
+        scheduler.step(val_macro_f1)
+
+        print(
+            f"   Train Loss: {train_loss:.4f}, Train Acc: {train_accuracy:.4f}, Train F1: {train_macro_f1:.4f}, Val Acc: {val_accuracy:.4f}, Val F1: {val_macro_f1:.4f}"
+        )
+
+        # save best model
+        if val_macro_f1 > best_val_f1:
+            best_val_f1 = val_macro_f1
+            torch.save(model, "best_ser_whisper.pkl")
+    

text/.gitignore

@@ -0,0 +1,155 @@
+dataset_my/
+old_dataset/
+
+### Database ###
+*.sqlite3
+
+### bin ###
+*.bin
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Environments
+*.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+### venv ###
+pyvenv.cfg
+pip-selfcheck.json
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+

text/README.md

@@ -0,0 +1,64 @@
+### API Documentation: Text Sentiment Analysis
+
+This API uses a fine-tuned GPT-3.5-turbo model to predict the sentiment of a given text. It accepts a short paragraph and returns the predicted sentiment (`positive`, `negative`, or `neutral`). Run `python .\test.py` to start the FastAPI server in [http://0.0.0.0:8000](http://0.0.0.0:8000).
+
+---
+
+### Base URL
+```
+http://0.0.0.0:8000
+```
+
+---
+
+### Endpoints
+
+#### 1. **Predict Sentiment**
+This endpoint accepts a short paragraph of text and returns the predicted sentiment.
+
+- **Endpoint**: `POST /predict-sentiment/`
+- **Request Body**:
+  ```json
+  {
+    "text": "I absolutely loved the product! It exceeded all my expectations."
+  }
+  ```
+- **Response**:
+  - **Success (200 OK)**:
+    ```json
+    {
+      "sentiment": "positive"
+    }
+    ```
+  - **Error (500 Internal Server Error)**:
+    ```json
+    {
+      "detail": "Error message describing the issue."
+    }
+    ```
+
+---
+
+### Example Usage
+
+#### Request
+```bash
+curl -X POST "http://0.0.0.0:8000/predict-sentiment/" \
+-H "Content-Type: application/json" \
+-d '{"text": "I absolutely loved the product! It exceeded all my expectations."}'
+```
+
+#### Response
+```json
+{
+  "sentiment": "positive"
+}
+```
+
+---
+
+### Notes
+- The API uses a fine-tuned GPT-3.5-turbo model for sentiment analysis.
+- The `text` field in the request body should contain a short paragraph.
+- The response will always include a `sentiment` field with one of the following values: `positive`, `negative`, or `neutral`.
+- Replace value of the `api_key` parameter with the actual OpenAI key

text/create_dataset.py

@@ -0,0 +1,68 @@
+import pandas as pd
+from sklearn.model_selection import train_test_split
+import json
+
+# Load the dataset
+def load_dataset(filepath):
+    return pd.read_csv(filepath, encoding='latin-1')
+
+def preprocess_dataset(df):
+    """
+    Preprocess the dataset:
+    1. Drop rows where 'review' is NaN.
+    2. Drop rows where 'review-label' is NaN.
+    3. Drop rows where 'review' is an empty string.
+    """
+    # Drop rows with NaN in 'review' or 'review-label'
+    df = df.dropna(subset=["review", "review-label"])
+
+    # Drop rows where 'review' is an empty string
+    df = df[df["review"].str.strip() != ""]
+
+    return df
+
+# Map review-label to sentiment
+def map_label_to_sentiment(label):
+    if label > 3:
+        return "positive"
+    elif label < 3:
+        return "negative"
+    else:
+        return "neutral"
+
+# Convert dataset to JSONL format
+def convert_to_jsonl(df, output_file):
+    with open(output_file, "w") as f:
+        for _, row in df.iterrows():
+            review = row["review"]
+            sentiment = map_label_to_sentiment(row["review-label"])
+            jsonl_entry = {
+                "messages": [
+                    {"role": "system", "content": "You are a sentiment analysis assistant. Classify the sentiment of the user's message as 'positive', 'negative' or 'neutral'."},
+                    {"role": "user", "content": review},
+                    {"role": "assistant", "content": sentiment}
+                ]
+            }
+            f.write(json.dumps(jsonl_entry) + "\n")
+
+# Main function
+def process_dataset(filepath, train_output="dataset_my\\train.jsonl", val_output="dataset_my\\val.jsonl", test_output="dataset_my\\test.jsonl", test_size=0.2, val_size=0.1):
+    # Load the dataset
+    df = load_dataset(filepath)
+    
+    # Preprocess dataset
+    df = preprocess_dataset(df)
+
+    # Split the dataset into train, validation, and test sets
+    train_df, test_df = train_test_split(df, test_size=test_size, random_state=42)
+    train_df, val_df = train_test_split(train_df, test_size=val_size, random_state=42)
+
+    # Convert each split to JSONL format
+    convert_to_jsonl(train_df, train_output)
+    convert_to_jsonl(val_df, val_output)
+    convert_to_jsonl(test_df, test_output)
+
+    print(f"Dataset processed and saved to {train_output}, {val_output}, and {test_output}")
+
+filepath = "dataset_my\\TeePublic_review.csv"
+process_dataset(filepath)

text/test.py

@@ -0,0 +1,62 @@
+from datetime import timedelta
+from tqdm import tqdm
+from sklearn.metrics import accuracy_score
+import json
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from openai import OpenAI
+from datetime import timedelta
+from openai import OpenAI
+import uvicorn
+import os
+
+app = FastAPI()
+
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+fine_tune_id = "ftjob-2nIK3dvxJf4QiAb6DQQ4A1GN"
+ft_job_details = client.fine_tuning.jobs.retrieve(fine_tune_id)
+fine_tuned_model = ft_job_details.fine_tuned_model
+
+class SentimentRequest(BaseModel):
+    text: str  # Input text (sentence / short paragraph)
+
+class SentimentResponse(BaseModel):
+    sentiment: str  # Predicted sentiment (positive/negative)
+
+# Endpoint to predict sentiment
+@app.post("/predict-sentiment/", response_model=SentimentResponse)
+async def predict_sentiment(request: SentimentRequest):
+    try:
+        # System prompt (customize based on your fine-tuning)
+        system_prompt = "You are a sentiment analysis model. Analyze the sentiment of the following text and respond with 'positive' or 'negative'."
+
+        # Get the input text
+        input_text = request.text
+
+        # Call the fine-tuned model
+        response = client.chat.completions.create(
+            model=fine_tuned_model,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": input_text}
+            ]
+        )
+
+        # Extract the predicted sentiment
+        predicted_sentiment = response.choices[0].message.content.strip().lower()
+
+        # Determine the most likely sentiment
+        if 'positive' in predicted_sentiment:
+            sentiment = "positive"
+        elif 'negative' in predicted_sentiment:
+            sentiment = "negative"
+        else:
+            sentiment = "neutral"
+
+        return {"sentiment": sentiment}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error predicting sentiment: {str(e)}")
+
+if __name__ == '__main__':
+    uvicorn.run(app, host="0.0.0.0", port=8000)

text/train.py

@@ -0,0 +1,31 @@
+from openai import OpenAI
+from dotenv import load_dotenv
+load_dotenv()
+client = OpenAI(api_key="")
+
+train_file = client.files.create(
+  file=open("dataset_my\\train.jsonl", "rb"),
+  purpose="fine-tune"
+)
+
+print(f"train file created with id : {train_file.id}")
+
+validation_file = client.files.create(
+  file=open("dataset_my\\val.jsonl", "rb"),
+  purpose="fine-tune"
+)
+
+print(f"validation file created with id : {validation_file.id}")
+
+fine_tune_response = client.fine_tuning.jobs.create(
+    training_file=train_file.id,
+    model="gpt-4o-mini-2024-07-18",
+    suffix='shopper_review_sentiment',
+    validation_file=validation_file.id,
+    seed=88
+)
+
+fine_tune_id = fine_tune_response.id
+status = client.fine_tuning.jobs.retrieve(fine_tune_id)
+print(f"Fine-tuning status: {status}")
+

vision/.gitignore

@@ -0,0 +1,154 @@
+best_fer_clip_vit.pkl
+
+### Database ###
+*.sqlite3
+
+### bin ###
+*.bin
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Environments
+*.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+### venv ###
+pyvenv.cfg
+pip-selfcheck.json
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+

vision/README.md

@@ -0,0 +1,103 @@
+### API Documentation (Markdown)
+
+### **Endpoints**
+
+#### 1. **Classify Emotion**
+This endpoint accepts an [image file](./dataset_my/pilot_test) and returns the predicted emotion probabilities and the most likely emotion. To create the api on localhost, run [test.py](./test.py) using ```python test.py``` command.
+
+- **Endpoint**: `POST /classify-emotion/`
+- **Request Body**:
+  - **Key**: `image_file`
+  - **Value**: The image file to analyze (e.g., `.jpg`, `.png`).
+- **Response**:
+  - **Success (200 OK)**:
+    ```json
+    {
+      "most_likely_emotion": "happy",
+      "emotion_probabilities": {
+        "angry": 0.01,
+        "disgust": 0.02,
+        "fear": 0.03,
+        "happy": 0.8,
+        "sad": 0.05,
+        "surprise": 0.04,
+        "neutral": 0.05
+      }
+    }
+    ```
+  - **Error (500 Internal Server Error)**:
+    ```json
+    {
+      "detail": "Error message describing the issue."
+    }
+    ```
+
+---
+
+### Instructions for API Testing with Postman
+
+#### Step 1: Set Up the Request
+1. Open Postman.
+2. Set the request type to `POST`.
+3. Enter the URL:
+   ```
+   http://127.0.0.1:8000/classify-emotion/
+   ```
+
+#### Step 2: Configure the Body
+1. Go to the **Body** tab.
+2. Select **form-data**.
+
+#### Step 3: Add the Image
+1. In the **Key** field, enter `image_file` (this must match the parameter name in the API).
+2. Hover over the **Value** field, and click the dropdown that says **Text**. Change it to **File**.
+3. Click **Choose File** and select your image file (e.g., `test_image.jpg`).
+
+#### Step 4: Send the Request
+1. Click the **Send** button to submit the request.
+
+---
+
+### Example Postman Configuration
+
+#### **Headers**:
+- Postman automatically sets the `Content-Type` to `multipart/form-data` when you use the **form-data** option. You don’t need to manually add this header.
+
+#### **Body**:
+| Key         | Value              
+|-------------|--------------------
+| `image_file`| `angry.jpeg.jpg`   
+
+---
+
+### Example Workflow in Postman
+
+1. **Set Up the Request**:
+   - Method: `POST`
+   - URL: `http://127.0.0.1:8000/classify-emotion/`
+   - Body: `form-data`
+   - Key: `image_file`
+   - Value: `angry.jpeg` (select as **File**)
+
+2. **Send the Request**:
+   - Click **Send**.
+
+3. **Check the Response**:
+   - If successful, you’ll see the predicted emotion and probabilities:
+     ```json
+     {
+       "most_likely_emotion": "happy",
+       "emotion_probabilities": {
+         "angry": 0.01,
+         "disgust": 0.02,
+         "fear": 0.03,
+         "happy": 0.8,
+         "sad": 0.05,
+         "surprise": 0.04,
+         "neutral": 0.05
+       }
+     }
+     ```
+   - If there’s an error, check the error message and debug accordingly.
+
+---

vision/test.py

@@ -0,0 +1,87 @@
+from fastapi import FastAPI, File, UploadFile, HTTPException
+from fastapi.responses import JSONResponse
+from PIL import Image
+import torch
+from torchvision import transforms
+import io
+import uvicorn
+from transformers import CLIPProcessor, CLIPModel
+import warnings
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+model_id = "openai/clip-vit-base-patch32"
+clip_model = CLIPModel.from_pretrained(model_id)
+processor = CLIPProcessor.from_pretrained(model_id)
+NUM_CLASSES = 7
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+class EmotionRecognitionModel(torch.nn.Module):
+    def __init__(self, clip_model, num_classes):
+        super(EmotionRecognitionModel, self).__init__()
+        self.clip_vision_model = clip_model.vision_model
+        self.fc = torch.nn.Linear(768, num_classes)
+
+    def forward(self, images):
+        with torch.no_grad():
+            vision_outputs = self.clip_vision_model(pixel_values=images)
+            embeddings = vision_outputs.last_hidden_state[:, 0, :]  # CLS token embeddings (shape: [batch_size, 768])
+        outputs = self.fc(embeddings)  # Shape: [batch_size, num_classes]
+        return outputs
+
+model = EmotionRecognitionModel(clip_model, NUM_CLASSES).to(DEVICE)
+
+app = FastAPI()
+
+# Load the pre-trained model
+# model = torch.load("best_fer_clip_vit.pkl", map_location=DEVICE)
+model.load_state_dict(torch.load("F:\\sociofi_my\\projects\\sentiment_analysis\\vision\\best_fer_clip_vit.pkl", map_location=DEVICE, weights_only=False))
+model.eval()
+
+# Define the image transformation pipeline
+transform = transforms.Compose([
+    transforms.Resize((224, 224)),  # Resize to match model input size
+    transforms.ToTensor(),          # Convert to tensor
+    transforms.Normalize(           # Normalize
+        mean=[0.485, 0.456, 0.406],
+        std=[0.229, 0.224, 0.225]
+    )
+])
+
+# Emotion labels
+emotion_labels = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"]
+
+# Function to classify emotion
+def classify_emotion(input_image):
+    transformed_image = transform(input_image).unsqueeze(0).to(DEVICE)
+
+    with torch.no_grad():
+        outputs = model(transformed_image)
+        probabilities = torch.softmax(outputs, dim=1).squeeze(0)
+
+    prob_dict = {emotion_labels[i]: probabilities[i].item() for i in range(len(emotion_labels))}
+    most_likely_emotion_idx = torch.argmax(probabilities).item()
+    most_likely_emotion = emotion_labels[most_likely_emotion_idx]
+
+    return prob_dict, most_likely_emotion
+
+# Endpoint to classify emotion
+@app.post("/classify-emotion/")
+async def classify_emotion_endpoint(image_file: UploadFile = File(...)):
+    try:
+        # Read the uploaded image file
+        image_bytes = await image_file.read()
+        image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
+
+        # Classify the emotion
+        prob_dict, most_likely_emotion = classify_emotion(image)
+
+        return JSONResponse({
+            "most_likely_emotion": most_likely_emotion,
+            "emotion_probabilities": prob_dict
+        })
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error processing image: {str(e)}")
+
+# Run the FastAPI app
+if __name__ == '__main__':
+    uvicorn.run(app, host="0.0.0.0", port=8000)