Upload 76 files

.gitattributes

@@ -107,3 +107,5 @@ genre_finder/find_genre_different_apporoch/features_dataset.csv filter=lfs diff=
 genre_finder/find_genre_different_apporoch/all_genre.csv filter=lfs diff=lfs merge=lfs -text
 rocketship-ml-model-train/assets/all_genres_clean.csv filter=lfs diff=lfs merge=lfs -text
 rocketship-ml-model-train/assets/features_dataset.csv filter=lfs diff=lfs merge=lfs -text
+all_combine_code/stage_extra_files/dev/sample.txt filter=lfs diff=lfs merge=lfs -text
+all_combine_code/stage_extra_files/output.json filter=lfs diff=lfs merge=lfs -text

all_combine_code/.env

@@ -0,0 +1,59 @@
+# GOOGLE_API_KEY=AIzaSyAzo1_bhEC8ASX5V-POgXQYArc12eS7Ln4 #AIzaSyAnXCwQEpRmekZOH_l6Pbpm2ccQpEWxMEw
+# GOOGLE_CX=479d684fcf07c49ed #e01019bc2a8e944dc
+# OPENAI_API_KEY=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
+
+#LUSA_API_KEY=bd3e6a99-874e-474a-b238-3d2a9e8bf6bf
+GOOGLE_API_KEY=AIzaSyAnXCwQEpRmekZOH_l6Pbpm2ccQpEWxMEw
+GOOGLE_CX=479d684fcf07c49ed
+# OPENAI_API_KEY=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
+OPENAI_API_KEY=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
+OPENROUTER_API_KEY=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
+OPENROUTER_URL=https://openrouter.ai/api/v1
+GEMINI_API_KEY=AIzaSyAwpQ8oPIc9AywvbrT8Jdd-FMzjLzosfyI
+GROQ_API_KEY=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
+TOOL_CALL_MODEL=openai/gpt-5.1-chat
+#CODE_CREATE_MODEL=x-ai/grok-code-fast-1
+CODE_CREATE_MODEL=gpt-4.1-2025-04-14
+
+AI_CHAT_HISTORY_S3=delvant-ai-all-models
+HISTORY_MAX_FOLDER_COUNT=500
+PL_HISTORY_MAX_FOLDER_COUNT=200
+
+MONGO_URI=mongodb+srv://dev:zzVIduQiBkc8Wstz@delveant.wg3g6m.mongodb.net/
+SUB_DOMAIN=kovaioncrmstage94320
+
+FLOW_URL=https://integrate.delveant.cloud/api/v1/flow/executeflow
+CORE_API_URL=https://coreapi.delveant.cloud/api/v1
+FLOW_URL_API=https://coreapi.delveant.cloud/api/v1/automatelead
+
+TENANT_ID=a758fb60-2fef-4a98-be9d-a1820297556b
+CLIENT_ID=686034a6-11ee-4659-a225-5b56915828e0
+CLIENT_SECRET=F-X8Q~CAScz7.2imThjYs7inSAStZ8cFLYtxGceR
+
+SANITIZER_SALT=e7576e0f19400cd8347050bcd67a2dd583286c909aac0a822a2bbc65ddb2b22e
+SANITIZER_SECRET_KEY=wuo1phSbKySaacnzy-iS_Jji2TiGnEpnb5lvN6uOsrQ=
+
+EMAIL_API_MICROSOFT=https://coreapi.delveant.cloud/api/v1/oauth/microsoft/send-email
+EMAIL_API_GOOGLE=https://coreapi.delveant.cloud/api/v1/oauth/google/send-email
+
+AUNTHENTICATE_URL=https://coreapi.delveant.cloud/api/v1/oauth/authenticated-email
+SCHEDULE_MICROSOFT=https://coreapi.delveant.cloud/api/v1/oauth/microsoft/schedule-meeting
+SCHEDULE_GOOGLE=https://coreapi.delveant.cloud/api/v1/oauth/google/schedule-meeting
+LEAD_API_URL=https://integrate.kovaion.cloud/api/v1/permission/updatedUpload
+
+AWS_ACCESS_KEY=AKIAZEBAZEFJRQURGGYZ
+AWS_SECRET_KEY=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
+AWS_REGION=ap-south-1
+BEDROCK_API_KEY=ABSKQmVkcm9ja0FQSUtleS13bnd6LWF0LTYyNzEzMzk4MTAxMTpZOTRJaEp0R3RCOHRTMzY2S1hzOTNuVXB3b2FvZ3JBdkRHQTh2Vi9hWlFPNWppV0hYT1N0S1I1dmpEST0=
+
+FINANCE_MONGO_URI=mongodb+srv://preprod:pfPDxshbRQ1p6qYV@cluster0.nnian.mongodb.net/
+FINANCE_DB_NAME="engage"
+FINANCE_COLLECTION_NAME="FinanceHub_PL Report"
+PL_HISTORY_S3=delveant-ai
+
+SPORTZIA_MONGO_URI=mongodb+srv://preprod:48Kzz9AwoKvZqn4C@preprod.wg3g6m.mongodb.net/
+SPORTZIA_DB_NAME=builderpreprod
+SPORTZI_COLLECTION_NMAE=Sportzia_ChatBot
+
+VITE_RESPONSE_ENCRYPTION_KEY=default_secret_key_32_chars_long!!
+VITE_PAYLOAD_ENCRYPTION_KEY=default_secret_key_32_chars_long!!

all_combine_code/.gitignore

@@ -0,0 +1,28 @@
+# Python compiled files
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# Ignore pycache specifically in tools_ folder
+tools_/__pycache__/
+
+# Virtual environment
+venv/
+.venv
+
+# IDE files (PyCharm)
+.idea/
+*.iml
+
+# Logs
+*.log
+
+# OS files
+.DS_Store
+Thumbs.db
+
+
+#git rm -r --cached .
+#git add .
+

all_combine_code/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.10-bullseye
+
+WORKDIR /app
+
+RUN mkdir -p \
+    /app/tmp \
+    /app/vector_dbs \
+    && chmod -R 775 /app
+
+#ENV TORCHINDUCTOR_CACHE_DIR=/app/tmp/torchinductor
+
+COPY . /app
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 5000
+
+CMD ["gunicorn","app:app","-k","uvicorn.workers.UvicornWorker","-w","8","--preload","--timeout","300","--keep-alive","10","--bind","0.0.0.0:5000"]
+

all_combine_code/agents_/delveant_prompts/crm_leads_agents.py

@@ -0,0 +1,157 @@
+import json
+
+
+def mongodb_query_generator_leads(
+        input_message,
+        data,
+        user_id,
+        data_with_datatype,
+        db_name,
+        collection_name, source
+):
+    return f"""
+You are a MongoDB query expert.
+
+Your task:
+Generate a valid PyMongo query JSON based on the user request and collection sample documents.
+
+- All records inside `isDeleted` available, **isDeleted=true means it's deleted records**, Your developed all pipeline code inside must Not consider this records.
+    - Strictly use all pipeline inside, follow concepts = {{isDeleted:{{$ne:true}}}}
+- NEVER output Markdown, text, or comments.
+
+
+### INPUTS
+
+User Query:
+{input_message}
+
+Database: {db_name}
+Collection: {collection_name}
+
+FIELDS & TYPES (STRICT – CASE SENSITIVE):
+{data_with_datatype}
+
+SCHEMA SAMPLE (2 records only, may not be complete):
+{data}
+
+SOURCE field(key) inside present names:
+{source}
+
+*Note source related queries clearly understand to make the response.*
+
+
+### GENERAL RULES:
+
+1) Always return ONLY a valid JSON object.
+2) Do NOT include explanations.
+3) Do NOT wrap response in markdown.
+4) JSON must be directly parsable by json.loads().
+5) If aggregation is required, use "operation": "aggregate" and provide pipeline.
+6) If simple filtering is enough, use "operation": "find".
+
+-----------------------------------
+DATE RELATED QUERIES (CRITICAL):
+
+- Must follow the rules **date related queries inside** year not mention strictly use current year.
+    eg . 23 march create lead names  (date 23, month march, year not mention use current year)
+
+If the user request involves:
+- last N days
+- last month
+- this month
+- today
+- yesterday
+- specific date range
+- between two dates
+
+You MUST:
+
+1) Use this import:
+   from datetime import datetime, timedelta, timezone
+
+2) Always define:
+   now = datetime.now(timezone.utc)
+
+3) Dynamically calculate date ranges using timedelta.
+
+4) NEVER:
+   - Compare date fields with raw strings
+   - Use ISODate()
+   - Use $dateSubtract
+   - Hardcode year unless user explicitly provides year
+   - Generate invalid python
+
+5) If multiple date fields exist:
+   - MUST use "creationdate" if present
+   - Otherwise fallback in this order:
+     contacteddate → engageddate → closeddate
+
+6) Date condition must be placed:
+   - inside "filter" (for find)
+   - inside "$match" (for aggregate)
+
+7) For date queries:
+   - python_code MUST define variables: start_date, end_date, now
+   - In filter/pipeline, reference them as plain strings: "start_date", "end_date"
+   - These will be resolved at runtime by the caller
+   - Example:
+     python_code: "from datetime import datetime, timedelta, timezone\\nnow = datetime.now(timezone.utc)\\nstart_date = datetime(now.year, 2, 23, tzinfo=timezone.utc)\\nend_date = datetime(now.year, 2, 27, tzinfo=timezone.utc)"
+     filter: {{"creationdate": {{"$gte": "start_date", "$lt": "end_date"}}
+
+
+8) TOP / BEST LEADS RULE (CRITICAL):
+
+- Any query containing **“top / best / top N leads”, Must follow the below pipeline  concept.
+    - pipeline = [{{'$addFields': {{'leadscore_num': {{'$convert': {{'input': {{'$trim': {{'input': {{'$toString': '$leadscore'}}}}}}, 'to': 'double', 'onError': None, 'onNull': None}}}}, 'firstname_str': {{'$trim': {{'input': {{'$toString': {{'$ifNull': ['$firstname', '']}}}}}}}}, 'lastname_str': {{'$trim': {{'input': {{'$toString': {{'$ifNull': ['$lastname', '']}}}}}}}}}}}}, {{'$addFields': {{'full_name': {{'$trim': {{'input': {{'$concat': [{{'$ifNull': ['$firstname_str', '']}}, {{'$cond': [{{'$and': [{{'$ne': ['$firstname_str', '']}}, {{'$ne': ['$lastname_str', '']}}]}}, ' ', '']}}, {{'$ifNull': ['$lastname_str', '']}}]}}}}}}}}}}, {{'$sort': {{'leadscore_num': -1}}}}, {{'$limit': 3}}, {{'$project': {{'_id': 0, 'full_name': 1, 'email': 1, 'phone': 1}}}}]
+
+9) pipeline or find query inside , user message need to set the limits.
+
+10) DATE OUTPUT (STRICT – MUST FOLLOW)
+    - NEVER project date fields directly (e.g., "creationdate": 1 ❌)
+    - ALWAYS use:
+      "$dateToString": {{ "format": "%d-%m-%Y", "date": "$<field>" }}
+    - Output must be string only.
+    - Your response inside all date must present "%d-%m-%Y" this format only.
+
+11) All response inside strictly avoid `_id` this field.
+
+
+### OUTPUT FORMAT:
+
+If NOT a date query:
+
+{{
+  "operation": "find" | "aggregate",
+  "python_code" : null
+  "filter": {{}},
+  "projection": {{}},
+  "pipeline": [],
+  "sort": {{}}, 
+  "limit": null or int value 
+}}
+
+If it IS a date query:
+
+{{
+  "operation": "find" | "aggregate",
+  "python_code": "from datetime import datetime, timedelta, timezone\\nnow = datetime.now(timezone.utc)\\n# date calculation here",
+  "filter": {{}},
+  "projection": {{}},
+  "pipeline": [],
+  "sort": {{}}, 
+  "limit": null or int value  
+}}
+
+-----------------------------------
+STRICT REQUIREMENTS:
+
+- Output must be valid JSON
+- Do not include comments
+- Do not include explanation text
+- Do not include markdown formatting
+- Field names must match exactly
+- Use double quotes only
+- Ensure JSON is syntactically correct
+
+Return ONLY the JSON object.
+"""

all_combine_code/agents_/delveant_prompts/db_collection_name_finder.py

@@ -0,0 +1,113 @@
+def db_collection_find(input, user_id, person_id, collection_keywords, priveous_chat_history_query) -> str:
+    return f"""
+You are the **Accurate CRM Query Formatter and Database Collection Name Finder Agent**.
+
+---
+
+### USER INPUT
+{input}
+
+### PREVIOUS CHAT HISTORY QUERY:
+{priveous_chat_history_query}
+
+
+## PRIMARY OBJECTIVE
+
+Your task is to generate:
+1. User input query clearly understand any spell mistake or sentence mistake present time correct it. otherwise return the same user query.
+2. The **correct CRM database collection name**, based on the user’s intent and query context.
+3. If the user’s input query is **self-related** (e.g., contains words like *my*, *personal*, etc.) — for example, “show my details” or “please give my details” — then `{user_id}` or `{person_id}` this two id's any one added to make the professional questions format.
+4. **User Input** suppose not clear format/meaning, kindly use **Previous chat history query**
+---
+
+## STEP-BY-STEP LOGIC
+
+### 1. Understand USER QUERY
+- Interpret the intent of the user query carefully.
+- Rephrase it to a **professional CRM-style analytic question** (e.g., "Show me my best lead source" → "Identify the best-performing lead source for user {user_id} based on lead scoring metrics.").
+
+---
+
+## 2. DATABASE COLLECTION MAPPING LOGIC:
+{collection_keywords}
+---
+
+## 3. Self-Related (Me / My) Handling:
+ **Decision Table**
+
+   | Step | Check                     | Requirement                                                                                    | If Fails        |
+    | ---- | ------------------------- | ---------------------------------------------------------------------------------------------- | --------------- |
+    | 1    | Self reference present    | `my`, `mine`, `myself`, `for me`, `about me`, `show me my`, `give me my`, `tell me my`, or `I` | ❌ Do NOT inject |
+    | 2    | Other identifiers absent  | No names, no user IDs, no `by/for/assigned to <person>`                                        | ❌ Do NOT inject |
+    | 3    | Personal ownership intent | Refers to user's own records (tasks, deals, contacts, opportunities, etc.)                     | ❌ Do NOT inject |
+    | ✅    | All 3 pass                | Inject `{user_id}`                                                                             | ✅ Inject        |
+
+
+ **Examples**
+
+  | Query                              | Inject `{user_id}`? | Output                                        |
+    | ---------------------------------- | ------------------- | --------------------------------------------- |
+    | show my deals                      | ✅ Yes               | Show the deals for user `{user_id}`           |
+    | give me my tasks for today         | ✅ Yes               | Give the tasks for today for user `{user_id}` |
+    | list my opportunities              | ✅ Yes               | List the opportunities for user `{user_id}`   |
+    | show deals closed by Satheesh      | ❌ No                | No injection                                  |
+    | list top 10 deals                  | ❌ No                | No injection                                  |
+    | show opportunities for account ABC | ❌ No                | No injection                                  |
+    | Show me deals won in the last 6 months | ❌ No                | No injection                                  |
+    | Please give the details            | ❌ No                | No injection                                  |
+
+
+
+## 4.Database Collection Classification (Strict):
+   | Priority | Intent Type | Keywords / Signals | Collection |
+    |----------|-------------|-------------------|-------------|
+    | 1 | Audit / Logs / Tracking | meeting, schedule, participants, logs, email status, sent, failed, email logs, not contacted | admin.audit.CRM_View |
+    | 2 | Tasks / Activities / Work | task, tasks, follow-up, followups, activities, to-do, reminder, pending, assigned, today, overdue | CRM_Task_View |
+    | 3 | Revenue / Deals / Sales | revenue, amount, payment, deal, deals, quote, quotation, pipeline, opportunity, sales | CRM_Opportunity_View |
+    | 4 | Accounts / Companies | account, accounts, company, companies, client, customer, firm, organization, business | CRM_Accounts_View |
+    | 5 | Contacts / Events | contact, contacts, event, event name, event register | CRM_Contacts_View |
+    | 6 | Leads / Person Specific | person name, leads info, personal email id | CRM_Leads_View |
+    | 7 | Comparative (Dynamic) | best, top, highest, maximum, most | decide by entity |
+
+ Apply in this exact order:
+
+    1. If meeting/email logs/audit → admin.audit.CRM_View (ALWAYS override)
+    2. If task/activity keywords → CRM_Task_View
+    3. If revenue/deal keywords → CRM_Opportunity_View
+    4. If account/company keywords → CRM_Accounts_View
+    5. If contact/event keywords → CRM_Contacts_View
+    6. If person name or leads data info → CRM_Leads_View
+
+
+ EXTRA EMAIL RULES (STRICT)
+
+    • Email ID of a person → CRM_Leads_View (eg . show my mail id, show ravi mail id)
+    • Contact details (contains word "contact") → CRM_Contacts_View
+    • Email status/log/count → admin.audit.CRM_View
+
+### 5. OUTPUT RULES
+
+- Return **only valid JSON**, formatted exactly as below.  
+- Do **not** include explanations, markdown, or extra text.  
+- Do **not** repeat input or chat history in the output.
+- If query inside 'leads or lead' words mention, use the 'CRM_Leads_View' 
+
+**Output JSON structure:**
+
+```json
+{{
+  "query": "corrected query --> (keep in mind user query 98% don't change.only correct the spell mistake or self related query add {user_id}).",
+  "db_collection_name": "CRM_Collection_Name"
+}}
+```
+
+### Notes:
+- If the user input query is at least 45% clear and meaningful, use the current `user input` query.
+- If the user input query is less than 45% clear, incomplete, vague, or not meaningful, then use the Previous Chat History Query instead.
+- Query inside any mailid or name or comany name don't correct the spell mistake, because this is case-sensitive.
+    eg . user_input inside : `manoiiifecs@hgamil.com` this present some spell mistake, strictly this format don't correct it.
+
+- If a company name is mentioned in short form (e.g., CDSL, AI), DO NOT expand it to its full form. 
+   Always preserve the exact company name format as provided in the user input. 
+
+"""

all_combine_code/agents_/delveant_prompts/tool_clasification_agent.py

@@ -0,0 +1,327 @@
+from textwrap import dedent
+
+TOOLS = [
+    # 1. CRM Info
+    {
+        "type": "function",
+        "function": {
+            "name": "get_crm_info",
+            "description": "Retrieve CRM Database DATA/MongoDB information such as leads, contacts, tasks, meetings, opportunities, accounts, revenue,comapny details ,analytics, and reports.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "user_message": {"type": "string", "description": "The original user query"},
+                    "collection": {
+                        "type": "string",
+                        "enum": [
+                            "CRM_Task_View",
+                            "CRM_Leads_View",
+                            "CRM_Contacts_View",
+                            "CRM_Opportunity_View",
+                            "CRM_Accounts_View",
+                            "admin.audit.CRM_View"
+                        ],
+                        "description": "The CRM/MongoDB collection to query"
+                    }
+                },
+                "required": ["user_message", "collection"]
+            }
+        }
+    },
+
+    # 2. Send_email_api
+    {
+        "type": "function",
+        "function": {
+            "name": "send_email_api",
+            "description": "Send an email with subject, body, optional CC, and attachments.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "to_email": {"type": "string"},
+                    "subject": {"type": "string"},
+                    "body": {"type": "string"},
+                    "provider": {
+                        "type": "string",
+                        "description": "Email provider (google or microsoft)",
+                        "enum": ["google", "microsoft"],
+                    },
+                    "cc_email": {"type": "string"},
+                    "attachments": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "url": {"type": "string"},
+                                "filename": {"type": "string"},
+                                "mimeType": {"type": "string"}
+                            },
+                            "required": ["url", "filename", "mimeType"]
+                        }
+                    }
+                },
+                "required": ["to_email", "subject", "body", "provider"]
+            }
+        }
+    },
+    # 3. Create lead
+    {
+        "type": "function",
+        "function": {
+            "name": "create_lead",
+            "description": "Create a new lead in CRM. Requires email, company name, and the lead's requirement.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "email": {"type": "string"},
+                    "company_name": {"type": "string"},
+                    "requirement": {"type": "string"}
+                },
+                "required": ["email", "company_name", "requirement"]
+            }
+        }
+    },
+
+    # 4.Schedule_meeting_api
+    {
+        "type": "function",
+        "function": {
+            "name": "schedule_meeting_api",
+            "description": "Schedule a meeting.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "title": {"type": "string"},
+                    "recipient": {"type": "string"},
+                    "description": {"type": "string"},
+                    "start_time": {"type": "string"},
+                    "end_time": {"type": "string"},
+                    "time_zone": {"type": "string"}
+                },
+                "required": ["title", "recipient", "description", "start_time", "end_time"]
+            }
+        }
+    },
+    # 5. Update query
+    {
+        "type": "function",
+        "function": {
+            "name": "update_query",
+            "description": "Update an existing CRM record.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "field_name": {"type": "string"},
+                    "new_value": {"type": "string"},
+                    "collection": {"type": "string"}
+                },
+                "required": ["field_name", "new_value", "collection"]
+            }
+        }
+    },
+
+    # 6. Greetings
+    {
+        "type": "function",
+        "function": {
+            "name": "greetings_api",
+            "description": "Handle greetings, small talk, or identity questions.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "message": {"type": "string"}
+                },
+                "required": ["message"]
+            }
+        }
+    },
+
+    # 7. PDF File QA
+    {
+        "type": "function",
+        "function": {
+            "name": "pdf_file_api",
+            "description": "Extract or explain information from uploaded PDF files.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "question": {"type": "string"}
+                },
+                "required": ["question"]
+            }
+        }
+    },
+
+    # 8. General Chat
+    {
+        "type": "function",
+        "function": {
+            "name": "general_chatbot_api",
+            "description": "MANDATORY fallback tool. Must be called for any non-CRM, non-email, non-meeting, non-PDF, non-greeting question. Text responses are forbidden.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "question": {"type": "string"}
+                },
+                "required": ["question"]
+            }
+        }
+    }
+]
+
+
+def tool_classification_agent():
+    return """
+# You are a strict enterprise tool **Router** that analyzes intent, selects exactly one correct below given tool, never responds in text, and always outputs a valid tool call or asks for missing required fields draft text format.    
+
+**Available tools:**
+
+| **No.** | **Tool Name**                    | **When to Use / Description**                                                                                                                                                                                                                                                                                        |
+| ------- | -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 1       | **send_email_api**               | Use for any requests related to sending emails, including subject and content.                                                                                                                                                                                                                                       |
+| 2       | **get_crm_info**                 | Use for CRM /MongoDB/ database related queries such as: <br>• Leads, contacts, opportunities, tasks <br>• Salary, revenue, sales revenue <br>• Email or company lookups <br>• Events or task related queries <br>• Analytics, updates, or retrieving database info <br>• MongoDB data extraction, counts,sums,totals |
+| 3       | **create_lead**                  | Use for creating a new lead in the CRM system using email and company name.                                                                                                                                                                                                                                          |
+| 4       | **schedule_meeting_api**         | Use for scheduling meetings in the calendar.                                                                                                                                                                                                                                                                         |
+| 5       | **update_query**                 | Use for updating or changing existing CRM records.                                                                                                                                                                                                                                                                   |
+| 6       | **greetings_api**                | Use when the user input is a greeting, polite message, small talk, or asks about the assistant’s identity. <br>Examples: “hi”, “hello”, “how are you”, “what is your name”, “who are you”                                                                                                                            |
+| 7       | **pdf_file_api**                 | Use for questions related to extracting or retrieving information from PDF files.                                                                                                                                                                                                                                    |
+| 8       | **general_chatbot_api**          | General questions answering like (eg. what is java?, what is oops, how to train the ml models?                                                                                                                                                                                                                        |
+
+# **CRM Query Classification Table (6 Collections) this based in tool name is `get_crm_info`**
+
+| **Rule No** | **Query Type / Condition**        | **Keywords / Indicators**                                                                          | **Collection to Use**         | **Notes / Examples**                               |
+| ----------- | --------------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------- | -------------------------------------------------- |
+| **1**       | Task-related queries              | tasks, follow-ups, activities, current work, pending actions, to-do                                | **CRM_Task_View**             | "Show my pending tasks", "List today’s follow-ups" |
+| **2A**      | Person name mentioned             | Any human name or mobile, phone, contact number, mail, email details                               | **CRM_Leads_View**            | "Show details of Rahul", "Lead info for Anitha"    |
+| **2B**      | Communication-related queries     |  events, event register details                                                                    | **CRM_Contacts_View**         | Contacts or event extraction strictly uses this    |
+| **3**       | Meeting & email audit queries     | meeting schedule, next meeting, meeting time, participants, email status, sent/failed emails, logs | **admin.audit.CRM_View**      | Includes “leads not contacted in last 10 days”     |
+| **4**       | Opportunity & revenue queries     | revenue, amount, payment, deal, deals, quotation, quote, pipeline                                  | **CRM_Opportunity_View**      | "Show total revenue this month", "List open deals" |
+| **5**       | Best / top / highest analysis     | best, top, highest, maximum, most                                                                  | **Dynamic (based on entity)** | Compute using keys like score, amount, probability |
+| **6**       | Account / company-related queries | accounts, companies, organizations, clients, customers, firms, businesses, enterprises             | **CRM_Accounts_View**         | "How many accounts assigned in Jan 2026?"          |
+
+
+**Tool selection rules (VERY IMPORTANT)**
+- If the user’s request is about CRM data in any way (leads, contacts, opportunities, tasks, meetings, emails, accounts, metrics, reports), you MUST use 'get_crm_info' unless the user is clearly asking to create/update something."
+- Use 'get_crm_info' when the user asks to *summarize, list, show, get, fetch, report, analyze, understand, group, classify, categorize, segment, score, or suggest next actions* for meetings, leads, contacts, opportunities, tasks, emails, or any CRM-related data. 
+
+1)`get_crm_info`: (below rules understand to must return `get_crm_info`):
+
+    - If user queries is related to the CRM/MongoDB/Database data or information extraction releted strictly return this tool.
+    - Query word is "count,sum, leads name, event registrations, register records, analytics, meetings calls details, etc..
+    - If query is Task data extraction related, Lead data extractions related, Contacts data extractions related, Opportunities data extractions related, Acoounts data extractions related.
+    - **show, list, get, fetch, summarize, analyze, report, count, total, pending, upcoming, status, details, history, dashboard, metrics, performance, pipeline, revenue, sales, events, meetings (if not creating), emails (if not sending), tasks, leads, contacts, accounts, opportunities**
+    - **Important Note:** `What is my mail id or Show my mail id or show my details for U0006` this format self related queries must return "get_crm_info" tool.
+
+    Sample Example Questions:
+    * Summarize today’s meetings
+    * What is my mail ID / show my mail ID
+    * Show my pending tasks
+    * Give me lead status report
+    * Analyze my pipeline for this quarter
+    * Group my leads into cold, warm, and hot categories and suggest next actions for each
+    * Which leads should I follow up with today
+    * Tell me which opportunities are most likely to close this month
+    * Show the top 5 sources with count of contacts registered for the event *“Intelligent HR & Payroll for NextGen Workforce 2026”* based on source
+    * What is the total registrations for the event *“Intelligent HR & Payroll for NextGen Workforce 2026”*
+    * How many emails has been add from santhosh last year?
+    * Please give the `Martinez20 Technologies` company all detail like employee count,etc
+
+
+2)`send_email_api`: (below rules understand to must return `send_email_api`):
+
+    - User query related for send email ,email send, compose, or draft emails or this similar.
+    - **Mandatory fields are:** 'to_email', 'subject', and 'body'. Optional: 'cc_email', 'attachments'
+    - If above mention `Mandatory fields` are missing, ask ONLY for the missing ones in a single message, then generate the draft.
+    - Please provide the following details so I can draft the email for you: \n -**to_mail** \n  \n- **Subject** of the email  \n- **Body/content** of the email.
+        
+3)`create_lead`: (below rules understand to must return `create_lead`):
+
+    - Use 'create_lead' when the user asks to create, add, register, or open a new lead
+    - Mandatory fields: 'email', 'company_name', and 'requirement'
+    - If the user provides multiple fields in one request, extract ALL of them and call the API immediately
+    - If above mention `Mandatory fields` fields are missing, ask ONLY for the missing ones in a single message, then generate the draft..
+    - If the lead already exists, respond with details and provide the view link.
+    - Once all fields are collected, call the API immediately.
+    - Confirm success or failure clearly to the user.
+    - When the user message contains multiple data points (emails, company names, requirements).
+
+4)`schedule_meeting_api`:( below rules understand to must return `schedule_meeting_api`):
+
+    - When the user explicitly wants to schedule/arrange/book/assign a meeting (e.g., 'schedule a meeting', 'book a demo').
+    - **Mandatory fields:** 'title/summary', 'recipient/attendees', 'date', 'startTime', 'endTime'. Optional: 'description', 'time_zone'
+    - If above mention `Mandatory fields` are missing, ask ONLY for the missing ones in a single message, then generate the draft.
+
+    - CRITICAL: If the user mentions a timezone (e.g., 'Europe/London', 'time zone is Europe/London'), extract ONLY the timezone value (e.g., 'Europe/London') and pass it as the 'time_zone' parameter. DO NOT include timezone information in the 'description' field.\n"
+    - IMPORTANT: For dates in DD/MM/YYYY format (e.g., '05/12/2025'), interpret as day/month/year (5th December 2025, not May 12th). When passing to start_time and end_time, use clear format like '2025-12-05 1:00 PM'.\n"
+    - The 'description' field should contain meeting details but NEVER timezone information.
+    - If no timezone is mentioned, default to 'Asia/Kolkata'
+
+    - **IMPORTANT NOTE:** Do NOT use this tool when the user asks to summarize, list, or check meetings; in those cases, use 'get_crm_info'.
+
+    *Note: User sometimes give the very short format query like only `schedule meeting` or `book meeting` this similar very short format time,  above mention "mandatory fields" what is missing that used to send the draft message.*    
+
+5)`greetings_api`:
+
+    - If the user input is a greeting, polite message, small talk, or asks about the assistant’s identity.
+    - ALSO route here when the query specifically mentions "Delvant CRM" as a product/platform explanation request.
+
+    Special Condition:
+    - If the phrase "Delvant CRM" appears together in the query
+      AND the intent is explain/what/how/about/usage/guide/help,
+      ALWAYS use greetings_api.
+
+    Examples:
+    - What is Delvant CRM
+    - Explain Delvant CRM
+    - How to use Delvant CRM
+    - About Delvant CRM
+    - Help me understand Delvant CRM
+
+    Important:
+    - Only trigger when both words appear together: "Delvant CRM"
+    - If only "CRM" or only "Delvant" appears → follow normal routing rules.
+
+
+6)`pdf_file_api`: (below rules understand to must return `pdf_file_api`):
+
+    - ** **only** for questions that require information from the uploaded PDF files inside the data.
+
+    - The query **must explicitly or implicitly refer to PDF content**, such as:
+      * Asking for personal details found in the PDF (email, phone number, address, etc.)      
+      * Asking questions like:
+
+        * “Please give my email and phone number from the PDF”
+        * “What is a transformer according to my PDF data?”
+        * “Explain this topic using my PDF file”
+        * "What is my name use pdf file"
+
+    * If the question mentions:    
+      * *my pdf file*
+      * *pdf data*
+      * *pdf document*
+      * *this pdf*
+      * *use pdf*
+      * or asks to extract / explain information **from the PDF**
+
+      *Note: query inside `not use pdf` or `not consider pdf` or this similar word mention time strictly avoid `pdf_file_api`*
+
+7)`general_chatbot_api`: (below rules understand to must return `general_chatbot_api`):
+
+    - Use when the user asks general, informational, or explanatory questions (what is, how to, explain, define, usage, guide, integrate, examples).  
+    - If no other tool matches, ALWAYS call this tool.  
+    - NEVER answer in text.  
+    - Action: call general_chatbot_api({"question": "<user message>"}). 
+
+
+### VERY CRITICAL NOTES:
+1. If query contains "lead" AND ("create" OR "new" OR "add") → call create_lead
+2. Else if query contains other create intents → return draft message
+3. Lead creation has highest priority: if the request mentions "lead" with create/add/new, ALWAYS call `create_lead` tool and NEVER return the draft message.
+
+| Condition (User Intent)                                                              | Action                                                                                |
+| ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------- |
+| Request is **create new task / account / contact / any create except lead**          | Return exactly: `"Development is in progress by the technical team. Launching soon."` |
+| Request is **create lead / new lead / add lead / register lead / any lead creation** | **Do NOT return message** → strictly call `create_lead` tool                          |
+
+"""
+

all_combine_code/all_services_/delveant_core_services.py

@@ -0,0 +1,1011 @@
+import asyncio,time
+import json
+import warnings
+
+warnings.filterwarnings("ignore")
+import subprocess, sys, os, re
+import datetime,threading
+import logging,requests,httpx
+
+from
+from models_.delveant_llm_model_.multi_agent_llm import (MultiAgentLLM,
+                                                         CodeErrorFixer)
+from tools_.delveant_all_tools_.extracted_json_output import (extract_json_outputs)
+from tools_.delveant_all_tools_.s3_history_id_details import get_sorted_memory_ids
+
+from tools_.delveant_all_tools_.encrypt_and_decrypt import encrypt_payload,decrypt_payload
+
+from agents_.plot_finder import plot_finder_agent
+from agents_.coding_error_fix_agent import code_error_resolver
+from models_.delveant_llm_model_.tool_calling_model import tool_finder_llm
+from rag_method.chat_with_pdf import question_and_answering_rag
+
+from dotenv import load_dotenv
+load_dotenv()
+
+from pymongo import MongoClient
+import mimetypes
+from urllib.parse import urlparse
+import dateparser
+import pytz
+from datetime import datetime,timezone
+
+from textwrap import dedent
+
+MONGO_URI = os.getenv("MONGO_URI")
+client = MongoClient(MONGO_URI)
+
+def convert_mongo_python_repr(text):
+    text = re.sub(
+        r"\{\s*['\"]?\$oid['\"]?\s*:\s*['\"]([0-9a-fA-F]+)['\"]\s*\}",
+        r'"\1"',
+        text
+    )
+    text = re.sub(
+        r"ObjectId\(['\"]([0-9a-fA-F]+)['\"]\)",
+        r'"\1"',
+        text
+    )
+    def datetime_replacer(match):
+        parts = match.group(1).split(",")
+        parts = [int(p.strip()) for p in parts[:3]]
+        dt = datetime.date(*parts)
+        return '"' + dt.strftime("%Y-%m-%d") + '"'
+    text = re.sub(
+        r"datetime\.datetime\(([^)]+)\)",
+        datetime_replacer,
+        text
+    )
+    text = re.sub(
+        r"(['\"]?)([A-Za-z0-9_]+)\1\s*:",
+        r'"\2": ',
+        text
+    )
+    def value_replacer(match):
+        value = match.group(1).strip()
+        if re.match(r"^-?\d+(\.\d+)?$", value):
+            return value
+        if value in ["true", "false", "null"]:
+            return value
+        return '"' + value.strip("'\"") + '"'
+    text = re.sub(
+        r":\s*([^,{}\s][^,{}]*)",
+        lambda m: ": " + value_replacer(m),
+        text
+    )
+    return text
+
+async def plot_finder(prompt):
+    llm = MultiAgentLLM()
+    output = await llm.plot_finder(prompt)
+    return output
+
+async def greetings_api(prompt):
+    llm = MultiAgentLLM()
+    output = await llm.generate_ai_response_greetings(prompt)
+    return output
+
+async def pdf_file_api(query,id):
+    output = await question_and_answering_rag(query,id)
+    return output
+
+async def empty_responser(prompt):
+    llm = MultiAgentLLM()
+    output = await llm.empty_list_responser(prompt)
+    return output
+
+async def error_code_handler(prompt):
+    llm = CodeErrorFixer()
+    output = await llm.code_debuger(prompt)
+    return output
+
+async def general_chatbot_api(prompt):
+    llm = MultiAgentLLM()
+    output = await llm.general_questions_answer(prompt)
+    return output
+
+def extract_timezone_from_message(message):
+    """Extract timezone from user message"""
+    tz_patterns = [
+        r'time zone is ([A-Za-z/]+)',
+        r'timezone is ([A-Za-z/]+)',
+        r'in ([A-Za-z]+/[A-Za-z]+)',
+        r'([A-Za-z]+/[A-Za-z]+) time',
+    ]
+
+    for pattern in tz_patterns:
+        match = re.search(pattern, message, re.IGNORECASE)
+        if match:
+            return match.group(1)
+    return None
+
+def run_python_code(path, timeout=40):
+    """
+    Run Python code with timeout of 60 seconds (1 minute)
+    Returns dict with either 'answer' or 'error' key
+    """
+    try:
+        result = subprocess.run(
+            [sys.executable, path],
+            capture_output=True,
+            text=True,
+            timeout=80,
+            encoding='utf-8'
+        )
+        if result.returncode != 0:
+            error_message = result.stderr.strip() if result.stderr else "Unknown error"
+            return {"error": error_message}
+        final_responser = result.stdout.strip()
+        return {"answer": final_responser}
+    except subprocess.TimeoutExpired:
+        return {"error": "Script execution timeout (exceeded 1 minute)"}
+    except subprocess.CalledProcessError as e:
+        error_message = e.stderr.strip() if e.stderr else str(e)
+        return {"error": error_message}
+    except Exception as e:
+        return {"error": str(e)}
+
+async def process_successful_result(final_responser, corrected_query):
+    import regex as rex
+    matches = rex.findall(r'\{(?:[^{}]|(?R))*\}', str(final_responser), rex.DOTALL)
+    first_two_records = matches[:2]
+    plot_finder_prompt = plot_finder_agent(corrected_query, str(first_two_records))
+    decision_find = await plot_finder(plot_finder_prompt)
+    clean_final = extract_json_outputs(decision_find)
+    matches = re.findall(r'"(.*?)":"(.*?)"', clean_final)
+    matches_dict = dict(matches)
+    output_format = matches_dict["format_data"]
+    plot_output = matches_dict["plot"]
+
+    if output_format.strip() == "empty":
+        final_response_1 = await empty_responser(corrected_query)
+        return {"status": "success", "view": "empty", "output": final_response_1,
+                "plot": "not_required","corrected_query": corrected_query}
+
+    elif output_format.strip() == "card":
+        return {"status": "success", "view": "card", "output": final_responser,
+                "plot": "not_required","corrected_query": corrected_query}
+
+    elif output_format.strip() =="data":
+        return {"status": "success", "view": "data", "plot": plot_output,
+                "output": final_responser,"corrected_query": corrected_query}
+
+
+def parse_datetime_to_iso(dt_str, tz="Asia/Kolkata"):
+    if not dt_str:
+        return None
+
+    settings = {
+        "RETURN_AS_TIMEZONE_AWARE": True,
+        "TIMEZONE": tz,
+        "DATE_ORDER": "YMD",
+    }
+
+    dt = dateparser.parse(dt_str, settings=settings)
+
+    if not dt:
+        return None
+
+    if dt.tzinfo is None:
+        tz_obj = pytz.timezone(tz)
+        dt = tz_obj.localize(dt)
+    dt_utc = dt.astimezone(pytz.UTC)
+    return dt_utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
+def normalize_attendees(attendees):
+    if isinstance(attendees, list) and len(attendees) == 1 and "," in attendees[0]:
+        return [email.strip() for email in attendees[0].split(",")]
+    return attendees
+
+async def get_crm_info(input_query, previous_chat_history_query, db_name, user_id, person_id):
+    script_path = None
+    corrected_query = None
+    try:
+        obj = DelveAnt()
+        final_output, corrected_query, first_python_code, path = await obj.run_file(
+            input_query, previous_chat_history_query, db_name, user_id, person_id
+        )
+        if final_output == "successfully_file_created":
+            try:
+                script_path = path
+                result = run_python_code(script_path, timeout=40)
+                if "answer" in result:
+                    final_responser = result["answer"]
+                    return await process_successful_result(final_responser, corrected_query)
+                elif "error" in result:
+                    max_retries = 3
+                    for attempt in range(max_retries):
+                        print(f"Error fixing attempt {attempt + 1}/{max_retries}")
+                        with open(script_path, "r", encoding="utf-8") as f:
+                            file_contents = f.read()
+                        error_message = result["error"]
+                        prompt = code_error_resolver(file_contents, error_message, input_query)
+                        print(prompt)
+                        chat_response = await error_code_handler(prompt)
+                        filter_python_code = re.sub(r"```(?:python)?|```", "", chat_response).strip()
+                        with open(script_path, "w", encoding="utf-8") as f:
+                            f.write(filter_python_code)
+                        result = run_python_code(script_path, timeout=40)
+                        if "answer" in result:
+                            final_responser = result["answer"]
+                            return await process_successful_result(final_responser, corrected_query)
+                    return {
+                        "status": "error",
+                        "view": "text",
+                        "output": f"Failed to execute after {max_retries} attempts. Last error: {result.get('error', 'Unknown error')}",
+                        "plot": "not_required",
+                        "corrected_query": corrected_query,
+                    }
+            except Exception as e:
+                return {
+                    "status": "error",
+                    "view": "text",
+                    "output": f"Server error: {str(e)}",
+                    "plot": "not_required",
+                    "corrected_query": corrected_query if 'corrected_query' in locals() else "query not found error",
+                }
+
+            finally:
+                if script_path and os.path.exists(script_path):
+                    try:
+                        os.remove(path)
+                        print(f"Cleaned up: {script_path}")
+                    except Exception as e:
+                        print(f"Error deleting {script_path}:", e)
+
+        else:
+            return {"status": "success", "view": "text", "output": final_output, "plot": "not_required","corrected_query": corrected_query}
+
+    except Exception as e:
+        logging.exception("Error processing request")
+        return {
+            "status": "error",
+            "view": "text",
+            "output": f"Server error: {str(e)}",
+            "plot": "not_required",
+            "corrected_query": corrected_query if 'corrected_query' in locals() else "query not found error",
+        }
+
+async def create_lead(email, company_name, requirement, subdomain, user_email,headers):
+    flow_url = os.getenv("FLOW_URL_API")
+
+    payload = {
+        "widgetid": "6880c0ecb384e4a0fe7c901f",
+        "subdomain": subdomain,
+        "createtype": "manual",
+        "assignee": user_email,
+        "email": email,
+        "company": company_name,
+        "requirement": requirement
+    }
+
+    encrypted_string = await encrypt_payload(payload)
+
+    final_payload = json.dumps({
+        "payload": encrypted_string
+    })
+
+
+    timeout = httpx.Timeout(120.0)
+
+    for attempt in range(3):
+        try:
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                response = await client.post(
+                    flow_url,
+                    content=final_payload,  # or data=final_payload
+                    headers=headers
+                )
+                response.raise_for_status()
+                final_answer,_ = await decrypt_payload(response.text)
+                return {"success": final_answer}
+
+        except httpx.TimeoutException:
+            if attempt == 2:
+                answer = {
+                    "error": "Lead creation API timed out after retries"
+                }
+                return answer
+            await asyncio.sleep(2 ** attempt)
+
+        except httpx.HTTPStatusError as e:
+            answer = {
+                "error": f"Lead creation API failed: {e.response.text}"
+            }
+            return answer
+
+        except Exception as e:
+            answer = {
+                "error": f"Internal server error: {str(e)}"
+            }
+            return answer
+
+
+def build_attachments(attachment_links):
+    attachments = []
+
+    for item in attachment_links:
+        if isinstance(item, str):
+            # Just a raw URL string
+            url = item
+            parsed = urlparse(url)
+            filename = parsed.path.split("/")[-1]
+            mime_type, _ = mimetypes.guess_type(url)
+            mime_type = mime_type or "application/octet-stream"
+
+        elif isinstance(item, dict):
+            # Dictionary payload (preferred)
+            url = item.get("url")
+            filename = item.get("filename")
+            mime_type = item.get("mimeType")
+
+            # Fallbacks if not provided
+            if not filename and url:
+                parsed = urlparse(url)
+                filename = parsed.path.split("/")[-1]
+            if not mime_type and url:
+                mime_type, _ = mimetypes.guess_type(url)
+                mime_type = mime_type or "application/octet-stream"
+
+        else:
+            continue
+
+        if url:
+            attachments.append({
+                "filename": filename,
+                "url": url,
+                "mimetype": mime_type
+            })
+
+    return attachments
+
+
+async def send_email_api(
+        to_email,
+        subject,
+        body,
+        subdomain,
+        provider,
+        user_id,
+        lead_id,
+        entityname,
+        cc_email,
+        headers,
+        attachment_links,
+        bcc=None,
+):
+    def delayed_send_email(frommail, to_email, subject, subdomain):
+        time.sleep(10)
+        # send_email(from_email=frommail, to_email=to_email, keyword=subject, subdomain=subdomain)
+
+    try:
+        AUNTHENTICATE_URL = os.getenv("AUNTHENTICATE_URL")
+        EMAIL_API_MICROSOFT = os.getenv("EMAIL_API_MICROSOFT")
+        EMAIL_API_GOOGLE = os.getenv("EMAIL_API_GOOGLE")
+
+        if not AUNTHENTICATE_URL:
+            return {"error": "AUNTHENTICATE_URL environment variable not set"}
+        if not EMAIL_API_MICROSOFT or not EMAIL_API_GOOGLE:
+            return {"error": "Email API environment variables not set"}
+
+        timeout = httpx.Timeout(120.0)
+
+        sample_data = {
+            "subDomain": subdomain,
+            "systemUserId": user_id,
+        }
+
+        encrypted = await encrypt_payload(sample_data, compress=False)
+
+        final_payload = json.dumps({
+            "payload": encrypted
+        })
+
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            auth_response = await client.post(
+                AUNTHENTICATE_URL,
+                content=final_payload,
+                headers=headers
+            )
+
+        auth_response.raise_for_status()
+        encrypt_response = auth_response.text
+        decrypted_response, check_true_false = await decrypt_payload(encrypt_response)
+
+        if check_true_false == True:
+            resp_json = decrypted_response
+
+            if isinstance(resp_json, dict) and isinstance(resp_json.get("data"), list):
+                auth_list = resp_json["data"]
+            elif isinstance(resp_json, list):
+                auth_list = resp_json
+            else:
+                answer = {"error": "Invalid authentication response format", "auth_response": resp_json}
+                return answer
+
+            if not auth_list:
+                answer = {"error": "Authentication list is empty", "auth_response": resp_json}
+                return answer
+
+            first_auth = auth_list[0]
+
+            frommail = first_auth.get("email") or first_auth.get("userId")
+            provider = first_auth.get("provider")
+
+            if not frommail:
+                return {"error": "No authenticated email found", "auth_response": resp_json}
+
+            if not provider:
+                return {"error": "Provider missing in authentication response", "auth_response": resp_json}
+
+            provider = provider.lower()
+
+            if provider not in ("microsoft", "google"):
+                return {"error": f"Unsupported provider '{provider}'"}
+
+            payload = {
+                "userId": frommail,
+                "provider": provider,
+                "subDomain": subdomain,
+                "to": to_email,
+                "cc": cc_email,
+                "bcc": bcc,
+                "subject": subject,
+                "systemUserId": user_id,
+                "body": body,
+                "leadId": lead_id,
+                "attachments": attachment_links,
+                "entityName": entityname,
+            }
+
+            if cc_email:
+                payload["cc"] = cc_email
+            if attachment_links:
+                payload["attachments"] = attachment_links
+
+            url = EMAIL_API_MICROSOFT if provider.lower() == "microsoft" else EMAIL_API_GOOGLE
+
+            encrypted1 = await encrypt_payload(payload, compress=False)
+
+            final_payload1 = json.dumps({
+                "payload": encrypted1
+            })
+
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                response = await client.post(url, content=final_payload1, headers=headers)
+
+            response.raise_for_status()
+
+            threading.Thread(
+                target=delayed_send_email,
+                args=(frommail, to_email, subject, subdomain)
+            ).start()
+
+            response_json,check_true_false = await decrypt_payload(response.text)
+            if check_true_false == True:
+                return response_json
+            else:
+                return {"error": "response return JSONDecodeEror"}
+
+    except httpx.RequestError as e:
+        return {"error": f"Email sending failed: {str(e)}"}
+
+    except Exception as e:
+        return {"error": f"Unexpected error in send_email_api: {str(e)}"}
+
+
+async def schedule_meeting_api(title, description, recipient, start_time,
+                               end_time, provider, subdomain, user_id, headers ,time_zone="Asia/Kolkata",
+                               person_id=""):
+    try:
+        AUNTHENTICATE_URL = os.getenv("AUNTHENTICATE_URL")
+        SCHEDULE_MICROSOFT = os.getenv("SCHEDULE_MICROSOFT")
+        SCHEDULE_GOOGLE = os.getenv("SCHEDULE_GOOGLE")
+
+        if not AUNTHENTICATE_URL:
+            return {"error": "AUNTHENTICATE_URL environment variable not set"}
+        if not SCHEDULE_MICROSOFT or not SCHEDULE_GOOGLE:
+            return {"error": "SCHEDULE_MICROSOFT / SCHEDULE_GOOGLE environment variables not set"}
+        start_iso = parse_datetime_to_iso(start_time, tz=time_zone)
+        end_iso = parse_datetime_to_iso(end_time, tz=time_zone)
+        print(f"start_iso={start_iso}, end_iso={end_iso}")
+
+        if not start_iso or not end_iso:
+            return {"error": "Invalid start or end time format."}
+
+        auth_payload = {"subDomain": subdomain, "systemUserId": user_id}
+
+        encrypted = await encrypt_payload(auth_payload, compress=False)
+
+
+        final_payload = {
+            "payload": encrypted
+        }
+
+        try:
+            getfromemail = requests.post(
+                AUNTHENTICATE_URL,
+                json=final_payload,
+                headers=headers,
+                timeout=60
+            )
+
+        except Exception as e:
+            answer = {"error": f"Network error calling AUNTHENTICATE_URL: {e}"}
+            return answer
+
+        decrypt_response, check_status = await decrypt_payload(getfromemail.text)
+
+        if check_status == True:
+            resp_json = decrypt_response
+        else:
+            resp_json = None
+
+        print({"auth_response_json": resp_json})
+
+        if not getfromemail.ok:
+            return {
+                "error": "Authenticated email API returned an error",
+                "status_code": getfromemail.status_code,
+                "response_text": getfromemail.text,
+                "parsed_json": resp_json,
+                "sent_payload": auth_payload,
+            }
+
+        if isinstance(resp_json, dict) and isinstance(resp_json.get("data"), list):
+            auth_list = resp_json["data"]
+        elif isinstance(resp_json, list):
+            auth_list = resp_json
+        else:
+            return {"error": "Invalid authentication response format", "auth_response": resp_json}
+
+        if not auth_list:
+            return {"error": "Authentication list is empty", "auth_response": resp_json}
+
+        first_auth = auth_list[0]
+        frommail = first_auth.get("email") or first_auth.get("userId")
+        provider = first_auth.get("provider")
+
+        if not frommail:
+            return {"error": "No authenticated email found", "auth_response": resp_json}
+
+        if not provider:
+            return {"error": "Provider missing in authentication response", "auth_response": resp_json}
+
+        provider = provider.lower()
+        if provider not in ("microsoft", "google"):
+            return {"error": f"Unsupported provider '{provider}'"}
+
+        attendees = normalize_attendees(
+            [recipient] if isinstance(recipient, str) else recipient
+        )
+
+        payload = {
+            "userId": frommail,
+            "provider": provider,
+            "subDomain": subdomain,
+            "summary": title,
+            "description": description or "",
+            "startTime": start_iso,
+            "endTime": end_iso,
+            "systemUserId": user_id,
+            "personId": person_id or recipient,
+            "attendees": attendees,
+            "timeZone": time_zone,
+        }
+        encrypted1 = await encrypt_payload(payload, compress=False)
+
+        final_payload1 = {
+            "payload": encrypted1
+        }
+
+        url = SCHEDULE_MICROSOFT if provider == "microsoft" else SCHEDULE_GOOGLE
+        response = requests.post(url, json=final_payload1, timeout=120,headers=headers)
+        response.raise_for_status()
+        try:
+            response_, check_status = await decrypt_payload(response.text)
+            if check_status == True:
+                schedule_json = response_
+            else:
+                schedule_json = None
+
+        except ValueError:
+            schedule_json = None
+
+        if not response.ok:
+            return {
+                "error": "Meeting scheduling failed at schedule API",
+                "status_code": response.status_code,
+                "response_text": response.text,
+                "parsed_json": schedule_json,
+                "sent_payload": payload,
+            }
+
+        return schedule_json or {"message": "Meeting scheduled, but response was not JSON."}
+
+    except Exception as e:
+        answer = {"error": f"Network error: {e}"}
+        return answer
+
+
+class ToolClasifier:
+    def __init__(self):
+        pass
+
+    def ensure_dict(self,data):
+        if isinstance(data, dict):
+            return data
+        if isinstance(data, str):
+            return json.loads(data)
+        raise TypeError(f"Unsupported tool_args type: {type(data)}")
+
+    async def handle_crm_chat(self,input_query,
+            previous_chat_history_query,
+            db_name, user_id,person_id,
+            lead_id,leademail,leadname,
+            user_email, session_id,
+            loggedUserRole,
+            loggedUserMail,provider,
+            your_id,entityname,
+            attachments,rag_uuid,headers):
+
+        tool_name, tool_args = await tool_finder_llm(previous_chat_history_query)
+        logging.info(f"tool_name: {tool_name}")
+        logging.info(f"tool_args: {tool_args}")
+        steps = []
+        step_counter = 1
+        output_result = tool_args
+
+        def add_step(desc: str):
+            nonlocal step_counter
+            steps.append({f"step{step_counter}": desc})
+            step_counter += 1
+
+        add_step(f"User asked: '{input_query}'")
+        add_step(f"Before Chat based query: {previous_chat_history_query}")
+
+        all_names = ["send_email_api", "get_crm_info", "create_lead","schedule_meeting_api","update_query",
+                     "greetings_api","pdf_file_api","general_chatbot_api","no_tool_name","no_tool_args"
+                     ]
+        if tool_name in all_names:
+            add_step(f"AI selected tool: {tool_name}")
+            add_step(f"Tool called with arguments: {tool_args}")
+            logging.info(f"Tool called : {tool_name}")
+
+            if tool_name.strip().strip() == "get_crm_info":
+                answer = await get_crm_info(
+                    input_query=input_query,
+                    previous_chat_history_query=previous_chat_history_query,
+                    db_name=db_name,
+                    user_id=user_id,
+                    person_id=person_id
+                )
+                data = answer
+                if data.get("status") == "success":
+                    view = data.get("view")
+                    plot = data.get("plot")
+                    status = data.get("status")
+                    corrected_query = data.get("corrected_query")
+                    json_str = json.dumps(data, ensure_ascii=False)
+                    json_bytes = json_str.encode("utf-8")
+                    decoded_str = json_bytes.decode("utf-8")
+                    data = json.loads(decoded_str)
+                    tool_output = {
+                        "result": data["output"],
+                        "pipeline": None,
+                        "plot": plot,
+                        "view": view,
+                        "status": status
+                    }
+                    reply_text = json.dumps(tool_output, ensure_ascii=False)
+
+                elif data.get("status") == "error":
+                    tool_output = answer
+                    reply_text = json.dumps(tool_output, ensure_ascii=False)
+                add_step(f"Corrected query: {corrected_query}")
+                add_step(f"Received CRM response: {tool_output}")
+
+
+            elif tool_name.strip().lower() == "create_lead":
+                data_dict = json.loads(tool_args)
+                email = data_dict["email"] or "".strip()
+                company_name = data_dict["company_name"] or "".strip()
+                requirement = data_dict["requirement"] or "".strip()
+                missing = None
+                if not email:
+                    missing = "email"
+                elif not company_name:
+                    missing = "company_name"
+                elif not requirement:
+                    missing = "requirement"
+
+                if missing:
+                    field_labels = {
+                        "email": "lead's email address",
+                        "company_name": "lead's company name",
+                        "requirement": "the lead's requirement/request (e.g., 'Sportizia demo')"
+                    }
+                    prompt = f"Please provide {field_labels[missing]}."
+                    add_step(f"Missing '{missing}'. Asking user for it.")
+                    reply_text = prompt
+                    tool_output = None
+
+                else:
+                    tool_output = await create_lead(
+                        email=email,
+                        company_name=company_name,
+                        requirement=requirement,
+                        subdomain=db_name,
+                        user_email=user_email,
+                        headers=headers
+                    )
+                    add_step(
+                        f"Lead creation API called with payload {{'email': '{email}', 'company_name': '{company_name}', 'requirement': '{requirement}'}}")
+                    add_step(f"Received Lead creation response: {tool_output}")
+                    if tool_output.get("success"):
+                        reply_text = tool_output["success"]
+                        tool_output = tool_output["success"]
+                    else:
+                        reply_text = tool_output["error"]
+                        tool_output = tool_output["error"]
+
+            elif tool_name.strip().lower() == "send_email_api":
+                response_json =json.loads(tool_args)
+                to_email = response_json.get("to_email", "").strip()
+                subject = response_json.get("subject", "").strip()
+                body = response_json.get("body", "").strip()
+                cc_email = response_json.get("cc_email", "").strip()
+                db = client[db_name]
+                audit_collection = db["admin.audit.CRM"]
+                missing = None
+                if not to_email:
+                    missing = "to_email"
+                elif not subject:
+                    missing = "subject"
+                elif not body:
+                    missing = "body"
+
+                if missing:
+                    field_labels = {
+                        "to_email": "recipient email address",
+                        "subject": "email subject",
+                        "body": "email message/content"
+                    }
+                    prompt = f"Please provide {field_labels[missing]}."
+                    add_step(f"Missing '{missing}'. Asking user for it.")
+                    reply_text = prompt
+                    tool_output = None
+
+                else:
+                    tool_output = await send_email_api(
+                        to_email=to_email,
+                        subject=subject,
+                        body=body,
+                        subdomain=db_name,
+                        provider=provider,
+                        user_id=user_id,
+                        lead_id=lead_id,
+                        entityname =entityname,
+                        cc_email=cc_email,
+                        headers=headers,
+                        attachment_links=attachments,
+                    )
+
+                    add_step(
+                        f"Email API called with payload {{'to_email': '{to_email}', 'subject': '{subject}', 'body': '{body[:50]}...'}}")
+                    add_step(f"Received Email response: {tool_output}")
+
+                    if tool_output and not tool_output.get("error"):
+                        reply_text = dedent(f"""
+                        <div style="font-family: 'Inter', sans-serif;">
+                        <h3 style="color:#1e293b; margin:0; font-size:13.5px;">✅ Email Sent Successfully</h3>
+                        <p style="color:#1e293b; margin:8px 0; font-size:13.5px;">
+                        <span><b>To:</b> {to_email}</span>
+                        </p>
+                        <p style="color:#1e293b; margin:8px 0; font-size:13.5px;">
+                        <span><b>Subject:</b> {subject}</span>
+                        </p>
+                        {f'<p style="color:#1e293b; margin:8px 0; font-size:13.5px;"><b>CC:</b> {cc_email}</p>' if cc_email else ''}
+                        </div>
+                        """)
+                        if lead_id:
+                            audit_data = {
+                                "ApplicationName": "CRM",
+                                "formName": "Leads",
+                                "createdId": lead_id,
+                                "Action": "SENDEMAIL",
+                                "data": {
+                                    "emailStatus": "send",
+                                    "subject": subject,
+                                    "to": to_email,
+                                    "cc": cc_email,
+                                    "attachments": attachments,
+                                    "body": body,
+                                    "leadEmail":to_email,
+                                    "sentAt": datetime.now(timezone.utc)
+                                },
+                                "created_by": person_id,
+                                "created_at": datetime.now(timezone.utc)
+                            }
+                            print(audit_data, "audit_data")
+                            try:
+                                result = audit_collection.insert_one(audit_data)
+                            except Exception as e:
+                                return {"error": str(e)}
+                    else:
+                        reply_text = f"""
+                                        <div style="font-family: 'Inter', sans-serif;">
+                                        <h3 style="color:#f44336; margin:0; font-size:13.5px;">❌ Failed to Send Email</h3>
+                                        <p style="color:#1e293b; margin:8px 0; font-size:13.5px;">
+                                        {tool_output.get("error", "Unknown error")}
+                                        </p>
+                                        </div>
+                                        """
+
+            elif tool_name.strip().lower() == "schedule_meeting_api":
+                response_json = json.loads(tool_args)
+                recipient = response_json.get("recipient", "").strip()
+                subject = response_json.get("subject", "").strip()
+                title = response_json.get("title", "").strip()
+                description = response_json.get("description", "").strip()
+                start_time = response_json.get("start_time", "").strip()
+                end_time = response_json.get("end_time", "").strip()
+                time_zone = response_json.get("time_zone", "").strip()
+                if not time_zone:
+                    extracted_tz = extract_timezone_from_message(input_query)
+                    time_zone = extracted_tz if extracted_tz else "Asia/Kolkata"
+                db = client[db_name]
+                audit_collection = db["admin.audit.CRM"]
+                missing = None
+                if not recipient:
+                    missing = "recipient"
+                elif not title:
+                    missing = "title"
+                elif not start_time or not end_time:
+                    missing = "datetime"
+
+                if missing:
+                    prompts = {
+                        "recipient": "recipient email",
+                        "title": "meeting title",
+                        "datetime": "meeting start and end time (e.g., 'Sep 5 2025 10:00pm to 11:00pm')"
+                    }
+                    reply_text = f"Please provide {prompts[missing]}."
+                    tool_output = None
+
+                else:
+                    tool_output = await schedule_meeting_api(
+                        title, description, recipient, start_time, end_time, provider,
+                        db_name, user_id, headers ,time_zone, person_id,
+                    )
+                    add_step(
+                        f"Meeting API called with payload {{'recipient': '{recipient}', 'title': '{title}'}}")
+
+                    add_step(f"Received Meeting response: {tool_output}")
+
+                    if tool_output and not tool_output.get("error"):
+                        reply_text = f"""
+                                    <div style="font-family: 'Inter', sans-serif;">
+                                    <h3 style="color:#1e293b; margin:0; font-size:13.5px;">✅ Meeting Scheduled Successfully</h3>
+                                    </div>
+                                    """
+                        if lead_id:
+                            audit_data = {
+                                "ApplicationName": "CRM",
+                                "formName": "Leads",
+                                "createdId": lead_id,
+                                "Action": "SCHEDULEMEETING",
+                                "data": {
+                                    "summary": title,
+                                    "description": description,
+                                    "startTime": start_time,
+                                    "endTime": end_time,
+                                    "attendees": recipient,
+                                    "timeZone": time_zone,
+                                    "leadEmail": leademail,
+                                    "leadName": leadname
+                                },
+                                "created_by": person_id,
+                                "created_at": datetime.now(timezone.utc)
+                            }
+
+                            audit_collection.insert_one(audit_data)
+                    else:
+                        reply_text = f"""
+                                    <div style="font-family: 'Inter', sans-serif;">
+                                        <h3 style="color:#f44336; margin:0; font-size:13.5px;">
+                                            Failed to Schedule Meeting
+                                        </h3>
+                                        <p style="color:#6b7280; margin:5px 0; font-size:13.5px;">
+                                            We couldn't schedule your meeting because your email account is not connected.
+                                            Please connect your email in the Settings page to continue.
+                                        </p>
+                                       <!-- Error message hidden
+                                        <p style="color:#6b7280; margin:6px 0; font-size:13px;">
+                                            Error: {tool_output.get("error", "Unknown error")}
+                                        </p>
+                                        -->
+
+                                        <a data-action="navigate-settings" data-tab="email"
+                                        style="display:inline-block; margin-top:8px; padding:6px 10px; background:#875255; 
+                                        color:white; text-decoration:none; border-radius:6px; font-size:13px; cursor:pointer;">
+                                        Go to Settings
+                                        </a>
+
+                                    </div>
+                                    """
+
+            elif tool_name.strip().lower() == "update_query":
+                response_json = self.ensure_dict(tool_args)
+                field_name = response_json.get("field_name", "").strip()
+                new_value = response_json.get("new_value", "").strip()
+                collection_name = response_json.get("collection", "").strip()
+                missing = None
+                if not field_name:
+                    missing = "field_name"
+                elif not new_value:
+                    missing = "new_value"
+
+                if missing:
+                    field_labels = {
+                        "field_name": "the field name (e.g., status, lead_name, company_name)",
+                        "new_value": "the new value for that field"
+                    }
+                    prompt = f"Please provide {field_labels[missing]}."
+                    add_step(f"Missing '{missing}'. Asking user for it.")
+                    reply_text = prompt
+                    tool_output = None
+                else:
+                    tool_output = "Our technical team is actively working on the development. The update will be released soon."
+                    reply_text = "Our technical team is actively working on the development. The update will be released soon."
+
+            elif tool_name.strip().lower() == "greetings_api":
+                ans = await greetings_api(input_query)
+                tool_output = {"status": "success", "view": "text", "plot": "not_required", "output": ans}
+                reply_text = ans
+
+            elif tool_name.strip().lower() == "pdf_file_api":
+                output_result_ans = await pdf_file_api(input_query,rag_uuid)
+                tool_output = {"status": "success", "view": "text", "plot": "not_required", "output": output_result_ans}
+                reply_text = output_result_ans
+
+            elif tool_name.strip().lower() == "general_chatbot_api":
+                answer = await general_chatbot_api(input_query)
+                tool_output = {"status": "success", "view": "text", "plot": "not_required", "output": answer}
+                reply_text = answer
+
+            elif tool_name.strip().lower() == "no_tool_name" and tool_args != "no_tool_args":
+                answer = tool_args
+                tool_output = {"status": "success", "view": "text", "plot": "not_required", "output": answer}
+                reply_text = answer
+
+            elif  tool_name.strip().lower() == "no_tool_name" and tool_args == "no_tool_args":
+                answer = "Could you please clarify your requirement? You may request CRM-related data (leads, tasks, meetings, accounts) or actions such as sending an email or creating a lead."
+                tool_output = {"status": "success", "view": "text", "plot": "not_required", "output": answer}
+                reply_text = answer
+
+        else:
+            tool_name = "no_tool_name"
+            tool_output = {
+                "status": "success",
+                "view": "text",
+                "plot": "not_required",
+                "output": "No tool action was needed. The query has been processed and a textual response is provided."
+            }
+            reply_text = "No tool action was needed. The query has been processed and a textual response is provided."
+
+        s3_bucket_ids = get_sorted_memory_ids()
+        return {
+            "session_id": session_id,
+            "selected_tool": tool_name,
+            "tool_args": tool_args,
+        "result": tool_output,
+        "chat_reply": reply_text,
+        "needs_input":  (
+            (tool_name is not None and tool_output is None) or
+            (tool_name is None and tool_output is None)
+        ),
+        "steps": steps,
+        "your_id":your_id,
+        "history_all_ids":s3_bucket_ids,
+       }

all_combine_code/app.py

@@ -0,0 +1,21 @@
+import os, sys, uuid
+
+os.environ.setdefault("TORCHINDUCTOR_CACHE_DIR", "/app/tmp/torchinductor")
+os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
+
+import warnings, shutil
+
+warnings.simplefilter("ignore", FutureWarning)
+from fastapi import FastAPI, Request, HTTPException
+from fastapi.responses import PlainTextResponse
+from fastapi import UploadFile, File
+from fastapi.middleware.cors import CORSMiddleware
+import logging
+from memory_.chat_history_handle import generate_response, store_final_chat
+from tools_.delveant_all_tools_.encrypt_and_decrypt import encrypt_payload, decrypt_payload
+
+from all_services_.delveant_core_services import ToolClasifier
+
+print("successfully imported")
+
+

all_combine_code/azure-pipelines.yml

@@ -0,0 +1,176 @@
+trigger:
+  branches:
+    include:
+      - dev
+
+# pool:
+#   vmImage: ubuntu-latest
+pool:
+  name: Default
+  demands:
+    - agent.name -equals stage1-newdevtest
+
+variables:
+  AWS_DEFAULT_REGION: 'ap-south-1'
+  AWS_ACCOUNT_ID: '627133981011'
+  IMAGE_REPO_NAME: 'delveant_agent'
+
+steps:
+  # Install AWS CLI (if not already available on your agent)
+  - script: |
+      curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+      unzip awscliv2.zip
+      sudo ./aws/install --update
+    displayName: 'Install AWS CLI'
+    condition: not(succeeded())
+
+  # Configure AWS credentials and login to ECR
+  - script: |
+      aws configure set aws_access_key_id $(AWS_ACCESS_KEY_ID)
+      aws configure set aws_secret_access_key $(AWS_SECRET_ACCESS_KEY)
+      aws configure set default.region $(AWS_DEFAULT_REGION)
+      
+      # Get ECR login token and login to Docker
+      aws ecr get-login-password --region $(AWS_DEFAULT_REGION) | docker login --username AWS --password-stdin $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_DEFAULT_REGION).amazonaws.com
+    displayName: 'Configure AWS and Login to ECR'
+  # Create reports directory
+  - script: |
+      mkdir -p $(Build.SourcesDirectory)/security-reports
+    displayName: 'Create Reports Directory'
+
+  # Secret detection scan with Gitleaks
+  - script: |
+      echo "🔍 Scanning for secrets in code..."
+      docker run --rm -v $(Build.SourcesDirectory):/code zricethezav/gitleaks:latest detect --source="/code" -f json -r /code/security-reports/gitleaks-report.json
+      if [ -s /code/security-reports/gitleaks-report.json ]; then
+        echo "❌ CRITICAL: Secrets detected in code! Build failed for security."
+        exit 1
+      else
+        echo "✅ No secrets found in code."
+      fi
+    displayName: '🔐 Secret Detection Scan'
+    continueOnError: false
+
+  # Dependency vulnerability scan with Trivy
+  - script: |
+      echo "🔍 Scanning dependencies for vulnerabilities with Trivy..."
+      mkdir -p /tmp/trivy-cache
+      
+      docker run --rm \
+        -v $(Build.SourcesDirectory):/workspace \
+        -v /tmp/trivy-cache:/root/.cache/trivy \
+        aquasec/trivy:latest fs \
+        --format json \
+        --output /workspace/security-reports/dependency-vulnerabilities.json \
+        --severity HIGH,CRITICAL \
+        /workspace
+      
+      docker run --rm \
+        -v $(Build.SourcesDirectory):/workspace \
+        -v /tmp/trivy-cache:/root/.cache/trivy \
+        aquasec/trivy:latest fs \
+        --format template \
+        --template "@contrib/html.tpl" \
+        --output /workspace/security-reports/dependency-vulnerabilities.html \
+        --severity HIGH,CRITICAL \
+        /workspace
+      
+      VULN_COUNT=$(docker run --rm \
+        -v $(Build.SourcesDirectory):/workspace \
+        -v /tmp/trivy-cache:/root/.cache/trivy \
+        aquasec/trivy:latest fs \
+        --format json \
+        --severity HIGH,CRITICAL \
+        --quiet \
+        /workspace | jq -r '.Results[]?.Vulnerabilities // [] | length' | awk '{sum += $1} END {print sum+0}')
+      
+      if [ "$VULN_COUNT" -gt 0 ]; then
+        echo "❌ CRITICAL: $VULN_COUNT high/critical dependency vulnerabilities found! Build failed."
+        exit 1
+      else
+        echo "✅ No critical dependency vulnerabilities found."
+      fi
+    displayName: '📦 Dependency Vulnerability Scan (Trivy)'
+    continueOnError: true
+
+  # Static Application Security Testing (SAST) - CORRECTED VERSION
+  - script: |
+      echo "🔍 Running Static Application Security Testing with Semgrep..."
+      
+      # Create output directory
+      mkdir -p $(Build.SourcesDirectory)/security-reports
+      
+      # Run Semgrep with explicit entrypoint
+      docker run --rm \
+        -v $(Build.SourcesDirectory):/src \
+        --entrypoint semgrep \
+        returntocorp/semgrep \
+        --config=auto \
+        --json \
+        --output=/src/security-reports/sast-report.json \
+        /src
+      
+      # Check if report was generated and analyze results
+      if [ -f $(Build.SourcesDirectory)/security-reports/sast-report.json ]; then
+        CRITICAL_COUNT=$(jq '[.results[] | select(.extra.severity == "ERROR")] | length' $(Build.SourcesDirectory)/security-reports/sast-report.json 2>/dev/null || echo "0")
+        
+        echo "📊 SAST Analysis Results:"
+        echo "   Critical Issues Found: $CRITICAL_COUNT"
+        
+        if [ "$CRITICAL_COUNT" -gt 0 ]; then
+          echo "❌ CRITICAL: $CRITICAL_COUNT critical security issues found!"
+          exit 1
+        else
+          echo "✅ No critical security issues found in static analysis."
+        fi
+      else
+        echo "⚠️ SAST report not generated. Check Semgrep execution."
+      fi
+    displayName: '🔍 Static Application Security Testing (SAST)'
+    continueOnError: true  # Keep true for testing
+
+  # Publish security reports - ALWAYS RUN EVEN IF BUILD FAILS
+  - task: PublishBuildArtifacts@1
+    condition: always()  # ← This ensures reports are published even if scans fail
+    inputs:
+      pathToPublish: '$(Build.SourcesDirectory)/security-reports'
+      artifactName: 'security-reports-$(Build.BuildId)'
+      publishLocation: 'Container'
+    displayName: '📊 Publish Security Reports'
+
+  # Build Docker image
+  - script: |
+      docker build -t $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_DEFAULT_REGION).amazonaws.com/$(IMAGE_REPO_NAME):$(Build.BuildId) .
+    displayName: 'Build Docker Image'
+
+  # Container scan (only if image built successfully)
+  # - script: |
+  #     docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
+  #       -v $(Build.SourcesDirectory):/reports \
+  #       aquasec/trivy:latest image \
+  #       --exit-code 1 \
+  #       --severity CRITICAL \
+  #       $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_DEFAULT_REGION).amazonaws.com/$(IMAGE_REPO_NAME):$(Build.BuildId)
+  #   displayName: '🐳 Container Vulnerability Scan'
+  # List images for verification
+  - script: |
+      docker images
+    displayName: 'List Docker Images'
+
+  # Push to ECR
+  - script: |
+      docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_DEFAULT_REGION).amazonaws.com/$(IMAGE_REPO_NAME):$(Build.BuildId)
+    displayName: 'Push Docker Image to ECR'
+
+  # Update deployment YAML
+  - script: |
+      git config --global user.email "devops@kovaion.ai"
+      git config --global user.name "Azure DevOps"
+      git clone https://$(AZURE_PAT)@dev.azure.com/kovaionai/Deployment/_git/Deployment
+      cd Deployment
+      git checkout delveant_cloud
+      sed -i 's|627133981011.dkr.ecr.ap-south-1.amazonaws.com/delveant_agent:.*|627133981011.dkr.ecr.ap-south-1.amazonaws.com/delveant_agent:$(Build.BuildId)|' agents.yaml
+      git add agents.yaml
+      git commit -m "update yaml with build ids"
+      git push origin delveant_cloud
+    displayName: 'Update YAML with Latest Image Tag and Push'

all_combine_code/genre_classification/genre_.ipynb

@@ -0,0 +1,611 @@
+{
+ "cells": [
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T13:35:25.148983400Z",
+     "start_time": "2026-03-27T13:35:25.121163900Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "#! pip install xgboost\n",
+    "#! pip install lightgbm"
+   ],
+   "id": "989cd81ab35b2992",
+   "outputs": [],
+   "execution_count": 17
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T14:16:27.830605300Z",
+     "start_time": "2026-03-27T14:15:36.993019800Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import re\n",
+    "import ast\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.preprocessing import StandardScaler, MultiLabelBinarizer\n",
+    "from sklearn.multioutput import MultiOutputClassifier\n",
+    "from xgboost import XGBClassifier\n",
+    "\n",
+    "path = r\"C:\\Users\\VigneshSubramani\\Downloads\\model_ready_dataset_final.csv\"\n",
+    "df = pd.read_csv(path)\n",
+    "\n",
+    "def parse_subgenres(text):\n",
+    "    if pd.isna(text) or text == \"\":\n",
+    "        return []\n",
+    "    match = re.search(r\"\\[(.*?)\\]\", str(text))\n",
+    "    if match:\n",
+    "        try:\n",
+    "            genre_list = ast.literal_eval(\"[\" + match.group(1) + \"]\")\n",
+    "            return [g.strip() for g in genre_list if g.strip()]\n",
+    "        except:\n",
+    "            return []\n",
+    "    return []\n",
+    "\n",
+    "df['subgenres_list'] = df['genre_subgenre'].apply(parse_subgenres)\n",
+    "\n",
+    "mlb = MultiLabelBinarizer()\n",
+    "y = mlb.fit_transform(df['subgenres_list'])\n",
+    "\n",
+    "drop_cols = ['_id', 'genre', 'genre_subgenre', 'subgenres_list', 'labels']\n",
+    "feature_cols = [c for c in df.columns if c not in drop_cols]\n",
+    "X = df[feature_cols].copy()\n",
+    "\n",
+    "X = X.select_dtypes(include=[np.number]).fillna(0)\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
+    "\n",
+    "scaler = StandardScaler()\n",
+    "X_train_scaled = scaler.fit_transform(X_train)\n",
+    "X_test_scaled = scaler.transform(X_test)\n",
+    "\n",
+    "model = MultiOutputClassifier(\n",
+    "    XGBClassifier(\n",
+    "        n_estimators=600,\n",
+    "        learning_rate=0.08,\n",
+    "        max_depth=6,\n",
+    "        subsample=0.85,\n",
+    "        colsample_bytree=0.85,\n",
+    "        random_state=42,\n",
+    "        eval_metric='logloss'\n",
+    "    ),\n",
+    "    n_jobs=-1\n",
+    ")\n",
+    "\n",
+    "model.fit(X_train_scaled, y_train)\n",
+    "\n",
+    "def predict_genre_subgenre(idx):\n",
+    "    pred = model.predict(X_test_scaled[idx].reshape(1, -1))[0]\n",
+    "    subgenres = mlb.inverse_transform(pred.reshape(1, -1))[0]\n",
+    "    if len(subgenres) == 0:\n",
+    "        return \"Rock -- None\"\n",
+    "    return f\"Rock -- {', '.join(subgenres)}\"\n",
+    "\n",
+    "print(\"FINAL PREDICTIONS:\")\n",
+    "for i in range(10):\n",
+    "    print(predict_genre_subgenre(i))"
+   ],
+   "id": "f1eebe48c0a482c3",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "FINAL PREDICTIONS:\n",
+      "Rock -- Contemporary Singer Songwriter\n",
+      "Rock -- None\n",
+      "Rock -- None\n",
+      "Rock -- None\n",
+      "Rock -- Classic Alternative Rock\n",
+      "Rock -- None\n",
+      "Rock -- Indie Rock\n",
+      "Rock -- Americana\n",
+      "Rock -- Indie Rock\n",
+      "Rock -- None\n"
+     ]
+    }
+   ],
+   "execution_count": 68
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T14:19:25.780735200Z",
+     "start_time": "2026-03-27T14:18:17.557680700Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import re\n",
+    "import ast\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.preprocessing import StandardScaler, MultiLabelBinarizer\n",
+    "from sklearn.multioutput import MultiOutputClassifier\n",
+    "from xgboost import XGBClassifier\n",
+    "\n",
+    "path = r\"C:\\Users\\VigneshSubramani\\Downloads\\model_ready_dataset_final.csv\"\n",
+    "df = pd.read_csv(path)\n",
+    "\n",
+    "def parse_subgenres(text):\n",
+    "    if pd.isna(text) or text == \"\":\n",
+    "        return []\n",
+    "    match = re.search(r\"\\[(.*?)\\]\", str(text))\n",
+    "    if match:\n",
+    "        try:\n",
+    "            genre_list = ast.literal_eval(\"[\" + match.group(1) + \"]\")\n",
+    "            return [g.strip() for g in genre_list if g.strip()]\n",
+    "        except:\n",
+    "            return []\n",
+    "    return []\n",
+    "\n",
+    "df['subgenres_list'] = df['genre_subgenre'].apply(parse_subgenres)\n",
+    "\n",
+    "mlb = MultiLabelBinarizer()\n",
+    "y = mlb.fit_transform(df['subgenres_list'])\n",
+    "\n",
+    "drop_cols = ['_id', 'genre', 'genre_subgenre', 'subgenres_list', 'labels']\n",
+    "feature_cols = [c for c in df.columns if c not in drop_cols]\n",
+    "X = df[feature_cols].copy()\n",
+    "\n",
+    "X = X.select_dtypes(include=[np.number]).fillna(0)\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
+    "\n",
+    "scaler = StandardScaler()\n",
+    "X_train_scaled = scaler.fit_transform(X_train)\n",
+    "X_test_scaled = scaler.transform(X_test)\n",
+    "\n",
+    "model = MultiOutputClassifier(\n",
+    "    XGBClassifier(\n",
+    "        n_estimators=800,\n",
+    "        learning_rate=0.05,\n",
+    "        max_depth=7,\n",
+    "        subsample=0.8,\n",
+    "        colsample_bytree=0.8,\n",
+    "        random_state=42,\n",
+    "        eval_metric='logloss'\n",
+    "    ),\n",
+    "    n_jobs=-1\n",
+    ")\n",
+    "\n",
+    "model.fit(X_train_scaled, y_train)"
+   ],
+   "id": "b73adeba8b23236b",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "FINAL PREDICTIONS:\n",
+      "Rock -- Contemporary Singer Songwriter\n",
+      "Rock -- Indie Rock\n",
+      "Rock -- Indie Rock\n",
+      "Rock -- Indie Pop\n",
+      "Rock -- Classic Alternative Rock, Indie Pop, Modern Alternative Rock\n",
+      "Rock -- Indie Rock\n",
+      "Rock -- Classic Alternative Rock, Indie Rock\n",
+      "Rock -- Americana\n",
+      "Rock -- Classic Alternative Rock, Indie Rock\n",
+      "Rock -- Contemporary Country\n",
+      "Rock -- Indie Rock\n",
+      "Rock -- Indie Pop, Modern Alternative Rock\n",
+      "Rock -- Contemporary Singer Songwriter, Indie Pop\n",
+      "Rock -- Contemporary Singer Songwriter, Indie Electronic, Indie Pop\n",
+      "Rock -- Classic Alternative Rock, Indie Pop, Indie Rock\n"
+     ]
+    }
+   ],
+   "execution_count": 69
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T14:23:46.993341800Z",
+     "start_time": "2026-03-27T14:23:36.236284900Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def get_accuracy(X_scaled, y_true, name=\"\"):\n",
+    "    y_pred = model.predict(X_scaled)\n",
+    "    subset_acc = np.mean(np.all(y_pred == y_true, axis=1))\n",
+    "    micro_f1 = f1_score(y_true, y_pred, average='micro')\n",
+    "    print(f\"{name} Subset Accuracy: {subset_acc:.4f}\")\n",
+    "    print(f\"{name} Micro F1 Score: {micro_f1:.4f}\")\n",
+    "\n",
+    "print(\"=== MODEL ACCURACY ===\")\n",
+    "get_accuracy(X_train_scaled, y_train, \"Train\")\n",
+    "get_accuracy(X_test_scaled, y_test, \"Test\")"
+   ],
+   "id": "5f4d018a7e7c5f21",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== MODEL ACCURACY ===\n",
+      "Train Subset Accuracy: 0.9568\n",
+      "Train Micro F1 Score: 0.9862\n",
+      "Test Subset Accuracy: 0.2026\n",
+      "Test Micro F1 Score: 0.2830\n"
+     ]
+    }
+   ],
+   "execution_count": 72
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T14:24:41.561225300Z",
+     "start_time": "2026-03-27T14:24:41.503715100Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def manual_predict(row_features):\n",
+    "    input_df = pd.DataFrame([row_features])\n",
+    "    input_numeric = input_df.select_dtypes(include=[np.number]).fillna(0)\n",
+    "    for col in X.columns:\n",
+    "        if col not in input_numeric.columns:\n",
+    "            input_numeric[col] = 0\n",
+    "    input_numeric = input_numeric[X.columns]\n",
+    "    input_scaled = scaler.transform(input_numeric)\n",
+    "    probas = model.predict_proba(input_scaled)\n",
+    "    threshold = 0.25\n",
+    "    pred_binary = np.zeros((1, y.shape[1]), dtype=int)\n",
+    "    for i in range(len(probas)):\n",
+    "        if probas[i][0][1] > threshold:\n",
+    "            pred_binary[0, i] = 1\n",
+    "    subgenres = mlb.inverse_transform(pred_binary)[0]\n",
+    "    if len(subgenres) == 0:\n",
+    "        return \"None\"\n",
+    "    return {', '.join(sorted(subgenres))}"
+   ],
+   "id": "d759399759dbe475",
+   "outputs": [],
+   "execution_count": 73
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T14:27:13.980046900Z",
+     "start_time": "2026-03-27T14:27:13.695114900Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "print(\"\\n=== FIRST ROW PREDICTION ===\")\n",
+    "first_row = df.iloc[0].to_dict()\n",
+    "pred_first = manual_predict(first_row)\n",
+    "print(pred_first)\n",
+    "\n",
+    "print(\"\\n=== LAST ROW PREDICTION ===\")\n",
+    "last_row = df.iloc[-1].to_dict()\n",
+    "pred_last = manual_predict(last_row)\n",
+    "print(pred_last)"
+   ],
+   "id": "59c4c35de1bde5ee",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "=== FIRST ROW PREDICTION ===\n",
+      "Rock -- Classic Alternative Rock, Indie Rock, Noise Pop\n",
+      "\n",
+      "=== LAST ROW PREDICTION ===\n",
+      "Rock -- Gothic Metal, Heavy Metal, Metalcore\n"
+     ]
+    }
+   ],
+   "execution_count": 76
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T14:25:23.755520400Z",
+     "start_time": "2026-03-27T14:25:23.711767500Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df.tail(1)",
+   "id": "ed40ccd2d8a6769e",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "                            _id  genre  \\\n",
+       "41934  698c479e0fc160cc5e51bc64  Metal   \n",
+       "\n",
+       "                                          genre_subgenre  \\\n",
+       "41934  Metal --- ['Gothic Metal', 'Heavy Metal', 'Met...   \n",
+       "\n",
+       "       trimmed_audio_duration_sec  syllable_count  word_count  \\\n",
+       "41934                   112.72127           573.0       455.0   \n",
+       "\n",
+       "       spectral_contrast_mean (mix)  chroma_mean (mix)  \\\n",
+       "41934                     19.482281           0.559816   \n",
+       "\n",
+       "       melody_variability (vocals)  rhythm_onset_rate (mix)  ...  \\\n",
+       "41934                     0.693639                  5.48255  ...   \n",
+       "\n",
+       "       vocab_richness  loudness_integrated_lufs custom (mix)  \\\n",
+       "41934           0.495                              -14.21983   \n",
+       "\n",
+       "       readability_score  energy_essentia (mix)  energy_librosa (mix)  \\\n",
+       "41934               16.4               0.354772              0.558767   \n",
+       "\n",
+       "       rms_energy_mean (mix)  sentiment_score  melody_complexity (vocals)  \\\n",
+       "41934               0.562382           -0.002                    2.916667   \n",
+       "\n",
+       "       avg_word_length                          subgenres_list  \n",
+       "41934             3.63  [Gothic Metal, Heavy Metal, Metalcore]  \n",
+       "\n",
+       "[1 rows x 29 columns]"
+      ],
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>_id</th>\n",
+       "      <th>genre</th>\n",
+       "      <th>genre_subgenre</th>\n",
+       "      <th>trimmed_audio_duration_sec</th>\n",
+       "      <th>syllable_count</th>\n",
+       "      <th>word_count</th>\n",
+       "      <th>spectral_contrast_mean (mix)</th>\n",
+       "      <th>chroma_mean (mix)</th>\n",
+       "      <th>melody_variability (vocals)</th>\n",
+       "      <th>rhythm_onset_rate (mix)</th>\n",
+       "      <th>...</th>\n",
+       "      <th>vocab_richness</th>\n",
+       "      <th>loudness_integrated_lufs custom (mix)</th>\n",
+       "      <th>readability_score</th>\n",
+       "      <th>energy_essentia (mix)</th>\n",
+       "      <th>energy_librosa (mix)</th>\n",
+       "      <th>rms_energy_mean (mix)</th>\n",
+       "      <th>sentiment_score</th>\n",
+       "      <th>melody_complexity (vocals)</th>\n",
+       "      <th>avg_word_length</th>\n",
+       "      <th>subgenres_list</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>41934</th>\n",
+       "      <td>698c479e0fc160cc5e51bc64</td>\n",
+       "      <td>Metal</td>\n",
+       "      <td>Metal --- ['Gothic Metal', 'Heavy Metal', 'Met...</td>\n",
+       "      <td>112.72127</td>\n",
+       "      <td>573.0</td>\n",
+       "      <td>455.0</td>\n",
+       "      <td>19.482281</td>\n",
+       "      <td>0.559816</td>\n",
+       "      <td>0.693639</td>\n",
+       "      <td>5.48255</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.495</td>\n",
+       "      <td>-14.21983</td>\n",
+       "      <td>16.4</td>\n",
+       "      <td>0.354772</td>\n",
+       "      <td>0.558767</td>\n",
+       "      <td>0.562382</td>\n",
+       "      <td>-0.002</td>\n",
+       "      <td>2.916667</td>\n",
+       "      <td>3.63</td>\n",
+       "      <td>[Gothic Metal, Heavy Metal, Metalcore]</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1 rows × 29 columns</p>\n",
+       "</div>"
+      ]
+     },
+     "execution_count": 75,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 75
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T14:32:46.590643700Z",
+     "start_time": "2026-03-27T14:32:46.540506300Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df.columns",
+   "id": "f581dd69d03fec27",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['_id', 'genre', 'genre_subgenre', 'trimmed_audio_duration_sec',\n",
+       "       'syllable_count', 'word_count', 'spectral_contrast_mean (mix)',\n",
+       "       'chroma_mean (mix)', 'melody_variability (vocals)',\n",
+       "       'rhythm_onset_rate (mix)', 'spectral_centroid_mean custom (mix)',\n",
+       "       'repetition_score custom (mix)', 'pitch_std (mix)',\n",
+       "       'tempo_bpm_corrected (mix)', 'tempo_bpm_original (mix)',\n",
+       "       'mfcc_mean_1 (mix)', 'mfcc_mean_2 (mix)',\n",
+       "       'loudness_range_lu custom (mix)', 'zero_crossing_rate (mix)',\n",
+       "       'vocab_richness', 'loudness_integrated_lufs custom (mix)',\n",
+       "       'readability_score', 'energy_essentia (mix)', 'energy_librosa (mix)',\n",
+       "       'rms_energy_mean (mix)', 'sentiment_score',\n",
+       "       'melody_complexity (vocals)', 'avg_word_length', 'subgenres_list'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 77,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 77
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "9b8e0e646877dffd"
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "a2c6e3303920e7b1"
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "f9be1b4975b13c44"
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "561bf73932a24d3a"
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "ed7e357983c3b734"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-29T11:07:41.453680300Z",
+     "start_time": "2026-03-29T11:07:41.256698100Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import ast, re\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.preprocessing import StandardScaler, MultiLabelBinarizer\n",
+    "from sklearn.multioutput import MultiOutputClassifier\n",
+    "from xgboost import XGBClassifier\n",
+    "from sklearn.metrics import f1_score\n",
+    "import pandas as pd\n",
+    "\n",
+    "path = r\"C:\\Users\\VigneshSubramani\\Downloads\\model_ready_dataset_final.csv\"\n",
+    "df = pd.read_csv(path)\n",
+    "\n",
+    "def parse_subgenres_fast(text):\n",
+    "    if pd.isna(text) or text == \"\":\n",
+    "        return []\n",
+    "\n",
+    "    text = str(text)\n",
+    "    parts = text.split('---')\n",
+    "    main_genre = parts[0].strip()\n",
+    "\n",
+    "    if len(parts) > 1:\n",
+    "        sub = parts[1].strip()\n",
+    "        sub = sub.strip('[]')\n",
+    "        if sub:\n",
+    "            subgenres = [s.strip().strip(\"'\").strip('\"') for s in sub.split(',')]\n",
+    "        else:\n",
+    "            subgenres = []\n",
+    "    else:\n",
+    "        subgenres = []\n",
+    "    return [main_genre] + subgenres\n",
+    "\n",
+    "df['subgenres_list'] = df['genre_subgenre'].apply(parse_subgenres_fast)"
+   ],
+   "id": "573c53470f0ef01f",
+   "outputs": [],
+   "execution_count": 8
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-29T11:07:43.398574800Z",
+     "start_time": "2026-03-29T11:07:43.351108Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df['subgenres_list'][0]",
+   "id": "1b59c31bd0b780e0",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Rock', 'Noise Pop', 'Classic Alternative Rock', 'Indie Rock']"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 9
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "9b1fa1d3b08915da"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

all_combine_code/genre_classification/main_file.ipynb

@@ -0,0 +1,2110 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2026-03-30T08:55:12.053860800Z",
+     "start_time": "2026-03-30T08:55:06.050743500Z"
+    }
+   },
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import re\n",
+    "import ast\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.preprocessing import StandardScaler, MultiLabelBinarizer\n",
+    "from sklearn.multioutput import MultiOutputClassifier\n",
+    "from xgboost import XGBClassifier\n",
+    "from sklearn.metrics import f1_score\n",
+    "pd.set_option(\"display.max_columns\", None)\n",
+    "path = r\"C:\\Users\\VigneshSubramani\\Downloads\\model_ready_dataset_final.csv\"\n",
+    "df = pd.read_csv(path)\n",
+    "df.sample(3)"
+   ],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "                            _id    genre  \\\n",
+       "27949  69b29e6c05f5ce265b882008     Rock   \n",
+       "40690  69b12fabebd9b3c1ca39e859  Country   \n",
+       "18977  692969b1c5db350c7bdaf13d     Rock   \n",
+       "\n",
+       "                                          genre_subgenre  \\\n",
+       "27949  Rock --- ['Indie Folk', 'Country Folk', 'Indie...   \n",
+       "40690  Country --- ['Americana', 'Country Rock', 'Alt...   \n",
+       "18977  Rock --- ['Noise Pop', 'Classic Alternative Ro...   \n",
+       "\n",
+       "       trimmed_audio_duration_sec  syllable_count  word_count  \\\n",
+       "27949                  191.332000           287.0       197.0   \n",
+       "40690                  215.146667           245.0       200.0   \n",
+       "18977                  203.948458           145.0       118.0   \n",
+       "\n",
+       "       spectral_contrast_mean (mix)  chroma_mean (mix)  \\\n",
+       "27949                     19.647192           0.453619   \n",
+       "40690                     19.253367           0.484467   \n",
+       "18977                     18.226969           0.532658   \n",
+       "\n",
+       "       melody_variability (vocals)  rhythm_onset_rate (mix)  \\\n",
+       "27949                     0.479049                 1.927778   \n",
+       "40690                     0.600583                 4.220377   \n",
+       "18977                     0.647673                 1.694444   \n",
+       "\n",
+       "       spectral_centroid_mean custom (mix)  repetition_score custom (mix)  \\\n",
+       "27949                             0.093420                       0.006936   \n",
+       "40690                             0.111674                       0.005806   \n",
+       "18977                             0.124794                       0.006610   \n",
+       "\n",
+       "       pitch_std (mix)  tempo_bpm_corrected (mix)  tempo_bpm_original (mix)  \\\n",
+       "27949        56.905636                 123.635239                123.635239   \n",
+       "40690        70.522621                  84.574928                 84.574928   \n",
+       "18977       145.266922                  85.543114                 85.543114   \n",
+       "\n",
+       "       mfcc_mean_1 (mix)  mfcc_mean_2 (mix)  loudness_range_lu custom (mix)  \\\n",
+       "27949         194.688721          -3.499274                        6.624297   \n",
+       "40690         178.628433         -50.666416                        3.653197   \n",
+       "18977         164.830765         -53.442635                        1.909515   \n",
+       "\n",
+       "       zero_crossing_rate (mix)  vocab_richness  \\\n",
+       "27949                  0.032766           0.325   \n",
+       "40690                  0.049714           0.345   \n",
+       "18977                  0.059000           0.542   \n",
+       "\n",
+       "       loudness_integrated_lufs custom (mix)  readability_score  \\\n",
+       "27949                             -16.190006               76.6   \n",
+       "40690                             -13.794414                2.1   \n",
+       "18977                             -11.325106                1.4   \n",
+       "\n",
+       "       energy_essentia (mix)  energy_librosa (mix)  rms_energy_mean (mix)  \\\n",
+       "27949               0.259487              0.465622               0.466699   \n",
+       "40690               0.330070              0.547306               0.547923   \n",
+       "18977               0.439053              0.604893               0.661765   \n",
+       "\n",
+       "       sentiment_score  melody_complexity (vocals)  avg_word_length  \n",
+       "27949            0.035                    2.666667             4.67  \n",
+       "40690            0.336                    3.250000             3.76  \n",
+       "18977           -0.013                    2.916667             3.42  "
+      ],
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>_id</th>\n",
+       "      <th>genre</th>\n",
+       "      <th>genre_subgenre</th>\n",
+       "      <th>trimmed_audio_duration_sec</th>\n",
+       "      <th>syllable_count</th>\n",
+       "      <th>word_count</th>\n",
+       "      <th>spectral_contrast_mean (mix)</th>\n",
+       "      <th>chroma_mean (mix)</th>\n",
+       "      <th>melody_variability (vocals)</th>\n",
+       "      <th>rhythm_onset_rate (mix)</th>\n",
+       "      <th>spectral_centroid_mean custom (mix)</th>\n",
+       "      <th>repetition_score custom (mix)</th>\n",
+       "      <th>pitch_std (mix)</th>\n",
+       "      <th>tempo_bpm_corrected (mix)</th>\n",
+       "      <th>tempo_bpm_original (mix)</th>\n",
+       "      <th>mfcc_mean_1 (mix)</th>\n",
+       "      <th>mfcc_mean_2 (mix)</th>\n",
+       "      <th>loudness_range_lu custom (mix)</th>\n",
+       "      <th>zero_crossing_rate (mix)</th>\n",
+       "      <th>vocab_richness</th>\n",
+       "      <th>loudness_integrated_lufs custom (mix)</th>\n",
+       "      <th>readability_score</th>\n",
+       "      <th>energy_essentia (mix)</th>\n",
+       "      <th>energy_librosa (mix)</th>\n",
+       "      <th>rms_energy_mean (mix)</th>\n",
+       "      <th>sentiment_score</th>\n",
+       "      <th>melody_complexity (vocals)</th>\n",
+       "      <th>avg_word_length</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>27949</th>\n",
+       "      <td>69b29e6c05f5ce265b882008</td>\n",
+       "      <td>Rock</td>\n",
+       "      <td>Rock --- ['Indie Folk', 'Country Folk', 'Indie...</td>\n",
+       "      <td>191.332000</td>\n",
+       "      <td>287.0</td>\n",
+       "      <td>197.0</td>\n",
+       "      <td>19.647192</td>\n",
+       "      <td>0.453619</td>\n",
+       "      <td>0.479049</td>\n",
+       "      <td>1.927778</td>\n",
+       "      <td>0.093420</td>\n",
+       "      <td>0.006936</td>\n",
+       "      <td>56.905636</td>\n",
+       "      <td>123.635239</td>\n",
+       "      <td>123.635239</td>\n",
+       "      <td>194.688721</td>\n",
+       "      <td>-3.499274</td>\n",
+       "      <td>6.624297</td>\n",
+       "      <td>0.032766</td>\n",
+       "      <td>0.325</td>\n",
+       "      <td>-16.190006</td>\n",
+       "      <td>76.6</td>\n",
+       "      <td>0.259487</td>\n",
+       "      <td>0.465622</td>\n",
+       "      <td>0.466699</td>\n",
+       "      <td>0.035</td>\n",
+       "      <td>2.666667</td>\n",
+       "      <td>4.67</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>40690</th>\n",
+       "      <td>69b12fabebd9b3c1ca39e859</td>\n",
+       "      <td>Country</td>\n",
+       "      <td>Country --- ['Americana', 'Country Rock', 'Alt...</td>\n",
+       "      <td>215.146667</td>\n",
+       "      <td>245.0</td>\n",
+       "      <td>200.0</td>\n",
+       "      <td>19.253367</td>\n",
+       "      <td>0.484467</td>\n",
+       "      <td>0.600583</td>\n",
+       "      <td>4.220377</td>\n",
+       "      <td>0.111674</td>\n",
+       "      <td>0.005806</td>\n",
+       "      <td>70.522621</td>\n",
+       "      <td>84.574928</td>\n",
+       "      <td>84.574928</td>\n",
+       "      <td>178.628433</td>\n",
+       "      <td>-50.666416</td>\n",
+       "      <td>3.653197</td>\n",
+       "      <td>0.049714</td>\n",
+       "      <td>0.345</td>\n",
+       "      <td>-13.794414</td>\n",
+       "      <td>2.1</td>\n",
+       "      <td>0.330070</td>\n",
+       "      <td>0.547306</td>\n",
+       "      <td>0.547923</td>\n",
+       "      <td>0.336</td>\n",
+       "      <td>3.250000</td>\n",
+       "      <td>3.76</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18977</th>\n",
+       "      <td>692969b1c5db350c7bdaf13d</td>\n",
+       "      <td>Rock</td>\n",
+       "      <td>Rock --- ['Noise Pop', 'Classic Alternative Ro...</td>\n",
+       "      <td>203.948458</td>\n",
+       "      <td>145.0</td>\n",
+       "      <td>118.0</td>\n",
+       "      <td>18.226969</td>\n",
+       "      <td>0.532658</td>\n",
+       "      <td>0.647673</td>\n",
+       "      <td>1.694444</td>\n",
+       "      <td>0.124794</td>\n",
+       "      <td>0.006610</td>\n",
+       "      <td>145.266922</td>\n",
+       "      <td>85.543114</td>\n",
+       "      <td>85.543114</td>\n",
+       "      <td>164.830765</td>\n",
+       "      <td>-53.442635</td>\n",
+       "      <td>1.909515</td>\n",
+       "      <td>0.059000</td>\n",
+       "      <td>0.542</td>\n",
+       "      <td>-11.325106</td>\n",
+       "      <td>1.4</td>\n",
+       "      <td>0.439053</td>\n",
+       "      <td>0.604893</td>\n",
+       "      <td>0.661765</td>\n",
+       "      <td>-0.013</td>\n",
+       "      <td>2.916667</td>\n",
+       "      <td>3.42</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 1
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T08:56:11.085130700Z",
+     "start_time": "2026-03-30T08:56:10.592907600Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def parse_subgenres(text):\n",
+    "    if pd.isna(text) or text == \"\":\n",
+    "        return []\n",
+    "    match = re.search(r\"\\[(.*?)\\]\", str(text))\n",
+    "    if match:\n",
+    "        try:\n",
+    "            genre_list = ast.literal_eval(\"[\" + match.group(1) + \"]\")\n",
+    "            return [g.strip() for g in genre_list if g.strip()]\n",
+    "        except:\n",
+    "            return []\n",
+    "    return []\n",
+    "\n",
+    "df['subgenres_list'] = df['genre_subgenre'].apply(parse_subgenres)\n",
+    "\n"
+   ],
+   "id": "67098b6deadbc3ef",
+   "outputs": [],
+   "execution_count": 2
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T08:56:23.372487600Z",
+     "start_time": "2026-03-30T08:56:23.315805900Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df['subgenres_list'][0]",
+   "id": "e0fc9a9de1291e67",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Noise Pop', 'Classic Alternative Rock', 'Indie Rock']"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 3
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": [
+    "mlb = MultiLabelBinarizer()\n",
+    "y = mlb.fit_transform(df['subgenres_list'])\n",
+    "\n",
+    "drop_cols = ['_id', 'genre', 'genre_subgenre', 'subgenres_list', 'labels',\n",
+    "             'trimmed_audio_duration_sec', 'syllable_count', 'word_count',\n",
+    "             'sentiment_score', 'avg_word_length']\n",
+    "\n",
+    "feature_cols = [c for c in df.columns if c not in drop_cols]\n",
+    "X = df[feature_cols].copy()\n",
+    "X = X.select_dtypes(include=[np.number]).fillna(0)\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
+    "\n",
+    "scaler = StandardScaler()\n",
+    "X_train_scaled = scaler.fit_transform(X_train)\n",
+    "X_test_scaled = scaler.transform(X_test)\n",
+    "\n",
+    "model = MultiOutputClassifier(\n",
+    "    XGBClassifier(\n",
+    "        n_estimators=1500,\n",
+    "        learning_rate=0.03,\n",
+    "        max_depth=8,\n",
+    "        subsample=0.85,\n",
+    "        colsample_bytree=0.8,\n",
+    "        random_state=42,\n",
+    "        eval_metric='logloss'\n",
+    "    ),\n",
+    "    n_jobs=-1\n",
+    ")\n",
+    "\n",
+    "model.fit(X_train_scaled, y_train)\n",
+    "\n",
+    "\n",
+    "def get_accuracy(X_scaled, y_true, name=\"\"):\n",
+    "    y_pred = model.predict(X_scaled)\n",
+    "    subset_acc = np.mean(np.all(y_pred == y_true, axis=1))\n",
+    "    micro_f1 = f1_score(y_true, y_pred, average='micro')\n",
+    "    print(f\"{name} Subset Accuracy: {subset_acc:.4f}\")\n",
+    "    print(f\"{name} Micro F1 Score: {micro_f1:.4f}\")\n",
+    "\n",
+    "print(\"=== MODEL ACCURACY ===\")\n",
+    "get_accuracy(X_train_scaled, y_train, \"Train\")\n",
+    "get_accuracy(X_test_scaled, y_test, \"Test\")\n",
+    "\n",
+    "def manual_predict(row):\n",
+    "    input_df = pd.DataFrame([row])\n",
+    "    input_numeric = input_df.select_dtypes(include=[np.number]).fillna(0)\n",
+    "\n",
+    "    for col in X.columns:\n",
+    "        if col not in input_numeric.columns:\n",
+    "            input_numeric[col] = 0\n",
+    "\n",
+    "    input_numeric = input_numeric[X.columns]\n",
+    "    input_scaled = scaler.transform(input_numeric)\n",
+    "\n",
+    "    probas = model.predict_proba(input_scaled)\n",
+    "    threshold = 0.20\n",
+    "    pred_binary = np.zeros((1, y.shape[1]), dtype=int)\n",
+    "\n",
+    "    for i in range(len(probas)):\n",
+    "        if probas[i][0][1] > threshold:\n",
+    "            pred_binary[0, i] = 1\n",
+    "\n",
+    "    subgenres = mlb.inverse_transform(pred_binary)[0]\n",
+    "    #main_genre = row.get('genre', 'Rock')\n",
+    "\n",
+    "    if len(subgenres) == 0:\n",
+    "        return f\"None\"\n",
+    "    return f\"{', '.join(sorted(subgenres))}\""
+   ],
+   "id": "c1086565f89c486b"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T14:45:34.721003500Z",
+     "start_time": "2026-03-27T14:45:34.665870Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "X.head()",
+   "id": "7036f929b47b9932",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "   spectral_contrast_mean (mix)  chroma_mean (mix)  \\\n",
+       "0                     18.938621           0.462013   \n",
+       "1                     18.361412           0.515950   \n",
+       "2                     17.545991           0.505095   \n",
+       "3                     19.065528           0.532723   \n",
+       "4                     18.549622           0.560616   \n",
+       "\n",
+       "   melody_variability (vocals)  rhythm_onset_rate (mix)  \\\n",
+       "0                     0.887239                 0.550994   \n",
+       "1                     0.553844                 1.785610   \n",
+       "2                     0.887161                 1.691063   \n",
+       "3                     0.917722                 4.237960   \n",
+       "4                     0.967337                 0.847676   \n",
+       "\n",
+       "   spectral_centroid_mean custom (mix)  repetition_score custom (mix)  \\\n",
+       "0                             0.124953                       0.015320   \n",
+       "1                             0.178827                       0.010278   \n",
+       "2                             0.115442                       0.010617   \n",
+       "3                             0.134564                       0.009808   \n",
+       "4                             0.179751                       0.008122   \n",
+       "\n",
+       "   pitch_std (mix)  tempo_bpm_corrected (mix)  tempo_bpm_original (mix)  \\\n",
+       "0       170.196686                 117.459671                117.459671   \n",
+       "1       174.526428                  93.132881                 93.132881   \n",
+       "2        98.435074                 105.041779                105.041779   \n",
+       "3       144.853897                  94.973419                 94.973419   \n",
+       "4        94.283890                 108.527031                108.527031   \n",
+       "\n",
+       "   mfcc_mean_1 (mix)  mfcc_mean_2 (mix)  loudness_range_lu custom (mix)  \\\n",
+       "0         159.208038         -47.135399                       17.099110   \n",
+       "1         124.982330         -54.048897                       10.356828   \n",
+       "2         178.284927         -41.860600                        4.583549   \n",
+       "3         166.110214         -79.556442                        3.334763   \n",
+       "4         118.163094         -54.811043                        0.855955   \n",
+       "\n",
+       "   zero_crossing_rate (mix)  vocab_richness  \\\n",
+       "0                  0.065750           0.842   \n",
+       "1                  0.115698           0.312   \n",
+       "2                  0.050672           0.462   \n",
+       "3                  0.085350           0.273   \n",
+       "4                  0.104765           0.350   \n",
+       "\n",
+       "   loudness_integrated_lufs custom (mix)  readability_score  \\\n",
+       "0                             -17.450430                1.6   \n",
+       "1                             -14.291489                5.2   \n",
+       "2                             -14.021948               18.8   \n",
+       "3                             -11.496668                1.7   \n",
+       "4                             -10.031396               52.0   \n",
+       "\n",
+       "   energy_essentia (mix)  energy_librosa (mix)  rms_energy_mean (mix)  \\\n",
+       "0               0.400498              0.541834               0.545006   \n",
+       "1               0.286286              0.491097               0.492117   \n",
+       "2               0.330273              0.539636               0.541554   \n",
+       "3               0.525465              0.693286               0.702081   \n",
+       "4               0.498986              0.696552               0.698920   \n",
+       "\n",
+       "   melody_complexity (vocals)  \n",
+       "0                    2.833333  \n",
+       "1                    2.333333  \n",
+       "2                    3.083333  \n",
+       "3                    3.166667  \n",
+       "4                    3.500000  "
+      ],
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>spectral_contrast_mean (mix)</th>\n",
+       "      <th>chroma_mean (mix)</th>\n",
+       "      <th>melody_variability (vocals)</th>\n",
+       "      <th>rhythm_onset_rate (mix)</th>\n",
+       "      <th>spectral_centroid_mean custom (mix)</th>\n",
+       "      <th>repetition_score custom (mix)</th>\n",
+       "      <th>pitch_std (mix)</th>\n",
+       "      <th>tempo_bpm_corrected (mix)</th>\n",
+       "      <th>tempo_bpm_original (mix)</th>\n",
+       "      <th>mfcc_mean_1 (mix)</th>\n",
+       "      <th>mfcc_mean_2 (mix)</th>\n",
+       "      <th>loudness_range_lu custom (mix)</th>\n",
+       "      <th>zero_crossing_rate (mix)</th>\n",
+       "      <th>vocab_richness</th>\n",
+       "      <th>loudness_integrated_lufs custom (mix)</th>\n",
+       "      <th>readability_score</th>\n",
+       "      <th>energy_essentia (mix)</th>\n",
+       "      <th>energy_librosa (mix)</th>\n",
+       "      <th>rms_energy_mean (mix)</th>\n",
+       "      <th>melody_complexity (vocals)</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>18.938621</td>\n",
+       "      <td>0.462013</td>\n",
+       "      <td>0.887239</td>\n",
+       "      <td>0.550994</td>\n",
+       "      <td>0.124953</td>\n",
+       "      <td>0.015320</td>\n",
+       "      <td>170.196686</td>\n",
+       "      <td>117.459671</td>\n",
+       "      <td>117.459671</td>\n",
+       "      <td>159.208038</td>\n",
+       "      <td>-47.135399</td>\n",
+       "      <td>17.099110</td>\n",
+       "      <td>0.065750</td>\n",
+       "      <td>0.842</td>\n",
+       "      <td>-17.450430</td>\n",
+       "      <td>1.6</td>\n",
+       "      <td>0.400498</td>\n",
+       "      <td>0.541834</td>\n",
+       "      <td>0.545006</td>\n",
+       "      <td>2.833333</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>18.361412</td>\n",
+       "      <td>0.515950</td>\n",
+       "      <td>0.553844</td>\n",
+       "      <td>1.785610</td>\n",
+       "      <td>0.178827</td>\n",
+       "      <td>0.010278</td>\n",
+       "      <td>174.526428</td>\n",
+       "      <td>93.132881</td>\n",
+       "      <td>93.132881</td>\n",
+       "      <td>124.982330</td>\n",
+       "      <td>-54.048897</td>\n",
+       "      <td>10.356828</td>\n",
+       "      <td>0.115698</td>\n",
+       "      <td>0.312</td>\n",
+       "      <td>-14.291489</td>\n",
+       "      <td>5.2</td>\n",
+       "      <td>0.286286</td>\n",
+       "      <td>0.491097</td>\n",
+       "      <td>0.492117</td>\n",
+       "      <td>2.333333</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>17.545991</td>\n",
+       "      <td>0.505095</td>\n",
+       "      <td>0.887161</td>\n",
+       "      <td>1.691063</td>\n",
+       "      <td>0.115442</td>\n",
+       "      <td>0.010617</td>\n",
+       "      <td>98.435074</td>\n",
+       "      <td>105.041779</td>\n",
+       "      <td>105.041779</td>\n",
+       "      <td>178.284927</td>\n",
+       "      <td>-41.860600</td>\n",
+       "      <td>4.583549</td>\n",
+       "      <td>0.050672</td>\n",
+       "      <td>0.462</td>\n",
+       "      <td>-14.021948</td>\n",
+       "      <td>18.8</td>\n",
+       "      <td>0.330273</td>\n",
+       "      <td>0.539636</td>\n",
+       "      <td>0.541554</td>\n",
+       "      <td>3.083333</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>19.065528</td>\n",
+       "      <td>0.532723</td>\n",
+       "      <td>0.917722</td>\n",
+       "      <td>4.237960</td>\n",
+       "      <td>0.134564</td>\n",
+       "      <td>0.009808</td>\n",
+       "      <td>144.853897</td>\n",
+       "      <td>94.973419</td>\n",
+       "      <td>94.973419</td>\n",
+       "      <td>166.110214</td>\n",
+       "      <td>-79.556442</td>\n",
+       "      <td>3.334763</td>\n",
+       "      <td>0.085350</td>\n",
+       "      <td>0.273</td>\n",
+       "      <td>-11.496668</td>\n",
+       "      <td>1.7</td>\n",
+       "      <td>0.525465</td>\n",
+       "      <td>0.693286</td>\n",
+       "      <td>0.702081</td>\n",
+       "      <td>3.166667</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>18.549622</td>\n",
+       "      <td>0.560616</td>\n",
+       "      <td>0.967337</td>\n",
+       "      <td>0.847676</td>\n",
+       "      <td>0.179751</td>\n",
+       "      <td>0.008122</td>\n",
+       "      <td>94.283890</td>\n",
+       "      <td>108.527031</td>\n",
+       "      <td>108.527031</td>\n",
+       "      <td>118.163094</td>\n",
+       "      <td>-54.811043</td>\n",
+       "      <td>0.855955</td>\n",
+       "      <td>0.104765</td>\n",
+       "      <td>0.350</td>\n",
+       "      <td>-10.031396</td>\n",
+       "      <td>52.0</td>\n",
+       "      <td>0.498986</td>\n",
+       "      <td>0.696552</td>\n",
+       "      <td>0.698920</td>\n",
+       "      <td>3.500000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 6
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T14:55:56.357032800Z",
+     "start_time": "2026-03-27T14:55:56.275652600Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "X.shape",
+   "id": "f7d2011c6b5f8699",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(41935, 20)"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 15
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:10:02.865313500Z",
+     "start_time": "2026-03-27T15:10:02.807027900Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df[[\"subgenres_list\",\"genre\"]].iloc[502,]",
+   "id": "d462801f323e88a",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "subgenres_list    [Modern Alternative Rock, Indie Rock]\n",
+       "genre                                              Rock\n",
+       "Name: 502, dtype: object"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 43
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:13:50.114563800Z",
+     "start_time": "2026-03-27T15:13:50.060507800Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "pd.set_option('display.max_columns', None)\n",
+    "df[(df[\"genre\"]==\"Metal\")].head(2)"
+   ],
+   "id": "1fbd1d7876fb8996",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "                            _id  genre  \\\n",
+       "41856  691b00908e8610886093475b  Metal   \n",
+       "41857  691c4b322bf3d72bd0be023e  Metal   \n",
+       "\n",
+       "                                          genre_subgenre  \\\n",
+       "41856  Metal --- ['Black Metal', 'Death Metal', 'Doom...   \n",
+       "41857          Metal --- ['Heavy Metal', 'Thrash Metal']   \n",
+       "\n",
+       "       trimmed_audio_duration_sec  syllable_count  word_count  \\\n",
+       "41856                  185.602971           185.0       158.0   \n",
+       "41857                  131.819683           353.0       253.0   \n",
+       "\n",
+       "       spectral_contrast_mean (mix)  chroma_mean (mix)  \\\n",
+       "41856                     20.152063           0.408171   \n",
+       "41857                     19.495984           0.517214   \n",
+       "\n",
+       "       melody_variability (vocals)  rhythm_onset_rate (mix)  \\\n",
+       "41856                     0.374856                 2.027778   \n",
+       "41857                     0.475671                 5.803382   \n",
+       "\n",
+       "       spectral_centroid_mean custom (mix)  repetition_score custom (mix)  \\\n",
+       "41856                             0.093899                       0.007882   \n",
+       "41857                             0.168338                       0.007993   \n",
+       "\n",
+       "       pitch_std (mix)  tempo_bpm_corrected (mix)  tempo_bpm_original (mix)  \\\n",
+       "41856       108.278755                 104.605293                104.605293   \n",
+       "41857       134.086761                 138.027847                138.027847   \n",
+       "\n",
+       "       mfcc_mean_1 (mix)  mfcc_mean_2 (mix)  loudness_range_lu custom (mix)  \\\n",
+       "41856         163.308655          22.928051                        7.131201   \n",
+       "41857         113.692780         -26.532072                        2.994879   \n",
+       "\n",
+       "       zero_crossing_rate (mix)  vocab_richness  \\\n",
+       "41856                  0.026933           0.582   \n",
+       "41857                  0.069337           0.352   \n",
+       "\n",
+       "       loudness_integrated_lufs custom (mix)  readability_score  \\\n",
+       "41856                             -18.566132               27.8   \n",
+       "41857                             -13.850066                3.0   \n",
+       "\n",
+       "       energy_essentia (mix)  energy_librosa (mix)  rms_energy_mean (mix)  \\\n",
+       "41856               0.153738              0.335069               0.346289   \n",
+       "41857               0.266081              0.441120               0.467461   \n",
+       "\n",
+       "       sentiment_score  melody_complexity (vocals)  avg_word_length  \\\n",
+       "41856            0.050                    1.916667             3.97   \n",
+       "41857            0.128                    2.250000             4.24   \n",
+       "\n",
+       "                                          subgenres_list  \n",
+       "41856  [Black Metal, Death Metal, Doom Metal, Heavy M...  \n",
+       "41857                        [Heavy Metal, Thrash Metal]  "
+      ],
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>_id</th>\n",
+       "      <th>genre</th>\n",
+       "      <th>genre_subgenre</th>\n",
+       "      <th>trimmed_audio_duration_sec</th>\n",
+       "      <th>syllable_count</th>\n",
+       "      <th>word_count</th>\n",
+       "      <th>spectral_contrast_mean (mix)</th>\n",
+       "      <th>chroma_mean (mix)</th>\n",
+       "      <th>melody_variability (vocals)</th>\n",
+       "      <th>rhythm_onset_rate (mix)</th>\n",
+       "      <th>spectral_centroid_mean custom (mix)</th>\n",
+       "      <th>repetition_score custom (mix)</th>\n",
+       "      <th>pitch_std (mix)</th>\n",
+       "      <th>tempo_bpm_corrected (mix)</th>\n",
+       "      <th>tempo_bpm_original (mix)</th>\n",
+       "      <th>mfcc_mean_1 (mix)</th>\n",
+       "      <th>mfcc_mean_2 (mix)</th>\n",
+       "      <th>loudness_range_lu custom (mix)</th>\n",
+       "      <th>zero_crossing_rate (mix)</th>\n",
+       "      <th>vocab_richness</th>\n",
+       "      <th>loudness_integrated_lufs custom (mix)</th>\n",
+       "      <th>readability_score</th>\n",
+       "      <th>energy_essentia (mix)</th>\n",
+       "      <th>energy_librosa (mix)</th>\n",
+       "      <th>rms_energy_mean (mix)</th>\n",
+       "      <th>sentiment_score</th>\n",
+       "      <th>melody_complexity (vocals)</th>\n",
+       "      <th>avg_word_length</th>\n",
+       "      <th>subgenres_list</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>41856</th>\n",
+       "      <td>691b00908e8610886093475b</td>\n",
+       "      <td>Metal</td>\n",
+       "      <td>Metal --- ['Black Metal', 'Death Metal', 'Doom...</td>\n",
+       "      <td>185.602971</td>\n",
+       "      <td>185.0</td>\n",
+       "      <td>158.0</td>\n",
+       "      <td>20.152063</td>\n",
+       "      <td>0.408171</td>\n",
+       "      <td>0.374856</td>\n",
+       "      <td>2.027778</td>\n",
+       "      <td>0.093899</td>\n",
+       "      <td>0.007882</td>\n",
+       "      <td>108.278755</td>\n",
+       "      <td>104.605293</td>\n",
+       "      <td>104.605293</td>\n",
+       "      <td>163.308655</td>\n",
+       "      <td>22.928051</td>\n",
+       "      <td>7.131201</td>\n",
+       "      <td>0.026933</td>\n",
+       "      <td>0.582</td>\n",
+       "      <td>-18.566132</td>\n",
+       "      <td>27.8</td>\n",
+       "      <td>0.153738</td>\n",
+       "      <td>0.335069</td>\n",
+       "      <td>0.346289</td>\n",
+       "      <td>0.050</td>\n",
+       "      <td>1.916667</td>\n",
+       "      <td>3.97</td>\n",
+       "      <td>[Black Metal, Death Metal, Doom Metal, Heavy M...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>41857</th>\n",
+       "      <td>691c4b322bf3d72bd0be023e</td>\n",
+       "      <td>Metal</td>\n",
+       "      <td>Metal --- ['Heavy Metal', 'Thrash Metal']</td>\n",
+       "      <td>131.819683</td>\n",
+       "      <td>353.0</td>\n",
+       "      <td>253.0</td>\n",
+       "      <td>19.495984</td>\n",
+       "      <td>0.517214</td>\n",
+       "      <td>0.475671</td>\n",
+       "      <td>5.803382</td>\n",
+       "      <td>0.168338</td>\n",
+       "      <td>0.007993</td>\n",
+       "      <td>134.086761</td>\n",
+       "      <td>138.027847</td>\n",
+       "      <td>138.027847</td>\n",
+       "      <td>113.692780</td>\n",
+       "      <td>-26.532072</td>\n",
+       "      <td>2.994879</td>\n",
+       "      <td>0.069337</td>\n",
+       "      <td>0.352</td>\n",
+       "      <td>-13.850066</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>0.266081</td>\n",
+       "      <td>0.441120</td>\n",
+       "      <td>0.467461</td>\n",
+       "      <td>0.128</td>\n",
+       "      <td>2.250000</td>\n",
+       "      <td>4.24</td>\n",
+       "      <td>[Heavy Metal, Thrash Metal]</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ]
+     },
+     "execution_count": 50,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 50
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:24:32.000214700Z",
+     "start_time": "2026-03-27T15:24:31.938664600Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "X.iloc[41856].to_dict()",
+   "id": "37ef9304bdae300c",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'spectral_contrast_mean (mix)': 20.1520629356514,\n",
+       " 'chroma_mean (mix)': 0.408170998096466,\n",
+       " 'melody_variability (vocals)': 0.3748557548841115,\n",
+       " 'rhythm_onset_rate (mix)': 2.0277777777777777,\n",
+       " 'spectral_centroid_mean custom (mix)': 0.0938994687064515,\n",
+       " 'repetition_score custom (mix)': 0.0078822056596288,\n",
+       " 'pitch_std (mix)': 108.27875518798828,\n",
+       " 'tempo_bpm_corrected (mix)': 104.60529327392578,\n",
+       " 'tempo_bpm_original (mix)': 104.60529327392578,\n",
+       " 'mfcc_mean_1 (mix)': 163.30865478515625,\n",
+       " 'mfcc_mean_2 (mix)': 22.928050994873047,\n",
+       " 'loudness_range_lu custom (mix)': 7.131200790405273,\n",
+       " 'zero_crossing_rate (mix)': 0.02693298086524,\n",
+       " 'vocab_richness': 0.582,\n",
+       " 'loudness_integrated_lufs custom (mix)': -18.566131591796875,\n",
+       " 'readability_score': 27.8,\n",
+       " 'energy_essentia (mix)': 0.1537376961168162,\n",
+       " 'energy_librosa (mix)': 0.335069430512918,\n",
+       " 'rms_energy_mean (mix)': 0.3462888819034169,\n",
+       " 'melody_complexity (vocals)': 1.9166666666666667}"
+      ]
+     },
+     "execution_count": 58,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 58
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:32:09.375740200Z",
+     "start_time": "2026-03-27T15:32:08.891439Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "print(\"\\n=== FIRST ROW PREDICTION ===\")\n",
+    "print(manual_predict(X.iloc[508].to_dict()))\n",
+    "\n",
+    "print(\"\\n=== LAST ROW PREDICTION ===\")\n",
+    "print(manual_predict(X.iloc[-1].to_dict()))"
+   ],
+   "id": "fef35c101a9a6ad",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "=== FIRST ROW PREDICTION ===\n",
+      "Indie Pop, Indie Rock, Modern Alternative Rock\n",
+      "\n",
+      "=== LAST ROW PREDICTION ===\n",
+      "Gothic Metal, Heavy Metal, Metalcore\n"
+     ]
+    }
+   ],
+   "execution_count": 63
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:31:39.900012500Z",
+     "start_time": "2026-03-27T15:31:39.638999300Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "audio_features_ = {\n",
+    "    'spectral_contrast_mean (mix)': 17.245,\n",
+    "    'chroma_mean (mix)': 0.482,\n",
+    "    'melody_variability (vocals)': 0.612,\n",
+    "    'rhythm_onset_rate (mix)': 2.145,\n",
+    "    'spectral_centroid_mean custom (mix)': 0.165,\n",
+    "    'repetition_score custom (mix)': 0.015,\n",
+    "    'pitch_std (mix)': 120.75,\n",
+    "    'tempo_bpm_corrected (mix)': 98.34,\n",
+    "    'tempo_bpm_original (mix)': 100.12,\n",
+    "    'mfcc_mean_1 (mix)': 128.56,\n",
+    "    'mfcc_mean_2 (mix)': -35.78,\n",
+    "    'loudness_range_lu custom (mix)': 8.45,\n",
+    "    'zero_crossing_rate (mix)': 0.089,\n",
+    "    'vocab_richness': 0.29,\n",
+    "    'loudness_integrated_lufs custom (mix)': -12.85,\n",
+    "    'readability_score': 48.5,\n",
+    "    'energy_essentia (mix)': 0.41,\n",
+    "    'energy_librosa (mix)': 0.52,\n",
+    "    'rms_energy_mean (mix)': 0.50,\n",
+    "    'melody_complexity (vocals)': 2.75\n",
+    "}\n",
+    "\n",
+    "answer = manual_predict(audio_features_)\n",
+    "print(answer)"
+   ],
+   "id": "6d8f315863783a68",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Indie Pop, Indie Rock, Modern Alternative Rock\n"
+     ]
+    }
+   ],
+   "execution_count": 61
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:31:41.650948200Z",
+     "start_time": "2026-03-27T15:31:41.421694900Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "fake_audio_features_2 = {'spectral_contrast_mean (mix)': 20.1520629356514,\n",
+    " 'chroma_mean (mix)': 0.408170998096466,\n",
+    " 'melody_variability (vocals)': 0.3748557548841115,\n",
+    " 'rhythm_onset_rate (mix)': 2.0277777777777777,\n",
+    " 'spectral_centroid_mean custom (mix)': 0.0938994687064515,\n",
+    " 'repetition_score custom (mix)': 0.0078822056596288,\n",
+    " 'pitch_std (mix)': 108.27875518798828,\n",
+    " 'tempo_bpm_corrected (mix)': 104.60529327392578,\n",
+    " 'tempo_bpm_original (mix)': 104.60529327392578,\n",
+    " 'mfcc_mean_1 (mix)': 163.30865478515625,\n",
+    " 'mfcc_mean_2 (mix)': 22.928050994873047,\n",
+    " 'loudness_range_lu custom (mix)': 7.131200790405273,\n",
+    " 'zero_crossing_rate (mix)': 0.02693298086524,\n",
+    " 'vocab_richness': 0.582,\n",
+    " 'loudness_integrated_lufs custom (mix)': -18.566131591796875,\n",
+    " 'readability_score': 27.8,\n",
+    " 'energy_essentia (mix)': 0.1537376961168162,\n",
+    " 'energy_librosa (mix)': 0.335069430512918,\n",
+    " 'rms_energy_mean (mix)': 0.3462888819034169,\n",
+    " 'melody_complexity (vocals)': 1.9166666666666667}\n",
+    "answer = manual_predict(fake_audio_features_2)\n",
+    "print(answer)"
+   ],
+   "id": "b998289b1d156dc2",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Black Metal, Death Metal, Doom Metal, Heavy Metal, Metalcore, Thrash Metal\n"
+     ]
+    }
+   ],
+   "execution_count": 62
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:24:02.082478Z",
+     "start_time": "2026-03-27T15:24:01.848007Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "fake_metal_audio = {\n",
+    "    'spectral_contrast_mean (mix)': 24.85,\n",
+    "    'chroma_mean (mix)': 0.38,\n",
+    "    'melody_variability (vocals)': 0.72,\n",
+    "    'rhythm_onset_rate (mix)': 4.95,\n",
+    "    'spectral_centroid_mean custom (mix)': 0.29,\n",
+    "    'repetition_score custom (mix)': 0.006,\n",
+    "    'pitch_std (mix)': 210.45,\n",
+    "    'tempo_bpm_corrected (mix)': 145.60,\n",
+    "    'tempo_bpm_original (mix)': 147.20,\n",
+    "    'mfcc_mean_1 (mix)': 142.75,\n",
+    "    'mfcc_mean_2 (mix)': -18.34,\n",
+    "    'loudness_range_lu custom (mix)': 6.25,\n",
+    "    'zero_crossing_rate (mix)': 0.18,\n",
+    "    'vocab_richness': 0.27,\n",
+    "    'loudness_integrated_lufs custom (mix)': -8.95,\n",
+    "    'readability_score': 42.0,\n",
+    "    'energy_essentia (mix)': 0.78,\n",
+    "    'energy_librosa (mix)': 0.82,\n",
+    "    'rms_energy_mean (mix)': 0.80,\n",
+    "    'melody_complexity (vocals)': 3.95\n",
+    "}\n",
+    "\n",
+    "answer = manual_predict(fake_metal_audio)\n",
+    "print(answer)"
+   ],
+   "id": "b265ee987709c2a3",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Rock -- Americana\n"
+     ]
+    }
+   ],
+   "execution_count": 57
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:38:00.278309800Z",
+     "start_time": "2026-03-27T15:37:57.734946600Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "",
+   "id": "228a47b066312e75",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Model saved as genre_model.pkl\n"
+     ]
+    }
+   ],
+   "execution_count": 64
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:49:22.832888800Z",
+     "start_time": "2026-03-27T15:49:20.228927800Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import joblib\n",
+    "bundle = {\n",
+    "    \"model\": model,\n",
+    "    \"scaler\": scaler,\n",
+    "    \"mlb\": mlb,\n",
+    "    \"feature_cols\": X.columns.tolist()\n",
+    "}\n",
+    "joblib.dump(bundle, \"genre_model1.pkl\")\n",
+    "print(\"Model saved successfully!\")"
+   ],
+   "id": "694614771e1c62a9",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model saved successfully!\n"
+     ]
+    }
+   ],
+   "execution_count": 68
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:49:54.380812800Z",
+     "start_time": "2026-03-27T15:49:51.308892Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import joblib\n",
+    "bundle = joblib.load(\"genre_model1.pkl\")\n",
+    "model = bundle[\"model\"]\n",
+    "scaler = bundle[\"scaler\"]\n",
+    "mlb = bundle[\"mlb\"]\n",
+    "feature_cols = bundle[\"feature_cols\"]"
+   ],
+   "id": "bb9f24b4e0947c22",
+   "outputs": [],
+   "execution_count": 70
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:51:05.220558400Z",
+     "start_time": "2026-03-27T15:51:05.157646Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def predict_from_loaded(row, model, scaler, mlb, feature_cols):\n",
+    "    import pandas as pd\n",
+    "    import numpy as np\n",
+    "    input_df = pd.DataFrame([row])\n",
+    "    input_numeric = input_df.select_dtypes(include=[np.number]).fillna(0)\n",
+    "    for col in feature_cols:\n",
+    "        if col not in input_numeric.columns:\n",
+    "            input_numeric[col] = 0\n",
+    "    input_numeric = input_numeric[feature_cols]\n",
+    "    input_scaled = scaler.transform(input_numeric)\n",
+    "    probas = model.predict_proba(input_scaled)\n",
+    "    threshold = 0.20\n",
+    "    pred_binary = np.zeros((1, len(probas)), dtype=int)\n",
+    "    for i in range(len(probas)):\n",
+    "        if probas[i][0][1] > threshold:\n",
+    "            pred_binary[0, i] = 1\n",
+    "    subgenres = mlb.inverse_transform(pred_binary)[0]\n",
+    "    if len(subgenres) == 0:\n",
+    "        return \"None\"\n",
+    "    return \", \".join(sorted(subgenres))"
+   ],
+   "id": "5391f7697e93bcad",
+   "outputs": [],
+   "execution_count": 71
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-27T15:51:42.665671400Z",
+     "start_time": "2026-03-27T15:51:42.509361Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "audio_features_ = {'spectral_contrast_mean (mix)': 20.1520629356514,\n",
+    " 'chroma_mean (mix)': 0.408170998096466,\n",
+    " 'melody_variability (vocals)': 0.3748557548841115,\n",
+    " 'rhythm_onset_rate (mix)': 2.0277777777777777,\n",
+    " 'spectral_centroid_mean custom (mix)': 0.0938994687064515,\n",
+    " 'repetition_score custom (mix)': 0.0078822056596288,\n",
+    " 'pitch_std (mix)': 108.27875518798828,\n",
+    " 'tempo_bpm_corrected (mix)': 104.60529327392578,\n",
+    " 'tempo_bpm_original (mix)': 104.60529327392578,\n",
+    " 'mfcc_mean_1 (mix)': 163.30865478515625,\n",
+    " 'mfcc_mean_2 (mix)': 22.928050994873047,\n",
+    " 'loudness_range_lu custom (mix)': 7.131200790405273,\n",
+    " 'zero_crossing_rate (mix)': 0.02693298086524,\n",
+    " 'vocab_richness': 0.582,\n",
+    " 'loudness_integrated_lufs custom (mix)': -18.566131591796875,\n",
+    " 'readability_score': 27.8,\n",
+    " 'energy_essentia (mix)': 0.1537376961168162,\n",
+    " 'energy_librosa (mix)': 0.335069430512918,\n",
+    " 'rms_energy_mean (mix)': 0.3462888819034169,\n",
+    " 'melody_complexity (vocals)': 1.9166666666666667}\n",
+    "\n",
+    "result = predict_from_loaded(audio_features_, model, scaler, mlb, feature_cols)\n",
+    "print(result)"
+   ],
+   "id": "cbc80439273b814f",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Black Metal, Death Metal, Doom Metal, Heavy Metal, Metalcore, Thrash Metal\n"
+     ]
+    }
+   ],
+   "execution_count": 73
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "### 1) All Data",
+   "id": "9a576cffaecfd507"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T09:59:36.204702500Z",
+     "start_time": "2026-03-30T09:59:34.858890200Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import pandas as pd\n",
+    "path = r\"C:\\Users\\VigneshSubramani\\Downloads\\features_dataset.csv\"\n",
+    "pd.set_option(\"display.max_columns\", None)\n",
+    "data = pd.read_csv(path,low_memory=False)\n",
+    "data"
+   ],
+   "id": "16dcec42226f1c5d",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "                                                track_url        genre  \\\n",
+       "0       https://open.spotify.com/track/38RJG8nLAwGycit...      Country   \n",
+       "1       https://open.spotify.com/track/2uqYupMHANxnwge...      Country   \n",
+       "2       https://open.spotify.com/track/48X5k2vce5rXckg...      Country   \n",
+       "3       https://open.spotify.com/track/3SaS0Va6o6jYkWv...      Country   \n",
+       "4       https://open.spotify.com/track/09hVlIVHtEE1nLU...      Country   \n",
+       "...                                                   ...          ...   \n",
+       "138685  https://open.spotify.com/track/6rZno3nqeT7hv2P...  Latin Urban   \n",
+       "138686  https://open.spotify.com/track/0O4ZJKR2QDLGrBZ...  Latin Urban   \n",
+       "138687  https://open.spotify.com/track/4oV82xuYkYgyM2n...  Latin Urban   \n",
+       "138688  https://open.spotify.com/track/3jztoVZMBEaTX0b...  Latin Urban   \n",
+       "138689  https://open.spotify.com/track/3mUr9hl2VhqorCu...  Latin Urban   \n",
+       "\n",
+       "                       name  features.melody_complexity (vocals)  \\\n",
+       "0             Forever To Me                             3.500000   \n",
+       "1                    Austin                             3.250000   \n",
+       "2                Dirt Cheap                             2.750000   \n",
+       "3             I Am Not Okay                             2.333333   \n",
+       "4              Burn It Down                             3.166667   \n",
+       "...                     ...                                  ...   \n",
+       "138685            YOGURCITO                                  NaN   \n",
+       "138686  Me Arriesgo Contigo                                  NaN   \n",
+       "138687              Vocales                                  NaN   \n",
+       "138688               Tímida                                  NaN   \n",
+       "138689      Romeo Y Julieta                                  NaN   \n",
+       "\n",
+       "        features.melody_range (vocals)  features.melody_variability (vocals)  \\\n",
+       "0                                 41.0                              0.751713   \n",
+       "1                                 35.0                              0.715152   \n",
+       "2                                 34.0                              0.453325   \n",
+       "3                                 34.0                              0.436993   \n",
+       "4                                 38.0                              0.531330   \n",
+       "...                                ...                                   ...   \n",
+       "138685                             NaN                                   NaN   \n",
+       "138686                             NaN                                   NaN   \n",
+       "138687                             NaN                                   NaN   \n",
+       "138688                             NaN                                   NaN   \n",
+       "138689                             NaN                                   NaN   \n",
+       "\n",
+       "        features.tempo_bpm_original (mix)  features.danceability custom (mix)  \\\n",
+       "0                              141.384491                            0.580731   \n",
+       "1                              105.666534                            0.747039   \n",
+       "2                              110.124130                            0.694952   \n",
+       "3                              141.088547                            0.602134   \n",
+       "4                              148.782501                            0.603468   \n",
+       "...                                   ...                                 ...   \n",
+       "138685                                NaN                                 NaN   \n",
+       "138686                                NaN                                 NaN   \n",
+       "138687                                NaN                                 NaN   \n",
+       "138688                                NaN                                 NaN   \n",
+       "138689                                NaN                                 NaN   \n",
+       "\n",
+       "        features.loudness_integrated_lufs custom (mix)  \\\n",
+       "0                                           -15.467463   \n",
+       "1                                           -12.137867   \n",
+       "2                                           -14.214710   \n",
+       "3                                           -12.991043   \n",
+       "4                                           -12.433587   \n",
+       "...                                                ...   \n",
+       "138685                                             NaN   \n",
+       "138686                                             NaN   \n",
+       "138687                                             NaN   \n",
+       "138688                                             NaN   \n",
+       "138689                                             NaN   \n",
+       "\n",
+       "        features.loudness_range_lu custom (mix)  \\\n",
+       "0                                     11.282878   \n",
+       "1                                      3.665868   \n",
+       "2                                      8.500771   \n",
+       "3                                      3.841528   \n",
+       "4                                      5.801809   \n",
+       "...                                         ...   \n",
+       "138685                                      NaN   \n",
+       "138686                                      NaN   \n",
+       "138687                                      NaN   \n",
+       "138688                                      NaN   \n",
+       "138689                                      NaN   \n",
+       "\n",
+       "        features.energy_librosa (mix)  features.energy_librosa_std (mix)  \\\n",
+       "0                            0.419014                           0.191760   \n",
+       "1                            0.469624                           0.169444   \n",
+       "2                            0.427623                           0.194988   \n",
+       "3                            0.524690                           0.185032   \n",
+       "4                            0.566157                           0.145061   \n",
+       "...                               ...                                ...   \n",
+       "138685                            NaN                                NaN   \n",
+       "138686                            NaN                                NaN   \n",
+       "138687                            NaN                                NaN   \n",
+       "138688                            NaN                                NaN   \n",
+       "138689                            NaN                                NaN   \n",
+       "\n",
+       "        features.energy_essentia (mix)  features.energy_essentia_std (mix)  \\\n",
+       "0                             0.225257                            0.167297   \n",
+       "1                             0.255954                            0.162414   \n",
+       "2                             0.226500                            0.159222   \n",
+       "3                             0.321317                            0.172458   \n",
+       "4                             0.355688                            0.138871   \n",
+       "...                                ...                                 ...   \n",
+       "138685                             NaN                                 NaN   \n",
+       "138686                             NaN                                 NaN   \n",
+       "138687                             NaN                                 NaN   \n",
+       "138688                             NaN                                 NaN   \n",
+       "138689                             NaN                                 NaN   \n",
+       "\n",
+       "        features.energy_combined (mix)  \\\n",
+       "0                             0.322135   \n",
+       "1                             0.362789   \n",
+       "2                             0.327062   \n",
+       "3                             0.423004   \n",
+       "4                             0.460922   \n",
+       "...                                ...   \n",
+       "138685                             NaN   \n",
+       "138686                             NaN   \n",
+       "138687                             NaN   \n",
+       "138688                             NaN   \n",
+       "138689                             NaN   \n",
+       "\n",
+       "        features.spectral_centroid_mean custom (mix)  \\\n",
+       "0                                           0.128541   \n",
+       "1                                           0.121481   \n",
+       "2                                           0.120138   \n",
+       "3                                           0.140651   \n",
+       "4                                           0.127909   \n",
+       "...                                              ...   \n",
+       "138685                                           NaN   \n",
+       "138686                                           NaN   \n",
+       "138687                                           NaN   \n",
+       "138688                                           NaN   \n",
+       "138689                                           NaN   \n",
+       "\n",
+       "        features.mfcc_mean_1 (mix)  features.mfcc_mean_2 (mix)  \\\n",
+       "0                       144.864594                   -7.119251   \n",
+       "1                       160.318756                   -3.738415   \n",
+       "2                       156.815781                  -13.236840   \n",
+       "3                       143.763428                  -21.374804   \n",
+       "4                       156.998581                  -24.430223   \n",
+       "...                            ...                         ...   \n",
+       "138685                         NaN                         NaN   \n",
+       "138686                         NaN                         NaN   \n",
+       "138687                         NaN                         NaN   \n",
+       "138688                         NaN                         NaN   \n",
+       "138689                         NaN                         NaN   \n",
+       "\n",
+       "        features.chroma_mean (mix)  features.spectral_contrast_mean (mix)  \\\n",
+       "0                         0.482208                              19.678119   \n",
+       "1                         0.501811                              19.144755   \n",
+       "2                         0.452644                              19.641929   \n",
+       "3                         0.434028                              19.570383   \n",
+       "4                         0.493009                              19.436073   \n",
+       "...                            ...                                    ...   \n",
+       "138685                         NaN                                    NaN   \n",
+       "138686                         NaN                                    NaN   \n",
+       "138687                         NaN                                    NaN   \n",
+       "138688                         NaN                                    NaN   \n",
+       "138689                         NaN                                    NaN   \n",
+       "\n",
+       "        features.repetition_score custom (mix)  features.pitch_mean (mix)  \\\n",
+       "0                                     0.007355                 188.741852   \n",
+       "1                                     0.007224                 225.259796   \n",
+       "2                                     0.006500                 184.371185   \n",
+       "3                                     0.006950                 232.608765   \n",
+       "4                                     0.005922                 241.072388   \n",
+       "...                                        ...                        ...   \n",
+       "138685                                     NaN                        NaN   \n",
+       "138686                                     NaN                        NaN   \n",
+       "138687                                     NaN                        NaN   \n",
+       "138688                                     NaN                        NaN   \n",
+       "138689                                     NaN                        NaN   \n",
+       "\n",
+       "        features.pitch_std (mix)  features.rms_energy_mean (mix)  \\\n",
+       "0                      54.632015                        0.432170   \n",
+       "1                      90.800995                        0.477567   \n",
+       "2                      49.438427                        0.436157   \n",
+       "3                      60.514282                        0.537289   \n",
+       "4                      98.557098                        0.579285   \n",
+       "...                          ...                             ...   \n",
+       "138685                       NaN                             NaN   \n",
+       "138686                       NaN                             NaN   \n",
+       "138687                       NaN                             NaN   \n",
+       "138688                       NaN                             NaN   \n",
+       "138689                       NaN                             NaN   \n",
+       "\n",
+       "        features.rms_energy_std (mix)  features.zero_crossing_rate (mix)  \\\n",
+       "0                            0.194504                           0.042678   \n",
+       "1                            0.166947                           0.040959   \n",
+       "2                            0.192781                           0.046456   \n",
+       "3                            0.180169                           0.056903   \n",
+       "4                            0.140620                           0.053897   \n",
+       "...                               ...                                ...   \n",
+       "138685                            NaN                                NaN   \n",
+       "138686                            NaN                                NaN   \n",
+       "138687                            NaN                                NaN   \n",
+       "138688                            NaN                                NaN   \n",
+       "138689                            NaN                                NaN   \n",
+       "\n",
+       "                  sub_genre[0]          sub_genre[1]  sub_genre[2]  \\\n",
+       "0         Contemporary Country                   NaN           NaN   \n",
+       "1                          NaN                   NaN           NaN   \n",
+       "2       Modern Neo-Traditional  Contemporary Country           NaN   \n",
+       "3                    Americana          Country Rock  Country Folk   \n",
+       "4                          NaN                   NaN           NaN   \n",
+       "...                        ...                   ...           ...   \n",
+       "138685             Latin Urban                   NaN           NaN   \n",
+       "138686             Latin Urban                   NaN           NaN   \n",
+       "138687             Latin Urban                   NaN           NaN   \n",
+       "138688             Latin Urban                   NaN           NaN   \n",
+       "138689             Latin Urban                   NaN           NaN   \n",
+       "\n",
+       "                sub_genre[3] sub_genre[4] sub_genre[5] sub_genre[6]  \\\n",
+       "0                        NaN          NaN          NaN          NaN   \n",
+       "1                        NaN          NaN          NaN          NaN   \n",
+       "2                        NaN          NaN          NaN          NaN   \n",
+       "3       Contemporary Country          NaN          NaN          NaN   \n",
+       "4                        NaN          NaN          NaN          NaN   \n",
+       "...                      ...          ...          ...          ...   \n",
+       "138685                   NaN          NaN          NaN          NaN   \n",
+       "138686                   NaN          NaN          NaN          NaN   \n",
+       "138687                   NaN          NaN          NaN          NaN   \n",
+       "138688                   NaN          NaN          NaN          NaN   \n",
+       "138689                   NaN          NaN          NaN          NaN   \n",
+       "\n",
+       "       sub_genre[7] sub_genre[8] sub_genre[9] sub_genre[10] sub_genre[11]  \\\n",
+       "0               NaN          NaN          NaN           NaN           NaN   \n",
+       "1               NaN          NaN          NaN           NaN           NaN   \n",
+       "2               NaN          NaN          NaN           NaN           NaN   \n",
+       "3               NaN          NaN          NaN           NaN           NaN   \n",
+       "4               NaN          NaN          NaN           NaN           NaN   \n",
+       "...             ...          ...          ...           ...           ...   \n",
+       "138685          NaN          NaN          NaN           NaN           NaN   \n",
+       "138686          NaN          NaN          NaN           NaN           NaN   \n",
+       "138687          NaN          NaN          NaN           NaN           NaN   \n",
+       "138688          NaN          NaN          NaN           NaN           NaN   \n",
+       "138689          NaN          NaN          NaN           NaN           NaN   \n",
+       "\n",
+       "       sub_genre[12] sub_genre[13] sub_genre[14] sub_genre[15]  \n",
+       "0                NaN           NaN           NaN           NaN  \n",
+       "1                NaN           NaN           NaN           NaN  \n",
+       "2                NaN           NaN           NaN           NaN  \n",
+       "3                NaN           NaN           NaN           NaN  \n",
+       "4                NaN           NaN           NaN           NaN  \n",
+       "...              ...           ...           ...           ...  \n",
+       "138685           NaN           NaN           NaN           NaN  \n",
+       "138686           NaN           NaN           NaN           NaN  \n",
+       "138687           NaN           NaN           NaN           NaN  \n",
+       "138688           NaN           NaN           NaN           NaN  \n",
+       "138689           NaN           NaN           NaN           NaN  \n",
+       "\n",
+       "[138690 rows x 42 columns]"
+      ],
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>track_url</th>\n",
+       "      <th>genre</th>\n",
+       "      <th>name</th>\n",
+       "      <th>features.melody_complexity (vocals)</th>\n",
+       "      <th>features.melody_range (vocals)</th>\n",
+       "      <th>features.melody_variability (vocals)</th>\n",
+       "      <th>features.tempo_bpm_original (mix)</th>\n",
+       "      <th>features.danceability custom (mix)</th>\n",
+       "      <th>features.loudness_integrated_lufs custom (mix)</th>\n",
+       "      <th>features.loudness_range_lu custom (mix)</th>\n",
+       "      <th>features.energy_librosa (mix)</th>\n",
+       "      <th>features.energy_librosa_std (mix)</th>\n",
+       "      <th>features.energy_essentia (mix)</th>\n",
+       "      <th>features.energy_essentia_std (mix)</th>\n",
+       "      <th>features.energy_combined (mix)</th>\n",
+       "      <th>features.spectral_centroid_mean custom (mix)</th>\n",
+       "      <th>features.mfcc_mean_1 (mix)</th>\n",
+       "      <th>features.mfcc_mean_2 (mix)</th>\n",
+       "      <th>features.chroma_mean (mix)</th>\n",
+       "      <th>features.spectral_contrast_mean (mix)</th>\n",
+       "      <th>features.repetition_score custom (mix)</th>\n",
+       "      <th>features.pitch_mean (mix)</th>\n",
+       "      <th>features.pitch_std (mix)</th>\n",
+       "      <th>features.rms_energy_mean (mix)</th>\n",
+       "      <th>features.rms_energy_std (mix)</th>\n",
+       "      <th>features.zero_crossing_rate (mix)</th>\n",
+       "      <th>sub_genre[0]</th>\n",
+       "      <th>sub_genre[1]</th>\n",
+       "      <th>sub_genre[2]</th>\n",
+       "      <th>sub_genre[3]</th>\n",
+       "      <th>sub_genre[4]</th>\n",
+       "      <th>sub_genre[5]</th>\n",
+       "      <th>sub_genre[6]</th>\n",
+       "      <th>sub_genre[7]</th>\n",
+       "      <th>sub_genre[8]</th>\n",
+       "      <th>sub_genre[9]</th>\n",
+       "      <th>sub_genre[10]</th>\n",
+       "      <th>sub_genre[11]</th>\n",
+       "      <th>sub_genre[12]</th>\n",
+       "      <th>sub_genre[13]</th>\n",
+       "      <th>sub_genre[14]</th>\n",
+       "      <th>sub_genre[15]</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>https://open.spotify.com/track/38RJG8nLAwGycit...</td>\n",
+       "      <td>Country</td>\n",
+       "      <td>Forever To Me</td>\n",
+       "      <td>3.500000</td>\n",
+       "      <td>41.0</td>\n",
+       "      <td>0.751713</td>\n",
+       "      <td>141.384491</td>\n",
+       "      <td>0.580731</td>\n",
+       "      <td>-15.467463</td>\n",
+       "      <td>11.282878</td>\n",
+       "      <td>0.419014</td>\n",
+       "      <td>0.191760</td>\n",
+       "      <td>0.225257</td>\n",
+       "      <td>0.167297</td>\n",
+       "      <td>0.322135</td>\n",
+       "      <td>0.128541</td>\n",
+       "      <td>144.864594</td>\n",
+       "      <td>-7.119251</td>\n",
+       "      <td>0.482208</td>\n",
+       "      <td>19.678119</td>\n",
+       "      <td>0.007355</td>\n",
+       "      <td>188.741852</td>\n",
+       "      <td>54.632015</td>\n",
+       "      <td>0.432170</td>\n",
+       "      <td>0.194504</td>\n",
+       "      <td>0.042678</td>\n",
+       "      <td>Contemporary Country</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>https://open.spotify.com/track/2uqYupMHANxnwge...</td>\n",
+       "      <td>Country</td>\n",
+       "      <td>Austin</td>\n",
+       "      <td>3.250000</td>\n",
+       "      <td>35.0</td>\n",
+       "      <td>0.715152</td>\n",
+       "      <td>105.666534</td>\n",
+       "      <td>0.747039</td>\n",
+       "      <td>-12.137867</td>\n",
+       "      <td>3.665868</td>\n",
+       "      <td>0.469624</td>\n",
+       "      <td>0.169444</td>\n",
+       "      <td>0.255954</td>\n",
+       "      <td>0.162414</td>\n",
+       "      <td>0.362789</td>\n",
+       "      <td>0.121481</td>\n",
+       "      <td>160.318756</td>\n",
+       "      <td>-3.738415</td>\n",
+       "      <td>0.501811</td>\n",
+       "      <td>19.144755</td>\n",
+       "      <td>0.007224</td>\n",
+       "      <td>225.259796</td>\n",
+       "      <td>90.800995</td>\n",
+       "      <td>0.477567</td>\n",
+       "      <td>0.166947</td>\n",
+       "      <td>0.040959</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>https://open.spotify.com/track/48X5k2vce5rXckg...</td>\n",
+       "      <td>Country</td>\n",
+       "      <td>Dirt Cheap</td>\n",
+       "      <td>2.750000</td>\n",
+       "      <td>34.0</td>\n",
+       "      <td>0.453325</td>\n",
+       "      <td>110.124130</td>\n",
+       "      <td>0.694952</td>\n",
+       "      <td>-14.214710</td>\n",
+       "      <td>8.500771</td>\n",
+       "      <td>0.427623</td>\n",
+       "      <td>0.194988</td>\n",
+       "      <td>0.226500</td>\n",
+       "      <td>0.159222</td>\n",
+       "      <td>0.327062</td>\n",
+       "      <td>0.120138</td>\n",
+       "      <td>156.815781</td>\n",
+       "      <td>-13.236840</td>\n",
+       "      <td>0.452644</td>\n",
+       "      <td>19.641929</td>\n",
+       "      <td>0.006500</td>\n",
+       "      <td>184.371185</td>\n",
+       "      <td>49.438427</td>\n",
+       "      <td>0.436157</td>\n",
+       "      <td>0.192781</td>\n",
+       "      <td>0.046456</td>\n",
+       "      <td>Modern Neo-Traditional</td>\n",
+       "      <td>Contemporary Country</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>https://open.spotify.com/track/3SaS0Va6o6jYkWv...</td>\n",
+       "      <td>Country</td>\n",
+       "      <td>I Am Not Okay</td>\n",
+       "      <td>2.333333</td>\n",
+       "      <td>34.0</td>\n",
+       "      <td>0.436993</td>\n",
+       "      <td>141.088547</td>\n",
+       "      <td>0.602134</td>\n",
+       "      <td>-12.991043</td>\n",
+       "      <td>3.841528</td>\n",
+       "      <td>0.524690</td>\n",
+       "      <td>0.185032</td>\n",
+       "      <td>0.321317</td>\n",
+       "      <td>0.172458</td>\n",
+       "      <td>0.423004</td>\n",
+       "      <td>0.140651</td>\n",
+       "      <td>143.763428</td>\n",
+       "      <td>-21.374804</td>\n",
+       "      <td>0.434028</td>\n",
+       "      <td>19.570383</td>\n",
+       "      <td>0.006950</td>\n",
+       "      <td>232.608765</td>\n",
+       "      <td>60.514282</td>\n",
+       "      <td>0.537289</td>\n",
+       "      <td>0.180169</td>\n",
+       "      <td>0.056903</td>\n",
+       "      <td>Americana</td>\n",
+       "      <td>Country Rock</td>\n",
+       "      <td>Country Folk</td>\n",
+       "      <td>Contemporary Country</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>https://open.spotify.com/track/09hVlIVHtEE1nLU...</td>\n",
+       "      <td>Country</td>\n",
+       "      <td>Burn It Down</td>\n",
+       "      <td>3.166667</td>\n",
+       "      <td>38.0</td>\n",
+       "      <td>0.531330</td>\n",
+       "      <td>148.782501</td>\n",
+       "      <td>0.603468</td>\n",
+       "      <td>-12.433587</td>\n",
+       "      <td>5.801809</td>\n",
+       "      <td>0.566157</td>\n",
+       "      <td>0.145061</td>\n",
+       "      <td>0.355688</td>\n",
+       "      <td>0.138871</td>\n",
+       "      <td>0.460922</td>\n",
+       "      <td>0.127909</td>\n",
+       "      <td>156.998581</td>\n",
+       "      <td>-24.430223</td>\n",
+       "      <td>0.493009</td>\n",
+       "      <td>19.436073</td>\n",
+       "      <td>0.005922</td>\n",
+       "      <td>241.072388</td>\n",
+       "      <td>98.557098</td>\n",
+       "      <td>0.579285</td>\n",
+       "      <td>0.140620</td>\n",
+       "      <td>0.053897</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>138685</th>\n",
+       "      <td>https://open.spotify.com/track/6rZno3nqeT7hv2P...</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>YOGURCITO</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>138686</th>\n",
+       "      <td>https://open.spotify.com/track/0O4ZJKR2QDLGrBZ...</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>Me Arriesgo Contigo</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>138687</th>\n",
+       "      <td>https://open.spotify.com/track/4oV82xuYkYgyM2n...</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>Vocales</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>138688</th>\n",
+       "      <td>https://open.spotify.com/track/3jztoVZMBEaTX0b...</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>Tímida</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>138689</th>\n",
+       "      <td>https://open.spotify.com/track/3mUr9hl2VhqorCu...</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>Romeo Y Julieta</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Latin Urban</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>138690 rows × 42 columns</p>\n",
+       "</div>"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 2
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "a73b0a2127420c43"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

all_combine_code/genre_classification/test1_file.ipynb

@@ -0,0 +1,325 @@
+{
+ "cells": [
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T08:57:26.406625300Z",
+     "start_time": "2026-03-30T08:57:24.333887600Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import re\n",
+    "import ast\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.preprocessing import StandardScaler, MultiLabelBinarizer\n",
+    "from sklearn.multioutput import MultiOutputClassifier\n",
+    "from xgboost import XGBClassifier\n",
+    "from sklearn.metrics import f1_score\n",
+    "\n",
+    "path = r\"C:\\Users\\VigneshSubramani\\Downloads\\model_ready_dataset_final.csv\"\n",
+    "df = pd.read_csv(path)\n",
+    "\n"
+   ],
+   "id": "89e4f464fdeba6a0",
+   "outputs": [],
+   "execution_count": 1
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T08:57:32.746112300Z",
+     "start_time": "2026-03-30T08:57:32.668847500Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def parse_subgenres_fast(text):\n",
+    "    if pd.isna(text) or text == \"\":\n",
+    "        return []\n",
+    "    text = str(text)\n",
+    "    parts = text.split('---')\n",
+    "    main_genre = parts[0].strip()\n",
+    "    if len(parts) > 1:\n",
+    "        sub = parts[1].strip()\n",
+    "        sub = sub.strip('[]')\n",
+    "        if sub:\n",
+    "            subgenres = [s.strip().strip(\"'\").strip('\"') for s in sub.split(',')]\n",
+    "        else:\n",
+    "            subgenres = []\n",
+    "    else:\n",
+    "        subgenres = []\n",
+    "    return [main_genre] + subgenres\n",
+    "\n",
+    "df['subgenres_list'] = df['genre_subgenre'].apply(parse_subgenres_fast)\n",
+    "\n"
+   ],
+   "id": "a0e5f938aaea7104",
+   "outputs": [],
+   "execution_count": 2
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T08:57:56.751354200Z",
+     "start_time": "2026-03-30T08:57:56.682503800Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df['subgenres_list'][0]",
+   "id": "a361e82d41548901",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Rock', 'Noise Pop', 'Classic Alternative Rock', 'Indie Rock']"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 3
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T08:58:43.550962900Z",
+     "start_time": "2026-03-30T08:58:43.487876100Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "mlb = MultiLabelBinarizer()\n",
+    "y = mlb.fit_transform(df['subgenres_list'])\n",
+    "y"
+   ],
+   "id": "895aab114eb128a5",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[0, 0, 0, ..., 0, 0, 0],\n",
+       "       [0, 0, 0, ..., 0, 0, 0],\n",
+       "       [0, 0, 0, ..., 0, 0, 0],\n",
+       "       ...,\n",
+       "       [0, 0, 0, ..., 0, 0, 0],\n",
+       "       [0, 0, 0, ..., 1, 0, 0],\n",
+       "       [0, 0, 0, ..., 0, 0, 0]], shape=(41935, 112))"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 4
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T09:02:12.058921100Z",
+     "start_time": "2026-03-30T08:59:11.763576600Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "drop_cols = ['_id', 'genre', 'genre_subgenre', 'subgenres_list', 'labels',\n",
+    "             'trimmed_audio_duration_sec', 'syllable_count', 'word_count',\n",
+    "             'sentiment_score', 'avg_word_length']\n",
+    "\n",
+    "feature_cols = [c for c in df.columns if c not in drop_cols]\n",
+    "X = df[feature_cols].copy()\n",
+    "X = X.select_dtypes(include=[np.number]).fillna(0)\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
+    "\n",
+    "scaler = StandardScaler()\n",
+    "X_train_scaled = scaler.fit_transform(X_train)\n",
+    "X_test_scaled = scaler.transform(X_test)\n",
+    "\n",
+    "model = MultiOutputClassifier(\n",
+    "    XGBClassifier(\n",
+    "        n_estimators=1500,\n",
+    "        learning_rate=0.03,\n",
+    "        max_depth=8,\n",
+    "        subsample=0.85,\n",
+    "        colsample_bytree=0.8,\n",
+    "        random_state=42,\n",
+    "        eval_metric='logloss'\n",
+    "    ),\n",
+    "    n_jobs=-1\n",
+    ")\n",
+    "\n",
+    "model.fit(X_train_scaled, y_train)\n",
+    "\n",
+    "def get_accuracy(X_scaled, y_true, name=\"\"):\n",
+    "    y_pred = model.predict(X_scaled)\n",
+    "    subset_acc = np.mean(np.all(y_pred == y_true, axis=1))\n",
+    "    micro_f1 = f1_score(y_true, y_pred, average='micro')\n",
+    "    print(f\"{name} Subset Accuracy: {subset_acc:.4f}\")\n",
+    "    print(f\"{name} Micro F1 Score: {micro_f1:.4f}\")\n",
+    "\n",
+    "print(\"=== MODEL ACCURACY ===\")\n",
+    "get_accuracy(X_train_scaled, y_train, \"Train\")\n",
+    "get_accuracy(X_test_scaled, y_test, \"Test\")\n",
+    "\n",
+    "def manual_predict(row):\n",
+    "    input_df = pd.DataFrame([row])\n",
+    "    input_numeric = input_df.select_dtypes(include=[np.number]).fillna(0)\n",
+    "\n",
+    "    for col in X.columns:\n",
+    "        if col not in input_numeric.columns:\n",
+    "            input_numeric[col] = 0\n",
+    "    input_numeric = input_numeric[X.columns]\n",
+    "    input_scaled = scaler.transform(input_numeric)\n",
+    "    probas = model.predict_proba(input_scaled)\n",
+    "    threshold = 0.20\n",
+    "    pred_binary = np.zeros((1, y.shape[1]), dtype=int)\n",
+    "    for i in range(len(probas)):\n",
+    "        if probas[i][0][1] > threshold:\n",
+    "            pred_binary[0, i] = 1\n",
+    "    subgenres = mlb.inverse_transform(pred_binary)[0]\n",
+    "    if len(subgenres) == 0:\n",
+    "        return f\"None\"\n",
+    "    return f\"{', '.join(sorted(subgenres))}\""
+   ],
+   "id": "8ff23ad3daa3533f",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== MODEL ACCURACY ===\n",
+      "Train Subset Accuracy: 0.9961\n",
+      "Train Micro F1 Score: 0.9992\n",
+      "Test Subset Accuracy: 0.1909\n",
+      "Test Micro F1 Score: 0.5527\n"
+     ]
+    }
+   ],
+   "execution_count": 5
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T09:03:23.140843Z",
+     "start_time": "2026-03-30T09:03:23.087981400Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df[[\"subgenres_list\",\"genre\"]].iloc[501,]",
+   "id": "d8574c668e09217f",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "subgenres_list    [Rock, Classic Alternative Rock]\n",
+       "genre                                         Rock\n",
+       "Name: 501, dtype: object"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 8
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T09:03:31.570593500Z",
+     "start_time": "2026-03-30T09:03:31.234859400Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "print(\"\\n=== FIRST ROW PREDICTION ===\")\n",
+    "print(manual_predict(X.iloc[501].to_dict()))\n",
+    "\n",
+    "print(\"\\n=== LAST ROW PREDICTION ===\")\n",
+    "print(manual_predict(X.iloc[-1].to_dict()))"
+   ],
+   "id": "d311cd5e9a4aa105",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "=== FIRST ROW PREDICTION ===\n",
+      "Modern Alternative Rock, Rock\n",
+      "\n",
+      "=== LAST ROW PREDICTION ===\n",
+      "Gothic Metal, Heavy Metal, Metal, Metalcore\n"
+     ]
+    }
+   ],
+   "execution_count": 9
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2026-03-30T09:04:23.092797900Z",
+     "start_time": "2026-03-30T09:04:20.630659500Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import joblib\n",
+    "bundle = {\n",
+    "    \"model\": model,\n",
+    "    \"scaler\": scaler,\n",
+    "    \"mlb\": mlb,\n",
+    "    \"feature_cols\": X.columns.tolist()\n",
+    "}\n",
+    "joblib.dump(bundle, \"genre_model1.pkl\")\n",
+    "print(\"Model saved successfully!\")"
+   ],
+   "id": "58b9f6372caf42f",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model saved successfully!\n"
+     ]
+    }
+   ],
+   "execution_count": 10
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "c3b93bc28cd98983"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

all_combine_code/genreclassification/.env

@@ -0,0 +1 @@
+VIBERATE_API=rot6LA7vWzUCdb2ZxmjOQSiUYyPuXcZI

all_combine_code/genreclassification/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Copy requirements.txt and install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the entire project
+COPY . .
+
+# Ensure the songs_data directory exists and has write permissions
+RUN mkdir -p songs_data && chmod -R 755 songs_data
+
+# Set environment variable for unbuffered output
+ENV PYTHONUNBUFFERED=1
+
+# Expose port
+EXPOSE 5003
+
+# Command to run the app
+CMD ["uvicorn", "main_app:app", "--host", "0.0.0.0", "--port", "5003"]

all_combine_code/genreclassification/chartmetric_summary_generator3.py

@@ -0,0 +1,264 @@
+import os, re, json
+import pandas as pd
+import numpy as np
+import difflib
+
+from viberate_api_report_generator2 import (
+    potential_fanbase_count,
+    similarity_songs_report,
+)
+
+
+def chartmetric_manual_generator(task_id: str, session_dir: str):
+    songs_list, df = similarity_songs_report(task_id)
+    if df is None or df.empty:
+        return None, None
+
+    excel_path = potential_fanbase_count(songs_list, session_dir)
+
+    if not isinstance(excel_path, str) or not excel_path.endswith(".xlsx"):
+        return None, None
+
+    if os.path.exists(excel_path) and os.path.getsize(excel_path) > 0:
+        try:
+            pd.ExcelFile(excel_path)
+            return excel_path, df
+        except Exception:
+            return None, None
+
+    return None, None
+
+
+def final_score_calculation(task_id: str, session_dir: str) -> int | None:
+    workbook_path, similarity_dataframe = chartmetric_manual_generator(task_id, session_dir)
+    if workbook_path is None:
+        return None
+
+    ALIASES = {
+        "spotify_streams":    ["spotify streams", "spotify stream", "spotify s"],
+        "youtube_views":      ["youtube views", "youtube view", "youtube video"],
+        "soundcloud_streams": ["soundcloud streams", "soundcloud plays", "sound"],
+    }
+    RPF_LOW, RPF_MID, RPF_HIGH = 4.0, 2.5, 1.5
+    USE_DECAY      = True
+    HALF_LIFE_YRS  = 5
+    LIFT_WEIGHTS   = {"youtube_views": 0.60, "soundcloud_streams": 0.40}
+    LAMBDA         = 0.10
+    LIFT_MIN_MAX   = (0.7, 1.5)
+    OVERLAP_BOUNDS = (0.30, 0.60)
+    OVERLAP_PENALTY_K  = 2.5
+    RECENCY_EXPONENT   = 1.0
+    AUDIENCE_SMOOTHING_LAMBDA = 0.15
+
+    # ── helpers ─────────────────────────────────────────────────────────────
+    def _norm(s):
+        return re.sub(r"\s+", " ", str(s).replace("\n", " ").strip().lower())
+
+    def resolve_col(cols, aliases):
+        norm_cols  = {c: _norm(c) for c in cols}
+        alias_norms = [_norm(a) for a in aliases]
+        for c, nc in norm_cols.items():
+            for a in alias_norms:
+                if nc == a or nc.startswith(a) or a in nc:
+                    return c
+        best_col, best_ratio = None, 0.0
+        for c, nc in norm_cols.items():
+            for a in alias_norms:
+                r = difflib.SequenceMatcher(None, nc, a).ratio()
+                if r > best_ratio:
+                    best_ratio, best_col = r, c
+        return best_col if best_ratio >= 0.60 else None
+
+    def last_non_null(series):
+        s = pd.to_numeric(series, errors="coerce").dropna()
+        return s.iloc[-1] if not s.empty else np.nan
+
+    def robust_z(series):
+        s = pd.to_numeric(series, errors="coerce")
+        x = s.dropna()
+        if len(x) < 3:
+            return pd.Series(0.0, index=s.index)
+        med = x.median()
+        mad = (x - med).abs().median()
+        if mad == 0 or np.isnan(mad):
+            return pd.Series(0.0, index=s.index)
+        return (0.6745 * (s - med) / mad).clip(-5, 5).fillna(0.0)
+
+    def simpson_overlap(weights, bounds=(0.30, 0.60)):
+        w = pd.to_numeric(weights, errors="coerce").fillna(0.0).values
+        if w.sum() <= 0:
+            return float(np.mean(bounds))
+        w = w / w.sum()
+        K  = len(w)
+        if K <= 1:
+            return float(bounds[1])
+        C    = float((w ** 2).sum())
+        base = 1.0 / K
+        low, high = bounds
+        return float(np.clip(low + (high - low) * ((C - base) / (1 - base)), low, high))
+
+    def ensure_weights(sel_df, use_decay=True, half_life=5):
+        sel = sel_df.copy()
+        if "release_date" in sel.columns:
+            sel["release_date"] = (
+                pd.to_datetime(sel["release_date"], errors="coerce").dt.tz_localize(None)
+            )
+            age_years = (pd.Timestamp.today().normalize() - sel["release_date"]).dt.days / 365.25
+            if use_decay and sel["release_date"].notna().any():
+                sel["recency_decay"] = (
+                    np.exp(-0.693 * (age_years.fillna(age_years.median()) / half_life))
+                    ** RECENCY_EXPONENT
+                )
+            else:
+                sel["recency_decay"] = 1.0
+        else:
+            sel["recency_decay"] = 1.0
+
+        if "raw_weight" not in sel.columns:
+            sel["raw_weight"] = sel["final_boosted_score"] * sel["recency_decay"]
+
+        tot = sel["raw_weight"].sum()
+        sel["weight_norm"] = sel["raw_weight"] / tot if tot > 0 else 1.0 / len(sel)
+        sel = sel.sort_values("weight_norm", ascending=False)
+        sel["cum_weight"] = sel["weight_norm"].cumsum()
+        return sel
+
+    all_tracks = pd.read_excel(workbook_path, sheet_name=None)
+    rows = []
+    for sheet_name, df in all_tracks.items():
+        df.columns = (
+            df.columns.astype(str)
+            .str.replace("\n", " ", regex=True)
+            .str.replace(r"\s+", " ", regex=True)
+            .str.strip()
+        )
+        col_spotify    = resolve_col(df.columns, ALIASES["spotify_streams"])
+        col_youtube    = resolve_col(df.columns, ALIASES["youtube_views"])
+        col_soundcloud = resolve_col(df.columns, ALIASES["soundcloud_streams"])
+
+        spotify_streams    = last_non_null(df[col_spotify])    if col_spotify    else np.nan
+        youtube_views      = last_non_null(df[col_youtube])    if col_youtube    else np.nan
+        soundcloud_streams = last_non_null(df[col_soundcloud]) if col_soundcloud else np.nan
+
+        if pd.notna(spotify_streams):
+            listeners_low  = spotify_streams / RPF_LOW
+            listeners_mid  = spotify_streams / RPF_MID
+            listeners_high = spotify_streams / RPF_HIGH
+        else:
+            listeners_low = listeners_mid = listeners_high = np.nan
+
+        rows.append({
+            "track":            sheet_name,
+            "spotify_streams":  spotify_streams,
+            "listeners_low":    listeners_low,
+            "listeners_mid":    listeners_mid,
+            "listeners_high":   listeners_high,
+            "youtube_views":    youtube_views,
+            "soundcloud_streams": soundcloud_streams,
+        })
+
+    summary_df = pd.DataFrame(rows)
+
+    # ── part 3 – weighted score ──────────────────────────────────────────────
+    selected_tracks = similarity_dataframe.copy()
+    selected_tracks.columns = (
+        selected_tracks.columns.astype(str)
+        .str.replace("\n", " ", regex=True)
+        .str.replace(r"\s+", " ", regex=True)
+        .str.strip()
+    )
+
+    sel = ensure_weights(selected_tracks, use_decay=USE_DECAY, half_life=HALF_LIFE_YRS)
+    sel = sel.sort_values("weight_norm", ascending=False).reset_index(drop=True)
+    sel["rank"] = np.arange(1, len(sel) + 1)
+
+    p2 = summary_df.copy()
+    p2["rank"] = p2["track"].str.extract(r"^(\d+)").astype(float).astype("Int64")
+    if p2["rank"].isna().any():
+        seq = np.arange(1, len(p2) + 1)
+        p2.loc[p2["rank"].isna(), "rank"] = seq[p2["rank"].isna()]
+    p2["rank"] = p2["rank"].astype(int)
+
+    cols_keep_sel = [
+        "rank", "artist_names", "name", "album", "final_boosted_score",
+        "release_date", "weight_norm", "cum_weight", "track_url",
+    ]
+    merged = pd.merge(
+        sel[cols_keep_sel],
+        p2[["rank", "track",
+            "spotify_streams", "listeners_low", "listeners_mid", "listeners_high",
+            "youtube_views", "soundcloud_streams"]],
+        on="rank", how="inner",
+    )
+
+    for col in ["youtube_views", "soundcloud_streams"]:
+        merged[f"z_{col}"] = robust_z(merged[col])
+
+    def weighted_z(row):
+        num = den = 0.0
+        for col, w in LIFT_WEIGHTS.items():
+            z = row.get(f"z_{col}", 0.0)
+            if pd.notna(z):
+                num += w * z
+                den += w
+        return num / den if den > 0 else 0.0
+
+    merged["z_weighted"] = merged.apply(weighted_z, axis=1)
+    lo, hi = LIFT_MIN_MAX
+    merged["lift"] = (1.0 + LAMBDA * merged["z_weighted"]).clip(lower=lo, upper=hi)
+
+    def safe_mul(a, b, c):
+        if pd.isna(b): b = 0.0
+        return float(a) * float(b) * float(c)
+
+    merged["contrib_low"]  = merged.apply(lambda r: safe_mul(r["weight_norm"], r["listeners_low"],  r["lift"]), axis=1)
+    merged["contrib_mid"]  = merged.apply(lambda r: safe_mul(r["weight_norm"], r["listeners_mid"],  r["lift"]), axis=1)
+    merged["contrib_high"] = merged.apply(lambda r: safe_mul(r["weight_norm"], r["listeners_high"], r["lift"]), axis=1)
+
+    raw = {
+        "low":  merged["contrib_low"].sum(),
+        "mid":  merged["contrib_mid"].sum(),
+        "high": merged["contrib_high"].sum(),
+    }
+
+    analog_totals   = (merged["listeners_mid"] * merged["lift"]).fillna(0)
+    AUDIENCE_BASELINE = int(np.median(analog_totals) * 20)
+
+    overlap         = simpson_overlap(merged["weight_norm"], OVERLAP_BOUNDS)
+    overlap_penalty = min(1.0, OVERLAP_PENALTY_K * overlap)
+    final           = {k: v * (1.0 - overlap_penalty) for k, v in raw.items()}
+
+    pred_audience = (
+        (1 - AUDIENCE_SMOOTHING_LAMBDA) * final["mid"] +
+        AUDIENCE_SMOOTHING_LAMBDA * AUDIENCE_BASELINE
+    )
+
+    summary_df.to_csv(os.path.join(session_dir, "part2_chartmetric_summary.csv"), index=False)
+
+    detail_cols = [
+        "rank", "artist_names", "name", "track", "weight_norm",
+        "listeners_low", "listeners_mid", "listeners_high",
+        "youtube_views", "soundcloud_streams",
+        "z_youtube_views", "z_soundcloud_streams",
+        "z_weighted", "lift", "contrib_low", "contrib_mid", "contrib_high",
+    ]
+    detail_df = merged[detail_cols].sort_values("rank")
+    for col in ["listeners_low", "listeners_mid", "listeners_high",
+                "contrib_low", "contrib_mid", "contrib_high"]:
+        detail_df[col] = detail_df[col].round(0).astype("Int64")
+    detail_df["weight_norm"] = detail_df["weight_norm"].round(4)
+    detail_df["lift"]        = detail_df["lift"].round(3)
+    detail_df["z_weighted"]  = detail_df["z_weighted"].round(3)
+    detail_df.to_csv(os.path.join(session_dir, "part3_contributions_detail.csv"), index=False)
+
+    summary_out = {
+        "overlap_weights_based": float(round(overlap, 6)),
+        "totals_raw":            raw,
+        "totals_final":          final,
+        "AUDIENCE_BASELINE":     AUDIENCE_BASELINE,
+        "pred_audience_smooth":  int(pred_audience),
+    }
+    with open(os.path.join(session_dir, "part3_summary.json"), "w") as f:
+        json.dump(summary_out, f, indent=2)
+
+    return int(pred_audience)

all_combine_code/genreclassification/main_app.py

@@ -0,0 +1,71 @@
+import gc
+import logging
+import os
+import shutil
+import tempfile
+import time
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+
+from chartmetric_summary_generator3 import final_score_calculation
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+SESSIONS_ROOT = os.path.join(BASE_DIR, "sessions")   # all per-request dirs live here
+os.makedirs(SESSIONS_ROOT, exist_ok=True)
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+def _cleanup_session(session_dir: str) -> None:
+    """Remove the entire per-request temp directory, tolerating locked files."""
+    if not os.path.exists(session_dir):
+        return
+    try:
+        gc.collect()
+        time.sleep(0.1)
+        shutil.rmtree(session_dir, ignore_errors=True)
+        logger.info(f"Cleaned up session dir: {session_dir}")
+    except Exception as e:
+        logger.warning(f"Could not fully clean up {session_dir}: {e}")
+
+
+@app.get("/")
+def read_root():
+    return {"Hello": "200 ok"}
+
+
+@app.post("/fanbase_score")
+async def fanbase_score(request: Request):
+    session_dir = tempfile.mkdtemp(dir=SESSIONS_ROOT)
+    logger.info(f"Created session dir: {session_dir}")
+
+    try:
+        data     = await request.json()
+        task_id  = data["track_id"].strip()
+        logger.info(f"Processing start – task_id={task_id}  session={session_dir}")
+
+        fanbase_score_count = final_score_calculation(task_id, session_dir)
+        logger.info(f"Fan Base Count: {fanbase_score_count}")
+
+        return {"fanbase_count": fanbase_score_count}
+
+    except Exception as e:
+        logger.error(f"Request failed: {e}")
+        return {"error": str(e)}
+
+    finally:
+        _cleanup_session(session_dir)

all_combine_code/genreclassification/requirements.txt

@@ -0,0 +1,8 @@
+fastapi==0.119.0
+uvicorn==0.37.0
+pandas==2.3.3
+XlsxWriter==3.2.9
+aiohttp==3.12.15
+python-dotenv== 1.1.1
+openpyxl==3.1.5
+requests==2.32.5

all_combine_code/genreclassification/songs_report_generator1.py

@@ -0,0 +1,66 @@
+import warnings
+warnings.filterwarnings("ignore")
+
+import requests
+import pandas as pd
+
+
+def fetch_similar_tracks_df(task_id):
+    """
+    Fetch similar tracks for a given task_id and return as a pandas DataFrame.
+    """
+    url = f"https://aimusic-reprocessing.kovaionplay.com/api/v1/progress/task/{task_id}"
+    headers = {
+        "x-api-key": "$2b$12$VXkJYUlA2ZMxX25dZ47g7uNVfERnghpvd690vgpy0wVHef/KSDMPq",
+        "x-user-id": "6819fa6f63df2ba55d256ee1"
+    }
+    response = requests.get(url, headers=headers)
+
+    if response.status_code == 404:
+        url = f"https://aimusic-backend-stage.kovaionplay.com/api/v1/progress/task/{task_id}"
+        response = requests.get(url, headers=headers)
+
+    if response.status_code != 200:
+        raise Exception(f"Error: Received status code {response.status_code}, {response.text}")
+
+    data = response.json()
+    similar_tracks = data.get("similar_tracks", [])
+
+    if not similar_tracks:
+        return pd.DataFrame()
+
+    rows = []
+    for track in similar_tracks:
+        track_info = track.get("track_info", {})
+        similarity_scores = track.get("similarity_scores", {})
+        artist_info_list = track.get("artist_info", [])
+        row = {
+            "final_boosted_score": track.get("final_boosted_score", "-"),
+            "album": track_info.get("album", "-"),
+            "track_url": track_info.get("track_url", "-"),
+            "name": track_info.get("name", "-"),
+            "release_date": track_info.get("release_date", "-"),
+            "top_song": track_info.get("top_song", "-"),
+            "artist_names": "|".join(artist.get("artist_name", "-") for artist in artist_info_list),
+            "chord_progression_similarity": similarity_scores.get("chord_progression_similarity", "-"),
+            "general_audio_similarity": similarity_scores.get("general_audio_similarity", "-"),
+            "lyrics_similarity": similarity_scores.get("lyrics_similarity", "-"),
+            "timber_similarity": similarity_scores.get("timber_similarity", "-"),
+            "vocal_melody_similarity": similarity_scores.get("vocal_melody_similarity", "-"),
+            "rhythm_similarity": similarity_scores.get("rhythm_similarity", "-"),
+            "harmonic_similarity": similarity_scores.get("harmonic_similarity", "-"),
+        }
+        rows.append(row)
+
+    df = pd.DataFrame(rows)
+    return df
+
+# if __name__ == "__main__":
+#     task_id = "69afc81760ed587a34dfc57a"
+#     df = fetch_similar_tracks_df(task_id)
+#     print(df)
+#
+#     print(df.columns)
+#
+#
+#     print(df["name"].tolist())

all_combine_code/genreclassification/viberate_api_report_generator2.py

@@ -0,0 +1,205 @@
+import os, re, logging
+import pandas as pd
+import numpy as np
+import aiohttp
+import asyncio
+from datetime import datetime, timedelta
+from concurrent.futures import ThreadPoolExecutor
+from dotenv import load_dotenv
+
+load_dotenv()
+
+pd.set_option('display.max_columns', None)
+
+api_token = os.getenv("VIBERATE_API")
+date_to = datetime.today().date()
+date_from = date_to - timedelta(days=365)
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+from songs_report_generator1 import fetch_similar_tracks_df
+
+
+def similarity_songs_report(task_id):
+    df = fetch_similar_tracks_df(task_id)
+    if df.empty:
+        return {"message": "No similar tracks found"}, None
+    return df["name"].tolist(), df
+
+
+class ViberateAPI:
+    """
+    All file I/O is scoped to `session_dir` so concurrent requests
+    never collide on the filesystem.
+    """
+
+    def __init__(self, session_dir: str):
+        self.uuid_cache = {}
+        self.base_folder = os.path.join(session_dir, "songs_data")
+        os.makedirs(self.base_folder, exist_ok=True)
+
+    async def get_uuid_from_song_name(self, session, song_name):
+        if song_name in self.uuid_cache:
+            return self.uuid_cache[song_name]
+        try:
+            url = "https://data.viberate.com/api/v1/track/search"
+            params = {"q": song_name, "limit": 5, "offset": 0}
+            headers = {"Access-Key": api_token}
+            async with session.get(url, params=params, headers=headers) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    if not data or "data" not in data or not isinstance(data["data"], list) or not data["data"]:
+                        return {"error": "Empty or invalid response"}
+                    first_track = data["data"][0]
+                    uuid = first_track["uuid"]
+                    artist_name = first_track.get("artist_name", "Unknown Artist")
+                    self.uuid_cache[song_name] = {"uuid": uuid, "name": artist_name}
+                    return {"uuid": uuid, "name": artist_name}
+                return {"error": "Invalid response"}
+        except Exception as e:
+            return {"error": str(e)}
+
+    def is_valid_uuid(self, uuid_string):
+        uuid_pattern = re.compile(
+            r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+        )
+        return uuid_string if uuid_pattern.fullmatch(uuid_string) else "UUID NOT VALID"
+
+    async def fetch_platform_data(self, session, uuid, artist_name, platform):
+        try:
+            url_map = {
+                "spotify":    "https://data.viberate.com/api/v1/track/{uuid}/spotify/streams-historical",
+                "soundcloud": "https://data.viberate.com/api/v1/track/{uuid}/soundcloud/plays-historical",
+                "youtube":    "https://data.viberate.com/api/v1/track/{uuid}/youtube/views-historical",
+                "beatport":   "https://data.viberate.com/api/v1/track/{uuid}/beatport/ranks-historical",
+            }
+            column_name_map = {
+                "spotify":    "spotify_streams",
+                "soundcloud": "soundcloud_streams",
+                "youtube":    "youtube_streams",
+                "beatport":   "beatport_streams",
+            }
+            column_name = column_name_map.get(platform, f"{platform}_streams")
+
+            if self.is_valid_uuid(uuid) == "UUID NOT VALID":
+                return {"error": "UUID NOT VALID"}, artist_name
+
+            url = url_map[platform].format(uuid=uuid)
+            headers = {"Accept": "application/json", "Access-Key": api_token}
+            params = {"date-from": date_from.isoformat(), "date-to": date_to.isoformat()}
+
+            async with session.get(url, headers=headers, params=params) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    if data and "data" in data and "data" in data["data"]:
+                        streams_data = data["data"]["data"]
+                        df = pd.DataFrame(list(streams_data.items()), columns=["date", column_name])
+                        df["date"] = pd.to_datetime(df["date"])
+                        return df, artist_name
+                    return pd.DataFrame(columns=["date", column_name]), artist_name
+                return {"error": "Invalid response"}, artist_name
+        except Exception as e:
+            return {"error": str(e)}, artist_name
+
+    async def final_result_from_song(self, song_name):
+        try:
+            async with aiohttp.ClientSession() as session:
+                uuid_data = await self.get_uuid_from_song_name(session, song_name)
+                if "error" in uuid_data:
+                    return {"error": uuid_data["error"]}
+
+                uuid = uuid_data["uuid"]
+                artist_name = uuid_data["name"]
+                platforms = ["spotify", "youtube", "soundcloud"]
+                tasks = [
+                    self.fetch_platform_data(session, uuid, artist_name, p)
+                    for p in platforms
+                ]
+                results = await asyncio.gather(*tasks, return_exceptions=True)
+
+                final_output = None
+                for result, _ in results:
+                    if isinstance(result, dict) and "error" in result:
+                        return {"error": result["error"]}
+                    if isinstance(result, pd.DataFrame):
+                        if final_output is None:
+                            final_output = result
+                        else:
+                            final_output = final_output.merge(result, on='date', how='outer')
+
+                if final_output is None:
+                    return {"error": "No data retrieved"}
+
+                return final_output.sort_values(by='date').reset_index(drop=True)
+        except Exception as e:
+            return {"error": str(e)}
+
+    def process_song(self, song_name):
+        try:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            df = loop.run_until_complete(self.final_result_from_song(song_name))
+            loop.close()
+
+            if isinstance(df, dict) and "error" in df:
+                return song_name, df
+
+            safe_song_name = re.sub(r'[\\/*?:"<>|]', "", song_name)
+            csv_path = os.path.join(self.base_folder, f"{safe_song_name}.csv")
+            df.to_csv(csv_path, index=False)
+            return song_name, {"success": csv_path}
+        except Exception as e:
+            return song_name, {"error": str(e)}
+
+
+def combine_csv_to_excel(songs_list: list, session_dir: str) -> str:
+    """
+    Downloads per-song CSVs into `session_dir/songs_data/` and writes
+    `session_dir/songs_data_combined.xlsx`.  Returns the xlsx path on
+    success or an error string.
+    """
+    try:
+        api = ViberateAPI(session_dir=session_dir)
+
+        if not songs_list:
+            return "No songs to process"
+
+        results = list(
+            ThreadPoolExecutor(max_workers=5).map(api.process_song, songs_list)
+        )
+
+        errors = [
+            f"Error for {song}: {res['error']}"
+            for song, res in results if "error" in res
+        ]
+        if errors:
+            logger.error("\n".join(errors))
+
+        successes = [res["success"] for _, res in results if "success" in res]
+        if not successes:
+            return "No CSV files generated"
+
+        excel_path = os.path.join(session_dir, "songs_data_combined.xlsx")
+        with pd.ExcelWriter(excel_path, engine='openpyxl') as writer:
+            for csv_file in successes:
+                df = pd.read_csv(csv_file, encoding='utf-8')
+                if df.empty:
+                    continue
+                sheet_name = os.path.splitext(os.path.basename(csv_file))[0][:31]
+                df.to_excel(writer, sheet_name=sheet_name, index=False)
+
+        return excel_path          # ← return path instead of a bare string
+
+    except Exception as e:
+        logger.error(f"combine_csv_to_excel failed: {e}")
+        return f"Process failed: {str(e)}"
+
+
+def potential_fanbase_count(songs_list: list, session_dir: str):
+    """Returns the xlsx path on success or an error string."""
+    try:
+        os.makedirs(session_dir, exist_ok=True)
+        return combine_csv_to_excel(songs_list, session_dir)
+    except Exception as e:
+        return f"Process failed: {str(e)}"

all_combine_code/local_model_to_s3/bucket_inside_file_delete.py

@@ -0,0 +1,30 @@
+import os
+import boto3
+from dotenv import load_dotenv
+
+load_dotenv()
+
+AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY")
+AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_KEY")
+AWS_REGION = os.getenv("AWS_REGION")
+
+BUCKET_NAME = "delvant-ai-all-models"
+
+s3 = boto3.client(
+    "s3",
+    aws_access_key_id=AWS_ACCESS_KEY_ID,
+    aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+    region_name=AWS_REGION
+)
+
+# List and delete all objects
+def delete_all_objects(bucket):
+    paginator = s3.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=bucket):
+        if "Contents" in page:
+            for obj in page["Contents"]:
+                print(f"Deleting {obj['Key']}")
+                s3.delete_object(Bucket=bucket, Key=obj["Key"])
+    print("✅ All objects deleted")
+
+delete_all_objects(BUCKET_NAME)

all_combine_code/local_model_to_s3/local_model_to_s3_bucket.py

@@ -0,0 +1,28 @@
+from dotenv import load_dotenv
+load_dotenv()
+import os
+import boto3
+
+
+AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY")
+AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_KEY")
+AWS_REGION = os.getenv("AWS_REGION")
+
+s3 = boto3.client(
+    "s3",
+    aws_access_key_id=AWS_ACCESS_KEY_ID,
+    aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+    region_name=AWS_REGION
+)
+
+folder = r"C:\Users\VigneshSubramani\PycharmProjects\delveant-ai-crm\all-MiniLM-L6-v2.pt"
+bucket = "delvant-ai-all-models"
+
+for root, dirs, files in os.walk(folder):
+    for file in files:
+        full_path = os.path.join(root, file)
+        s3_path = os.path.relpath(full_path, folder)
+
+        s3.upload_file(full_path, bucket, s3_path)
+
+print("Upload complete 🚀")

all_combine_code/memory_/chat_history_handle.py

@@ -0,0 +1,194 @@
+import os,json,re
+import tempfile
+import shutil
+from uuid import uuid4
+from typing import Dict, List
+import boto3
+import tiktoken
+from datetime import datetime
+from dotenv import load_dotenv
+
+from langchain_core.messages import HumanMessage, AIMessage, SystemMessage
+from models_.delveant_llm_model_.multi_agent_llm import ChatSynthesizer
+from .prompt_agents import chat_history
+
+load_dotenv()
+
+AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY")
+AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_KEY")
+AWS_REGION = os.getenv("AWS_REGION")
+
+S3_BUCKET = os.getenv("AI_CHAT_HISTORY_S3")
+S3_BASE_PREFIX = "long_term_memory"
+
+MAX_HISTORY = 25
+MAX_FOLDERS = int(os.getenv("HISTORY_MAX_FOLDER_COUNT"))
+
+ENCODING = tiktoken.get_encoding("o200k_base")
+
+s3 = boto3.client(
+    "s3",
+    aws_access_key_id=AWS_ACCESS_KEY_ID,
+    aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+    region_name=AWS_REGION
+)
+
+user_sessions: Dict[str, List] = {}
+
+
+
+def truncate_to_tokens(text: str, max_tokens: int = 50) -> str:
+    if not text:
+        return ""
+    if not isinstance(text, str):
+        text = json.dumps(text, ensure_ascii=False)
+    tokens = ENCODING.encode(text)
+    return ENCODING.decode(tokens[:max_tokens])
+
+
+async def llm(prompt):
+    llm = ChatSynthesizer()
+    return await llm.chat_synthesizer(prompt)
+
+
+def messages_s3_key(user_id: str):
+    return f"{S3_BASE_PREFIX}/{user_id}/messages.json"
+
+
+def new_session_id():
+    return str(uuid4())
+
+
+def get_user_messages(user_id: str):
+    if user_id not in user_sessions:
+        user_sessions[user_id] = [SystemMessage(content="You are a helpful AI assistant.")]
+        load_messages_from_s3(user_id)
+    return user_sessions[user_id]
+
+
+def trim_history(messages):
+    system_msg = messages[0]
+    history = messages[-MAX_HISTORY:]
+    return [system_msg] + history
+
+
+def save_messages_to_s3(user_id: str):
+    temp = tempfile.mkdtemp()
+    path = os.path.join(temp, "messages.json")
+
+    user_sessions[user_id] = trim_history(user_sessions[user_id])
+
+    with open(path, "w") as f:
+        json.dump(
+            [{"type": m.__class__.__name__, "content": m.content} for m in user_sessions[user_id]],
+            f
+        )
+
+    s3.upload_file(path, S3_BUCKET, messages_s3_key(user_id))
+    shutil.rmtree(temp)
+
+
+def load_messages_from_s3(user_id: str):
+    try:
+        temp = tempfile.mkdtemp()
+        path = os.path.join(temp, "messages.json")
+
+        s3.download_file(S3_BUCKET, messages_s3_key(user_id), path)
+
+        with open(path) as f:
+            data = json.load(f)
+
+        for m in data[1:]:
+            if m["type"] == "HumanMessage":
+                user_sessions[user_id].append(HumanMessage(content=m["content"]))
+            elif m["type"] == "AIMessage":
+                user_sessions[user_id].append(AIMessage(content=m["content"]))
+
+        shutil.rmtree(temp)
+    except:
+        pass
+
+
+def delete_prefix(prefix: str):
+    paginator = s3.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=S3_BUCKET, Prefix=prefix):
+        if "Contents" not in page:
+            continue
+        s3.delete_objects(
+            Bucket=S3_BUCKET,
+            Delete={"Objects": [{"Key": obj["Key"]} for obj in page["Contents"]]}
+        )
+
+
+def cleanup_old_user_folders():
+    paginator = s3.get_paginator("list_objects_v2")
+
+    folders = []
+    for page in paginator.paginate(Bucket=S3_BUCKET, Prefix=f"{S3_BASE_PREFIX}/", Delimiter="/"):
+        for p in page.get("CommonPrefixes", []):
+            prefix = p["Prefix"]
+            resp = s3.list_objects_v2(Bucket=S3_BUCKET, Prefix=prefix)
+            if "Contents" in resp:
+                last_modified = max(obj["LastModified"] for obj in resp["Contents"])
+                folders.append((last_modified, prefix))
+
+    folders.sort(reverse=True)
+
+    for _, prefix in folders[MAX_FOLDERS:]:
+        delete_prefix(prefix)
+
+
+def delete_user_history(user_id: str):
+    delete_prefix(f"{S3_BASE_PREFIX}/{user_id}/")
+    user_sessions.pop(user_id, None)
+
+def create_runtime_user_id() -> str:
+    return f"{uuid4()}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+
+async def generate_response(user_input: str, user_id: str):
+    try:
+        if user_id == "new_chat":
+            cleanup_old_user_folders()
+            user_id = create_runtime_user_id()
+
+        elif re.fullmatch(r"[a-f0-9\-]{36}_\d{8}_\d{6}", user_id, re.IGNORECASE):
+            pass
+
+        else:
+            cleanup_old_user_folders()
+            user_id = create_runtime_user_id()
+
+    except Exception:
+        cleanup_old_user_folders()
+        user_id = create_runtime_user_id()
+
+    messages = get_user_messages(user_id)
+    messages.append(HumanMessage(content=user_input))
+    user_sessions[user_id] = trim_history(messages)
+    #print( "trim messages:" ,user_sessions[user_id])
+    prompt = chat_history(user_input, user_sessions[user_id])
+    response_content = await llm(prompt)
+    return {"user_id": user_id, "chat_history_query": response_content}
+
+
+async def store_final_chat(user_id: str, user_input: str, final_reply: str):
+    messages = get_user_messages(user_id)
+    truncated_reply = truncate_to_tokens(final_reply)
+    messages.append(HumanMessage(content=user_input))
+    messages.append(AIMessage(content=truncated_reply))
+    user_sessions[user_id] = trim_history(messages)
+    save_messages_to_s3(user_id)
+
+
+# async def chat():#
+#     user_id = "703af383-6b70-4b68-979c-f005d7ff866f"#
+#     while True:
+#         user_input = input("You: ")#
+#         if user_input.lower() == "exit":
+#             break#
+#         res = await generate_response(user_input, user_id)#
+#         user_id = res["user_id"]
+#         reply = res["chat_history_query"]#
+#         print("Bot:", reply)#
+#         await store_final_chat(user_id, user_input, reply)#
+# asyncio.run(chat())

all_combine_code/memory_/prompt_agents.py

@@ -0,0 +1,82 @@
+from textwrap import dedent
+def chat_history(user_input, history):
+    prompt = dedent(f"""
+You are a 20+ award-winning elite Conversational Intent Synthesizer and Question Generated Architect.
+
+You are given:
+- The full previous conversation history
+- The current user input
+
+---
+
+## PREVIOUS CONVERSATION HISTORY:
+{history}
+
+
+## CURRENT USER INPUT:
+{user_input}
+
+---
+
+Your task is to generate exactly ONE precise, fully-qualified, executable command-style query that completely and accurately captures the user's active intent by intelligently synthesizing the current input with all relevant prior conversation context, without omission, paraphrasing of intent verbs, explanation, or additional commentary.
+
+**Core behavior:**
+
+1. Deeply understand the `user input` and all Conversional history inside intent. Accurately find relation user input and previous chat history to make 100% required response. 
+2. If the current user input does NOT introduce a clear new intent, assume it continues the previous active intent.
+3. Extract and accumulate all relevant entities and details across conversation turns (names, companies, attributes, requirements, conditions, constraints, etc.).
+4. Merge the active intent and all accumulated entities into a single, complete command-style query.
+5. Analyze the current user query in the context of previous conversation history to fully understand their relationship and intent before generating a response.
+
+
+**Intent handling rules:**
+
+- If the user clearly provides a new intent, discard the previous intent and start fresh.
+- If the user input only provides details, corrections, or values, apply them to the existing intent.
+- Do NOT drop previously collected information unless the user explicitly corrects or removes it.
+- **If the user query contains explicit indicators like this is "new chat", "new query", "new question", or similar phrases, treat it as a completely new intent query and do NOT use Previous Chat History Query.**
+    eg. total lead count new chat ( or use new chat/ not consider previous chat history ot this similar)
+    your answer : total lead count? 
+
+**Output rules:**
+
+- Preserve the original intent wording as much as possible.
+- Do NOT paraphrase intent verbs.
+- Do NOT generate explanations, summaries, or conversational responses.
+- Do NOT ask questions.
+- Do NOT output multiple options.
+- Do NOT include system text or reasoning.
+- Return ONLY the final synthesized query.
+
+**Greeting and small-talk handling:**
+
+- If the user input is a greeting, acknowledgement, or small talk
+  (e.g., hi, hello, hey, good morning, thanks, ok),
+  treat it as a NEW intent.
+- In this case, return the greeting exactly as provided.
+- Do NOT reuse or continue any previous intent.
+
+**Special cases:**
+
+- If the current input is already a complete standalone query, return it as-is.
+- If the current input is unrelated to the previous conversation, treat it as a new intent.
+- If names or entities were previously changed or corrected, always use the latest valid value.  
+
+**Very strict rule:**
+- Before whole chat history and current message clearly understand to make the smart and professional query format.  
+
+
+Your output must always be a single, professional, command-style query that reflects the full conversation context.
+
+*(NOTE: query inside any spell mistake, please correct it.)*
+
+### **VERY IMPORTANT NOTES:**
+
+- Query inside mention fresh or new not consider history, this similar consider that is **new one query**.
+- Query inside "name" or "mail" mention time return same mail and name. Never assume field name like (firstname, lastname, email 1).
+- Strictly  Follow this : always prioritize the most recent/last user message while accurately synthesizing it with relevant prior conversation context, unless explicitly instructed to treat it as a new query.
+- Strictly avoid the SQL/MySQL format query.
+
+Return ONLY the final query with plain/text.
+""")
+    return prompt

all_combine_code/models_/code_generation_model.py

@@ -0,0 +1,90 @@
+import os, re, sys, json,httpx
+from openai import AsyncOpenAI
+from dotenv import load_dotenv
+import asyncio
+load_dotenv()
+
+client = AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY"),
+                     timeout=240.0)
+
+class CodeGenerationLLM:
+    async def generate_response(self,prompt: str) -> str:
+        full_text = []
+        response = await client.chat.completions.create(
+            model="gpt-5-chat-latest",
+            temperature=0.1,
+            messages=[
+                {"role": "system", "content": "You are an expert in PyMongo query generation with valid JSON format."},
+                {"role": "user", "content": prompt}
+            ],
+            stream=True,
+            max_tokens=2500,
+            #max_completion_tokens=3800
+            response_format={"type": "json_object"}
+        )
+        async for chunk in response:
+            delta = chunk.choices[0].delta
+            if delta and delta.content:
+                token = delta.content
+                print(token, end="", flush=True)
+                full_text.append(token)
+        return "".join(full_text).strip()
+
+    async def generate_final_output(self,prompt) -> str:
+        llm_output = await self.generate_response(prompt)
+        return llm_output
+
+
+class OpenRouterClient:
+    def __init__(self):
+        load_dotenv()
+
+        self.api_key = os.getenv("OPENROUTER_API_KEY")
+
+        self.url = "https://openrouter.ai/api/v1/chat/completions"
+
+        self.headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    async def stream_chat(self, prompt):
+        payload = {
+            "model": "mistralai/codestral-2508", # anthropic/claude-sonnet-4.6 ,mistralai/codestral-2508
+            "stream": True,
+            "max_tokens": 3000,
+            "temperature": 0.1,
+            "messages": [
+                {"role": "system", "content": "You are an expert in PyMongo query generation with valid JSON format."},
+                {"role": "user", "content": prompt}
+            ]
+        }
+
+        full_text = ""
+
+        async with httpx.AsyncClient(timeout=60) as client:
+            async with client.stream(
+                    "POST",
+                    self.url,
+                    headers=self.headers,
+                    json=payload
+            ) as r:
+
+                async for line in r.aiter_lines():
+
+                    if not line or not line.startswith("data:"):
+                        continue
+
+                    data = line.removeprefix("data: ").strip()
+
+                    if data == "[DONE]":
+                        break
+
+                    chunk = json.loads(data)
+                    delta = chunk["choices"][0]["delta"].get("content")
+
+                    if delta:
+                        print(delta, end="", flush=True)
+                        full_text += delta
+
+        return full_text

all_combine_code/models_/delveant_llm_model_/multi_agent_llm.py

@@ -0,0 +1,264 @@
+import os, re, sys, json, gc
+import asyncio,logging,random,time
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+load_dotenv()
+
+from textwrap import dedent
+LLM_SEMAPHORE = asyncio.Semaphore(8)
+OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
+OPENROUTER_URL = os.getenv("OPENROUTER_URL")
+REQUEST_TIMEOUT = 40.0
+
+client = AsyncOpenAI(
+    #base_url=OPENROUTER_URL,
+    api_key=os.getenv("OPENAI_API_KEY"),
+    timeout=120,
+)
+
+
+class MultiAgentLLM:
+    async def general_questions_answer(self,prompt):
+        response = await client.chat.completions.create(
+            model="gpt-3.5-turbo",
+            messages=[
+                {"role": "system", "content": "You are very helful AI assistant.Following Questions understand to make the smart response"},
+                {"role": "user", "content": prompt}
+            ],
+        )
+        message = response.choices[0].message.content
+        return message
+
+
+    async def generate_ai_response_greetings(self,user_prompt) -> str:
+        async with LLM_SEMAPHORE:
+            for attempt in range(3):
+                try:
+                    system_prompt = dedent("""
+                    You are DelveAnt CRM’s AI assistant developed by kovaion.ai.
+
+                    Rules:
+                    - Keep responses short, professional, and helpful.
+                    - If the query mentions "Delvant CRM" or asks about the product,
+                      give a brief 2–3 line overview of DelveAnt CRM, its purpose, and key benefits.
+                    - Do not give long explanations.
+                    - Do not sound casual or chatty.
+                    - Always use correct spelling: "DelveAnt".
+                    """).strip()
+                    response = await client.chat.completions.create(
+                        model="gpt-3.5-turbo",
+                        messages=[
+                            {"role": "system", "content":system_prompt},
+                            {"role": "user", "content": user_prompt},
+                        ])
+                    return response.choices[0].message.content
+
+                except Exception as e:
+                    logging.info(f"LLM attempt {attempt + 1} failed: {e}")
+                    if attempt == 2:
+                        raise
+                    await asyncio.sleep(2 ** attempt + random.uniform(0.2, 0.6))
+
+    async def delveant_chatbot(self,user_prompt) -> str:
+        async with LLM_SEMAPHORE:
+            for attempt in range(3):
+                try:
+                    system_prompt = dedent("""
+                    You are the owner and official AI assistant of DelveAnt CRM.
+                    Rules:
+                    - Answer the user question briefly and professionally.
+                    - Maximum 3 short sentences.
+                    - Do NOT include lists, steps, or examples.
+                    - Do NOT use line breaks.
+                    - End the response with a full stop.
+                    - Always use correct spelling: "DelveAnt".
+                     """).strip()
+                    response = await client.chat.completions.create(
+                        model="gpt-4o-mini",
+                        messages=[
+                            {"role": "system", "content": system_prompt},
+                            {"role": "user", "content": user_prompt},
+                        ],
+                        temperature=0.0,
+                        max_tokens=150,
+                    )
+                    return response.choices[0].message.content
+
+                except Exception as e:
+                    logging.info(f"LLM attempt {attempt + 1} failed: {e}")
+                    if attempt == 2:
+                        raise
+                    await asyncio.sleep(2 ** attempt + random.uniform(0.2, 0.6))
+
+
+    async def plot_finder(self,user_prompt) -> str:
+        async with LLM_SEMAPHORE:
+            for attempt in range(3):
+                try:
+                    system_prompt = "You are very highly Rule based **Decision Find Agent**."
+                    response = await client.chat.completions.create(
+                        model="gpt-5-chat-latest",
+                        messages=[
+                            {"role": "system", "content": system_prompt},
+                            {"role": "user", "content": user_prompt},
+                        ],
+                        temperature=0.0,
+                        max_tokens=150,
+                    )
+                    return response.choices[0].message.content
+
+                except Exception as e:
+                    logging.info(f"LLM attempt {attempt + 1} failed: {e}")
+                    if attempt == 2:
+                        raise
+                    await asyncio.sleep(2 ** attempt + random.uniform(0.2, 0.6))
+
+    async def empty_list_responser(self,user_prompt) -> str:
+        async with LLM_SEMAPHORE:
+            prompt = dedent(f"""
+            # You are a professional and well-structured responder. small or medium format text not long enough.
+
+            **User Input:**  
+            - {user_prompt}
+
+            **MongoDB Query Output:**  
+            []
+
+            **System Context:**  
+            - The MongoDB query for the above user input returned an empty list ("[]").  
+            - This means that no matching data was found in the database for the given query.
+
+            **Your Task:**  
+            - Provide a clear, professional, and informative response to the user **based on the query and the empty result**.  
+            - Clearly explain that no data was found for the requested query.         
+            - If the user’s query is unclear or incomplete, politely suggest a refined version of the query or possible next steps.  
+            - Keep the response concise, neutral, and easy to understand.  
+            - Do **not** include any code, technical details, or system-related text in your final response.  
+            - Your response must directly relate to the user’s query and the fact that the result was empty. 
+            - Your response inside don't mention "Unfortunately" this word.          
+             """)
+            for attempt in range(3):
+                try:
+                    response = await client.chat.completions.create(
+                        model="gpt-4o-mini",
+                        messages=[
+                            {"role": "system", "content": "You are a helpful AI assistant."},
+                            {"role": "user", "content": prompt},
+                        ],
+                        temperature=0.0,
+                        max_tokens=100,
+                    )
+                    return response.choices[0].message.content
+
+                except Exception as e:
+                    logging.info(f"LLM attempt {attempt + 1} failed: {e}")
+                    if attempt == 2:
+                        raise
+                    await asyncio.sleep(2 ** attempt + random.uniform(0.2, 0.6))
+
+
+class CodeErrorFixer:
+    def __init__(self):
+        self.system_prompt = dedent("""
+You are a senior Python engineer and runtime error fixer.
+Your job is to:
+1. Analyze the error
+2. Identify the root cause
+3. Fix the error.
+4. Return corrected code.
+5. Do not add explanations unless asked
+6. Return corrected python code.
+""")
+
+    async def code_debuger(self,prompt: str):
+        full_text = []
+        stream = await client.chat.completions.create(
+            model="gpt-4.1-2025-04-14",
+            messages=[
+                {"role": "system", "content": self.system_prompt},
+                {"role": "user", "content": prompt}
+            ],
+            max_tokens=4000,
+            stream=True,
+        )
+        async for chunk in stream:
+            delta = chunk.choices[0].delta
+            if delta and delta.content:
+                token = delta.content
+                print(token, end="", flush=True)
+                full_text.append(token)
+        return "".join(full_text)
+
+
+class ChatSynthesizer:
+    def __init__(self):
+        self.model = "gpt-5.1"
+    async def chat_synthesizer(self,user_prompt) -> str:
+        async with LLM_SEMAPHORE:
+            for attempt in range(3):
+                try:
+                    response = await client.chat.completions.create(
+                        model=self.model,
+                        messages=[
+                            {"role": "system", "content":"You are 20+ award-winning Intelligent Conversational Intent Synthesizer"},
+                            {"role": "user", "content": user_prompt},
+                        ],
+                        temperature=0,
+                        top_p=1,
+                        #reasoning_effort=None,
+                        max_completion_tokens=300,
+                    )
+                    return response.choices[0].message.content
+
+                except Exception as e:
+                    logging.info(f"LLM attempt {attempt + 1} failed: {e}")
+                    if attempt == 2:
+                        raise
+                    await asyncio.sleep(2 ** attempt + random.uniform(0.2, 0.6))
+
+class CollectionAndQueryFinder:
+    def __init__(self):
+        self.model = "gpt-5-chat-latest"
+        self.system_prompt = dedent("""
+        You are 20+ years award winning Accurate CRM Query Formatter and Database Collection Name Finder Agent
+        """)
+
+    async def generate_response(self,prompt,agents,user_id,person_id,collection_keywords,priveous_chat_history_query) -> str:
+        async with LLM_SEMAPHORE:
+            for attempt in range(3):
+                try:
+                    formatted_prompt = agents(prompt, user_id, person_id, collection_keywords,
+                                              priveous_chat_history_query)
+                    response = await client.chat.completions.create(
+                        model=self.model,
+                        messages=[
+                            {"role": "system", "content":self.system_prompt},
+                            {"role": "user", "content": formatted_prompt},
+                        ],
+                        temperature=0.1,
+                        max_tokens=300,
+                        #reasoning_effort="none"
+                    )
+                    return response.choices[0].message.content
+
+                except Exception as e:
+                    logging.info(f"LLM attempt {attempt + 1} failed: {e}")
+                    if attempt == 2:
+                        raise
+                    await asyncio.sleep(2 ** attempt + random.uniform(0.2, 0.6))
+
+
+# async def main():
+#     chat = ChatSynthesizer()
+#     user_prompt = "what is java?"
+#
+#     try:
+#         response = await chat.chat_synthesizer(user_prompt)
+#         print("LLM Response:", response)
+#     except Exception as e:
+#         print("Error:", e)
+#
+#
+# # Run the async main function
+# if __name__ == "__main__":
+#     asyncio.run(main())

all_combine_code/models_/delveant_llm_model_/tool_calling_model.py

@@ -0,0 +1,117 @@
+import warnings
+warnings.filterwarnings("ignore")
+
+import os
+import json
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+from agents_.delveant_prompts.tool_clasification_agent import TOOLS, tool_classification_agent
+load_dotenv()
+
+OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
+OPENROUTER_BASE_URL = os.getenv("OPENROUTER_URL")
+
+
+client = AsyncOpenAI(
+    api_key=OPENROUTER_API_KEY,
+    base_url=OPENROUTER_BASE_URL,
+    default_headers={"X-Title": "CRM-Tool-Agent"},
+    timeout=30.0
+)
+
+async def classify_and_call_tool(user_query):
+    response = await client.chat.completions.create(
+        model="anthropic/claude-sonnet-4.6",
+        messages=[
+            {"role": "system", "content": tool_classification_agent()},
+            {"role": "user", "content": user_query},
+        ],
+        tools=TOOLS,
+        tool_choice="auto",
+        temperature=0.1,
+    )
+    return response
+
+
+def parse_tool_response(response):
+    message = response.choices[0].message
+    if hasattr(message, "tool_calls") and message.tool_calls:
+        tool_calls = []
+        for call in message.tool_calls:
+            args = call.function.arguments
+            if isinstance(args, str):
+                try:
+                    args1 = json.loads(args)
+                except json.JSONDecodeError:
+                    args1 = {}
+
+            tool_calls.append({
+                "tool_name": call.function.name,
+                "arguments": args,
+            })
+        return tool_calls, None
+    return None, message.content
+
+
+async def tool_finder_llm1(previous_chat_history_query):
+    try:
+        response = await classify_and_call_tool(previous_chat_history_query)
+        tool_calls, content = parse_tool_response(response)
+        if tool_calls:
+            for call in tool_calls:
+                tool_name = call["tool_name"]
+                tool_args = call["arguments"]
+                return tool_name, tool_args
+
+        else:
+            if content:
+                return "no_tool_name", content
+
+            else:
+                return "no_tool_name", "no_tool_args"
+
+    except Exception as e:
+        return "no_tool_name", "Our techinical team working this related process, update soon."
+
+from openai import AsyncOpenAI
+from dotenv import load_dotenv
+load_dotenv()
+import os
+
+client = AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+async def tool_finder_llm(previous_chat_history_query):
+    response = await client.chat.completions.create(
+        model="gpt-5.2",
+        messages=[
+            {"role": "system", "content": tool_classification_agent()},
+            {"role": "user", "content": previous_chat_history_query}
+        ],
+        tools=TOOLS,
+        tool_choice="auto",
+        #temperature=0.0,
+        timeout=120.0,
+        reasoning_effort='none',
+    )
+    message = response.choices[0].message
+    if message.tool_calls and len(message.tool_calls) > 0:
+        tool_call = message.tool_calls[0]
+        tool_name = tool_call.function.name
+        tool_args = tool_call.function.arguments
+        return tool_name, tool_args
+    else:
+        content = message.content
+        if content:
+            return "no_tool_name", content
+
+        else:
+            return "no_tool_name", "no_tool_args"
+
+
+# async def main():
+#     query = "create lead with email 'vicky@gmail.com', IBM"
+#     tool_name, tool_args = await tool_finder_llm(query)
+#     print("Tool:", tool_name)
+#     print("Arguments:", tool_args)
+#
+# asyncio.run(main())

all_combine_code/mongodb_data_chat/delveant_chatbot.py

@@ -0,0 +1,152 @@
+import os
+import sys
+
+from pymongo import MongoClient
+from dotenv import load_dotenv
+
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(project_root)
+
+from tools_.delveant_all_tools_.crm_data_extraction import (extract_crm_lead)
+
+from models_.code_generation_model import CodeGenerationLLM
+from models_.delveant_llm_model_.multi_agent_llm import CollectionAndQueryFinder
+
+load_dotenv()
+
+MONGO_URI = os.getenv("MONGO_URI")
+mongo_db_client = MongoClient(MONGO_URI)
+
+class Delveant:
+    def __init__(self):
+        self.mongo_db_client = mongo_db_client
+
+    def extract_keywords(self,db_name):
+        db = self.mongo_db_client[db_name]
+        crm_leads_collection = db["CRM_AICollection_Keys"]
+        first_two = list(crm_leads_collection.find().sort([("_id", 1)]).limit(2))
+        last_two = list(crm_leads_collection.find().sort([("_id", -1)]).limit(2))
+        documents = first_two + last_two
+        formatted_output = []
+        for doc in documents:
+            for item in doc.get("keywords", []):
+                collection_name = item.get("collection_name", "")
+                keywords = item.get("keywords", [])
+
+                formatted_output.append({
+                    "collection_name": collection_name,
+                    "keywords": keywords
+                })
+        return formatted_output
+
+    def extract_lead_data_(self,input_message,collection_name,db_name,user_id,perso_id):
+        documents, field_name_datatypes, status, source = extract_crm_lead(
+            collection_name=collection_name,
+            db_name=db_name,
+        )
+        prompt = mongodb_query_generator_leads(input_message,documents,user_id,
+                                               field_name_datatypes,db_name,collection_name,source)
+        return prompt
+
+    def resolve_date_variables(self,final_json) -> dict:
+        python_code = final_json.get("python_code")
+        if not python_code:
+            return final_json
+        local_vars = {}
+        exec(python_code, {"datetime": datetime, "timedelta": timedelta, "timezone": timezone}, local_vars)
+
+        def replace_vars(obj):
+            if isinstance(obj, dict):
+                return {k: replace_vars(v) for k, v in obj.items()}
+            elif isinstance(obj, list):
+                return [replace_vars(i) for i in obj]
+            elif isinstance(obj, str) and obj in local_vars:
+                return local_vars[obj]
+            return obj
+
+        final_json["filter"] = replace_vars(final_json.get("filter", {}))
+        final_json["pipeline"] = replace_vars(final_json.get("pipeline", []))
+        return final_json
+
+    async def final_response_executer(self,prompt):
+        try:
+            answer = generate_response(prompt)
+            final_json = json.loads(answer)
+            final_json = resolve_date_variables(final_json)
+
+            pipeline = final_json.get("pipeline", [])
+            filter_ = final_json.get("filter", {})
+            operation = final_json.get("operation", "find")
+            projection = final_json.get("projection", {})
+            sort = final_json.get("sort", {})
+            limit = final_json.get("limit", 0)
+
+            if operation.lower() == "find":
+                cursor = crm_leads_collection.find(filter_, projection)
+                if sort:
+                    cursor = cursor.sort(list(sort.items()))
+                if limit:
+                    cursor = cursor.limit(limit)
+                results = list(cursor)
+
+            elif operation.lower() == "aggregate":
+                if sort and not any("$sort" in s for s in pipeline):
+                    pipeline.append({"$sort": sort})
+                if limit and not any("$limit" in s for s in pipeline):
+                    pipeline.append({"$limit": limit})
+                results = list(crm_leads_collection.aggregate(pipeline))
+
+            else:
+                print("Unknown operation:", operation)
+                results = []
+
+            json_data = json.loads(json_util.dumps(results))
+            cleaned_documents = []
+            for doc in json_data:
+                cleaned_doc = {
+                    k: v
+                    for k, v in doc.items()
+                    if v not in (None, '') and k != "_id"
+                }
+                if cleaned_doc:
+                    cleaned_documents.append(cleaned_doc)
+            return cleaned_documents
+
+        except Exception as e:
+            return "error"
+
+
+        async def generate_response(self,prompt,model="openai"):
+            if model == "openai":
+                print("openai model call")
+                llm = CodeGenerationLLM()
+                response = await llm.generate_final_output(prompt)
+
+            if model == "openrouter":
+                print("openrouter model call")
+                llm = OpenRouterClient()
+                response = await llm.stream_chat(prompt)
+
+            return response
+
+    async def collection_query_finader(self,prompt,agents,user_id,person_id,collection_keywords,priveous_chat_history_query):
+        llm = CollectionAndQueryFinder()
+        return await llm.generate_response(prompt,agents,user_id,person_id,collection_keywords,priveous_chat_history_query)
+
+    async def db_and_collection(self,db_name,collection_name):
+        db = self.mongo_db_client[db_name]
+        collection_name= db[collection_name]
+        return db, collection_name
+
+    async def final_output_extractor_and_responser(self, prompt,priveous_chat_history_query ,agents,db_name,user_id,person_id):
+        pass
+
+    async def run_file(self,user_input,priveous_chat_history_query,db_name, user_id, person_id):
+        answer =  await self.final_output_extractor_and_responser(
+            user_input,priveous_chat_history_query, db_collection_find, db_name, user_id, person_id
+        )
+
+
+
+
+

all_combine_code/recommedation_engine/app.py

@@ -0,0 +1,85 @@
+import os
+import uuid
+import asyncio
+from typing import List
+from fastapi import FastAPI, UploadFile, File, HTTPException
+from dotenv import load_dotenv
+
+from binary_classifier.classify import GeminiSportsClassifier
+from embeddings_ import process_inputs
+from s3_file_uploader import upload_to_s3
+
+load_dotenv()
+
+app = FastAPI(title="Sports Classification API")
+classifier = GeminiSportsClassifier()
+
+UPLOAD_DIR = "temp_uploads"
+os.makedirs(UPLOAD_DIR, exist_ok=True)
+
+@app.post("/classify")
+async def classify_media(files: List[UploadFile] = File(...)):
+    if not files:
+        raise HTTPException(status_code=400, detail="No files uploaded")
+    file_paths = []
+    path_to_original = {}
+    try:
+        for file in files:
+            ext = os.path.splitext(file.filename)[1]
+            temp_name = f"{uuid.uuid4()}{ext}"
+            temp_path = os.path.join(UPLOAD_DIR, temp_name)
+            with open(temp_path, "wb") as f:
+                content = await file.read()
+                f.write(content)
+            file_paths.append(temp_path)
+            path_to_original[temp_path] = file.filename
+        first_level_results = await classifier.classify_batch(file_paths)
+        allowed = {"not_sports", "error"}
+        filtered_items = []
+
+        for item in first_level_results:
+            if item.get("answer") not in allowed:
+                path = item["file"]
+                file_name = path_to_original.get(path)
+                ext = os.path.splitext(path)[1].lower()
+
+                media_type = "video" if ext in [".mp4", ".avi", ".mov"] else "image"
+
+                filtered_items.append({
+                    "file_path": path,
+                    "file_name": file_name,
+                    "media_type": media_type,
+                    "answer": item.get("answer")
+                })
+
+        if not filtered_items:
+            return []
+
+        embed_paths = [item["file_path"] for item in filtered_items]
+        embedding_result = process_inputs(embed_paths)
+        image_embeddings = embedding_result.get("images", [])
+        video_embeddings = embedding_result.get("videos", [])
+        img_idx = 0
+        vid_idx = 0
+
+        for item in filtered_items:
+            if item["media_type"] == "image":
+                item["embedding"] = image_embeddings[img_idx].tolist() if len(image_embeddings) > 0 else None
+                img_idx += 1
+            else:
+                item["embedding"] = video_embeddings[vid_idx].tolist() if len(video_embeddings) > 0 else None
+                vid_idx += 1
+
+        async def upload_single(item):
+            url = await upload_to_s3(item["file_path"], item["file_name"])
+            item["s3_url"] = url
+            return item
+
+        uploaded_results = await asyncio.gather(*[upload_single(item) for item in filtered_items])
+
+        return uploaded_results
+
+    finally:
+        for path in file_paths:
+            if os.path.exists(path):
+                os.remove(path)

all_combine_code/recommedation_engine/binary_classifier/classify.py

@@ -0,0 +1,100 @@
+import os, cv2,json
+import asyncio
+from PIL import Image
+from google import genai
+from google.genai import types
+from textwrap import dedent
+
+from dotenv import load_dotenv
+load_dotenv()
+
+client = genai.Client(api_key=os.getenv("GEMINI_API_KEY"))
+
+class GeminiSportsClassifier:
+    def __init__(self):
+        self.categories = [
+            "a photo of a marathon race on city roads", "a photo of long distance marathon runners with bib numbers",
+            "a photo of a large group running marathon event", "a photo of Olympic marathon race",
+            "a photo of sprint running race on track", "a photo of Olympic track and field athletics",
+            "a photo of relay race baton passing", "a photo of hurdles race on track",
+            "a photo of high jump Olympic event",
+            "a photo of pole vault Olympic event", "a photo of long jump Olympic event",
+            "a photo of javelin throw Olympic event",
+            "a photo of discus throw Olympic event", "a photo of shot put Olympic event",
+            "a photo of Olympic swimming competition",
+            "a photo of Olympic cycling road race", "a photo of Olympic track cycling",
+            "a photo of Olympic gymnastics performance",
+            "a photo of Olympic boxing match", "a photo of Olympic wrestling match",
+            "a photo of Olympic weightlifting competition",
+            "a photo of cricket match on stadium", "a photo of cricket players batting and bowling",
+            "a photo of football match in stadium",
+            "a photo of soccer players playing match", "a photo of football goal scoring moment",
+            "a photo of basketball game in indoor court",
+            "a photo of volleyball match on court", "a photo of tennis match on court",
+            "a photo of badminton match indoor stadium"
+        ]
+    def _prepare_image(self, img):
+        img.thumbnail((512, 512))
+        return img
+
+    def _extract_frames(self, video_path, num_frames=2):
+        cap = cv2.VideoCapture(video_path)
+        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        frames = []
+        for i in range(num_frames):
+            pos = int((i + 1) * total_frames / (num_frames + 1))
+            cap.set(cv2.CAP_PROP_POS_FRAMES, pos)
+            ret, frame = cap.read()
+            if ret:
+                frame = cv2.resize(frame, (480, 270))
+                img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+                frames.append(img)
+        cap.release()
+        return frames
+
+    async def classify(self, file_path):
+        ext = os.path.splitext(file_path)[1].lower()
+        try:
+            media = []
+            if ext in ['.jpg', '.jpeg', '.png', ".webp"]:
+                media.append(self._prepare_image(Image.open(file_path)))
+            elif ext in ['.mp4', '.avi', '.mov']:
+                media = self._extract_frames(file_path)
+            else:
+                return {"file": file_path, "answer": "not_sports"}
+
+            prompt = dedent(f"""
+        Analyze the provided media (image or video frames).
+        1. If it matches one of these categories, return the category: {self.categories}
+        2. If it is not related to these sports, return "not_sports".
+        Output strictly in JSON: {{"answer": "category_name"}}
+        """)
+
+            response = await client.aio.models.generate_content(
+                model="gemini-2.0-flash",
+                contents=[prompt, *media],
+                config=types.GenerateContentConfig(
+                    response_mime_type='application/json'
+                )
+            )
+
+            data = json.loads(response.text)
+            if isinstance(data, dict):
+                return {"file": file_path, **data}
+            else:
+                return {"file": file_path, "answer": str(data)}
+
+        except Exception as e:
+            return {"file": file_path, "answer": "error", "details": str(e)}
+
+    async def classify_batch(self, file_paths, max_concurrent=5):
+        semaphore = asyncio.Semaphore(max_concurrent)
+
+        async def classify_with_limit(path):
+            async with semaphore:
+                return await self.classify(path)
+
+        tasks = [classify_with_limit(path) for path in file_paths]
+        return await asyncio.gather(*tasks)
+
+

all_combine_code/recommedation_engine/embeddings_.py

@@ -0,0 +1,98 @@
+import torch
+import clip
+import cv2
+from PIL import Image
+from concurrent.futures import ThreadPoolExecutor
+
+device = "cpu"
+
+model, preprocess = clip.load("ViT-B/16", device=device)
+model.eval()
+
+SUPPORTED_IMAGE_EXTS = (".jpg", ".jpeg", ".png")
+SUPPORTED_VIDEO_EXTS = (".mp4", ".avi", ".mov")
+
+def preprocess_image(path):
+    img = Image.open(path).convert("RGB")
+    return preprocess(img)
+
+def load_images_parallel(image_paths, max_workers=4):
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        images = list(executor.map(preprocess_image, image_paths))
+    return torch.stack(images)
+
+def images_to_embeddings_cpu(image_paths, batch_size=32):
+    outputs = []
+
+    for i in range(0, len(image_paths), batch_size):
+        batch_paths = image_paths[i:i + batch_size]
+        images = load_images_parallel(batch_paths)
+
+        with torch.no_grad():
+            emb = model.encode_image(images)
+            emb = emb / emb.norm(dim=-1, keepdim=True)
+        outputs.append(emb)
+    return torch.cat(outputs).numpy()
+
+def extract_frames(video_path, sample_rate=1):
+    cap = cv2.VideoCapture(video_path)
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    interval = int(max(1, fps * sample_rate))
+
+    frames = []
+    count = 0
+
+    while cap.isOpened():
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        if count % interval == 0:
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            frames.append(Image.fromarray(frame))
+
+        count += 1
+
+    cap.release()
+    return frames
+
+
+def video_to_embedding_cpu(video_path):
+    frames = extract_frames(video_path)
+
+    if not frames:
+        return None
+
+    images = torch.stack([preprocess(f) for f in frames])
+
+    with torch.no_grad():
+        emb = model.encode_image(images)
+        emb = emb / emb.norm(dim=-1, keepdim=True)
+
+    return emb.mean(dim=0).numpy()
+
+
+def process_videos_parallel(video_files, max_workers=2):
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        return list(executor.map(video_to_embedding_cpu, video_files))
+
+
+def process_inputs(files):
+    image_files = []
+    video_files = []
+
+    for f in files:
+        f_lower = f.lower()
+        if f_lower.endswith(SUPPORTED_IMAGE_EXTS):
+            image_files.append(f)
+        elif f_lower.endswith(SUPPORTED_VIDEO_EXTS):
+            video_files.append(f)
+
+    results = {}
+    if image_files:
+        results["images"] = images_to_embeddings_cpu(image_files)
+    if video_files:
+        results["videos"] = process_videos_parallel(video_files)
+    return results
+
+