Update s3/meta_data_creation_from_s3.py

s3/meta_data_creation_from_s3.py

@@ -1,167 +1,168 @@
-import os
-import io
-import json
-import datetime
-import pandas as pd
-from langchain.chat_models import ChatOpenAI
-from langchain.prompts import PromptTemplate
-import boto3
-from io import StringIO
-from botocore.exceptions import ClientError
-import dotenv
-import os
-import sys
-
-from pathlib import Path
-from typing import List
-PROJECT_ROOT = Path(__file__).resolve().parents[1]
-if str(PROJECT_ROOT) not in sys.path:
-    sys.path.insert(0, str(PROJECT_ROOT))
-from retrieve_secret import *
-
-# Load environment variables from .env file if present
-dotenv.load_dotenv()
-
-# Get the OPENAI_API_KEY from the environment variable
-# print(f"API_KEY: {OPENAI_API_KEY}")
-
-# Set your OpenAI API Key (better: use Colab "Secrets" to avoid hardcoding)
-
-
-
-# === CONFIG ===
-ENDPOINT_URL = "https://s3.us-west-1.idrivee2.com"
-ACCESS_KEY = "rNuPBAQetemqpEeBospZ"
-SECRET_KEY = "BU4FccUYxzXVqiWjPSJM1CWEX1cNhBqbU9NeGidE"
-BUCKET_NAME = "accusagas3"  # replace with your actual bucket
-
-# === INITIALIZE CLIENT ===
-s3 = boto3.client(
-    "s3",
-    endpoint_url=ENDPOINT_URL,
-    aws_access_key_id=ACCESS_KEY,
-    aws_secret_access_key=SECRET_KEY,
-)
-
-
-def read_csv_from_s3(path: str) -> pd.DataFrame:
-    """
-    Reads a CSV file from iDrive e2 (S3) directly into a Pandas DataFrame.
-
-    Args:
-        path (str): The S3 key/path, e.g. "vatsav_123/reports/Gold Futures Historical Data.csv"
-
-    Returns:
-        pd.DataFrame: Loaded DataFrame from the CSV
-    """
-    try:
-        # Fetch object from bucket
-        response = s3.get_object(Bucket=BUCKET_NAME, Key=path)
-
-        # Read the content
-        csv_data = response["Body"].read().decode("utf-8")
-
-        # Convert to DataFrame
-        df = pd.read_csv(StringIO(csv_data))
-
-        print(f"✅ Successfully loaded {path} into DataFrame")
-        return df
-
-    except ClientError as e:
-        if e.response["Error"]["Code"] == "NoSuchKey":
-            print(f"❌ File not found in S3: {path}")
-        else:
-            print(f"❌ S3 error: {e}")
-        return pd.DataFrame()  # return empty dataframe on failure
-    except Exception as e:
-        print(f"❌ Unexpected error: {e}")
-        return pd.DataFrame()
-
-
-def create_file_metadata_from_df(df: pd.DataFrame, file_name: str, file_path: str) -> dict:
-    """
-    Generate metadata from a DataFrame (directly from S3).
-    """
-    # --- Basic File Info ---
-    file_type = "csv"
-    file_size = len(df)
-    created_date = datetime.datetime.now().strftime("%Y-%m-%d")
-    modified_date = created_date  # For this example, we use current date as a placeholder
-
-    # --- Extract Structured Info from DataFrame ---
-    all_columns_list = df.columns.tolist()
-    data_types = {col: str(df[col].dtype) for col in df.columns}
-
-    sheet_info = [{
-        "sheet_name": "CSV",
-        "num_rows": len(df),
-        "num_columns": len(df.columns),
-        "columns": df.columns.tolist(),
-        "sample_data": df.head(3).to_dict(orient="records")
-    }]
-
-    # --- Text preview for LLM ---
-    file_preview = df.head(5).to_string()  # Preview the first 5 rows as a text snippet
-
-    # --- OpenAI LLM for summary + tags ---
-    llm = ChatOpenAI(model="gpt-4o-mini", openai_api_key=os.environ["OPENAI_API_KEY"])
-
-    prompt = PromptTemplate(
-        input_variables=["file_name", "file_type", "columns", "sheets", "preview"],
-        template="""
-You are a metadata extractor for data files.
-Given this file:
-- File Name: {file_name}
-- File Type: {file_type}
-- Sheets: {sheets}
-- Columns: {columns}
-- Preview Content: {preview}
-
-Generate strictly valid JSON with keys:
-- summary_text: A concise 2-3 sentence description of the file.
-- tags: 8-12 recommended metadata search keywords. These tags should be:
-  * Derived from the file name, sheet names, column names, and content
-  * Consider the inferred data domain (healthcare, finance, etc.)
-  * Focus on analysis use cases (e.g., "trends", "forecasting", "customer behavior")
-"""
-    )
-
-    ai_response = llm.predict(prompt.format(file_name=file_name, file_type=file_type, columns=all_columns_list, sheets=sheet_info, preview=file_preview))
-
-    # Try parsing the AI response to valid JSON
-    try:
-        ai_json = json.loads(ai_response)
-    except:
-        # In case the AI response is not valid JSON, fallback to simple text
-        ai_json = {"summary_text": ai_response.strip(), "tags": ["data", "file", "metadata"]}
-
-    # --- Build Final Metadata JSON ---
-    metadata = {
-        "file_name": file_name,
-        "file_path": file_path,
-        "file_type": file_type,
-        "file_size_bytes": file_size,
-        "num_sheets": len(sheet_info),
-        "sheets": sheet_info,
-        "all_columns_list": list(set(all_columns_list)),
-        "data_types": data_types,
-        "tags": ai_json.get("tags", []),
-        "summary_text": ai_json.get("summary_text", ""),
-        "created_date": created_date,
-        "modified_date": modified_date,
-        "creator": "system_user"
-    }
-
-    return metadata
-
-
-# Example usage:
-# path = "vatsav/csv/Gold Futures Historical Data.csv"
-# df = read_csv_from_s3(path)
-# file_name = "Gold Futures Historical Data.csv"
-# file_path = path  # In case you want to keep the S3 path
-
-# # Now we use the new function to generate metadata
-# metadata = create_file_metadata_from_df(df, file_name, file_path)
-# # print("+"*60)
-# print(json.dumps(metadata, indent=2))
+import os
+import io
+import json
+import datetime
+import pandas as pd
+from langchain_openai import OpenAIEmbeddings
+
+from langchain.prompts import PromptTemplate
+import boto3
+from io import StringIO
+from botocore.exceptions import ClientError
+import dotenv
+import os
+import sys
+
+from pathlib import Path
+from typing import List
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+from retrieve_secret import *
+
+# Load environment variables from .env file if present
+dotenv.load_dotenv()
+
+# Get the OPENAI_API_KEY from the environment variable
+# print(f"API_KEY: {OPENAI_API_KEY}")
+
+# Set your OpenAI API Key (better: use Colab "Secrets" to avoid hardcoding)
+
+
+
+# === CONFIG ===
+ENDPOINT_URL = "https://s3.us-west-1.idrivee2.com"
+ACCESS_KEY = "rNuPBAQetemqpEeBospZ"
+SECRET_KEY = "BU4FccUYxzXVqiWjPSJM1CWEX1cNhBqbU9NeGidE"
+BUCKET_NAME = "accusagas3"  # replace with your actual bucket
+
+# === INITIALIZE CLIENT ===
+s3 = boto3.client(
+    "s3",
+    endpoint_url=ENDPOINT_URL,
+    aws_access_key_id=ACCESS_KEY,
+    aws_secret_access_key=SECRET_KEY,
+)
+
+
+def read_csv_from_s3(path: str) -> pd.DataFrame:
+    """
+    Reads a CSV file from iDrive e2 (S3) directly into a Pandas DataFrame.
+
+    Args:
+        path (str): The S3 key/path, e.g. "vatsav_123/reports/Gold Futures Historical Data.csv"
+
+    Returns:
+        pd.DataFrame: Loaded DataFrame from the CSV
+    """
+    try:
+        # Fetch object from bucket
+        response = s3.get_object(Bucket=BUCKET_NAME, Key=path)
+
+        # Read the content
+        csv_data = response["Body"].read().decode("utf-8")
+
+        # Convert to DataFrame
+        df = pd.read_csv(StringIO(csv_data))
+
+        print(f"✅ Successfully loaded {path} into DataFrame")
+        return df
+
+    except ClientError as e:
+        if e.response["Error"]["Code"] == "NoSuchKey":
+            print(f"❌ File not found in S3: {path}")
+        else:
+            print(f"❌ S3 error: {e}")
+        return pd.DataFrame()  # return empty dataframe on failure
+    except Exception as e:
+        print(f"❌ Unexpected error: {e}")
+        return pd.DataFrame()
+
+
+def create_file_metadata_from_df(df: pd.DataFrame, file_name: str, file_path: str) -> dict:
+    """
+    Generate metadata from a DataFrame (directly from S3).
+    """
+    # --- Basic File Info ---
+    file_type = "csv"
+    file_size = len(df)
+    created_date = datetime.datetime.now().strftime("%Y-%m-%d")
+    modified_date = created_date  # For this example, we use current date as a placeholder
+
+    # --- Extract Structured Info from DataFrame ---
+    all_columns_list = df.columns.tolist()
+    data_types = {col: str(df[col].dtype) for col in df.columns}
+
+    sheet_info = [{
+        "sheet_name": "CSV",
+        "num_rows": len(df),
+        "num_columns": len(df.columns),
+        "columns": df.columns.tolist(),
+        "sample_data": df.head(3).to_dict(orient="records")
+    }]
+
+    # --- Text preview for LLM ---
+    file_preview = df.head(5).to_string()  # Preview the first 5 rows as a text snippet
+
+    # --- OpenAI LLM for summary + tags ---
+    llm = ChatOpenAI(model="gpt-4o-mini", openai_api_key=os.environ["OPENAI_API_KEY"])
+
+    prompt = PromptTemplate(
+        input_variables=["file_name", "file_type", "columns", "sheets", "preview"],
+        template="""
+You are a metadata extractor for data files.
+Given this file:
+- File Name: {file_name}
+- File Type: {file_type}
+- Sheets: {sheets}
+- Columns: {columns}
+- Preview Content: {preview}
+
+Generate strictly valid JSON with keys:
+- summary_text: A concise 2-3 sentence description of the file.
+- tags: 8-12 recommended metadata search keywords. These tags should be:
+  * Derived from the file name, sheet names, column names, and content
+  * Consider the inferred data domain (healthcare, finance, etc.)
+  * Focus on analysis use cases (e.g., "trends", "forecasting", "customer behavior")
+"""
+    )
+
+    ai_response = llm.predict(prompt.format(file_name=file_name, file_type=file_type, columns=all_columns_list, sheets=sheet_info, preview=file_preview))
+
+    # Try parsing the AI response to valid JSON
+    try:
+        ai_json = json.loads(ai_response)
+    except:
+        # In case the AI response is not valid JSON, fallback to simple text
+        ai_json = {"summary_text": ai_response.strip(), "tags": ["data", "file", "metadata"]}
+
+    # --- Build Final Metadata JSON ---
+    metadata = {
+        "file_name": file_name,
+        "file_path": file_path,
+        "file_type": file_type,
+        "file_size_bytes": file_size,
+        "num_sheets": len(sheet_info),
+        "sheets": sheet_info,
+        "all_columns_list": list(set(all_columns_list)),
+        "data_types": data_types,
+        "tags": ai_json.get("tags", []),
+        "summary_text": ai_json.get("summary_text", ""),
+        "created_date": created_date,
+        "modified_date": modified_date,
+        "creator": "system_user"
+    }
+
+    return metadata
+
+
+# Example usage:
+# path = "vatsav/csv/Gold Futures Historical Data.csv"
+# df = read_csv_from_s3(path)
+# file_name = "Gold Futures Historical Data.csv"
+# file_path = path  # In case you want to keep the S3 path
+
+# # Now we use the new function to generate metadata
+# metadata = create_file_metadata_from_df(df, file_name, file_path)
+# # print("+"*60)
+# print(json.dumps(metadata, indent=2))