Data Summary

app.py

@@ -15,11 +15,15 @@ warnings.filterwarnings("ignore", category=DeprecationWarning)
 
 # Height of the Tabs Text Area
 TAB_LINES = 8
-# Load Token
+
+
+#----------CONNECT TO DATABASE----------
 md_token = os.getenv('MD_TOKEN')
-# Connect to DB
 conn = duckdb.connect(f"md:my_db?motherduck_token={md_token}", read_only=True)
+#---------------------------------------
+
 
+#-------LOAD HUGGINGFACE-------
 models = ["Qwen/Qwen2.5-72B-Instruct","meta-llama/Meta-Llama-3-70B-Instruct",
           "meta-llama/Llama-3.1-70B-Instruct"]
 
@@ -35,13 +39,13 @@ for model in models:
       continue
   
 llm = ChatHuggingFace(llm=endpoint).bind_tools(tools=[], max_tokens=8192)
+#---------------------------------------
 
-
-
+#-----LOAD PROMPT FROM LANCHAIN HUB-----
 prompt_autogenerate = hub.pull("autogenerate-rules-testworkflow")
 prompt_user_input = hub.pull("usergenerate-rules-testworkflow")
 
-
+#--------------ALL UTILS----------------
 # Get Databases
 def get_schemas():
     schemas = conn.execute("""
@@ -67,10 +71,18 @@ def get_data_df(schema):
 
 
 def format_prompt(df):
-  return prompt_autogenerate.format_prompt(data=df.head().to_json(orient='records'))
-def format_user_prompt(df, user_description):
-  return prompt_user_input.format_prompt(data=df.head(2).to_json(orient='records'), user_description=user_description)
-
+    summary_df = pd.DataFrame({
+        "max": df.max(),
+        "min": df.min(),
+        "top": df.mode().iloc[0],
+        "nunique": df.nunique(),
+        "count": df.count(),
+        "dtype": df.dtypes.astype(str)
+        }).reset_index().rename(columns={"index": "column"})
+    return prompt_autogenerate.format_prompt(data=df.head().to_json(orient='records'),
+                                           summary=summary_df.to_json(orient='records'))
+def format_user_prompt(df):
+    return prompt_user_input.format_prompt(data=df.head().to_json(orient='records'))
 
 def process_inputs(inputs) :
     print(inputs)
@@ -167,6 +179,10 @@ def statistics(df):
     df_alerts = pd.DataFrame(alerts_list, columns=['Data Quality Issue', 'Category'])
 
     return df_statistics, df_alerts
+#---------------------------------------
+
+
+
 # Main Function
 def main(table):
     schema = get_table_schema(table)