ETL pipeline added

ETL pipeline now appears after pressing the Analyze button. Some stylistical changes to README

README.md

@@ -15,10 +15,13 @@
  </h2>
 
  <div align='center'>
- <a href="https://github.com/NexDatawork/data-agents/pulls"><img alt = "pull requests" src = "https://img.shields.io/github/issues-pr-closed/NexDatawork/data-agents?label=pull%20requests&labelColor=blue"/></a> 
- <a href="https://github.com/NexDatawork/data-agents/blob/main/LICENSE"><img alt = "LICENSE" src = "https://img.shields.io/badge/license-blue"/></a> 
- <a href = "https://discord.gg/Tb55tT5UtZ"><img src="https://img.shields.io/badge/Discord-Join%20Community-7289DA?logo=discord&logoColor=white" alt="Discord"></a>
+ <a href="https://github.com/NexDatawork/data-agents/pulls"><img alt = "pull requests" src = "https://img.shields.io/github/issues-pr-closed/NexDatawork/data-agents?label=pull%20requests&labelColor=rgba(56, 52, 182, 1)&color=rgb(90, 42, 184)"/></a> 
+ <a href="https://github.com/NexDatawork/data-agents/blob/main/LICENSE"><img alt = "LICENSE" src = "https://img.shields.io/badge/license-Apache%202.0-blueviolet?style=flat&color=rgb(90, 42, 184)&labelColor=rgba(56, 52, 182, 1)"/></a> 
+ <a href = "https://discord.gg/Tb55tT5UtZ"><img src="https://img.shields.io/badge/Discord-Join%20Community-7289DA?logo=discord&logoColor=white&color=rgb(90, 42, 184)&labelColor=rgba(56, 52, 182, 1)" alt="Discord"></a>
  <a href="https://github.com/NexDatawork/data-agents/stargazers"><img src="https://img.shields.io/github/stars/NexDatawork/data-agents?style=social" alt="GitHub Stars"></a>
+ <a href="https://huggingface.co/NexDatawork">
+  <img alt="Hugging Face" src="https://img.shields.io/badge/Hugging%20Face-Models%20%26%20Datasets?logo=huggingface&color=rgb(90, 42, 184)">
+</a>
 
 
  </div>

examples/data_agent_demo.ipynb

@@ -561,29 +561,39 @@
         "\n",
         "#llm is the agent that creates the etl pipeline\n",
         "#dataframe is a string with the name of the dataframe push through the etl process\n",
-        "def etl_pipeline(dataframe,history):\n",
-        "  tools = [preview_data, suggest_transformation, generate_python_code]\n",
-        "\n",
-        "  agent = initialize_agent(tools, model, agent='zero-shot-react-description',verbose=True)\n",
-        "\n",
-        "  input_prompt = f\"\"\"\n",
-        "  Preview the table {dataframe} and \\\n",
-        "      generate Python code to read the table, clean it, and finally write the \\\n",
-        "      dataframe into a table called {'Cleaned_'+dataframe}]. \\\n",
-        "      Do not stop the Python session\n",
-        "      \"\"\"\n",
-        "\n",
-        "  # Preview + suggest + generate code in a single run\n",
-        "  response = agent.run({\n",
-        "      \"input\": input_prompt,\n",
-        "      \"chat_history\": [],\n",
-        "      \"handle_parsing_errors\": True\n",
-        "  })\n",
-        "\n",
-        "  print(\"Generated Python Code:\\n\")\n",
-        "  print(response)\n",
-        "  response2 = response.strip('`').replace('python', '')\n",
-        "  return history + response2, response2\n"
+        "def etl_pipeline(dataframe,history=\"\"):\n",
+        "  print(\"=\"*10 + \"\\nETL_PIPELINE\\n\"+\"=\"*10)\n",
+        "\n",
+        "  try:\n",
+        "\n",
+        "      table_name = dataframe[0]\n",
+        "\n",
+        "\n",
+        "\n",
+        "      tools = [preview_data, suggest_transformation, generate_python_code]\n",
+        "\n",
+        "      agent = initialize_agent(tools, model, agent='zero-shot-react-description',verbose=True)\n",
+        "\n",
+        "      input_prompt = f\"\"\"\n",
+        "      Preview the table {table_name} and \\\n",
+        "          generate Python code to read the table, clean it, and finally write the \\\n",
+        "          dataframe into a table called {'Cleaned_'+table_name}]. \\\n",
+        "          Do not stop the Python session\n",
+        "          \"\"\"\n",
+        "\n",
+        "      # Preview + suggest + generate code in a single run\n",
+        "      response = agent.run({\n",
+        "          \"input\": input_prompt,\n",
+        "          \"chat_history\": [],\n",
+        "          \"handle_parsing_errors\": True\n",
+        "      })\n",
+        "\n",
+        "      print(\"Generated Python Code:\\n\")\n",
+        "      print(response)\n",
+        "      response2 = response.strip('`').replace('python', '')\n",
+        "      return history + response2, response2\n",
+        "  except Exception as e:\n",
+        "    return f'ETL pipeline error: {e}',f'ETL pipeline error: {e}'\n"
       ],
       "metadata": {
         "id": "n6sfgj8W4eAA"
@@ -730,13 +740,13 @@
         "\n",
         "    with gr.Column():\n",
         "\n",
-        "      result_display = gr.Markdown(label=\"📌 Report Output (Markdown)\")\n",
+        "      #result_display = gr.Markdown(label=\"📌 Report Output (Markdown)\")\n",
         "\n",
         "      with gr.Row():\n",
         "\n",
         "        trace_display = gr.Markdown(label=\"🛠️ Data Agent Reasoning - Your Explainable Agent\", elem_classes=[\"trace-markdown\"])\n",
         "\n",
-        "        sql_display = gr.Markdown(label='SQL Process')\n",
+        "        etl_display = gr.Markdown(label='ETL Process',elem_classes=[\"trace-markdown\"])\n",
         "\n",
         "\n",
         "      with gr.Row(equal_height=True):\n",
@@ -762,6 +772,7 @@
         "        scraping_button.click(fn=web_scraping,inputs=[question_input,history],outputs = [trace_display,history])\n",
         "\n",
         "        ask_button.click(fn=ask_agent,inputs=[file_input, question_input,history],outputs=[trace_display,history])\n",
+        "        ask_button.click(fn=etl_pipeline,inputs=[file_input ,history],outputs=[etl_display,history])\n",
         "\n",
         "demo.launch(share=True,debug=False)"
       ],