modify(1)

.gitignore

@@ -1,4 +1,5 @@
 .env
+.venv/
 __pycache__/
 .pytest_cache/
 *.pyc

src/app.py

@@ -2,13 +2,14 @@ import streamlit as st
 import os
 from services.queryService import QService
 from services.llm_client import LLMClient
+from services.get_file_status import check_lead_existance
 from sentence_transformers import SentenceTransformer
 from post_extraction_tools import (
     website_adder,
     clean_json,
     lead_scoring,
     data_quality_enhancer,
-    chart_data
+    chart_data,
 )
 from services.add_leads import add_leads_f
 import json
@@ -25,13 +26,16 @@ def load_model():
     os.environ["SENTENCE_TRANSFORMERS_HOME"] = "/tmp/huggingface/sentence_transformers"
 
     os.makedirs("/tmp/huggingface", exist_ok=True)
-    return SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2", cache_folder="/tmp/huggingface")
+    return SentenceTransformer(
+        "sentence-transformers/all-MiniLM-L6-v2", cache_folder="/tmp/huggingface"
+    )
 
 
 embedder = load_model()
 lead_scorer = lead_scoring.LeadScoring(llm, embedder)
 
 st.set_page_config(page_title="Caprae Capital Lead Generation Tool", layout="wide")
+main_lead_info_file = "src/data/all_cleaned_companies.json"
 # st.title("Lead Management Dashboard")
 # This is the navigation section
 if "page" not in st.session_state:
@@ -62,12 +66,13 @@ with st.sidebar:
             st.session_state.page = page_name
 
 if st.session_state.page == "Dashboard":
-    main_lead_info_file = "data/all_cleaned_companies.json"
     st.header("Welcome!!")
     st.text("Here you will find all about your leads.")
     if st.session_state.data_enhancement == True:
-        if os.path.isdir(main_lead_info_file):
-            fig_ind, fig_coun, fig_btype, fig_rev = chart_data.create_chart("data/all_cleaned_companies.json")
+        if check_lead_existance(main_lead_info_file):
+            fig_ind, fig_coun, fig_btype, fig_rev = chart_data.create_chart(
+                main_lead_info_file
+            )
             col1, col2, col3 = st.columns(3)
 
             with col1:
@@ -81,30 +86,36 @@ if st.session_state.page == "Dashboard":
             with col3:
                 st.subheader("Business type-wise Distribution")
                 st.plotly_chart(fig_btype, use_container_width=True)
-            
+
             st.subheader("Revenue-based Distribution")
             st.plotly_chart(fig_rev, use_container_width=True)
         else:
-            st.subheader("Do Data Enhancement first in order to view the diagramatic details!!")
-    if os.path.isdir(main_lead_info_file):
-        df_display = chart_data.df_creator_from_json_and_process(main_lead_info_file).sort_values(by="score", ascending=False).rename(columns={
-            "company_name": "Company Name",
-            "key_industry": "Industry Type",
-            "industry_type": "Speciality",
-            "street": "Street",
-            "city": "City",
-            "state": "State",
-            "country": "Country",
-            "phone": "Phone",
-            "email": "Email",
-            "company_size": "Number of Employees",
-            "approx_revenue": "Revenue",
-            "business_type": "Business Type",
-            "website_url": "Website",
-            "country": "Country"
-        })
+            st.subheader(
+                "Do Data Enhancement first in order to view the diagramatic details!!"
+            )
+
+    if check_lead_existance(main_lead_info_file):
+        df_display = chart_data.df_creator_from_json_and_process(main_lead_info_file).sort_values(by="score", ascending=False).rename(
+            columns={
+                "company_name": "Company Name",
+                "key_industry": "Industry Type",
+                "industry_type": "Speciality",
+                "street": "Street",
+                "city": "City",
+                "state": "State",
+                "country": "Country",
+                "phone": "Phone",
+                "email": "Email",
+                "company_size": "Number of Employees",
+                "approx_revenue": "Revenue",
+                "business_type": "Business Type",
+                "website_url": "Website",
+                "country": "Country",
+            }
+        )
         st.subheader("All Company Details")
         st.dataframe(df_display)
+
     else:
         # st.subheader("All Company Details")
         st.write("_There are no leads yet.Go to Data Enrichment to create leads!!_")
@@ -151,9 +162,15 @@ if st.session_state.page == "Enrich Companies":
                     key="manual_entry_b",
                 )
                 if manual_entry_button:
-                    if not (st.session_state.data_enhancement and st.session_state.intelliscore):
-                        st.warning("Complete the Data Enhancement and Intelliscore Lead Scoring first!!")
+                    if (check_lead_existance(main_lead_info_file)) and not (
+                        st.session_state.data_enhancement
+                        and st.session_state.intelliscore
+                    ):
+                        st.warning(
+                            "Complete the Data Enhancement and Intelliscore Lead Scoring first!!"
+                        )
                         pass
+
                     else:
                         lead_data = {
                             "company_name": lead_name,
@@ -174,9 +191,9 @@ if st.session_state.page == "Enrich Companies":
                         cleaned_data = clean_json.clean_json_f(lead_data)
                         cleaned_data_obj = json.loads(cleaned_data)
                         cleaned_data_obj = add_leads_f(
-                            "data/all_cleaned_companies.json", cleaned_data_obj
+                            main_lead_info_file, cleaned_data_obj
                         )
-                        with open("data/all_cleaned_companies.json", "w") as f:
+                        with open(main_lead_info_file, "w") as f:
                             json.dump(cleaned_data_obj, f, indent=2)
 
                         print("Cleaned JSON saved to all_cleaned_companies.json")
@@ -192,7 +209,6 @@ if st.session_state.page == "Enrich Companies":
                         st.session_state.data_enhancement = False
                         st.session_state.intelliscore = False
                         st.session_state.lead_conditions = False
-                    
 
     with tab2:
         st.subheader("Advanced Intelligent Scrapper and Data Completion")
@@ -258,9 +274,9 @@ if st.session_state.page == "Enrich Companies":
                     cleaned_data = clean_json.clean_json_f(data)
                     cleaned_data_obj = json.loads(cleaned_data)
                     cleaned_data_obj = add_leads_f(
-                        "data/all_cleaned_companies.json", cleaned_data_obj
+                        main_lead_info_file, cleaned_data_obj
                     )
-                    with open("data/all_cleaned_companies.json", "w") as f:
+                    with open(main_lead_info_file, "w") as f:
                         json.dump(cleaned_data_obj, f, indent=2)
 
                     print("Cleaned JSON saved to all_cleaned_companies.json")
@@ -278,7 +294,7 @@ if st.session_state.page == "Enrich Companies":
                         final_data, embedder
                     )[0]
 
-                    with open("data/all_cleaned_companies.json", "w") as f:
+                    with open(main_lead_info_file, "w") as f:
                         json.dump(enhanced_data, f, indent=2)
 
                     print(
@@ -291,7 +307,7 @@ if st.session_state.page == "Enrich Companies":
                     res = lead_scorer.scrape_and_augment(
                         own_comp_info, own_comp_web_url
                     )
-                    with open("data/lead_conditions.json", "w") as f:
+                    with open("src/data/lead_conditions.json", "w") as f:
                         json.dump(res, f, indent=2)
 
                     scored_leads = lead_scorer.score(enhanced_data, res)
@@ -308,39 +324,47 @@ if st.session_state.page == "Enhance Data Quality":
     st.text("This tool uses embedding model to ensure clean and reliable data quality.")
     with st.container(border=True):
         st.subheader("Your Current Data")
-        with open("data/all_cleaned_companies.json", "r") as f:
-            temp_data = json.load(f)
-        temp_df = pd.DataFrame(temp_data.get("companies", []))
-        st.dataframe(temp_df)
-        col13, col14, col15 = st.columns([1, 1, 1])
-        with col14:
-            enhance_data_b = st.button(
-                "Enhance Data", type="primary", use_container_width=True
-            )
-            if enhance_data_b and st.session_state.data_enhancement == False and st.session_state.pipeline_executed == False:
-                with st.spinner("Enhancing the data..."):
-                    enhancer_output = data_quality_enhancer.enhancer(
-                        temp_data, embedder
-                    )
-                    enhanced_data, duplicate_comps = (
-                        enhancer_output[0],
-                        enhancer_output[1]["duplicate_company_names"],
-                    )
-                    st.success("Enhancement Completed!!")
+        if check_lead_existance(main_lead_info_file):
+            with open(main_lead_info_file, "r") as f:
+                temp_data = json.load(f)
+
+            temp_df = pd.DataFrame(temp_data.get("companies", []))
+            st.dataframe(temp_df)
+            col13, col14, col15 = st.columns([1, 1, 1])
+            with col14:
+                enhance_data_b = st.button(
+                    "Enhance Data", type="primary", use_container_width=True
+                )
+                if (
+                    enhance_data_b
+                    and st.session_state.data_enhancement == False
+                    and st.session_state.pipeline_executed == False
+                ):
+                    with st.spinner("Enhancing the data..."):
+                        enhancer_output = data_quality_enhancer.enhancer(
+                            temp_data, embedder
+                        )
+                        enhanced_data, duplicate_comps = (
+                            enhancer_output[0],
+                            enhancer_output[1]["duplicate_company_names"],
+                        )
+                        st.success("Enhancement Completed!!")
 
-                with open("data/all_cleaned_companies.json", "w") as f:
-                    json.dump(enhanced_data, f, indent=2)
-                if duplicate_comps == []:
-                    st.text("No Duplicate Entries Found!!")
-                else:
-                    st.text(f"Removed {len(duplicate_comps)} duplicate companies!!")
-                    st.text("Removed Companies: ")
-                    for c in duplicate_comps:
-                        st.text(c)
-                st.session_state.data_enhancement = True
+                    with open(main_lead_info_file, "w") as f:
+                        json.dump(enhanced_data, f, indent=2)
+                    if duplicate_comps == []:
+                        st.text("No Duplicate Entries Found!!")
+                    else:
+                        st.text(f"Removed {len(duplicate_comps)} duplicate companies!!")
+                        st.text("Removed Companies: ")
+                        for c in duplicate_comps:
+                            st.text(c)
+                    st.session_state.data_enhancement = True
 
-            elif enhance_data_b and st.session_state.data_enhancement == True:
-                st.text("Already Enhanced!!")
+                elif enhance_data_b and st.session_state.data_enhancement == True:
+                    st.text("Already Enhanced!!")
+        else:
+            st.warning("No Leads Found! Go to Enrichment tool to add leads.")
 
 if st.session_state.page == "IntelliSCORE":
     st.subheader("Advanced Lead Scoring Tool")
@@ -374,7 +398,7 @@ if st.session_state.page == "IntelliSCORE":
             )
             if intelliscore_b:
                 if st.session_state.data_enhancement == True:
-                    with open("data/all_cleaned_companies.json", "r") as f:
+                    with open(main_lead_info_file, "r") as f:
                         leads = json.load(f)
 
                     if ask_scrap_per == "yes" or ask_scrap_per == None:
@@ -382,13 +406,13 @@ if st.session_state.page == "IntelliSCORE":
                             res = lead_scorer.scrape_and_augment(
                                 additional_info, comp_url
                             )
-                            with open("data/lead_conditions.json", "w") as f:
+                            with open("src/data/lead_conditions.json", "w") as f:
                                 json.dump(res, f, indent=2)
                             st.success("Scrapping Completed!")
                             if res and "error" not in res:
                                 st.session_state.lead_conditions = True
 
-                        with open("data/lead_conditions.json", "r") as f:
+                        with open("src/data/lead_conditions.json", "r") as f:
                             lead_cond = json.load(f)
                         with st.spinner("Scoring the leads..."):
                             scored_leads = lead_scorer.score(leads, lead_cond)
@@ -399,7 +423,7 @@ if st.session_state.page == "IntelliSCORE":
 
                     else:
                         st.text("Skipping url scrapping...")
-                        with open("data/lead_conditions.json", "r") as f:
+                        with open("src/data/lead_conditions.json", "r") as f:
                             lead_cond = json.load(f)
                         with st.spinner("Scoring the leads..."):
                             scored_leads = lead_scorer.score(leads, lead_cond)
@@ -409,4 +433,4 @@ if st.session_state.page == "IntelliSCORE":
                         st.session_state.intelliscore = True
 
                 else:
-                    st.warning("Complete the Data Enhancement first!!")
+                    st.warning("Complete the Data Enhancement first!!")

src/data/all_cleaned_companies.json

@@ -1,191 +0,0 @@
-{
-  "companies": [
-    {
-      "company_name": "Sprout Social",
-      "industry_type": "Social Media Management Software",
-      "location": "Chicago, Illinois, USA",
-      "company_size": "1001",
-      "street": "131 S Dearborn St",
-      "city": "Chicago",
-      "state": "IL",
-      "country": "USA",
-      "phone": "(312) 593-3600",
-      "email": null,
-      "approx_revenue": "$340.2 million",
-      "business_type": "B2B",
-      "website_url": "https://sproutsocial.com/",
-      "score": 2.16,
-      "key_industry": "Software & SaaS"
-    },
-    {
-      "company_name": "Oracle",
-      "industry_type": "Software Development, Cloud Computing, Database Technology",
-      "location": "Bangalore, India",
-      "company_size": "170000",
-      "street": "Bagmane Tech Park, Outer Ring Road, Doddanekundi Village, Mahadevapura",
-      "city": "Bangalore",
-      "state": "Karnataka",
-      "country": "India",
-      "phone": "91-80-41070000",
-      "email": null,
-      "approx_revenue": "$53 billion",
-      "business_type": "B2B",
-      "website_url": "https://www.oracle.com/",
-      "key_industry": "Cloud Computing & DevOps",
-      "score": 2.4699999999999998
-    },
-    {
-      "company_name": "Microsoft",
-      "industry_type": "Software Development, Cloud Computing, Operating Systems",
-      "location": "Bangalore, India",
-      "company_size": "221000",
-      "street": "Microsoft Signature Building, Embassy Golf Links Business Park, Off Intermediate Ring Road, Domlur",
-      "city": "Bangalore",
-      "state": "Karnataka",
-      "country": "India",
-      "phone": "080 4010 3000",
-      "email": null,
-      "approx_revenue": "$211.9 billion",
-      "business_type": "B2B",
-      "website_url": "https://www.microsoft.com/en-in",
-      "key_industry": "Cloud Computing & DevOps",
-      "score": 3.54
-    },
-    {
-      "company_name": "Accenture",
-      "industry_type": "IT Services, Consulting, Software Development",
-      "location": "Bangalore, India",
-      "company_size": "733000",
-      "street": "1, Old Madras Road, Bagmane Constellation Business Park, Doddanekundi, Marathahalli",
-      "city": "Bangalore",
-      "state": "Karnataka",
-      "country": "India",
-      "phone": "91-80-62150000",
-      "email": null,
-      "approx_revenue": "$64.1 billion",
-      "business_type": "B2B",
-      "website_url": "https://www.accenture.com/in-en",
-      "key_industry": "Consulting & Business Services",
-      "score": 2.52
-    },
-    {
-      "company_name": "Infosys",
-      "industry_type": "IT Services, Business Consulting, Software Development, Outsourcing",
-      "location": "Bangalore, India",
-      "company_size": "317000",
-      "street": "Plot No. 44 and 97A, Electronics City, Hosur Road",
-      "city": "Bangalore",
-      "state": "Karnataka",
-      "country": "India",
-      "phone": "080 2852 0261",
-      "email": null,
-      "approx_revenue": "$18.2 billion",
-      "business_type": "B2B",
-      "website_url": "https://www.infosys.com/",
-      "key_industry": "Consulting & Business Services",
-      "score": 2.52
-    },
-    {
-      "company_name": "Wipro",
-      "industry_type": "IT Services, Consulting, Business Process Services, Software Development",
-      "location": "Bangalore, India",
-      "company_size": "240000",
-      "street": "Doddakannelli, Sarjapur Road",
-      "city": "Bangalore",
-      "state": "Karnataka",
-      "country": "India",
-      "phone": "080 2844 0011",
-      "email": null,
-      "approx_revenue": "$10.8 billion",
-      "business_type": "B2B",
-      "website_url": "https://www.wipro.com/",
-      "key_industry": "Consulting & Business Services",
-      "score": 2.49
-    },
-    {
-      "company_name": "The Kraft Heinz Company",
-      "industry_type": "Food and Beverage",
-      "location": "Chicago, Illinois, USA",
-      "company_size": "36000",
-      "street": "Aon Center, 200 E Randolph St",
-      "city": "Chicago",
-      "state": "IL",
-      "country": "USA",
-      "phone": null,
-      "email": null,
-      "approx_revenue": "$26 billion",
-      "business_type": "Both",
-      "website_url": "https://www.kraftheinzcompany.com/",
-      "key_industry": "Food & Beverages",
-      "score": 2.2
-    },
-    {
-      "company_name": "Mondel\u00c4\u201cz International",
-      "industry_type": "Food and Beverage",
-      "location": "Chicago, Illinois, USA",
-      "company_size": "91000",
-      "street": "905 W Fulton Market",
-      "city": "Chicago",
-      "state": "IL",
-      "country": "USA",
-      "phone": "+1 847 943 4000",
-      "email": null,
-      "approx_revenue": "$36.01 billion",
-      "business_type": "Both",
-      "website_url": "https://www.mondelezinternational.com/",
-      "key_industry": "Food & Beverages",
-      "score": 2.2
-    },
-    {
-      "company_name": "US Foods",
-      "industry_type": "Foodservice Distributor",
-      "location": "Rosemont, Illinois, USA",
-      "company_size": "29000",
-      "street": "9399 W. Higgins Road, Suite 100",
-      "city": "Rosemont",
-      "state": "IL",
-      "country": "USA",
-      "phone": "847-720-8000",
-      "email": null,
-      "approx_revenue": "$35 billion",
-      "business_type": "B2B",
-      "website_url": "https://www.usfoods.com/",
-      "key_industry": "Logistics, Supply Chain & Warehousing",
-      "score": 3.1799999999999997
-    },
-    {
-      "company_name": "The Quaker Oats Company",
-      "industry_type": "Food and Beverage",
-      "location": "Chicago, Illinois, USA",
-      "company_size": "2500",
-      "street": "433 W Van Buren St",
-      "city": "Chicago",
-      "state": "IL",
-      "country": "USA",
-      "phone": "+1 312 821 1000",
-      "email": null,
-      "approx_revenue": "$2.8 billion",
-      "business_type": "Both",
-      "website_url": "https://www.quakeroats.com/",
-      "key_industry": "Food & Beverages",
-      "score": 2.2
-    },
-    {
-      "company_name": "Conagra Brands, Inc.",
-      "industry_type": "Food and Beverage",
-      "location": "Chicago, Illinois, USA",
-      "company_size": "18000",
-      "street": "222 W Merchandise Mart Plaza, 13th Fl",
-      "city": "Chicago",
-      "state": "IL",
-      "country": "USA",
-      "phone": "+1 312 549 5000",
-      "email": null,
-      "approx_revenue": "$12.28 billion",
-      "business_type": "Both",
-      "website_url": "https://www.conagrabrands.com/",
-      "key_industry": "Food & Beverages",
-      "score": 2.2
-    }
-  ]
-}

src/post_extraction_tools/chart_data.py

@@ -21,7 +21,9 @@ def parse_revenue(rev_str):
 def df_creator_from_json_and_process(filepath: str):
     with open(filepath, "r") as f:
         data = json.load(f)["companies"]
-        
+    for c in data:
+        if "score" not in c:
+            c["score"] = None
     return pd.DataFrame(data)
 
 def create_chart(filepath: str):

src/post_extraction_tools/data_quality_enhancer.py

@@ -124,7 +124,7 @@ def enhancer(data: object, embedder) -> list:
 
 
 def add_ind_key(data: list, embedder) -> list:
-    with open("data/key_industry_embeddings.json", "r") as f:
+    with open("src/data/key_industry_embeddings.json", "r") as f:
         key_ind_embs = json.load(f)["industry_embeddings"]
     for c in data:
         if "key_industry" not in c:

src/services/get_file_status.py

@@ -0,0 +1,13 @@
+import os
+import json
+
+def check_lead_existance(filepath: str)->bool:
+    if os.path.isfile(filepath) and os.path.getsize(filepath):
+        with open(filepath, 'r') as f:
+            data = json.load(f)["companies"]
+        if len(data) > 0:
+            return True
+        else:
+            return False
+    else:
+        return False