Upload 7 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+lgd_database.db filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,612 @@
+import base64
+import streamlit as st
+import pandas as pd
+from pg_utils_fn import create_entity_name_list, fetch_block_mapping, fetch_gp_mapping
+from pg_utils_fn import fetch_district_mapping
+from pg_utils_fn import populate_entity_mapping
+from pg_utils_fn import load_file
+from utils import update_all_data
+from pg_utils_fn import create_village_mapped_dataset, populate_village_mapping, process_file
+from pg_utils_fn import create_sub_district_mapped_dataset, populate_sub_district_mapping
+from pg_utils_fn import create_gp_mapped_dataset, populate_gp_mapping
+from pg_utils_fn import create_block_mapped_dataset, populate_block_mapping
+from pg_utils_fn import generate_download_link
+from pg_utils_fn import update_variations
+from pg_utils_fn import create_district_mapped_dataset
+from pg_utils_fn import create_mapped_dataset, get_state_mappings
+from databasefn import insert_record
+
+
+# Set Streamlit configuration
+st.set_option('deprecation.showfileUploaderEncoding', False)
+def home_page():
+
+    # Add an attractive header
+    st.title('CodeYatra')
+    #st.image('mandala.jpg')  # Replace 'your_logo.png' with the path to your logo image
+
+    # Write a brief and engaging introduction
+    st.write(
+        "Welcome to the CodeYatra LGD Mapping App! 🌟 This app makes mapping a breeze, allowing "
+        "you to explore states, districts, sub-districts, blocks, gram panchayats, and villages LGD Mapping with ease. "
+        "Whether you're a researcher 👩‍🔬, a data analyst 📊, or simply curious about India's geography, this app is "
+        "the perfect resource for you. Say goodbye to tedious manual searches and say hello to accurate and "
+        "up-to-date mapping at your fingertips. Let's embark on an exciting journey of exploration and "
+        "discovery with the CodeYatra LGD Code Mapping App! 🗺️🚀"
+    )
+
+
+    st.warning("⚠️ **Important Notice:** ")
+    st.write(" To ensure smooth operation of the application, please update the column names as follows:")
+    st.write("- State: Change the state column name to 'state_name'")
+    st.write("- District: Change the district column name to 'district_name'")
+    st.write("- Sub District: Change the sub district column name to 'sub_district_name'")
+    st.write("- Block: Change the block column name to 'block_name'")
+    st.write("- Gram Panchayat (GP): Change the gp column name to 'gp_name'")
+    st.write("- Village: Change the village column name to 'village_name'")
+
+
+
+    # Add visually appealing images or illustrations to showcase the app's features
+    #st.image('your_app_features.png', use_column_width=True)  # Replace 'your_app_features.png' with the path to your app features image
+
+    st.subheader('Start Mapping Now')
+    st.write('Ready to explore and map LGD codes? Click the button below to get started.')
+    
+
+    if st.button('Start Mapping', key='sub_mapping_button'):
+        #redirect_to_subset_dataset_page()
+        redirect_to_state_mapping_page()
+
+
+def redirect_to_update_dataset_page():
+    # Modify the URL parameters to navigate to the state mapping page
+    state_mapping_url = st.experimental_set_query_params(page='update')
+    page_route()
+    st.experimental_rerun()
+def redirect_to_subset_dataset_page():
+    # Modify the URL parameters to navigate to the state mapping page
+    state_mapping_url = st.experimental_set_query_params(page='subset')
+    page_route()
+    st.experimental_rerun()
+def redirect_to_state_mapping_page():
+    # Modify the URL parameters to navigate to the state mapping page
+    state_mapping_url = st.experimental_set_query_params(page='state')
+    page_route()
+    st.experimental_rerun()
+
+def redirect_to_district_page():
+    # Modify the URL parameters to navigate to the district page
+    district_url = st.experimental_set_query_params(page='district')
+    page_route()
+    st.experimental_rerun()
+def redirect_to_block_page():
+    # Modify the URL parameters to navigate to the block page
+    block_url = st.experimental_set_query_params(page='block')
+    page_route()
+    st.experimental_rerun()
+
+def redirect_to_panchayat_page():
+    # Modify the URL parameters to navigate to the block page
+    panchayat_url = st.experimental_set_query_params(page='panchayat')
+    page_route()
+    st.experimental_rerun()
+
+def redirect_to_village_page():
+    # Modify the URL parameters to navigate to the block page
+    panchayat_url = st.experimental_set_query_params(page='village')
+    page_route()
+    st.experimental_rerun()
+def redirect_to_sub_district_page():
+    # Modify the URL parameters to navigate to the district page
+    subdistrict_url = st.experimental_set_query_params(page='subdistrict')
+    page_route()
+    st.experimental_rerun()
+
+
+def page_route():
+    query_params = st.experimental_get_query_params()
+    if "page" in query_params and query_params["page"][0] == "state":
+        state_mapping_page()
+    elif "page" in query_params and query_params["page"][0] == "subset":
+        subset_page()
+    elif "page" in query_params and query_params["page"][0] == "update":
+        update_data()
+    elif "page" in query_params and query_params["page"][0] == "district":
+        district_page()
+    elif "page" in query_params and query_params["page"][0] == "block":
+        block_page()
+    elif "page" in query_params and query_params["page"][0] == "panchayat":
+        gp_page()
+    elif "page" in query_params and query_params["page"][0] == "subdistrict":
+        sub_district_page()
+    elif "page" in query_params and query_params["page"][0] == "village":
+        village_page()
+    elif "page" in query_params and query_params["page"][0] == "insertRecord":
+        pass
+        #insert_record()
+    else:
+        home_page()
+
+# Main app logic
+def main():
+    st.set_page_config(
+        page_title="CodeYatra",
+        page_icon="🌟",
+        layout="centered",
+        initial_sidebar_state="expanded",
+    )
+    page_route()
+
+def update_data():
+     st.title("Update the Corpus of LGD codes")
+     if st.button('Update Corpus', key='corpus_button'):   
+        update_all_data()
+
+
+#sub set
+def subset_page():
+    st.title("Select Specific Columns from CSV/Excel File")
+    
+    # File uploader
+    file = st.file_uploader("Upload a CSV or Excel file", type=["csv", "xlsx"])
+    
+    if file is not None:
+        # Process the file
+        
+        df = process_file(file)
+        if df is not None:
+            # Display the processed DataFrame
+            st.write('Subsetted Columns')
+            st.write(df.head())
+            st.info("Important: To perform LGD mapping accurately, please download the subsetted file. It contains the selected columns required for mapping. Without the subsetted file, LGD mapping cannot be done effectively.")
+            st.warning("⚠️ **Important Notice:** Please upload the subsetted file on the next page for LGD mapping. It contains the required columns for accurate mapping.")
+
+            # Download button
+            csv = df.to_csv(index=False)
+            b64 = base64.b64encode(csv.encode()).decode()
+            href = f'<a href="data:file/csv;base64,{b64}" download="processed_data.csv">Download CSV</a>'
+            st.markdown(href, unsafe_allow_html=True)
+
+        if st.button('Start Mapping', key='mapping_button'):
+                    redirect_to_state_mapping_page()
+# State Mapping Code
+def state_mapping_page(dataset_selected=False):
+    """
+    Generates the state mapping page which allows users to upload a CSV or Excel file containing a 'state_name' column. 
+    It then creates a mapped dataset which contains a state_code column based on the state_name column in the uploaded dataset. 
+    It also allows users to update the state name variations and download the mapped dataset. 
+    If the district_mapping_button is clicked, it redirects to the district mapping page. 
+    
+    Parameters:
+    ----------
+    dataset_selected: bool, optional
+        Default value is False. If True, the 'dataset_file' is obtained from the session state. If False, it prompts the user to upload a dataset file.
+
+    Returns:
+    ----------
+    None
+    """
+    
+    st.title('State LGD Mapping')
+
+
+    if not dataset_selected:
+        dataset_file = st.file_uploader('Upload dataset', type=['csv', 'xlsx'])
+        if dataset_file is None:
+            st.warning("Please upload a dataset file.")
+            return
+    else:
+        dataset_file = st.session_state['dataset_file']
+
+    dataset = pd.read_csv(dataset_file) if dataset_file.name.endswith('.csv') else pd.read_excel(dataset_file)
+    if 'state_name' not in dataset.columns:
+        st.error("Error: The dataset does not contain the 'state_name' column.")
+        return
+    st.subheader("Before State LGD Mapping")
+    st.write(dataset.head())
+    with st.spinner("Processing..."):
+        state_mapping = get_state_mappings()
+        mapped_dataset = create_mapped_dataset(dataset, state_mapping)
+        unmatched_names = mapped_dataset[mapped_dataset['state_code'] == -2]['state_name']
+
+    if unmatched_names.empty:
+        st.success('No Unmatched State Names')
+        mapped_dataset.to_csv('data.csv', index=False)
+
+        st.subheader("After State LGD Mapping")
+        with st.spinner("Processing..."):
+            st.write(mapped_dataset.head())
+            generate_download_link(mapped_dataset)
+
+        if st.button('Start District Mapping', key='district_mapping_button'):
+            if 'district_name' not in mapped_dataset.columns:
+                st.error("Error: The dataset does not contain the 'district_name' column.")
+                return
+            redirect_to_district_page()
+    else:
+        st.subheader('Unmatched State Names')
+        st.write(unmatched_names.unique())
+        note = "Please provide the state name variations separated by commas or a single state name."
+        st.info(note)
+        st.subheader('Update State Name Variations')
+        state_mapping = get_state_mappings()
+        mapped_dataset = create_mapped_dataset(dataset, state_mapping)
+        unmatched_names = mapped_dataset[mapped_dataset['state_code'] == -2]['state_name']
+        entity_table_name = "states"
+        update_variations(unmatched_names.unique(), state_mapping, entity_table_name)
+        unmatched_names = mapped_dataset[mapped_dataset['state_code'] == -2]['state_name']
+        
+        if unmatched_names.empty:
+            st.write(mapped_dataset.head())
+            mapped_dataset.to_csv('data.csv', index=False)
+            generate_download_link(mapped_dataset)
+            if st.button('Start District Mapping', key='district_mapping_button'):
+                if 'district_name' not in mapped_dataset.columns:
+                    st.error("Error: The dataset does not contain the 'district_name' column.")
+                    return
+                redirect_to_district_page()
+
+def district_page():
+    """
+    This function is responsible for displaying the District LGD mapping page. It fetches the district mapping dataset,
+    maps the dataset, and downloads the mapped dataset. If there are any unmatched district names, it prompts the user to
+    provide the district name variations. Once the district name variations are provided, it updates the entityNameVariants
+    column in the SQLite table and generates a new mapped dataset for download. If the 'Start Sub-District/Block Mapping'
+    button is clicked, it does nothing. 
+    """
+    st.title('District LGD Mapping')
+    st.subheader("Before District LGD Mapping")
+    state_dataset = load_file()
+    st.write(state_dataset.head())
+    # Apply district mapping and create a new dataset
+    data  = fetch_district_mapping()
+    district_mapping = populate_entity_mapping(data,'district_name','state_code')
+    mapped_dataset = create_district_mapped_dataset(state_dataset, district_mapping)
+    # Check if there are any unmatched names
+    unmatched_names = mapped_dataset[mapped_dataset['district_code'] == -2]['district_name']
+
+    if unmatched_names.empty:
+        # Display a message if there are no unmatched names
+        st.success('No Unmatched district Names')
+        mapped_dataset.to_csv("data.csv",index=False)
+        # Create a CSV file in memory
+        st.subheader("After District LGD Mapping")
+        st.write(state_dataset.head())
+        generate_download_link(mapped_dataset)
+        condition = True 
+        if 'sub_district_name' not in mapped_dataset.columns:
+            condition = False
+
+        if st.button('Start Block Mapping', key='block_mapping_button'):
+            if 'block_name' not in mapped_dataset.columns:
+                    st.error("Error: The dataset does not contain the 'block_name' column.")
+                    return
+            redirect_to_block_page()
+
+        if condition:    
+            if st.button('Start Sub-District Mapping', key='sub-district_mapping_button'):
+                if 'sub_district_name' not in mapped_dataset.columns:
+                        st.error("Error: The dataset does not contain the 'sub_district_name' column.")
+                        return
+                redirect_to_sub_district_page()
+    else:
+            # Display the dataset with unmatched names
+            st.subheader('Unmatched District Names')
+            st.write(f'Unmatched District Count: '+str(len(unmatched_names.unique())))
+            st.write(unmatched_names.unique())
+            # Display the note
+            note = "Please provide the district name variations separated by commas or a single district name."
+            st.info(note)
+            # Accept comma-separated values or single value only
+            st.subheader('Update District Name Variations')
+            #district_mapping = populate_entity_mapping(data,'district_name','state_code')
+            #mapped_dataset = create_district_mapped_dataset(state_dataset, district_mapping)
+            # Check if there are any unmatched names
+            #unmatched_names = mapped_dataset[mapped_dataset['district_code'] == -2]['district_name']
+            district_names = [row[0] for row in data]
+            entity_table_name = "district"
+            update_variations(unmatched_names.unique(), district_names, entity_table_name)
+            #unmatched_names = mapped_dataset[mapped_dataset['district_code'] == -2]['district_name']
+            if unmatched_names.empty:
+                st.success('District Name Variations Updated Successfully.')
+                # Create a CSV file in memory
+                mapped_dataset.to_csv("data.csv",index=False)
+                st.subheader("After District LGD Mapping")
+                st.write(mapped_dataset.head())
+                generate_download_link(mapped_dataset)
+
+                if 'block_name' in mapped_dataset.columns:
+                    if st.button('Start Block Mapping', key='block_mapping_button'):
+                        if 'block_name' not in mapped_dataset.columns:
+                            st.error("Error: The dataset does not contain the 'block_name' column.")
+                            return
+                        redirect_to_block_page()
+
+                if 'sub_district_name' in mapped_dataset.columns:
+                    if st.button('Start Sub-District Mapping', key='sub-district_mapping_button'):
+                        if 'sub_district_name' not in mapped_dataset.columns:
+                            st.error("Error: The dataset does not contain the 'sub_district_name' column.")
+                            return
+                        redirect_to_sub_district_page()
+
+# Block mapping
+def block_page():
+    """
+    This function is responsible for displaying the block LGD mapping page. It fetches the block mapping dataset,
+    maps the dataset, and downloads the mapped dataset. If there are any unmatched block names, it prompts the user to
+    provide the block name variations. Once the block name variations are provided, it updates the entityNameVariants
+    column in the SQLite table and generates a new mapped dataset for download. If the 'Start Sub-block/Block Mapping'
+    button is clicked, it does nothing. 
+    """
+    st.title('Block LGD Mapping')
+    st.subheader("Before Block LGD Mapping")
+    block_dataset = pd.read_csv('data.csv')
+    st.write(block_dataset.head())
+    data = fetch_block_mapping()
+    # Apply block mapping and create a new dataset
+    block_mapping = populate_entity_mapping(data,'block_name','district_code')
+    mapped_dataset = create_block_mapped_dataset(block_dataset, block_mapping)
+    # Check if there are any unmatched names
+    unmatched_names = mapped_dataset[mapped_dataset['block_code'] == -2]['block_name']
+
+    if unmatched_names.empty:
+        # Display a message if there are no unmatched names
+        st.success('No Unmatched Block Names')
+
+        # Create a CSV file in memory
+        
+        st.subheader("After Block LGD Mapping")
+        mapped_dataset.to_csv("data.csv",index=False)
+        st.write(mapped_dataset.head())
+        generate_download_link(mapped_dataset)
+        if st.button('Start Panchayat Mapping', key='Panchayat_mapping_button'):
+            if 'gp_name' not in mapped_dataset.columns:
+                        st.error("Error: The dataset does not contain the 'gp_name' column.")
+                        return
+            redirect_to_panchayat_page()
+
+    else:
+        # Display the dataset with unmatched names
+        st.subheader('Unmatched block Names')
+        st.write(f'Unmatched block Count: '+str(len(unmatched_names.unique())))
+        st.write(unmatched_names.unique())
+
+        # Display the note
+        note = "Please provide the block name variations separated by commas or a single block name."
+        st.warning(note)
+
+        # Accept comma-separated values or single value only
+        st.subheader('Update Block Name Variations')
+
+        entity_table_name = "block"
+        block_mapping = populate_entity_mapping(data,'block_name','district_code')
+        mapped_dataset = create_block_mapped_dataset(block_dataset, block_mapping)
+        block_names = [row[0] for row in data]
+        unmatched_names = mapped_dataset[mapped_dataset['block_code'] == -2]['block_name']
+        unmatched_names = unmatched_names.unique()
+        update_variations(unmatched_names, block_names, entity_table_name)
+        # Display a success message
+        unmatched_names = mapped_dataset[mapped_dataset['block_code'] == -2]['block_name']
+        if unmatched_names.empty:
+            st.success('Block Name Variations Updated Successfully.')
+            # Create a CSV file in memory
+            st.subheader("After block LGD Mapping")
+            mapped_dataset.to_csv("data.csv",index=False)
+            st.write(mapped_dataset.head())
+            generate_download_link(mapped_dataset)
+
+            if st.button('Start Panchayat Mapping', key='Panchayat_mapping_button'):
+                if 'gp_name' not in mapped_dataset.columns:
+                            st.error("Error: The dataset does not contain the 'gp_name' column.")
+                            return
+                redirect_to_panchayat_page()
+
+#GP Mapping 
+
+
+
+def gp_page():
+    """
+    This function is responsible for displaying the gp LGD mapping page. It fetches the gp mapping dataset,
+    maps the dataset, and downloads the mapped dataset. If there are any unmatched gp names, it prompts the user to
+    provide the gp name variations. Once the gp name variations are provided, it updates the entityNameVariants
+    column in the SQLite table and generates a new mapped dataset for download. If the 'Start Sub-gp/gp Mapping'
+    button is clicked, it does nothing. 
+    """
+    st.title('GP LGD Mapping')
+    st.subheader("Before GP LGD Mapping")
+    gp_dataset = pd.read_csv('data.csv')
+    data= fetch_gp_mapping()
+    st.write(gp_dataset.head())
+    unmatched_names = None
+    # Apply gp mapping and create a new dataset
+    
+    #gp_mapping = populate_gp_mapping()
+    gp_mapping = populate_entity_mapping(data,'gp_name','block_code')
+    mapped_dataset = create_gp_mapped_dataset(gp_dataset, gp_mapping)
+
+    # Check if there are any unmatched names
+    unmatched_names = mapped_dataset[mapped_dataset['gp_code'] == -2]['gp_name']
+    if unmatched_names.empty:
+        # Display a message if there are no unmatched names
+        st.success('No Unmatched GP Names')
+
+        # Create a CSV file in memory
+        csv_file = mapped_dataset.to_csv(index=False)
+        st.subheader("After GP LGD Mapping")
+        st.write(mapped_dataset.head())
+
+        generate_download_link(mapped_dataset)
+
+        if st.button('Start Village Mapping', key='village_mapping_button'):
+            if 'village_name' not in mapped_dataset.columns:
+                        st.error("Error: The dataset does not contain the 'village_name' column.")
+                        return
+            redirect_to_panchayat_page()
+
+    else:
+        # Display the dataset with unmatched names
+        st.subheader('Unmatched GP Names')
+        st.write(f'Unmatched GP Count: '+str(len(unmatched_names.unique())))
+        st.write(unmatched_names.unique())
+        
+        # Display the note
+        note = "Please provide the GP name variations separated by commas or a single GP name."
+        st.info(note)
+
+        # Accept comma-separated values or single value only
+        st.subheader('Update GP Name Variations')
+        gp_mapping = populate_gp_mapping()
+        mapped_dataset = create_gp_mapped_dataset(gp_dataset, gp_mapping)
+        entity_name = create_entity_name_list()
+        # Check if there are any unmatched names
+        unmatched_names = mapped_dataset[mapped_dataset['gp_code'] == -2]['gp_name']
+        update_variations(unmatched_names.unique(), entity_name, "gp")
+        unmatched_names = mapped_dataset[mapped_dataset['gp_code'] == -2]['gp_name']
+        if unmatched_names.empty:
+            st.success('GP Name Variations Updated Successfully.')
+            # Create a CSV file in memory
+            mapped_dataset.to_csv("data.csv",index=False)
+            st.subheader("After GP LGD Mapping")
+            st.write(mapped_dataset.head())
+            # Generate download link for the CSV file
+            generate_download_link(mapped_dataset)
+            if st.button('Start Village Mapping', key='village_mapping_button'):
+                if 'village_name' not in mapped_dataset.columns:
+                            st.error("Error: The dataset does not contain the 'village_name' column.")
+                            return
+                redirect_to_panchayat_page()
+
+
+#village page
+
+def village_page():
+    """
+    This function is responsible for displaying the village LGD mapping page. It fetches the village mapping dataset,
+    maps the dataset, and downloads the mapped dataset. If there are any unmatched village names, it prompts the user to
+    provide the village name variations. Once the village name variations are provided, it updates the entityNameVariants
+    column in the SQLite table and generates a new mapped dataset for download. If the 'Start Sub-village/village Mapping'
+    button is clicked, it does nothing. 
+    """
+    st.title('village LGD Mapping')
+    st.subheader("Before village LGD Mapping")
+    village_dataset = pd.read_csv('data.csv')
+    st.write(village_dataset.head())
+    # Apply village mapping and create a new dataset
+    village_mapping = populate_village_mapping()
+
+    mapped_dataset = create_village_mapped_dataset(village_dataset, village_mapping)
+
+    # Check if there are any unmatched names
+    unmatched_names = mapped_dataset[mapped_dataset['village_code'] == -2]['village_name']
+
+    if unmatched_names.empty:
+        # Display a message if there are no unmatched names
+        st.success('No Unmatched Village Names')
+        # Create a CSV file in memory
+        st.subheader("After village LGD Mapping")
+        st.write(mapped_dataset.head())
+        generate_download_link(mapped_dataset)
+
+    else:
+        # Display the dataset with unmatched names
+        st.subheader('Unmatched village Names')
+        st.write(f'Unmatched villages Count: '+str(len(unmatched_names.unique())))
+        st.write(unmatched_names.unique())
+        # Display the note
+        note = "Please provide the village name variations separated by commas or a single village name."
+        st.info(note)
+        # Accept comma-separated values or single value only
+        st.subheader('Update village Name Variations')
+        village_mapping = populate_village_mapping()
+        mapped_dataset = create_village_mapped_dataset(village_dataset, village_mapping)
+        # Check if there are any unmatched names
+        unmatched_names = mapped_dataset[mapped_dataset['village_code'] == -2]['panchayat_name']
+        update_variations(unmatched_names.unique(), village_mapping, "villages")
+        unmatched_names = mapped_dataset[mapped_dataset['village_code'] == -2]['panchayat_name']
+        if unmatched_names.empty:
+            st.success('village Name Variations Updated Successfully.')
+            # Create a CSV file in memory
+            mapped_dataset.to_csv("data.csv",index=False)
+            st.subheader("After village LGD Mapping")
+            st.write(mapped_dataset.head())
+            # Generate download link for the CSV file
+            generate_download_link(mapped_dataset)
+            
+
+
+#sub-district mapping
+def sub_district_page():
+    """
+    This function is responsible for displaying the Sub-District LGD mapping page. It fetches the Sub-District mapping dataset,
+    maps the dataset, and downloads the mapped dataset. If there are any unmatched Sub-District names, it prompts the user to
+    provide the Sub-District name variations. Once the Sub-District name variations are provided, it updates the entityNameVariants
+    column in the SQLite table and generates a new mapped dataset for download. If the 'Start Sub-Sub-District/Sub-District Mapping'
+    button is clicked, it does nothing. 
+    """
+    st.title('Sub-District LGD Mapping')
+    st.subheader("Before Sub-District LGD Mapping")
+    sub_district_dataset = pd.read_csv('data.csv')
+    st.write(sub_district_dataset.head())
+    # Apply Sub-District mapping and create a new dataset
+    sub_district_mapping = populate_sub_district_mapping()
+
+    mapped_dataset = create_sub_district_mapped_dataset(sub_district_dataset, sub_district_mapping)
+
+    # Check if there are any unmatched names
+    unmatched_names = mapped_dataset[mapped_dataset['sub_district_code'] == -2]['sub_district_name']
+
+    if unmatched_names.empty:
+        # Display a message if there are no unmatched names
+        st.success('No Unmatched Sub-District Names')
+        mapped_dataset.to_csv("data.csv",index=False)
+        # Create a CSV file in memory
+        generate_download_link(mapped_dataset)
+        if st.button('Start Panchayat Mapping', key='Panchayat_mapping_button'):
+                if 'panchayat_name' not in mapped_dataset.columns:
+                            st.error("Error: The dataset does not contain the 'panchayat_name' column.")
+                            return
+                redirect_to_panchayat_page()
+
+    else:
+        # Display the dataset with unmatched names
+        st.subheader('Unmatched Sub-District Names')
+        st.write(f'Unmatched Sub-District Count: '+str(len(unmatched_names.unique())))
+        st.write(unmatched_names)
+
+        # Display the note
+        note = "Please provide the Sub-District name variations separated by commas or a single Sub-District name."
+        st.info(note)
+
+        # Accept comma-separated values or single value only
+        st.subheader('Update Sub-District Name Variations')
+        sub_district_mapping = populate_sub_district_mapping()
+        mapped_dataset = create_sub_district_mapped_dataset(sub_district_dataset, sub_district_mapping)
+        unmatched_names = mapped_dataset[mapped_dataset['sub_district_code'] == -2]['sub_district_name']
+        update_variations(unmatched_names.unique(), sub_district_mapping, "sub_district")
+        unmatched_names = mapped_dataset[mapped_dataset['sub_district_code'] == -2]['sub_district_name']
+        if unmatched_names.empty:
+            # Display a success message
+            st.success('Sub-District Name Variations Updated Successfully.')
+            # Create a CSV file in memory
+            csv_file = mapped_dataset.to_csv(index=False)
+            st.subheader("After Sub-District LGD Mapping")
+            st.write(mapped_dataset.head())
+            mapped_dataset.to_csv("data.csv",index=False)
+            st.write(mapped_dataset.head())
+            # Generate download link for the CSV file
+            generate_download_link(mapped_dataset)
+
+
+            if st.button('Start Panchayat Mapping', key='Panchayat_mapping_button'):
+                if 'panchayat_name' not in mapped_dataset.columns:
+                            st.error("Error: The dataset does not contain the 'panchayat_name' column.")
+                            return
+                redirect_to_panchayat_page()
+    
+
+
+
+if __name__ == "__main__":
+    main()

databasefn.py

@@ -0,0 +1,277 @@
+import streamlit as st
+import sqlite3
+import pandas as pd
+def insert_record(table_name):
+    """
+    Inserts a new record into the specified table in the LGD database.
+
+    Parameters:
+    - table_name (str): The name of the table to insert the record into.
+
+    Returns:
+    - None
+    """
+    st.header("Insert Record")
+    # Connect to the SQLite database
+    conn = sqlite3.connect("lgd_database.db")
+    cursor = conn.cursor()
+
+    # Retrieve the first 5 rows from the table
+    select_query = f"SELECT * FROM {table_name}"
+    cursor.execute(select_query)
+    rows = cursor.fetchall()
+
+    # Convert the rows to a DataFrame
+    columns = [description[0] for description in cursor.description]
+    df = pd.DataFrame(rows, columns=columns)
+    df = df.iloc[:, :-1]
+    # Display the retrieved rows in a table
+    st.subheader("Existing Records")
+    st.write(df)
+
+    # Set default values for level_name and level_code based on table_name
+    if table_name == "states":
+        level_name = "State"
+        level_code = 1
+    elif table_name == "district":
+        level_name = "District"
+        level_code = 2
+    elif table_name == "sub_district":
+        level_name = "Sub-district"
+        level_code = 3
+    elif table_name == "block":
+        level_name = "Block"
+        level_code = 4
+    elif table_name == "gp":
+        level_name = "Gram Panchayats"
+        level_code = 5
+    else:
+        level_name = ""
+        level_code = 0
+
+    # Input fields for entityLGDCode and levelCode
+    entity_lgd_code = st.number_input("Entity LGD Code", min_value=0, step=1)
+
+    # Input field for entityName
+    entity_name = st.text_input("Entity Name")
+
+    # Input field for entityNameVariants
+    entity_name_variants = st.text_input("Entity Name Variants")
+
+    # Input field for entityParent
+    entity_parent = st.number_input("Entity Parent", min_value=0, step=1)
+
+    if level_name and level_code:
+        st.text(f"Level Name: {level_name}")
+        st.text(f"Level Code: {level_code}")
+    else:
+        # Input fields for levelName and levelCode
+        level_name = st.text_input("Level Name")
+        level_code = st.number_input("Level Code", min_value=0, step=1)
+
+    # Insert button
+    if st.button("Insert"):
+        # Perform validation checks before inserting the record
+        errors = []
+
+        if entity_lgd_code == 0:
+            errors.append("Entity LGD Code cannot be zero.")
+
+        if level_code == 0:
+            errors.append("Level Code cannot be zero.")
+
+        if not entity_name or not isinstance(entity_name, str):
+            errors.append("Entity Name is required and must be a text.")
+
+        if not level_name or not isinstance(level_name, str):
+            errors.append("Level Name is required and must be a text.")
+
+        if not entity_name_variants or not isinstance(entity_name_variants, str):
+            errors.append("Entity Name Variants is required and must be a text.")
+
+        if not entity_parent or not isinstance(entity_parent, int):
+            errors.append("Entity Parent is required and must be an integer.")
+
+        if errors:
+            st.error("\n".join(errors))
+        else:
+            # Connect to the SQLite database
+            conn = sqlite3.connect("lgd_database.db")
+            cursor = conn.cursor()
+
+            # Check if the entityLGDCode already exists in the table
+            select_query = f"SELECT entityLGDCode FROM {table_name} WHERE entityLGDCode = ?"
+            cursor.execute(select_query, (entity_lgd_code,))
+            existing_code = cursor.fetchone()
+
+            if existing_code:
+                st.error("Entity LGD Code already exists in the table.")
+            else:
+                # Prepare the SQL query for creating the table if it doesn't exist
+                create_table_query = f"""
+                CREATE TABLE IF NOT EXISTS {table_name} (
+                    entityLGDCode INTEGER PRIMARY KEY,
+                    entityName TEXT,
+                    levelCode INTEGER,
+                    levelName TEXT,
+                    entityNameVariants TEXT,
+                    entityParent INTEGER
+                )
+                """
+                cursor.execute(create_table_query)
+
+                # Prepare the SQL query for inserting the record
+                insert_query = f"""
+                INSERT INTO {table_name} (entityLGDCode, entityName, levelCode, levelName, entityNameVariants, entityParent)
+                VALUES (?, ?, ?, ?, ?, ?)
+                """
+                values = (entity_lgd_code, entity_name, level_code, level_name, entity_name_variants, entity_parent)
+
+                try:
+                    # Execute the SQL query
+                    cursor.execute(insert_query, values)
+                    conn.commit()
+                    st.success("Record inserted successfully!")
+                except sqlite3.Error as e:
+                    st.error("An error occurred while inserting the record: {}".format(e))
+                    conn.rollback()
+                finally:
+                    # Close the database connection
+                    cursor.close()
+                    conn.close()
+
+                # Reset the input fields
+                entity_lgd_code = 0
+                level_code = 0
+                entity_name = ""
+                level_name = ""
+                entity_name_variants = ""
+                entity_parent = 0
+
+
+def update_record(table_name):
+    """
+    Update a record in a SQLite database table.
+
+    :param table_name: The name of the table to update the record in.
+    :type table_name: str
+
+    :return: None
+    :rtype: None
+    """
+    st.header("Update Record")
+
+    # Input field for entityLGDCode
+    entity_lgd_code = st.number_input("Entity LGD Code", min_value=0, step=1)
+
+    # Input field for entityName
+    entity_name = st.text_input("Entity Name")
+
+    # Input field for entityNameVariants
+    entity_name_variants = st.text_input("Entity Name Variants")
+
+    # Input field for entityParent
+    entity_parent = st.number_input("Entity Parent", min_value=0, step=1)
+
+    # Update button
+    if st.button("Update"):
+        # Perform validation checks before updating the record
+        errors = []
+
+        if entity_lgd_code == 0:
+            errors.append("Entity LGD Code cannot be zero.")
+
+        if not entity_name or not isinstance(entity_name, str):
+            errors.append("Entity Name is required and must be a text.")
+
+        if not entity_name_variants or not isinstance(entity_name_variants, str):
+            errors.append("Entity Name Variants is required and must be a text.")
+
+        if not entity_parent or not isinstance(entity_parent, int):
+            errors.append("Entity Parent is required and must be an integer.")
+
+        if errors:
+            st.error("\n".join(errors))
+        else:
+            # Connect to the SQLite database
+            conn = sqlite3.connect("lgd_database.db")
+            cursor = conn.cursor()
+
+            # Prepare the SQL query for updating the record
+            update_query = f"""
+            UPDATE {table_name}
+            SET entityName = ?, entityNameVariants = ?, entityParent = ?
+            WHERE entityLGDCode = ?
+            """
+            values = (entity_name, entity_name_variants, entity_parent, entity_lgd_code)
+
+            try:
+                # Execute the SQL query
+                cursor.execute(update_query, values)
+                conn.commit()
+                st.success("Record updated successfully!")
+            except sqlite3.Error as e:
+                st.error("An error occurred while updating the record: {}".format(e))
+                conn.rollback()
+            finally:
+                # Close the database connection
+                cursor.close()
+                conn.close()
+
+            # Reset the input fields
+            entity_lgd_code = 0
+            entity_name = ""
+            entity_name_variants = ""
+            entity_parent = 0
+
+def delete_record(table_name):
+    """
+    Deletes a record from the specified table in an SQLite database based on the entityLGDCode.
+    
+    :param table_name: Name of the table to delete the record from.
+    :type table_name: str
+    
+    :return: None
+    :rtype: None
+    """
+    st.header("Delete Record")
+
+    # Input field for entityLGDCode
+    entity_lgd_code = st.number_input("Entity LGD Code", min_value=0, step=1)
+
+    # Delete button
+    if st.button("Delete"):
+        # Connect to the SQLite database
+        conn = sqlite3.connect("lgd_database.db")
+        cursor = conn.cursor()
+
+        # Prepare the SQL query for deleting the record
+        delete_query = f"""
+        DELETE FROM {table_name}
+        WHERE entityLGDCode = ?
+        """
+        values = (entity_lgd_code,)
+
+        try:
+            # Execute the SQL query
+            cursor.execute(delete_query, values)
+            conn.commit()
+            st.success("Record deleted successfully!")
+        except sqlite3.Error as e:
+            st.error("An error occurred while deleting the record: {}".format(e))
+            conn.rollback()
+        finally:
+            # Close the database connection
+            cursor.close()
+            conn.close()
+
+        # Reset the input fields
+        entity_lgd_code = 0
+
+""" if __name__ == "__main__":
+    table_name = "block"  # Provide the table name here
+
+    # Usage examples
+    update_record(table_name)
+    delete_record(table) """
+

lgd_database.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94221cb8148c0db3ba3b4f238293c015df16813a73e4fd60ba75371a3ddac21c
+size 306753536

mapping.py

@@ -0,0 +1,89 @@
+import sqlite3
+import streamlit as st
+import pandas as pd
+from collections import defaultdict
+import base64
+import multiprocessing
+
+from pg_utils_fn import create_mapped_dataset, get_state_mappings
+
+
+
+def create_gp_mapped_dataset(dataset, mapping):
+    """
+    Create a mapped dataset by associating gp codes with gp names in the dataset.
+    """
+    dataset['panchayat_name'] = dataset['panchayat_name'].str.strip()
+    dataset['gp_code'] = dataset['panchayat_name'].str.lower().map(mapping)
+    dataset.loc[dataset['gp_code'].isnull(), 'gp_code'] = -2
+    return dataset
+
+def fetch_gp_mapping():
+    """
+    Fetch the gp mapping from the SQLite database.
+
+    Returns:
+    - A list of tuples containing the gp entity name, LGD code, name variants, and parent entity.
+    """
+    # Connect to the SQLite database
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Retrieve gp data from the 'gps' table
+    cursor.execute("SELECT entityName, entityLGDCode, entityNameVariants, entityParent FROM gp")
+    data = cursor.fetchall()
+
+    # Close the database connection
+    conn.close()
+
+    return data
+
+def populate_gp_mapping():
+    """
+    Populates a gp mapping dictionary using data from a database and a local file.
+
+    Returns:
+        A defaultdict containing the mapping of gp names to their respective codes.
+    """
+    state_dataset = pd.read_csv('data.csv')
+    data = fetch_gp_mapping()
+    unique_rows = state_dataset.drop_duplicates(subset=['panchayat_name'])
+    unique_rows_lower = unique_rows.apply(lambda x: (x['panchayat_name'].strip().lower(), x['block_code']), axis=1).tolist()
+
+    entity_mapping = {}
+    edname = "Not Available"
+    for entity_name, entity_code, entity_variants, parent_code in data:
+        for row in unique_rows_lower:
+            entity_name_lower = row[0]
+            state_code = row[1]
+            if entity_name_lower == entity_name.lower() :
+                if int(parent_code) == int(state_code) :
+                    entity_mapping[entity_name_lower.lower()] = entity_code
+                    if entity_variants:
+                        for variant in entity_variants.split(','):
+                            entity_mapping[variant.strip().lower()] = entity_code
+
+    return entity_mapping
+
+
+
+import pandas as pd
+
+
+def main_gp():
+    gp_dataset = pd.read_csv('data.csv')
+    gp_mapping = populate_gp_mapping()
+
+    mapped_dataset = create_gp_mapped_dataset(gp_dataset, gp_mapping)
+
+    unmatched_names = mapped_dataset[mapped_dataset['gp_code'] == -2]['panchayat_name']
+    return unmatched_names,mapped_dataset
+
+
+
+def main_state(dataset):
+    state_mapping = get_state_mappings()
+    mapped_dataset = create_mapped_dataset(dataset, state_mapping)
+    unmatched_names = mapped_dataset[mapped_dataset['state_code'] == -2]['state_name']
+
+    return unmatched_names,mapped_dataset

pg_utils_fn.py

@@ -0,0 +1,735 @@
+import sqlite3
+import streamlit as st
+import pandas as pd
+from collections import defaultdict
+import base64
+import multiprocessing
+
+
+
+def load_file():
+    data = pd.read_csv('data.csv')
+    return data
+
+def process_file(file):
+    try:
+        if file.type == 'text/csv':
+            df = pd.read_csv(file)
+        elif file.type == 'application/vnd.ms-excel':
+            df = pd.read_excel(file)
+        else:
+            raise ValueError("Invalid file format. Only CSV and Excel files are supported.")
+
+        # Get the present columns based on the intersection of specified columns and available columns
+        specified_columns = ["state_name", "district_name", "sub_district_name", "block_name", "gp_name", "village_name"]
+        present_columns = list(set(specified_columns).intersection(df.columns))
+        
+        if not present_columns:
+            raise ValueError("None of the specified columns are present in the file.")
+        
+        # Select only the present columns
+        st.write("Dataset Information:")
+        st.write(df.head())
+        df = df[present_columns]
+        
+        # Perform operations on the selected columns or use them as needed
+        # ...
+        
+        return df
+    
+    except Exception as e:
+        st.error("An error occurred during file processing: " + str(e))
+        return None
+
+# Streamlit app
+
+def get_state_mappings():
+    """
+    This function retrieves state names, codes, and variants from an SQLite database and returns a dictionary
+    containing the state mappings. No parameters are required. The keys of the dictionary are the state names and
+    variants in lowercase, and the values are the corresponding state codes. Returns a dictionary.
+    """
+    with sqlite3.connect('lgd_database.db') as conn:
+        cursor = conn.cursor()
+        cursor.execute("SELECT entityName, entityLGDCode, entityNameVariants FROM states")
+        data = cursor.fetchall()
+
+    mapping_dict = {}
+    for state_name, state_code, state_variants in data:
+        mapping_dict[state_name.lower()] = state_code
+        if state_variants:
+            for variant in state_variants.split(','):
+                mapping_dict[variant.strip().lower()] = state_code
+
+    return mapping_dict
+
+def create_mapped_dataset(dataset, mapping):
+
+    dataset['state_name'] = dataset['state_name'].str.strip()
+    dataset['state_code'] = dataset['state_name'].str.lower().map(mapping)
+    dataset.loc[dataset['state_code'].isnull(), 'state_code'] = -2
+    return dataset 
+
+
+
+def create_selectbox_widget(name, values):
+    """
+    Creates a select box widget using the Streamlit library. The select box widget allows the user to choose a value from a list of values.
+
+    :param name: The name of the select box widget.
+    :type name: str
+    :param values: A list of values to be displayed in the select box.
+    :type values: list
+    :return: The selected value from the select box.
+    :rtype: any
+    """
+    return st.selectbox(f'{name}', values, key=name)
+
+
+def query_state_data():
+    """
+    Connects to an SQLite database and retrieves all the entity names and their variants from the 'states' table.
+
+    :return: A list of tuples containing the entity names and their variants.
+    """
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+    cursor.execute("SELECT entityName, entityNameVariants FROM states")
+    state_data = cursor.fetchall()
+    conn.close()
+    return state_data
+
+def update_state_variants(state_name, name_variant, state_data):
+    """
+    Updates the variants of a state name in the state_data dictionary.
+
+    Parameters:
+        state_name (str): The name of the state to update the variants for.
+        name_variant (str): The new name variant to add.
+        state_data (dict): A dictionary containing state names and their variants.
+
+    Returns:
+        tuple or None: A tuple containing the updated state name and its variants if the state name is found and updated in the dictionary. Otherwise, returns None.
+    """
+    for state_name_db, entityNameVariants in state_data:
+        if state_name.lower() == state_name_db.lower():
+            new_variants = f"{entityNameVariants}, {name_variant}" if entityNameVariants else name_variant
+            conn = sqlite3.connect('lgd_database.db')
+            cursor = conn.cursor()
+            cursor.execute("UPDATE states SET entityNameVariants = ? WHERE entityName = ?", (new_variants, state_name_db))
+            conn.commit()
+            conn.close()
+            return state_name_db, new_variants
+    return None, None
+
+def process_unmatched_names(unmatched_names, state_mapping):
+    """
+    Process unmatched names and update state data.
+
+    Parameters:
+    - unmatched_names (list): A list of unmatched names.
+    - state_mapping (dict): A dictionary mapping state names to their data.
+
+    Returns:
+    None
+    """
+    state_data = query_state_data()
+    for unmatched_name_index, unmatched_name in enumerate(unmatched_names):
+        st.write(f'Unmatched Name: {unmatched_name}')
+        state_name = create_selectbox_widget(f'Enter state name {unmatched_name_index}:', list(state_mapping.keys()))
+        state_exists = state_name.lower() in state_mapping.keys()
+
+        if state_name and not state_exists:
+            st.error('State name not found in the table. Please enter a valid state name.')
+
+        name_variant = st.text_input(f'Enter name variant {unmatched_name_index}:')
+
+        if state_name and name_variant and state_exists:
+            state_name_db, new_variants = update_state_variants(state_name, name_variant, state_data)
+            if state_name_db and new_variants:
+                st.success(f'State Name: {state_name_db} Variations: {new_variants} Updated Successfully.')
+
+        st.write('---')
+
+
+def fetch_district_mapping():
+    """
+    Fetch the district mapping from the SQLite database.
+
+    Returns:
+    - A list of tuples containing the district entity name, LGD code, name variants, and parent entity.
+    """
+    # Connect to the SQLite database
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Retrieve district data from the 'districts' table
+    cursor.execute("SELECT entityName, entityLGDCode, entityNameVariants, entityParent FROM district")
+    data = cursor.fetchall()
+
+    # Close the database connection
+    conn.close()
+
+    return data
+
+def populate_entity_mapping(data,column_name,parent_column_name):
+    """
+    Populates a entity mapping dictionary using data from a database and a local file.
+
+    Returns:
+        A defaultdict containing the mapping of entity names to their respective codes.
+    """
+    # Load unique entity data
+    state_dataset = pd.read_csv('data.csv')
+    unique_rows = state_dataset.drop_duplicates(subset=[column_name])
+    unique_rows_lower = unique_rows.apply(lambda x: (x[column_name].strip().lower(), x[parent_column_name]), axis=1).tolist()
+
+    entity_mapping = {}
+    for entity_name, entity_code, entity_variants, parent_code in data:
+        for row in unique_rows_lower:
+            entity_name_lower = row[0]
+            state_code = row[1]
+            if int(parent_code) == int(state_code):
+                if entity_name_lower.strip() == entity_name.strip().lower():
+                    entity_mapping[entity_name_lower] = entity_code
+                    #print(entity_name_lower)
+                else:
+                    if entity_variants:
+                        for variant in entity_variants.split(','):
+                            if variant.strip().lower() == entity_name_lower.strip():
+                                entity_mapping[variant.strip().lower()] = entity_code
+                                print(variant.strip().lower())
+
+    return entity_mapping
+
+
+
+    """ 
+    def create_district_mapped_dataset(dataset, mapping):
+    
+    Create a mapped dataset by associating state codes with district names in the dataset.
+    
+    dataset['district_name'] = dataset['district_name'].str.strip().str.lower()
+    dataset['district_code'] = dataset['district_name'].str.lower().map(mapping)
+    dataset.loc[dataset['district_code'].isnull(), 'district_code'] = -1
+    return dataset 
+    """
+
+
+
+def process_district_name(district_name, mapping):
+    """
+    Process a district name by stripping, converting to lowercase, and mapping to a code.
+    """
+    district_name = district_name.strip().lower()
+    return district_name, mapping.get(district_name, -2)
+
+def create_district_mapped_dataset(dataset, mapping):
+    """
+    Create a mapped dataset by associating state codes with district names in the dataset.
+    """
+    pool = multiprocessing.Pool()
+    results = pool.starmap(process_district_name, zip(dataset['district_name'], [mapping]*len(dataset)))
+    dataset['district_name'], dataset['district_code'] = zip(*results)
+    return dataset
+
+
+import sqlite3
+
+def update_variations(unmatched_names, mapping, entity_table_name, chunk_size=1):
+    """
+    Updates the variations of unmatched names in the given mapping dictionary for a specific entity table.
+
+    Parameters:
+    - unmatched_names (list): A list of unmatched names to update the variations for.
+    - mapping (dict): A dictionary mapping entity names to their variations.
+    - entity_table_name (str): The name of the entity table to update the variations in.
+    - chunk_size (int): The size of each processing chunk.
+
+    Returns:
+    - str: The message "Done" indicating that the variations have been updated successfully.
+    """
+    try:
+        conn = sqlite3.connect('lgd_database.db')
+        cursor = conn.cursor()
+
+        num_unmatched = len(unmatched_names)
+        num_chunks = (num_unmatched + chunk_size - 1) // chunk_size
+
+        chunk_index = 0  # Initialize chunk index
+
+        while chunk_index < num_chunks:
+            start_idx = chunk_index * chunk_size
+            end_idx = min((chunk_index + 1) * chunk_size, num_unmatched)
+            current_chunk = unmatched_names[start_idx:end_idx]
+
+            for index, unmatched_name in enumerate(current_chunk):
+                entity_name = create_selectbox_widget(f'Select {entity_table_name} name {start_idx + index + 1}:', list(mapping))
+                if not entity_name:
+                    continue
+
+                entity_exists = entity_name in mapping
+                if not entity_exists:
+                    st.error(f'{entity_table_name} name not found in the table. Please enter a valid {entity_table_name} name.')
+                    continue
+
+                name_variant = st.text_input(f'Select an appropriate value for the following variant {unmatched_name}:')
+                if not name_variant:
+                    continue
+
+                cursor.execute(f"SELECT entityName, entityNameVariants, entityLGDCode FROM {entity_table_name}")
+                entity_data = cursor.fetchall()
+
+                for entity_name_db, entityNameVariants, entity_LGD_Code in entity_data:
+                    if entity_name.lower() == entity_name_db.lower():
+                        new_variants = f"{entityNameVariants.strip()}, {name_variant.strip()}" if entityNameVariants else name_variant
+                        cursor.execute(f"UPDATE {entity_table_name} SET entityNameVariants = ? WHERE entityLGDCode = ?", (new_variants.strip(), int(entity_LGD_Code)))
+                        st.success(f'{entity_name_db} Variation Updated Successfully.')
+                        break
+
+                conn.commit()
+                st.write('---')
+
+            chunk_index += 1  # Increment chunk index
+
+            if chunk_index < num_chunks:
+                unique_button_key = f"load_next_button_{chunk_index}"
+                load_next_button = st.button("Load Next Chunk", key=unique_button_key)
+                if not load_next_button:
+                    break  # Break the loop if button is not clicked
+
+        conn.close()
+
+    except Exception as e:
+        st.error(f"An error occurred: {str(e)}")
+    return "Done"
+
+
+
+
+
+
+
+
+
+def update_variationsold(unmatched_names, mapping, entity_table_name):
+    """
+    Updates the variations of unmatched names in the given mapping dictionary for a specific entity table.
+
+    Parameters:
+    - unmatched_names (list): A list of unmatched names to update the variations for.
+    - mapping (dict): A dictionary mapping entity names to their variations.
+    - entity_table_name (str): The name of the entity table to update the variations in.
+
+    Returns:
+    - str: The message "Done" indicating that the variations have been updated successfully.
+    """
+    try:
+        conn = sqlite3.connect('lgd_database.db')
+        cursor = conn.cursor()
+
+        for index, unmatched_name in enumerate(unmatched_names):
+            entity_name = create_selectbox_widget(f'Select {entity_table_name} name {index+1}:', list(mapping))
+            if not entity_name:
+                continue
+
+            entity_exists = entity_name in mapping
+            if not entity_exists:
+                st.error(f'{entity_table_name} name not found in the table. Please enter a valid {entity_table_name} name.')
+                continue
+
+            name_variant = st.text_input(f'Select an appropriate value for the following variant {unmatched_name}:')
+            if not name_variant:
+                continue
+
+            cursor.execute(f"SELECT entityName, entityNameVariants, entityLGDCode FROM {entity_table_name}")
+            entity_data = cursor.fetchall()
+
+            for entity_name_db, entityNameVariants, entity_LGD_Code in entity_data:
+                if entity_name.lower() == entity_name_db.lower():
+                    new_variants = f"{entityNameVariants.strip()}, {name_variant.strip()}" if entityNameVariants else name_variant
+                    cursor.execute(f"UPDATE {entity_table_name} SET entityNameVariants = ? WHERE entityLGDCode = ?", (new_variants.strip(), int(entity_LGD_Code)))
+                    st.success(f'{entity_name_db} Variation Updated Successfully.')
+                    break
+
+            conn.commit()
+            st.write('---')
+
+        conn.close()
+
+    except Exception as e:
+        st.error(f"An error occurred: {str(e)}")
+    return "Done"
+
+
+def update_variationso(unmatched_names, mapping, entity_table_name):
+    """
+    Updates the variations of a given entity in the database.
+
+    Parameters:
+    - unmatched_names (list): A list of unmatched names.
+    - mapping (dict): A dictionary mapping entity names to their corresponding values.
+    - entity_table_name (str): The name of the entity table.
+
+    Returns:
+    - str: The status message indicating the success of the function.
+    """
+    try:
+        entity_name = create_selectbox_widget(f'Select {entity_table_name} name :', list(mapping))
+        if not entity_name:
+            return
+
+        entity_exists = entity_name in mapping
+        if not entity_exists:
+            st.error(f'{entity_table_name} name not found in the table. Please enter a valid {entity_table_name} name.')
+            return
+
+        name_variant = st.text_input(f'Enter name variant {unmatched_names[0]}:')
+        if not name_variant:
+            return
+
+        conn = sqlite3.connect('lgd_database.db')
+        cursor = conn.cursor()
+
+        cursor.execute(f"SELECT entityName, entityNameVariants, entityLGDCode FROM {entity_table_name}")
+        entity_data = cursor.fetchall()
+
+        for entity_name_db, entityNameVariants, entity_LGD_Code in entity_data:
+            if entity_name.lower() == entity_name_db.lower():
+                new_variants = f"{entityNameVariants.strip()}, {name_variant.strip()}" if entityNameVariants else name_variant
+                cursor.execute(f"UPDATE {entity_table_name} SET entityNameVariants = ? WHERE entityLGDCode = ?", (new_variants.strip(), int(entity_LGD_Code)))
+                st.success(f'{entity_name_db} Variation Updated Successfully.')
+                break
+
+        conn.commit()
+        conn.close()
+        st.write('---')
+
+    except Exception as e:
+        st.error(f"An error occurred: {str(e)}")
+    return "Done"
+
+
+def update_variations_without_parent(unmatched_names, mapping, entity_table_name):
+
+    try:
+
+
+        entity_name = create_selectbox_widget(f'Select {entity_table_name} name :', list(mapping))
+
+
+ 
+
+        entity_exists = entity_name in mapping
+
+       
+
+        if entity_name and not entity_exists:
+
+            st.error(f'{entity_table_name} name not found in the table. Please enter a valid {entity_table_name} name.')
+
+ 
+
+        name_variant = st.text_input(f'Enter name variant {unmatched_names[0]}:')
+
+        if entity_name and name_variant and entity_exists:
+
+            conn = sqlite3.connect('lgd_database.db')
+
+            cursor = conn.cursor()
+
+            cursor.execute(f"SELECT entityName, entityNameVariants, entityLGDCode FROM {entity_table_name}")
+
+            entity_data = cursor.fetchall()
+
+            for entity_name_db, entityNameVariants, entity_LGD_Code in entity_data:
+
+                if entity_name.lower() == entity_name_db.lower():
+
+                    new_variants = f"{entityNameVariants.strip()}, {name_variant.strip()}" if entityNameVariants else name_variant
+
+                    cursor.execute(f"UPDATE {entity_table_name} SET entityNameVariants = ? WHERE entityLGDCode = ?", (new_variants.strip(), int(entity_LGD_Code)))
+
+                    st.success(f'{entity_name_db} Variation Updated Successfully.')
+
+                   
+
+ 
+
+            conn.commit()
+
+            conn.close()
+
+            st.write('---')
+
+ 
+
+    except Exception as e:
+
+        st.error(f"An error occurred: {str(e)}")
+
+    return "Done"
+
+
+
+
+import concurrent.futures
+
+
+
+def generate_download_link(mapped_dataset):
+
+    csv_file = mapped_dataset.to_csv(index=False)
+    b64 = base64.b64encode(csv_file.encode()).decode()
+    href = f'<a href="data:file/csv;base64,{b64}" download="mapped_dataset.csv">Download</a>'
+    st.success('Download Mapped Dataset')
+    st.markdown(href, unsafe_allow_html=True) 
+
+
+def fetch_block_mapping():
+    """
+    Fetch the block mapping from the SQLite database.
+
+    Returns:
+    - A list of tuples containing the block entity name, LGD code, name variants, and parent entity.
+    """
+    # Connect to the SQLite database
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Retrieve block data from the 'blocks' table
+    cursor.execute("SELECT entityName, entityLGDCode, entityNameVariants, entityParent FROM block")
+    data = cursor.fetchall()
+
+    # Close the database connection
+    conn.close()
+
+    return data
+
+def populate_block_mapping():
+
+    state_dataset = pd.read_csv('data.csv')
+    
+    unique_rows = state_dataset.drop_duplicates(subset=['block_name'])
+    unique_rows_lower = unique_rows.apply(lambda x: (x['block_name'].strip().lower(), x['district_code']), axis=1).tolist()
+
+    district_mapping = {}
+    edname = "Not Available"
+
+    for district_name, district_code, district_variants, parent_code in data:
+        for row in unique_rows_lower:
+            district_name_lower = row[0]
+            state_code = row[1]
+            if district_name_lower == district_name.lower():
+                if int(parent_code) == int(state_code):
+                    district_mapping[district_name.lower()] = district_code
+                    if district_variants:
+                        for variant in district_variants.split(','):
+                            district_mapping[variant.strip().lower()] = district_code
+    
+
+    for district_name, district_code, district_variants, parent_code in data:
+        if edname.lower() == district_name.lower():
+            if int(parent_code) == int(0):
+                district_mapping[district_name.lower()] = district_code
+                if district_variants:
+                    for variant in district_variants.split(','):
+                        district_mapping[variant.strip().lower()] = district_code
+    
+
+    return district_mapping
+
+
+
+def create_block_mapped_dataset(dataset, mapping):
+    """
+    Create a mapped dataset by associating block codes with block names in the dataset.
+    """
+    dataset['block_name'] = dataset['block_name'].str.strip()
+    dataset['block_code'] = dataset['block_name'].str.lower().map(mapping)
+    dataset.loc[dataset['block_code'].isnull(), 'block_code'] = -2
+    return dataset
+
+
+def fetch_gp_mapping():
+    """
+    Fetch the gp mapping from the SQLite database.
+
+    Returns:
+    - A list of tuples containing the gp entity name, LGD code, name variants, and parent entity.
+    """
+    # Connect to the SQLite database
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Retrieve gp data from the 'gps' table
+    cursor.execute("SELECT entityName, entityLGDCode, entityNameVariants, entityParent FROM gp")
+    data = cursor.fetchall()
+
+    # Close the database connection
+    conn.close()
+
+    return data
+
+def fetch_village_mapping():
+    """
+    Fetch the gp mapping from the SQLite database.
+
+    Returns:
+    - A list of tuples containing the gp entity name, LGD code, name variants, and parent entity.
+    """
+    # Connect to the SQLite database
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Retrieve gp data from the 'gps' table
+    cursor.execute("SELECT villageNameEnglish, villageCode, entityNameVariants FROM villages")
+    data = cursor.fetchall()
+
+    # Close the database connection
+    conn.close()
+
+    return data
+
+def create_entity_name_list():
+    data = fetch_gp_mapping()
+    entity_name_list = [entity_name for entity_name, _, _, _ in data]
+    return entity_name_list
+def populate_gp_mapping():
+    """
+    Populates a gp mapping dictionary using data from a database and a local file.
+
+    Returns:
+        A dictionary containing the mapping of gp names to their respective codes.
+    """
+    state_dataset = pd.read_csv('data.csv')
+    data = fetch_gp_mapping()
+    unique_rows = state_dataset.drop_duplicates(subset=['gp_name'])
+    unique_rows_lower = unique_rows.apply(lambda x: (str(x['gp_name']).strip().lower(), x['block_code']), axis=1).tolist()
+
+    entity_mapping = {}
+
+    # Populate mapping for entity name and variants
+    for entity_name, entity_code, entity_variants, parent_code in data:
+        for row in unique_rows_lower:
+            entity_name_lower = row[0]
+            state_code = row[1]
+            if entity_name_lower == entity_name.lower() and int(parent_code) == int(state_code):
+                entity_mapping[entity_name_lower] = entity_code
+                if entity_variants:
+                    for variant in entity_variants.split(','):
+                        entity_mapping[variant.strip().lower()] = entity_code
+
+    # Populate mapping for special case entity name
+    edname = "Not Available"
+    for entity_name, entity_code, entity_variants, parent_code in data:
+        if edname.lower() == entity_name.lower() and str(parent_code) == str(0):
+            entity_mapping[entity_name.lower()] = entity_code
+            if entity_variants:
+                for variant in entity_variants.split(','):
+                    entity_mapping[variant.strip().lower()] = entity_code
+
+    return entity_mapping
+
+
+def populate_village_mapping():
+    """
+    Populates a gp mapping dictionary using data from a database and a local file.
+
+    Returns:
+        A defaultdict containing the mapping of gp names to their respective codes.
+    """
+    state_dataset = pd.read_csv('data.csv')
+    data = fetch_village_mapping()
+    unique_rows = state_dataset.drop_duplicates(subset=['village_name'])
+    unique_rows_lower = unique_rows.apply(lambda x: (x['village_name'].strip().lower(), x['panchayat_name']), axis=1).tolist()
+
+    district_mapping = {}
+
+    for district_name, district_code, district_variants, parent_code in data:
+        for row in unique_rows_lower:
+            district_name_lower = row[0]
+            state_code = row[1]
+            if district_name_lower == district_name.lower():
+                if int(parent_code) == int(state_code):
+                    district_mapping[district_name_lower] = district_code
+                    if district_variants:
+                        for variant in district_variants.split(','):
+                            district_mapping[variant.strip().lower()] = district_code
+
+    return district_mapping
+
+
+
+def create_gp_mapped_dataset(dataset, mapping):
+    """
+    Create a mapped dataset by associating gp codes with gp names in the dataset.
+    """
+    dataset['gp_name'] = dataset['gp_name'].str.strip()
+    dataset['gp_code'] = dataset['gp_name'].str.lower().map(mapping)
+    dataset.loc[dataset['gp_code'].isnull(), 'gp_code'] = -2
+    return dataset
+
+def create_village_mapped_dataset(dataset, mapping):
+    """
+    Create a mapped dataset by associating gp codes with gp names in the dataset.
+    """
+    dataset['village_name'] = dataset['village_name'].str.strip()
+    dataset['village_code'] = dataset['village_name'].str.lower().map(mapping)
+    dataset.loc[dataset['village_code'].isnull(), 'village_code'] = -2
+    return dataset
+
+def fetch_sub_district_mapping():
+    """
+    Fetch the Sub-District mapping from the SQLite database.
+
+    Returns:
+    - A list of tuples containing the Sub-District entity name, LGD code, name variants, and parent entity.
+    """
+    # Connect to the SQLite database
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Retrieve Sub-District data from the 'Sub-Districts' table
+    cursor.execute("SELECT entityName, entityLGDCode, entityNameVariants, entityParent FROM sub_district")
+    data = cursor.fetchall()
+
+    # Close the database connection
+    conn.close()
+    return data
+
+def populate_sub_district_mapping():
+    """
+    Populates a sub_district mapping dictionary using data from a database and a local file.
+
+    Returns:
+        A defaultdict containing the mapping of sub_district names to their respective codes.
+    """
+    state_dataset = pd.read_csv('data.csv')
+    data = fetch_sub_district_mapping()
+    unique_rows = state_dataset.drop_duplicates(subset=['sub_district_name'])
+    unique_rows_lower = unique_rows.apply(lambda x: (x['sub_district_name'].strip().lower(), x['district_code']), axis=1).tolist()
+    district_mapping = {}
+    for district_name, district_code, district_variants, parent_code in data:
+        for row in unique_rows_lower:
+            district_name_lower = row[0]
+            state_code = row[1]
+            if district_name_lower == district_name.lower():
+                if int(parent_code) == int(state_code):
+                    district_mapping[district_name_lower] = district_code
+                    if district_variants:
+                        for variant in district_variants.split(','):
+                            district_mapping[variant.strip().lower()] = district_code
+
+    return district_mapping
+
+
+
+def create_sub_district_mapped_dataset(dataset, mapping):
+    """
+    Create a mapped dataset by associating Sub-District codes with Sub-District names in the dataset.
+    """
+    dataset['sub_district_name'] = dataset['sub_district_name'].str.strip()
+    dataset['sub_district_code'] = dataset['sub_district_name'].str.lower().map(mapping)
+    dataset.loc[dataset['sub_district_code'].isnull(), 'sub_district_code'] = -2
+    return dataset

requirements.txt

@@ -0,0 +1,4 @@
+sqlite3
+streamlit
+pandas
+numpy

utils.py

@@ -0,0 +1,546 @@
+import sqlite3
+import requests
+import sqlite3
+import hashlib
+
+def fetch_data_from_api(code,local_body_type_code):
+    url = f'https://lgdirectory.gov.in/webservices/lgdws/localBodyList?stateCode={code}&localbodyTypeCode={local_body_type_code}'
+    payload = {}
+    headers = {}
+    response = requests.request("POST", url, headers=headers, data=payload)
+    if response.status_code == 200:
+        data = response.json()
+        return data
+    else:
+        print("Failed to fetch data from the API.")
+        return None
+
+def calculate_hash(data):
+    hash_object = hashlib.sha256(str(data).encode())
+    return hash_object.hexdigest()
+
+
+def get_level_code(tablename):
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+    # Execute the SQL query
+    query = f"SELECT entityLGDCode FROM {tablename}"
+    cursor.execute(query)
+
+    # Fetch all the rows from the query result
+    rows = cursor.fetchall()
+
+    # Extract the entityLGDCode values into a Python list
+    result = [row[0] for row in rows]
+
+    # Close the cursor and connection
+    cursor.close()
+
+    # Print the final result
+    return result
+
+
+
+def check_and_update_data():
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/villageListWithHierarchy?subDistrictCode='
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/subdistrictList?districtCode='
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/districtList?stateCode='
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/blockList?districtCode='
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/getBlockwiseMappedGP?blockCode='
+
+    try:
+        result = get_level_code("states")
+        for code in result:
+            print(code)
+            for local_body_type_code in range(30):
+                try:
+                    data = fetch_data_from_api(str(code), local_body_type_code)
+                    if data:
+                        data_hash = calculate_hash(data)
+                        insert_local_body_data_in_database(data, data_hash, code)
+                    else:
+                        print("Failed to fetch data from the API.")
+                except Exception as e:
+                    print("An error occurred:", str(e))
+    except Exception as e:
+        print("An error occurred:", str(e))
+
+def insert_district_data_in_database(data,data_hash, code):
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Create a table if it doesn't exist
+    cursor.execute('''CREATE TABLE IF NOT EXISTS district (
+                        entityLGDCode INTEGER PRIMARY KEY,
+                        census2001Code TEXT,
+                        census2011Code TEXT,
+                        entityName TEXT,
+                        levelCode INTEGER,
+                        levelName TEXT,
+                        entityNameVariants TEXT,
+                        entityParent TEXT,
+                        dataHash TEXT
+                    )''')
+
+    try:
+        # Start a transaction
+        conn.execute("BEGIN TRANSACTION")
+
+        # Insert the data into the table
+        for item in data:
+            entity_lgd_code = item['districtCode']
+            census_2001_code = item['census2001Code']
+            census_2011_code = item['census2011Code']
+            entity_name = item['districtNameEnglish']
+            level_code = 2
+            level_name = "District"
+            entity_name_variants = item['districtNameLocal']
+            entity_parent = code
+
+            cursor.execute('''INSERT INTO district (
+                                entityLGDCode, census2001Code, census2011Code, entityName,
+                                levelCode, levelName, entityNameVariants, entityParent,dataHash
+                            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?,?)''',
+                           (
+                               entity_lgd_code, census_2001_code, census_2011_code, entity_name,
+                               level_code, level_name, entity_name_variants, entity_parent,data_hash
+                           ))
+            print(f"Data for districtCode: {entity_lgd_code} inserted successfully.")
+
+        # Commit the changes
+        conn.execute("COMMIT")
+        print("All data inserted successfully.")
+
+    except Exception as e:
+        # Rollback the transaction in case of any error
+        conn.execute("ROLLBACK")
+        print(f"Error occurred: {str(e)}")
+
+    finally:
+        # Close the connection
+        conn.close()
+
+import sqlite3
+
+def insert_sub_district_data_in_database(data,datahash ,code):
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Create a table if it doesn't exist
+    cursor.execute('''CREATE TABLE IF NOT EXISTS sub_district (
+                        entityLGDCode INTEGER PRIMARY KEY,
+                        census2001Code TEXT,
+                        census2011Code TEXT,
+                        entityName TEXT,
+                        levelCode INTEGER,
+                        levelName TEXT,
+                        entityNameVariants TEXT,
+                        entityParent TEXT,
+                        dataHash TEXT
+                    )''')
+
+    try:
+        # Start a transaction
+        conn.execute("BEGIN TRANSACTION")
+
+        # Insert the data into the table
+        for item in data:
+            entity_lgd_code = item['subdistrictCode']
+            census_2001_code = item['census2001Code']
+            census_2011_code = item['census2011Code']
+            entity_name = item['subdistrictNameEnglish']
+            level_code = 3
+            level_name = "sub_district"
+            entity_name_variants = item['subdistrictNameLocal']
+            entity_parent = code
+
+            cursor.execute('''INSERT INTO sub_district (
+                                entityLGDCode, census2001Code, census2011Code, entityName,
+                                levelCode, levelName, entityNameVariants, entityParent, dataHash
+                            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?,?)''',
+                           (
+                               entity_lgd_code, census_2001_code, census_2011_code, entity_name,
+                               level_code, level_name, entity_name_variants, entity_parent,datahash
+                           ))
+            print(f"Data for subdistrictCode: {entity_lgd_code} inserted successfully.")
+
+        # Commit the changes
+        conn.execute("COMMIT")
+        print("All data inserted successfully.")
+
+    except Exception as e:
+        # Rollback the transaction in case of any error
+        conn.execute("ROLLBACK")
+        print(f"Error occurred: {str(e)}")
+
+    finally:
+        # Close the connection
+        conn.close()
+
+import sqlite3
+
+def insert_block_data_in_database(data,datahash ,code):
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Create a table if it doesn't exist
+    cursor.execute('''CREATE TABLE IF NOT EXISTS block (
+                        entityLGDCode INTEGER PRIMARY KEY,
+                        entityName TEXT,
+                        levelCode INTEGER,
+                        levelName TEXT,
+                        entityNameVariants TEXT,
+                        entityParent TEXT,
+                        dataHash TEXT
+                    )''')
+
+    try:
+        # Start a transaction
+        conn.execute("BEGIN TRANSACTION")
+
+        # Insert the data into the table
+        for item in data:
+            entity_lgd_code = item['blockCode']
+            entity_name = item['blockNameEnglish']
+            level_code = 4
+            level_name = "block"
+            entity_name_variants = item['blockNameLocal']
+            entity_parent = code
+
+            cursor.execute('''INSERT INTO block (
+                                entityLGDCode, entityName,
+                                levelCode, levelName, entityNameVariants, entityParent, dataHash
+                            ) VALUES (?, ?, ?, ?, ?, ?, ?)''',
+                           (
+                               entity_lgd_code, entity_name,
+                               level_code, level_name, entity_name_variants, entity_parent,datahash
+                           ))
+            print(f"Data for block: {entity_lgd_code} inserted successfully.")
+
+        # Commit the changes
+        conn.execute("COMMIT")
+        print("All data inserted successfully.")
+
+    except Exception as e:
+        # Rollback the transaction in case of any error
+        conn.execute("ROLLBACK")
+        print(f"Error occurred: {str(e)}")
+
+    finally:
+        # Close the connection
+        conn.close()
+import sqlite3
+import urllib3
+
+def insert_gp_data_in_database(data, datahash, code):
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Create a table if it doesn't exist
+    cursor.execute('''CREATE TABLE IF NOT EXISTS gp (
+                        entityLGDCode INTEGER PRIMARY KEY,
+                        entityName TEXT,
+                        levelCode INTEGER,
+                        levelName TEXT,
+                        entityNameVariants TEXT,
+                        entityParent TEXT,
+                        entityParentName TEXT,
+                        dataHash TEXT
+                    )''')
+
+    try:
+        # Start a transaction
+        conn.execute("BEGIN TRANSACTION")
+
+        # Insert the data into the table
+        for item in data:
+            entity_lgd_code = item['localBodyCode']
+            entity_name = item['localBodyNameEnglish']
+            level_code = 5
+            level_name = "Gram Panchayats"
+            entity_name_variants = item['localBodyNameLocal']
+            entity_parent = code
+            entity_parent_name = "block"
+
+            cursor.execute('''INSERT INTO gp (
+                                entityLGDCode, entityName,
+                                levelCode, levelName, entityNameVariants, entityParent,entityParentName, dataHash
+                            ) VALUES (?, ?, ?, ?, ?, ?, ?,?)''',
+                           (
+                               entity_lgd_code, entity_name,
+                               level_code, level_name, entity_name_variants, entity_parent,entity_parent_name,datahash
+                           ))
+            print(f"Data for gp: {entity_lgd_code} inserted successfully.")
+
+        # Commit the changes
+        conn.execute("COMMIT")
+        print("All data inserted successfully.")
+
+    except urllib3.exceptions.TimeoutError as te:
+        print("Timeout error occurred while making the HTTP request.")
+        # Handle the timeout error here, e.g., retry the request or log the error.
+
+    except Exception as e:
+        # Rollback the transaction in case of any other error
+        conn.execute("ROLLBACK")
+        print(f"Error occurred: {str(e)}")
+
+    finally:
+        # Close the connection
+        conn.close()
+
+import sqlite3
+import urllib3
+
+def insert_local_body_data_in_database(data, datahash, code):
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Create a table if it doesn't exist
+    cursor.execute('''CREATE TABLE IF NOT EXISTS local_body (
+                        entityLGDCode INTEGER PRIMARY KEY,
+                        entityName TEXT,
+                        entitylocalBodyTypeName TEXT,
+                        levelCode INTEGER,
+                        levelName TEXT,
+                        entityNameVariants TEXT,
+                        entityParent TEXT,
+                        entityParentName TEXT,
+                        dataHash TEXT
+                    )''')
+
+    try:
+        # Start a transaction
+        conn.execute("BEGIN TRANSACTION")
+
+        # Insert the data into the table
+        for item in data:
+            entity_lgd_code = item['localBodyCode']
+            entity_name = item['localBodyNameEnglish']
+            entity_local_body_type_name = item['localBodyTypeName']
+            level_code = -1
+            level_name = "Local Body Type Name"
+            entity_name_variants = item['localBodyNameLocal']
+            entity_parent = code
+            entity_parent_name = "state"
+
+            cursor.execute('''INSERT INTO local_body (
+                                entityLGDCode, entityName,entitylocalBodyTypeName,
+                                levelCode, levelName, entityNameVariants, entityParent,entityParentName, dataHash
+                            ) VALUES (?, ?, ?, ?, ?, ?, ?,?,?)''',
+                           (
+                               entity_lgd_code, entity_name,entity_local_body_type_name,
+                               level_code, level_name, entity_name_variants, entity_parent,entity_parent_name,datahash
+                           ))
+            print(f"Data for gp: {entity_lgd_code} inserted successfully.")
+
+        # Commit the changes
+        conn.execute("COMMIT")
+        print("All data inserted successfully.")
+
+    except urllib3.exceptions.TimeoutError as te:
+        print("Timeout error occurred while making the HTTP request.")
+        # Handle the timeout error here, e.g., retry the request or log the error.
+
+    except Exception as e:
+        # Rollback the transaction in case of any other error
+        conn.execute("ROLLBACK")
+        print(f"Error occurred: {str(e)}")
+
+    finally:
+        # Close the connection
+        conn.close()
+
+
+import sqlite3
+
+def store_village_data_in_database(data, data_hash):
+    conn = sqlite3.connect('lgd_database.db')
+    cursor = conn.cursor()
+
+    # Create a table if it doesn't exist
+    cursor.execute('''CREATE TABLE IF NOT EXISTS villages (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        stateCode INTEGER,
+                        stateNameEnglish TEXT,
+                        districtCode INTEGER,
+                        districtNameEnglish TEXT,
+                        subDistrictCode INTEGER,
+                        subDistrictNameEnglish TEXT,
+                        blockCode INTEGER,
+                        blockNameEnglish TEXT,
+                        localBodyCode INTEGER,
+                        localBodyTypeCode INTEGER,
+                        localBodyNameEnglish TEXT,
+                        villageCode INTEGER,
+                        villageNameEnglish TEXT,
+                        villageStatus TEXT,
+                        dataHash TEXT
+                    )''')
+
+    try:
+        # Start a transaction
+        conn.execute("BEGIN TRANSACTION")
+
+        # Insert the data
+        for item in data:
+            state_code = item['stateCode']
+            state_name = item['stateNameEnglish']
+            district_code = item['districtCode']
+            district_name = item['districtNameEnglish']
+            subdistrict_code = item['subDistrictCode']
+            subdistrict_name = item['subDistrictNameEnglish']
+            block_code = item['blockCode']
+            block_name = item['blockNameEnglish']
+            local_body_code = item['localBodyCode']
+            local_body_type_code = item['localBodyTypeCode']
+            local_body_name = item['localBodyNameEnglish']
+            village_code = item['villageCode']
+            village_name = item['villageNameEnglish']
+            village_status = item['villageStatus']
+
+            # Insert a new row
+            cursor.execute('''INSERT INTO villages (
+                                stateCode, stateNameEnglish, districtCode, districtNameEnglish,
+                                subDistrictCode, subDistrictNameEnglish, blockCode, blockNameEnglish,
+                                localBodyCode, localBodyTypeCode, localBodyNameEnglish,
+                                villageCode, villageNameEnglish, villageStatus, dataHash
+                            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',
+                           (
+                               state_code, state_name, district_code, district_name,
+                               subdistrict_code, subdistrict_name, block_code, block_name,
+                               local_body_code, local_body_type_code, local_body_name,
+                               village_code, village_name, village_status, data_hash
+                           ))
+            print(f"Data for stateCode: {state_code}, villageCode: {village_code} inserted successfully.")
+
+        # Commit the changes
+        conn.execute("COMMIT")
+        print("All data inserted successfully.")
+
+    except Exception as e:
+        # Rollback the transaction in case of any error
+        conn.execute("ROLLBACK")
+        print(f"Error occurred: {str(e)}")
+
+    finally:
+        # Close the connection
+        conn.close()
+
+
+
+def check_write_and_update_data(data,table_name):
+    try:
+        data = data
+        if data:
+            conn = sqlite3.connect('lgd_database.db')
+            cursor = conn.cursor()
+
+            # Retrieve the stored data from the database
+            cursor.execute(f'SELECT entityLGDCode, entityName FROM {table_name}')
+            rows = cursor.fetchall()
+
+            changed_rows = set()
+
+            for item in data:
+                if table_name == 'states':
+                    entity_lgd_code = item['stateCode']
+                    entity_name = item['stateNameEnglish']
+                elif table_name == 'district':
+                    entity_lgd_code = item['districtCode']
+                    entity_name = item['districtNameEnglish']
+                elif table_name == 'sub_district':
+                    entity_lgd_code = item['subdistrictCode']
+                    entity_name = item['subdistrictNameEnglish']
+                elif table_name == 'block':
+                    entity_lgd_code = item['blockCode']
+                    entity_name = item['blockNameEnglish']
+                elif table_name == 'gp':
+                    entity_lgd_code = item['localBodyCode']
+                    entity_name = item['localBodyNameEnglish']
+                """ elif table_name == 'villages':
+                    entity_lgd_code = item['localBodyCode']
+                    entity_name = item['localBodyNameEnglish'] """
+
+
+                # Find the matching row in the database
+                matching_rows = [row for row in rows if row[0] == entity_lgd_code]
+
+                if matching_rows:
+                    # Check if entityLGDCode and entityName have changed
+                    row = matching_rows[0]
+                    if row[0] != entity_lgd_code or row[1] != entity_name:
+                        changed_rows.add((entity_lgd_code, entity_name))
+
+            if changed_rows:
+                print("The following rows have changed:")
+                for row in changed_rows:
+                    entity_lgd_code, entity_name = row
+                    print("entityLGDCode:", entity_lgd_code)
+                    print("entityName:", entity_name)
+                    print()
+                    # Update the values in the database
+                    cursor.execute(f"UPDATE {table_name} SET entityName = ? WHERE entityLGDCode = ?", (entity_name, entity_lgd_code))
+                    conn.commit()          
+                print("Data updated successfully!")
+
+            else:
+                print("Data has not changed.")
+
+            # Update the data hash in the database
+
+            conn.close()
+
+        else:
+            print("Failed to fetch data from the API.")
+
+    except Exception as e:
+        print("An error occurred:", str(e))
+
+
+def fetch_data_from_api_update(url):
+    payload = {}
+    headers = {}
+    response = requests.request("POST", url, headers=headers, data=payload)
+    if response.status_code == 200:
+        data = response.json()
+        return data
+    else:
+        print("Failed to fetch data from the API.")
+        return None
+    
+
+def update_all_data():
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/villageListWithHierarchy?subDistrictCode='
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/subdistrictList?districtCode='
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/districtList?stateCode='
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/blockList?districtCode='
+    #url = 'https://lgdirectory.gov.in/webservices/lgdws/getBlockwiseMappedGP?blockCode='
+    name_pair = {'district':'states','sub_district':'district','block':'district','gp':'block'}
+    for key, value in name_pair.items():
+        table_name = key
+        print('table name:', table_name)
+        try:
+            result = get_level_code(value)
+            for code in result:
+                try:
+                    if table_name == 'states':
+                        url = f'https://lgdirectory.gov.in/webservices/lgdws/stateList'
+                    elif table_name == 'district':
+                        url = f'https://lgdirectory.gov.in/webservices/lgdws/districtList?stateCode={code}'
+                    elif table_name == 'sub_district':
+                        url = f'https://lgdirectory.gov.in/webservices/lgdws/subdistrictList?districtCode={code}'
+                    elif table_name == 'block':
+                        url = f'https://lgdirectory.gov.in/webservices/lgdws/blockList?districtCode={code}'
+                    elif table_name == 'gp':
+                        url = f'https://lgdirectory.gov.in/webservices/lgdws/getBlockwiseMappedGP?blockCode={code}'
+                    data = fetch_data_from_api_update(url)
+
+                    if data:
+                        check_write_and_update_data(data,table_name)
+                    else:
+                        print("Failed to fetch data from the API.")
+                except Exception as e:
+                    print("An error occurred:", str(e))
+        except Exception as e:
+            print("An error occurred:", str(e))