app.py

# Importing Necessary Libraries
import streamlit as st
import requests
import json
from datetime import datetime
import time
from docx import Document
import tempfile
import pandas as pd
import zipfile
import os

# Function to Scrape the User Profile and Save the Result in the DOCx File
def process_dataset(api_token, input_url, email_id, phone_number):
    """
    Triggers a dataset and fetches the snapshot from BrightData API, saving the result to a DOCx file.
    
    Parameters:
        api_token (str): The authorization token for API access.
        input_url (str): The LinkedIn URL to be processed.
        email_id (str): The email ID to be added to the profile data.
        phone_number (str): The phone number to be added to the profile data.
    
    Returns:
        str: Path to the DOCx file.
    """
    print("Starting process_dataset")
    print(f"API Token: {api_token[:5]}***")  # Masked for security
    print(f"Input URL: {input_url}")
    print(f"Email ID: {email_id}, Phone Number: {phone_number}")

    headers = {
        "Authorization": f"Bearer {api_token}",
        "Content-Type": "application/json"
    }

    # Trigger the BrightData API to fetch the data
    trigger_url = "https://api.brightdata.com/datasets/v3/trigger"
    trigger_payload = [{"url": input_url}]
    trigger_params = {"dataset_id": "gd_l1viktl72bvl7bjuj0", "include_errors": "true"}

    try:
        print("Triggering BrightData API...")
        trigger_response = requests.post(trigger_url, json=trigger_payload, headers=headers, params=trigger_params)
        print(f"Trigger Response Status Code: {trigger_response.status_code}")
        if trigger_response.status_code == 200:
            trigger_data = trigger_response.json()
            print(f"Trigger Response Data: {trigger_data}")
            snapshot_id = trigger_data.get('snapshot_id', None)
            if not snapshot_id:
                print("Snapshot ID not found in response.")
                return "Error: Snapshot ID not found in the response"
        else:
            print("Failed to trigger dataset.")
            return f"Error: Failed to trigger dataset with status code {trigger_response.status_code}"
    except Exception as e:
        print(f"Exception during API trigger: {e}")
        return f"Error: {str(e)}"

    # Fetch the snapshot
    snapshot_url = f"https://api.brightdata.com/datasets/v3/snapshot/{snapshot_id}"
    snapshot_params = {"format": "json"}

    attempts = 0
    max_retries = 10
    retry_interval = 10  # seconds
    snapshot_data = None

    while attempts < max_retries:
        try:
            print(f"Fetching snapshot (Attempt {attempts + 1})...")
            snapshot_response = requests.get(snapshot_url, headers=headers, params=snapshot_params)
            print(f"Snapshot Response Status Code: {snapshot_response.status_code}")
            if snapshot_response.status_code == 200:
                snapshot_data = snapshot_response.json()
                print("Snapshot data retrieved successfully.")
                break
            elif snapshot_response.status_code == 202:
                print("Snapshot not ready yet. Retrying...")
                time.sleep(retry_interval)
                attempts += 1
            else:
                print("Failed to fetch snapshot.")
                return f"Error: Failed to fetch snapshot with status code {snapshot_response.status_code}"
        except Exception as e:
            print(f"Exception during snapshot fetch: {e}")
            return f"Error: {str(e)}"

    if not snapshot_data:
        print("Snapshot data could not be retrieved after retries.")
        return "Error: Snapshot data could not be retrieved after multiple retries"

    if isinstance(snapshot_data, list) and len(snapshot_data) > 0:
        profile_data = snapshot_data[0]
        print(f"Profile Data: {profile_data}")
    else:
        print("No data found in snapshot.")
        return "Error: No data found in snapshot"

    profile_data['Email_ID'] = email_id
    profile_data['Phone_Number'] = phone_number

    # Generate DOCx file from the scraped data
    print("Generating DOCx file...")
    doc = Document()
    doc.add_heading('LinkedIn Profile Data', 0)

    def add_json_to_word(data, parent=None):
        if isinstance(data, dict):
            for key, value in data.items():
                parent.add_paragraph(f"{key}:")
                add_json_to_word(value, parent=parent)
        elif isinstance(data, list):
            for item in data:
                add_json_to_word(item, parent=parent)
        else:
            parent.add_paragraph(f"{data}")

    add_json_to_word(profile_data, parent=doc)

    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".docx")
    doc.save(temp_file.name)

    print(f"DOCx file saved at {temp_file.name}")
    return temp_file

# Streamlit Interface
def create_streamlit_interface():
    st.title("LinkedIn Profile Scraper Using BrightData")

    api_token = st.text_input("BrightData Token", placeholder="Enter BrightData Token", type="password")
    uploaded_file = st.file_uploader("Upload CSV File", type=["csv"])

    if uploaded_file and api_token:
        print("CSV file and API token received.")
        df = pd.read_csv(uploaded_file)
        valid_rows = df[df['LinkedIn Link'].notna()]
        print(f"Number of valid rows: {len(valid_rows)}")

        temp_files = []  # List to store file paths

        for index, row in valid_rows.iterrows():
            try:
                email = row['Email Address'] if pd.notna(row['Email Address']) else "None"
                phone = row['Mobile Number'] if pd.notna(row['Mobile Number']) else "None"
                linkedin = row['LinkedIn Link']

                print(f"Processing LinkedIn Profile: {linkedin}")
                result_file = process_dataset(api_token, linkedin, email, phone)

                if isinstance(result_file, str) and result_file.startswith("Error"):
                    print(f"Error for {linkedin}: {result_file}")
                    st.error(f"Error for {linkedin}: {result_file}")
                    continue

                temp_files.append(result_file.name)
                print(f"Successfully processed {linkedin}")

                with open(result_file.name, "rb") as f:
                    st.download_button(
                        label=f"Download Resume for {row.get('Applicant Name', 'Unknown')}",
                        data=f,
                        file_name=f"Resume_{row.get('Applicant Name', 'Unknown')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.docx",
                        mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document"
                    )
            except Exception as e:
                print(f"Unexpected error for {row['LinkedIn Link']}: {e}")
                st.error(f"Unexpected error for {row['LinkedIn Link']}: {str(e)}")

        if temp_files:
            print("Creating ZIP file for all resumes...")
            zip_file_path = tempfile.NamedTemporaryFile(delete=False, suffix=".zip").name
            with zipfile.ZipFile(zip_file_path, 'w') as zipf:
                for file_path in temp_files:
                    arcname = os.path.basename(file_path)
                    zipf.write(file_path, arcname=arcname)
            print(f"ZIP file created at {zip_file_path}")

            # Provide the ZIP file for download
            with open(zip_file_path, "rb") as f:
                st.download_button(
                    label="Download All Resumes as ZIP",
                    data=f,
                    file_name=f"All_Resumes_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip",
                    mime="application/zip"
                )
    elif not api_token:
        print("API token not provided.")
        st.warning("Please enter your BrightData API Token.")
    elif not uploaded_file:
        print("CSV file not uploaded.")
        st.warning("Please upload a CSV file.")

# Run the Streamlit app
if __name__ == "__main__":
    print("Starting Streamlit application...")
    create_streamlit_interface()