insertGigWorkersIntoDB.py

import csv
import psycopg2
import os
from dotenv import load_dotenv

# Load environment variables from .env file
load_dotenv()

# Get database connection
def get_database_connection():
    database_url = os.getenv("DATABASE_URL")
    if not database_url:
        raise ValueError("DATABASE_URL is not set in environment variables.")
    return psycopg2.connect(database_url)

# Verify data
def verify_data(row):
    # Check if necessary fields are present
    required_fields = ["Name", "About (description after screening)", "Skills (after screening)", "Rating", "Trust Score", "Ninja level(chunin ,join ,kage )", "Task Experience", "Online or not or schedule at this time each day"]
    for field in required_fields:
        if field not in row or not row[field]:
            return False
    
    # Check if rating and trust_score are within valid ranges
    try:
        rating = float(row["Rating"])
        if rating < 0 or rating > 5:
            return False
    except ValueError:
        return False
    
    try:
        trust_score = int(row["Trust Score"])
        if trust_score < 0 or trust_score > 100:
            return False
    except ValueError:
        return False
    
    return True

# Insert data into database
def insert_data(row):
    skills_array = row["Skills (after screening)"].split(",")
    skills_array = [skill.strip() for skill in skills_array]
    ninja_level = row["Ninja level(chunin ,join ,kage )"]
    
    conn = get_database_connection()
    try:
        with conn.cursor() as curs:
            curs.execute("""
                INSERT INTO gig_workers (name, about, skills, rating, trust_score, ninja_level, task_experience, online_status, profile_pic)
                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
                RETURNING id
            """, (row["Name"], row["About (description after screening)"], skills_array, row["Rating"], row["Trust Score"], ninja_level, row["Task Experience"], row["Online or not or schedule at this time each day"], ''))
            worker_id = curs.fetchone()[0]
            conn.commit()
        print(f"Gig worker added with ID: {worker_id}")
    except Exception as e:
        conn.rollback()
        print(f"Error adding gig worker: {e}")
    finally:
        conn.close()

# Main function
def main():
    file_path = '/Users/vishal/WORK/DATA_GEN/synthetic_profiles_1000.csv'
    
    with open(file_path, newline='', encoding='utf-8') as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            if verify_data(row):
                insert_data(row)
            else:
                print(f"Invalid data: {row}")

if __name__ == "__main__":
    main()