Spaces:
Paused
Paused
File size: 2,679 Bytes
5b01a63 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
import csv
import psycopg2
import os
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
# Get database connection
def get_database_connection():
database_url = os.getenv("DATABASE_URL")
if not database_url:
raise ValueError("DATABASE_URL is not set in environment variables.")
return psycopg2.connect(database_url)
# Verify data
def verify_data(row):
# Check if necessary fields are present
required_fields = ["Name", "About (description after screening)", "Skills (after screening)", "Rating", "Trust Score", "Ninja level(chunin ,join ,kage )", "Task Experience", "Online or not or schedule at this time each day"]
for field in required_fields:
if field not in row or not row[field]:
return False
# Check if rating and trust_score are within valid ranges
try:
rating = float(row["Rating"])
if rating < 0 or rating > 5:
return False
except ValueError:
return False
try:
trust_score = int(row["Trust Score"])
if trust_score < 0 or trust_score > 100:
return False
except ValueError:
return False
return True
# Insert data into database
def insert_data(row):
skills_array = row["Skills (after screening)"].split(",")
skills_array = [skill.strip() for skill in skills_array]
ninja_level = row["Ninja level(chunin ,join ,kage )"]
conn = get_database_connection()
try:
with conn.cursor() as curs:
curs.execute("""
INSERT INTO gig_workers (name, about, skills, rating, trust_score, ninja_level, task_experience, online_status, profile_pic)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
""", (row["Name"], row["About (description after screening)"], skills_array, row["Rating"], row["Trust Score"], ninja_level, row["Task Experience"], row["Online or not or schedule at this time each day"], ''))
worker_id = curs.fetchone()[0]
conn.commit()
print(f"Gig worker added with ID: {worker_id}")
except Exception as e:
conn.rollback()
print(f"Error adding gig worker: {e}")
finally:
conn.close()
# Main function
def main():
file_path = '/Users/vishal/WORK/DATA_GEN/synthetic_profiles_1000.csv'
with open(file_path, newline='', encoding='utf-8') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
if verify_data(row):
insert_data(row)
else:
print(f"Invalid data: {row}")
if __name__ == "__main__":
main()
|