natasha1704 commited on
Commit
276fc0f
·
verified ·
1 Parent(s): e73e447

Delete interaction_db/data_loader.py

Browse files
Files changed (1) hide show
  1. interaction_db/data_loader.py +0 -85
interaction_db/data_loader.py DELETED
@@ -1,85 +0,0 @@
1
- import os
2
- import time
3
- import pandas as pd
4
- import mysql.connector
5
- from mysql.connector import Error
6
- from sqlalchemy import create_engine
7
- from dotenv import load_dotenv
8
- from interaction_db.data_sources import (
9
- DRUG_TO_DRUG_POSITIVE,
10
- DRUG_TO_DRUG_NEGATIVE,
11
- SINGLE_DRUG_DATA,
12
- )
13
-
14
- load_dotenv()
15
-
16
- USER = os.getenv("MYSQL_USER")
17
- PASSWORD = os.getenv("MYSQL_PASSWORD")
18
- HOST = os.getenv("MYSQL_HOST")
19
- PORT = int(os.getenv("MYSQL_PORT", 3306))
20
- DATABASE = os.getenv("MYSQL_DATABASE")
21
-
22
-
23
- def wait_for_mysql(host, user, password, port, timeout=60):
24
- print(f"⏳ Waiting for MySQL at {host}:{port}...")
25
- start_time = time.time()
26
- while True:
27
- try:
28
- conn = mysql.connector.connect(
29
- host=host, user=user, password=password, port=port
30
- )
31
- conn.close()
32
- print("MySQL is ready!")
33
- break
34
- except Error as e:
35
- if time.time() - start_time > timeout:
36
- raise TimeoutError("MySQL did not become ready in time.")
37
- time.sleep(2)
38
-
39
-
40
- wait_for_mysql(HOST, USER, PASSWORD, PORT)
41
-
42
- # Create the database if it doesn't exist
43
- conn = mysql.connector.connect(host=HOST, user=USER, password=PASSWORD)
44
- cursor = conn.cursor()
45
- cursor.execute(f"CREATE DATABASE IF NOT EXISTS {DATABASE}")
46
- print(f"Database '{DATABASE}' created or already exists.")
47
- cursor.close()
48
- conn.close()
49
-
50
- # Data
51
- file_paths = {
52
- "drug_to_drug_positive_controls": DRUG_TO_DRUG_POSITIVE,
53
- "drug_to_drug_negative_controls": DRUG_TO_DRUG_NEGATIVE,
54
- "single_drug_data": SINGLE_DRUG_DATA,
55
- }
56
-
57
- # MySQL
58
- engine = create_engine(
59
- f"mysql+mysqlconnector://{USER}:{PASSWORD}@{HOST}:{PORT}/{DATABASE}"
60
- )
61
-
62
- # Process data
63
- for table_key, file_path in file_paths.items():
64
- if not os.path.exists(file_path):
65
- print(f"File not found: {file_path}")
66
- continue
67
-
68
- if table_key == "single_drug_data":
69
- # Load positive and negative controls into separate tables
70
- sheet_map = {
71
- "Tab1 - Positive": "single_drug_positive_controls",
72
- "Tab2 - Negative": "single_drug_negative_controls",
73
- }
74
- for sheet_name, table_name in sheet_map.items():
75
- df = pd.read_excel(file_path, sheet_name=sheet_name)
76
- df = df.where(pd.notnull(df), None)
77
- df.to_sql(name=table_name, con=engine, if_exists="replace", index=False)
78
- print(f"Uploaded sheet '{sheet_name}' to table '{table_name}'")
79
- else:
80
- df = pd.read_excel(file_path, sheet_name=0)
81
- df = df.where(pd.notnull(df), None)
82
- df.to_sql(name=table_key, con=engine, if_exists="replace", index=False)
83
- print(f"Uploaded file '{file_path}' to table '{table_key}'")
84
-
85
- print("All Excel data uploaded to MySQL successfully.")