Spaces:
Sleeping
Sleeping
File size: 1,295 Bytes
8630e6c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 |
"""
This script reads all CSV files from the data/ folder and inserts them into MongoDB.
"""
import os
import pandas as pd
from pymongo import MongoClient
from dotenv import load_dotenv
# Load environment variables
load_dotenv()
# MongoDB config
MONGO_URI = os.getenv("MONGO_URI")
MONGO_DB = os.getenv("MONGO_DB")
# Initialize connection
client = MongoClient(MONGO_URI)
db = client[MONGO_DB]
def insert_csv_to_mongo(data_folder="../data"):
"""Read CSV files from data folder and insert into MongoDB."""
csv_files = [f for f in os.listdir(data_folder) if f.endswith(".csv")]
if not csv_files:
print("⚠️ No CSV files found in the data folder.")
return
all_docs = []
for file in csv_files:
path = os.path.join(data_folder, file)
print(f"📂 Reading: {path}")
df = pd.read_csv(path)
records = df.to_dict(orient="records")
all_docs.extend(records)
if all_docs:
collection = db["company_data"]
collection.insert_many(all_docs)
print(f"✅ Inserted {len(all_docs)} records into MongoDB collection 'company_data'.")
else:
print("⚠️ No data found to insert.")
if __name__ == "__main__":
insert_csv_to_mongo()
|