Spaces:
Sleeping
Sleeping
File size: 3,033 Bytes
984c70c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | from datasets import load_dataset
import pandas as pd
import os
import joblib
import logging
from config import DATASETS
from utils.text_processing import clean_text
# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def load_all_datasets():
all_data = pd.DataFrame()
# Load Hugging Face datasets
for ds_name in DATASETS:
if "/" in ds_name: # HF dataset
try:
logger.info(f"Loading Hugging Face dataset: {ds_name}")
dataset = load_dataset(ds_name)
if 'train' in dataset:
df = pd.DataFrame(dataset['train'])
else:
df = pd.DataFrame(dataset)
# Standardize column names
if 'resume' in df.columns and 'job_description' in df.columns:
df = df.rename(columns={'resume': 'text', 'job_description': 'jd_text'})
df['type'] = 'resume_jd_pair'
elif 'resume' in df.columns:
df = df.rename(columns={'resume': 'text'})
df['type'] = 'resume'
elif 'job_description' in df.columns:
df = df.rename(columns={'job_description': 'text'})
df['type'] = 'job_description'
all_data = pd.concat([all_data, df], ignore_index=True)
logger.info(f"Loaded {len(df)} records from {ds_name}")
except Exception as e:
logger.error(f"Error loading {ds_name}: {e}")
# Load Kaggle datasets
kaggle_path = "data/kaggle"
if os.path.exists(kaggle_path):
for file in os.listdir(kaggle_path):
if file.endswith('.csv'):
try:
file_path = os.path.join(kaggle_path, file)
logger.info(f"Loading Kaggle dataset: {file}")
df = pd.read_csv(file_path)
# Standardize columns
if 'Resume' in df.columns:
df = df.rename(columns={'Resume': 'text'})
df['type'] = 'resume'
elif 'Description' in df.columns:
df = df.rename(columns={'Description': 'text'})
df['type'] = 'job_description'
elif 'job_desc' in df.columns:
df = df.rename(columns={'job_desc': 'text'})
df['type'] = 'job_description'
all_data = pd.concat([all_data, df], ignore_index=True)
logger.info(f"Loaded {len(df)} records from {file}")
except Exception as e:
logger.error(f"Error loading Kaggle dataset {file}: {e}")
# Clean text
if not all_data.empty and 'text' in all_data.columns:
all_data['cleaned_text'] = all_data['text'].apply(clean_text)
return all_data |