Spaces:
Runtime error
Runtime error
File size: 2,495 Bytes
0104e91 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | """Shaping the raw DataFrame: label mapping, deduplication and balancing."""
import pandas as pd
from config.constants import (
LABEL_COLUMN,
RATING_COLUMN,
RATING_TO_SENTIMENT,
RAW_COLUMNS_TO_DROP,
RAW_TEXT_COLUMN_CANDIDATES,
SENTIMENT_COLUMN,
TEXT_COLUMN,
)
def resolve_raw_text_column(df, text_column=None):
"""Return the name of the raw text column present in ``df``."""
if text_column is not None:
return text_column
for candidate in RAW_TEXT_COLUMN_CANDIDATES:
if candidate in df.columns:
return candidate
raise KeyError(
'No raw text column found; expected one of %s, got %s'
% (list(RAW_TEXT_COLUMN_CANDIDATES), list(df.columns))
)
def prepare_dataframe(df, text_column=None):
"""Drop bookkeeping columns, map ratings to sentiments and rename columns."""
text_column = resolve_raw_text_column(df, text_column)
df = df.drop(RAW_COLUMNS_TO_DROP, axis=1)
# Replace numeric values in the 'Rating' column with corresponding sentiment labels
df[SENTIMENT_COLUMN] = df[RATING_COLUMN].replace(RATING_TO_SENTIMENT)
# Rename columns
df = df.rename(columns={text_column: TEXT_COLUMN})
df = df.drop(RATING_COLUMN, axis=1)
return df
def rename_sentiment_to_label(df):
"""Rename the ``Sentiment`` column to ``label``."""
return df.rename(columns={SENTIMENT_COLUMN: LABEL_COLUMN})
def drop_missing_and_duplicates(df):
"""Drop rows with missing values and rows whose text is a duplicate."""
df = df.dropna()
index = df[df[TEXT_COLUMN].duplicated() == True].index
df.drop(index, axis=0, inplace=True)
return df
def balance_labels(df, label_column=LABEL_COLUMN, random_state=None):
"""Downsample the majority labels to the size of the smallest one.
Returns:
A new, index-reset, class-balanced DataFrame.
"""
sentiment_counts = df[label_column].value_counts()
target_count = sentiment_counts.min()
# Downsample 'Neutral' and 'Negative' to match the target count
neutral_equalized = df[df[label_column] == 'Neutral'].sample(
target_count, replace=False, random_state=random_state)
negative_equalized = df[df[label_column] == 'Negative'].sample(
target_count, replace=False, random_state=random_state)
# Combine the equalized dataframes
return pd.concat(
[neutral_equalized, negative_equalized, df[df[label_column] == 'Positive']]
).reset_index(drop=True)
|