csv_clean.py

import sys
import pandas as pd
from collections import Counter

def majority_vote(labels):
    counter = Counter(labels)
    most_common = counter.most_common()
    if len(most_common) == 1:
        return most_common[0][0]
    elif len(most_common) > 1 and most_common[0][1] > most_common[1][1]:
        return most_common[0][0]
    return None  # Tie

def main():
    if len(sys.argv) != 2:
        print("Usage: python clean_majority.py <path_to_csv>")
        sys.exit(1)

    input_path = sys.argv[1]
    df = pd.read_csv(input_path)

    grouped = df.groupby('post_id')
    rows_to_keep = []

    for post_id, group in grouped:
        labels = group['label'].tolist()
        majority = majority_vote(labels)

        if majority is None:
            continue  # skip ties
        if majority.lower() == 'offensive':
            continue  # skip offensive

        # Keep only one row for this post_id with the majority label
        representative_row = group.iloc[0].copy()
        representative_row['label'] = majority
        rows_to_keep.append(representative_row)

    cleaned_df = pd.DataFrame(rows_to_keep)
    output_path = 'cleaned_output.csv'
    cleaned_df.to_csv(output_path, index=False)
    print(f"Cleaned data saved to: {output_path}")

if __name__ == "__main__":
    main()