Spaces:
Runtime error
Runtime error
File size: 3,867 Bytes
0f95818 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | # data/preprocess.py
import re
import pandas as pd
import psycopg2
import os
from dotenv import load_dotenv
load_dotenv()
# Common Hinglish/Hindi words for code-mix ratio detection
HINDI_WORDS = {
'bakwaas','bekar','mast','zabardast','yaar','bhai','ekdum','bilkul',
'nahi','nhi','haan','theek','accha','sahi','faltu','kya','hai','tha',
'thi','ho','kar','raha','rahi','wala','wali','bohot','bahut','thoda',
'zyada','abhi','phir','matlab','sach','acha','hoga','karega','karenge',
'karo','karna','dono','sabse','kitna','kyun','isliye','lekin','aur',
'paisa','vasool','khushi','aansu','match','coffee','thandi','taste',
'exam','clear','full','kal','yeh','ye','toh','to','bhi','se','pe',
'mein','mai','ne','ko','ka','ki','ke','log','kuch','sab','bas','ab'
}
EMOJI_MAP = {
'🔥': 'ekdum mast',
'💯': 'bilkul sahi',
'😂': 'bahut funny',
'😡': 'bohot gussa',
'👎': 'bilkul bekar',
'🙄': 'seriously kya',
'❤️': 'bohot pasand',
'😍': 'ekdum zabardast',
'😢': 'bahut dukh',
'👍': 'bahut accha',
'😤': 'bohot gussa',
'🤮': 'ekdum bekar',
}
def compute_code_mix_ratio(text):
"""What fraction of words are Hindi/Hinglish?"""
words = text.lower().split()
if not words:
return 0.0
hindi_count = sum(1 for w in words if w in HINDI_WORDS)
return round(hindi_count / len(words), 3)
def clean_text(text):
"""Full Hinglish-aware cleaning pipeline"""
if not isinstance(text, str):
return ""
# 1. Replace emojis with Hindi equivalents
for emoji, replacement in EMOJI_MAP.items():
text = text.replace(emoji, f' {replacement} ')
# 2. Remove URLs
text = re.sub(r'http\S+|www\S+', '', text)
# 3. Remove @mentions
text = re.sub(r'@\w+', '', text)
# 4. Remove hashtag symbol but keep the word
text = re.sub(r'#(\w+)', r'\1', text)
# 5. Remove special characters but keep Hindi-relevant punctuation
text = re.sub(r'[^\w\s]', ' ', text)
# 6. Remove extra whitespace
text = re.sub(r'\s+', ' ', text).strip()
# 7. Lowercase
text = text.lower()
return text
def preprocess_all():
"""Clean all tweets in DB and update cleaned_text + code_mix_ratio"""
conn = psycopg2.connect(os.getenv("DATABASE_URL"))
cur = conn.cursor()
# Fetch all unprocessed tweets
cur.execute("SELECT id, text FROM tweets WHERE cleaned_text IS NULL")
rows = cur.fetchall()
print(f"Processing {len(rows)} tweets...")
updated = 0
skipped = 0
for tweet_id, text in rows:
cleaned = clean_text(text)
if len(cleaned) < 5: # skip very short texts
skipped += 1
continue
ratio = compute_code_mix_ratio(cleaned)
cur.execute("""
UPDATE tweets
SET cleaned_text = %s, code_mix_ratio = %s
WHERE id = %s
""", (cleaned, ratio, tweet_id))
updated += 1
conn.commit()
cur.close()
conn.close()
print(f"✅ Updated {updated} rows")
print(f"⏭️ Skipped {skipped} rows (too short)")
# Show stats
conn = psycopg2.connect(os.getenv("DATABASE_URL"))
cur = conn.cursor()
cur.execute("""
SELECT
COUNT(*) as total,
ROUND(AVG(code_mix_ratio)::numeric, 3) as avg_hindi_ratio,
COUNT(CASE WHEN is_labelled THEN 1 END) as labelled,
COUNT(CASE WHEN NOT is_labelled THEN 1 END) as unlabelled
FROM tweets
WHERE cleaned_text IS NOT NULL
""")
stats = cur.fetchone()
cur.close()
conn.close()
print(f"\n📊 Dataset stats:")
print(f" Total processed : {stats[0]}")
print(f" Avg Hindi ratio : {stats[1]}")
print(f" Labelled : {stats[2]}")
print(f" Unlabelled : {stats[3]}")
if __name__ == "__main__":
preprocess_all()
|