File size: 1,492 Bytes
7e9cfd1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | """Quick cleanup for meme-y dialectal tweets: split hashtags back into words,
squash stretched-out letters, strip URLs. Keeps emojis and dialect intact --
just surfaces the words buried inside campaign hashtags. Used on both the
retrieval pool and the incoming tweets.
"""
import re
_URL = re.compile(r"https?://\S+")
_HASH = re.compile(r"#(\w+)")
_ELONG = re.compile(r"(.)\1{2,}")
_WS = re.compile(r"\s+")
def normalize_text(text):
t = str(text)
t = _URL.sub(" ", t)
# segment hashtags: #a_b_c -> a b c
t = _HASH.sub(lambda m: " " + m.group(1).replace("_", " ") + " ", t)
t = t.replace("_", " ")
# collapse 3+ repeats of any character to two (keeps some emphasis)
t = _ELONG.sub(r"\1\1", t)
return _WS.sub(" ", t).strip()
def main():
import argparse
import pandas as pd
ap = argparse.ArgumentParser()
ap.add_argument("--in", dest="inp", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--cols", default="text,tweet_text",
help="comma-separated text columns to normalise")
args = ap.parse_args()
df = pd.read_csv(args.inp, keep_default_na=False, encoding="utf-8-sig")
df.columns = [c.strip() for c in df.columns]
for c in args.cols.split(","):
if c in df.columns:
df[c] = df[c].map(normalize_text)
df.to_csv(args.out, index=False, encoding="utf-8-sig")
print(f"[normalize] {len(df)} rows -> {args.out}")
if __name__ == "__main__":
main()
|