Spaces:
Paused
Paused
File size: 1,760 Bytes
f66cd72 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | """
Comment preprocessing — MUST stay byte-for-byte identical to the training
notebook (backend/notebooks/kratt_bert_training.ipynb), or the model sees an
input distribution it was never trained on and quality collapses.
model input format:
niche <niche> . <likes bucket> . <replies bucket> . <cleaned text>
e.g.
niche low effort . few likes . no replies . first!!
"""
import re
# Invisible chars: ZWSP, directional marks, word-joiner family, BOM, soft hyphen.
# NOT stripped: U+200D (ZWJ glues family/skin-tone emoji) and U+FE0F (emoji
# variation selector) — "keep emojis" includes their invisible glue.
INVISIBLE_RE = re.compile("[]")
URL_RE = re.compile(r"(?:https?://|www\.)\S+", re.IGNORECASE)
USER_RE = re.compile(r"@[\w.\-]+")
TAG_RE = re.compile(r"#\w+")
def clean_text(text) -> str:
s = INVISIBLE_RE.sub("", str(text))
s = URL_RE.sub(" <URL> ", s) # URLs first: they can contain '#' and '@' fragments
s = USER_RE.sub(" <USER> ", s)
s = TAG_RE.sub(" <TAG> ", s)
# lowercase everything EXCEPT all-caps tokens (shouting is a signal); the
# <URL>/<USER>/<TAG> placeholders are all-caps so they survive untouched.
return " ".join(
tok if (len(tok) >= 2 and tok.isupper()) else tok.lower()
for tok in s.split()
)
def like_phrase(n: int) -> str:
return "many likes" if n >= 10 else ("some likes" if n >= 2 else "few likes")
def reply_phrase(n: int) -> str:
return "has replies" if n > 0 else "no replies"
def build_model_text(text, niche_tag: str, like_count: int, reply_count: int) -> str:
return (
f"niche {niche_tag.replace('-', ' ')} . "
f"{like_phrase(like_count)} . {reply_phrase(reply_count)} . {clean_text(text)}"
)
|