File size: 3,050 Bytes
b5989f0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Cheap syntactic features for intent classification, concatenated to the BGE embedding. The
embedding captures topic/paraphrase; these capture the interrogative-vs-declarative *syntax* that the
embedding blurs (so 'my policy is X' [fact] and 'what is my policy?' [question] separate cleanly).
Shared by evals/intent_train.py (training) and tiered_rag_mlx.py (inference) so they never drift."""
import re
_QWORD = ("what", "what's", "whats", "who", "who's", "whose", "when", "where", "which",
"how", "why", "is my", "are my", "do you", "did i", "can you")
_RECALL = re.compile(r"\b(remind me|recall|do you remember|tell me (my|where|what)|what was|again\b)", re.I)
_STORE = re.compile(r"\b(remember|note that|save this|keep in mind|for later|don't forget)\b", re.I)
_ASSERT = re.compile(r"\bmy\b.{0,40}?\b(is|are|=|named|called|costs?)\b|^\s*(i'?m|i am|i'?ll|i will|i live|i work|i drive|i prefer|i'?d like)\b", re.I)
_IMPER = re.compile(r"^\s*(make it|change|set |add |order|paint|use |scratch that|update|move )", re.I)
_CODE = re.compile(r"\b[A-Z]{2,}[- ]?\d|\b\d{3,}\b") # a code / number value present
# spelled-out arithmetic ('nine times five', 'fifteen percent of two hundred') — so math doesn't depend
# on a literal digit (the classifier was leaning on \d and missing word-numbers)
_NUMWORD = re.compile(r"\b(zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|"
r"fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|"
r"sixty|seventy|eighty|ninety|hundred|thousand|million|dozen|half|quarter|double)\b", re.I)
_OPWORD = re.compile(r"\b(plus|minus|times|divided|multiply|multiplied|subtract(ed)?|add(ed|s)?|percent|"
r"sum of|total of|product of|difference|average of|squared|cubed)\b", re.I)
# a declarative assertion ("the theme is space", "it costs $5 each", "deadline is Monday")
_DECL = re.compile(r"\b(the|this|that|it|we'?re|there|deadline|theme|venue|color|colour|price|name)\b.{0,30}?"
r"\b(is|are|'?re|'?s|costs?|moved|changed|named|called|set to|will be)\b", re.I)
def _declarative(t):
return not (t.endswith("?") or t.startswith(_QWORD)) and bool(_DECL.search(t))
def intent_feats(text):
t = text.strip().lower()
return [
float(t.endswith("?")),
float(t.startswith(_QWORD)),
float(bool(_RECALL.search(text))),
float(bool(_STORE.search(text))),
float(bool(_ASSERT.search(text))),
float(bool(_IMPER.search(text))),
float(bool(re.search(r"\d", text))),
float(bool(_CODE.search(text))),
float(_declarative(t)), # 'the theme is space', 'it costs $5'
float(bool(_NUMWORD.search(text))), # spelled-out number ('twelve', 'hundred')
float(bool(_OPWORD.search(text))), # arithmetic op word ('minus', 'percent')
min(len(t.split()) / 12.0, 1.5), # length (chit-chat tends short)
]
|