NVFP4 (ModelOpt) quantization of sensenova/SenseNova-Loop-9B-Preview @27abea77 with BF16 MTP head
a57aa9f verified | """Light-weight re-implementation of a SUBSET of the IFEval instruction checkers. | |
| Semantics follow google-research/google-research `instruction_following_eval/instructions.py` | |
| (Apache-2.0, (c) Google LLC), re-written from scratch with these documented deviations: | |
| * word tokenisation uses regex \\w+ (orig: nltk RegexpTokenizer(r"\\w+") -> identical) and | |
| capital-word counting uses whitespace/punct split instead of nltk.word_tokenize; | |
| * `length_constraints:number_sentences` and a few rarely used ids are NOT supported; the eval | |
| builder only selects prompts whose every instruction id is in SUPPORTED. | |
| * language detection uses `langdetect` (Apache-2.0) with a fixed seed, like the original. | |
| Only "strict" scoring is implemented (prompt-level: all instructions pass). | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import re | |
| import threading | |
| _LANG_LOCK = threading.Lock() # langdetect seeds the *global* random module -> not thread-safe | |
| try: | |
| import langdetect | |
| langdetect.DetectorFactory.seed = 0 | |
| except Exception: # pragma: no cover | |
| langdetect = None | |
| def _rel(count: int, relation: str, target: int) -> bool: | |
| if relation == "less than": | |
| return count < target | |
| if relation == "at least": | |
| return count >= target | |
| raise ValueError(relation) | |
| def _lang(text: str) -> str | None: | |
| if langdetect is None: | |
| return None | |
| try: | |
| with _LANG_LOCK: | |
| langdetect.DetectorFactory.seed = 0 | |
| return langdetect.detect(text) | |
| except Exception: | |
| return None | |
| def no_comma(r, **k): | |
| return "," not in r | |
| def number_highlighted_sections(r, num_highlights, **k): | |
| n = 0 | |
| for h in re.findall(r"\*[^\n\*]*\*", r): | |
| if h.strip("*").strip(): | |
| n += 1 | |
| for h in re.findall(r"\*\*[^\n\*]*\*\*", r): | |
| if h.removeprefix("**").removesuffix("**").strip(): | |
| n += 1 | |
| return n >= num_highlights | |
| def title(r, **k): | |
| return any(t.lstrip("<").rstrip(">").strip() for t in re.findall(r"<<[^\n]+>>", r)) | |
| def number_bullet_lists(r, num_bullets, **k): | |
| a = re.findall(r"^\s*\*[^\*].*$", r, flags=re.M) | |
| b = re.findall(r"^\s*-.*$", r, flags=re.M) | |
| return len(a) + len(b) == num_bullets | |
| def json_format(r, **k): | |
| v = r.strip() | |
| v = re.sub(r"^```(?:json|Json|JSON)?\s*", "", v) | |
| v = re.sub(r"\s*```$", "", v).strip() | |
| try: | |
| json.loads(v) | |
| return True | |
| except ValueError: | |
| return False | |
| def multiple_sections(r, section_spliter, num_sections, **k): | |
| pat = r"\s?" + re.escape(section_spliter.strip()) + r"\s?\d+\s?" | |
| return len(re.split(pat, r)) - 1 >= num_sections | |
| def constrained_response(r, **k): | |
| v = r.strip() | |
| return any(o in v for o in ("My answer is yes.", "My answer is no.", "My answer is maybe.")) | |
| def english_lowercase(r, **k): | |
| return r == r.lower() and _lang(r) == "en" | |
| def english_capital(r, **k): | |
| return r == r.upper() and _lang(r) == "en" | |
| def capital_word_frequency(r, capital_frequency, capital_relation, **k): | |
| words = re.findall(r"[A-Za-z][A-Za-z'\-]*", r) | |
| n = sum(1 for w in words if w.isupper()) | |
| return _rel(n, capital_relation, capital_frequency) | |
| def number_words(r, num_words, relation, **k): | |
| return _rel(len(re.findall(r"\w+", r)), relation, num_words) | |
| def number_paragraphs(r, num_paragraphs, **k): | |
| paras = re.split(r"\s?\*\*\*\s?", r) | |
| n = len(paras) | |
| for i, p in enumerate(paras): | |
| if not p.strip(): | |
| if i == 0 or i == len(paras) - 1: | |
| n -= 1 | |
| else: | |
| return False | |
| return n == num_paragraphs | |
| def nth_paragraph_first_word(r, num_paragraphs, nth_paragraph, first_word, **k): | |
| paras = re.split(r"\n\n", r) | |
| n = sum(1 for p in paras if p.strip()) | |
| paras_ne = [p for p in paras if p.strip()] | |
| if nth_paragraph > len(paras_ne): | |
| return False | |
| p = paras_ne[nth_paragraph - 1].strip() | |
| if not p: | |
| return False | |
| word = p.split()[0].strip() | |
| word = word.lstrip("'\"").rstrip(".,?!'\"").lower() | |
| # original strips leading/trailing punctuation char by char | |
| word = re.sub(r"[^\w]+$", "", re.sub(r"^[^\w]+", "", word)) | |
| return n == num_paragraphs and word == first_word.lower() | |
| def existence(r, keywords, **k): | |
| return all(re.search(kw, r, flags=re.I) for kw in keywords) | |
| def frequency(r, keyword, frequency, relation, **k): | |
| return _rel(len(re.findall(keyword, r, flags=re.I)), relation, frequency) | |
| def forbidden_words(r, forbidden_words, **k): | |
| return not any(re.search(r"\b" + w + r"\b", r, flags=re.I) for w in forbidden_words) | |
| def letter_frequency(r, letter, let_frequency, let_relation, **k): | |
| return _rel(r.lower().count(letter.lower()), let_relation, let_frequency) | |
| def end_checker(r, end_phrase, **k): | |
| return r.strip().strip('"').lower().endswith(end_phrase.strip().lower()) | |
| def quotation(r, **k): | |
| v = r.strip() | |
| return len(v) > 1 and v[0] == '"' and v[-1] == '"' | |
| def postscript(r, postscript_marker, **k): | |
| v = r.lower() | |
| m = postscript_marker.strip() | |
| if m == "P.P.S": | |
| pat = r"\s*p\.\s?p\.\s?s.*$" | |
| elif m == "P.S.": | |
| pat = r"\s*p\.\s?s\..*$" | |
| else: | |
| pat = r"\s*" + re.escape(m.lower()) + r".*$" | |
| return bool(re.findall(pat, v, flags=re.M)) | |
| def number_placeholders(r, num_placeholders, **k): | |
| return len(re.findall(r"\[.*?\]", r)) >= num_placeholders | |
| def repeat_prompt(r, prompt_to_repeat, **k): | |
| return r.strip().lower().startswith(prompt_to_repeat.strip().lower()) | |
| def two_responses(r, **k): | |
| parts = r.split("******") | |
| valid = [] | |
| for i, p in enumerate(parts): | |
| if not p.strip(): | |
| if i != 0 and i != len(parts) - 1: | |
| return False | |
| else: | |
| valid.append(p) | |
| return len(valid) == 2 and valid[0].strip() != valid[1].strip() | |
| def response_language(r, language, **k): | |
| return _lang(r) == language | |
| SUPPORTED = { | |
| "punctuation:no_comma": no_comma, | |
| "detectable_format:number_highlighted_sections": number_highlighted_sections, | |
| "detectable_format:title": title, | |
| "detectable_format:number_bullet_lists": number_bullet_lists, | |
| "detectable_format:json_format": json_format, | |
| "detectable_format:multiple_sections": multiple_sections, | |
| "detectable_format:constrained_response": constrained_response, | |
| "change_case:english_lowercase": english_lowercase, | |
| "change_case:english_capital": english_capital, | |
| "change_case:capital_word_frequency": capital_word_frequency, | |
| "length_constraints:number_words": number_words, | |
| "length_constraints:number_paragraphs": number_paragraphs, | |
| "length_constraints:nth_paragraph_first_word": nth_paragraph_first_word, | |
| "keywords:existence": existence, | |
| "keywords:frequency": frequency, | |
| "keywords:forbidden_words": forbidden_words, | |
| "keywords:letter_frequency": letter_frequency, | |
| "startend:end_checker": end_checker, | |
| "startend:quotation": quotation, | |
| "detectable_content:postscript": postscript, | |
| "detectable_content:number_placeholders": number_placeholders, | |
| "combination:repeat_prompt": repeat_prompt, | |
| "combination:two_responses": two_responses, | |
| "language:response_language": response_language, | |
| } | |
| def check(response: str, instruction_ids: list[str], kwargs_list: list[dict]) -> list[bool]: | |
| out = [] | |
| for iid, kw in zip(instruction_ids, kwargs_list): | |
| kw = {k: v for k, v in (kw or {}).items() if v is not None} | |
| fn = SUPPORTED[iid] | |
| try: | |
| out.append(bool(response.strip()) and bool(fn(response, **kw))) | |
| except Exception: | |
| out.append(False) | |
| return out | |