File size: 7,298 Bytes
7bd6c7e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 |
import re
import json
class EvalHandler:
def __init__(self):
self.rule_patterns = {
'comma_restriction': re.compile(r'no.*comma|without.*comma', re.IGNORECASE),
'placeholder_requirement': re.compile(r'placeholder.*\[.*\]|square.*bracket', re.IGNORECASE),
'lowercase_requirement': re.compile(r'lowercase|no.*capital|all.*lowercase', re.IGNORECASE),
'capital_frequency': re.compile(r'capital.*letter.*less.*than|capital.*word.*frequency', re.IGNORECASE),
'quotation_requirement': re.compile(r'wrap.*quotation|double.*quote', re.IGNORECASE),
'json_format': re.compile(r'json.*format|JSON.*output|format.*json', re.IGNORECASE),
'word_count': re.compile(r'less.*than.*word|word.*limit|maximum.*word', re.IGNORECASE),
'section_requirement': re.compile(r'section.*start|SECTION.*X', re.IGNORECASE),
'ending_requirement': re.compile(r'finish.*exact.*phrase|end.*phrase', re.IGNORECASE),
'forbidden_words': re.compile(r'not.*allowed|forbidden.*word|without.*word', re.IGNORECASE),
'capital_letters_only': re.compile(r'all.*capital|CAPITAL.*letter', re.IGNORECASE)
}
def detect_rules(self, instruction):
applicable_rules = []
if self.rule_patterns['comma_restriction'].search(instruction):
applicable_rules.append('CommaChecker')
if self.rule_patterns['placeholder_requirement'].search(instruction):
applicable_rules.append('PlaceholderChecker')
if self.rule_patterns['lowercase_requirement'].search(instruction):
applicable_rules.append('LowercaseLettersEnglishChecker')
if self.rule_patterns['capital_frequency'].search(instruction):
applicable_rules.append('CapitalWordFrequencyChecker')
if self.rule_patterns['quotation_requirement'].search(instruction):
applicable_rules.append('QuotationChecker')
if self.rule_patterns['json_format'].search(instruction):
applicable_rules.append('JsonFormat')
if self.rule_patterns['word_count'].search(instruction):
applicable_rules.append('NumberOfWords')
if self.rule_patterns['section_requirement'].search(instruction):
applicable_rules.append('SectionChecker')
if self.rule_patterns['ending_requirement'].search(instruction):
applicable_rules.append('EndChecker')
if self.rule_patterns['forbidden_words'].search(instruction):
applicable_rules.append('ForbiddenWords')
if self.rule_patterns['capital_letters_only'].search(instruction):
applicable_rules.append('CapitalLettersEnglishChecker')
return applicable_rules
def apply_rule_fix(self, response, rules, instruction= ""):
for rule in rules:
if rule == 'CommaChecker':
response = self._fix_commas(response, instruction)
elif rule == 'PlaceholderChecker':
response = self._fix_placeholders(response, instruction)
elif rule == 'LowercaseLettersEnglishChecker':
response = self._fix_lowercase(response)
elif rule == 'CapitalWordFrequencyChecker':
response = self._fix_capital_frequency(response, instruction)
elif rule == 'QuotationChecker':
response = self._fix_quotations(response)
elif rule == 'JsonFormat':
response = self._fix_json_format(response, instruction)
elif rule == 'NumberOfWords':
response = self._fix_word_count(response, instruction)
elif rule == 'SectionChecker':
response = self._fix_sections(response, instruction)
elif rule == 'EndChecker':
response = self._fix_ending(response, instruction)
elif rule == 'ForbiddenWords':
response = self._fix_forbidden_words(response, instruction)
elif rule == 'CapitalLettersEnglishChecker':
response = self._fix_all_capitals(response, instruction)
return response
def _fix_commas(self, response, instruction):
return response.replace(',', '')
def _fix_placeholders(self, response, instruction):
num_match = re.search(r'at least (\d+)', instruction, re.IGNORECASE)
if num_match:
target_count = int(num_match.group(1))
current_count = len(re.findall(r'\[.*?\]', response))
words = response.split()
for i in range(target_count - current_count):
if i < len(words):
words[i] = f'[{words[i]}]'
return ' '.join(words)
return response
def _fix_lowercase(self, response):
return response.lower()
def _fix_capital_frequency(self, response, instruction):
max_match = re.search(r'less than (\d+)', instruction, re.IGNORECASE)
if max_match:
max_capitals = int(max_match.group(1))
words = response.split()
capital_count = sum(1 for word in words if word.isupper())
if capital_count > max_capitals:
for i, word in enumerate(words):
if word.isupper() and capital_count > max_capitals:
words[i] = word.lower()
capital_count -= 1
return ' '.join(words)
return response
def _fix_quotations(self, response):
return f'"{response}"'
def _fix_json_format(self, response, instruction):
return json.dumps({"response": response}, indent=2)
def _fix_word_count(self, response, instruction):
limit_match = re.search(r'less than (\d+)', instruction, re.IGNORECASE)
if limit_match:
word_limit = int(limit_match.group(1))
words = response.split()
if len(words) > word_limit:
return ' '.join(words[:word_limit])
return response
def _fix_sections(self, response, instruction):
section_match = re.search(r'(\d+) section', instruction, re.IGNORECASE)
if section_match:
num_sections = int(section_match.group(1))
sections = []
for i in range(num_sections):
sections.append(f"SECTION {i+1}:")
sections.append("This section provides content here.")
return '\n\n'.join(sections)
return response
def _fix_ending(self, response, instruction):
end_match = re.search(r'finish.*with.*phrase[:\s]*([^.!?]*)', instruction, re.IGNORECASE)
if end_match:
required_ending = end_match.group(1).strip()
if not response.endswith(required_ending):
return response + " " + required_ending
return response
def _fix_forbidden_words(self, response, instruction):
forbidden_match = re.search(r'without.*word[:\s]*([^.!?]*)', instruction, re.IGNORECASE)
if forbidden_match:
forbidden_word = forbidden_match.group(1).strip().lower()
response = re.sub(re.escape(forbidden_word), '', response, flags=re.IGNORECASE)
return response.strip()
def _fix_all_capitals(self, response, instruction):
return response.upper()
|