import re from typing import List def _analyze(text: str) -> List[str]: text = text.lower() hyphenated = re.findall(r'\b\w+-\w+\b', text) print(f"Hyphenated found: {hyphenated}") for word in hyphenated: joined = word.replace('-', '') text += f" {joined}" tokens = re.findall(r'\b\w+\b', text) return [t for t in tokens if len(t) > 1] test_text = "The pro-rata amount is calculated." print(f"Tokens: {_analyze(test_text)}")