scratch/test_analyzer.py

import re
from typing import List

def _analyze(text: str) -> List[str]:
    text = text.lower()
    hyphenated = re.findall(r'\b\w+-\w+\b', text)
    print(f"Hyphenated found: {hyphenated}")
    for word in hyphenated:
        joined = word.replace('-', '')
        text += f" {joined}"
    tokens = re.findall(r'\b\w+\b', text)
    return [t for t in tokens if len(t) > 1]

test_text = "The pro-rata amount is calculated."
print(f"Tokens: {_analyze(test_text)}")