Spaces:
Running
Running
| """Unicode math-symbol sanitization for Python code blocks.""" | |
| from __future__ import annotations | |
| _UNICODE_REPLACEMENTS: dict[str, str] = { | |
| "\u222B": "integral", # Integral sign | |
| "\u2211": "sum", # Summation | |
| "\u221E": "inf", # Infinity | |
| "\u00B2": "**2", # Superscript 2 | |
| "\u00B3": "**3", # Superscript 3 | |
| "\u2074": "**4", # Superscript 4 | |
| "\u207B": "-", # Superscript minus | |
| "\u00B9": "**1", # Superscript 1 | |
| "\u27E8": "<", # Left angle bracket | |
| "\u27E9": ">", # Right angle bracket | |
| "\u2329": "<", # Left-pointing angle bracket | |
| "\u232A": ">", # Right-pointing angle bracket | |
| "\u2014": "-", # Em dash | |
| "\u2013": "-", # En dash | |
| "\u2018": "'", # Left single quote | |
| "\u2019": "'", # Right single quote | |
| "\u201C": '"', # Left double quote | |
| "\u201D": '"', # Right double quote | |
| "\u2264": "<=", # Less than or equal | |
| "\u2265": ">=", # Greater than or equal | |
| "\u2260": "!=", # Not equal | |
| "\u00D7": "*", # Multiplication sign | |
| "\u00F7": "/", # Division sign | |
| "\u2248": "==", # Almost equal | |
| "\u2245": "==", # Approximately equal | |
| "\u2261": "==", # Identical to | |
| "\u2192": "->", # Right arrow | |
| "\u2190": "<-", # Left arrow | |
| "\u221A": "sqrt", # Square root | |
| "\u03C0": "pi", # Pi | |
| } | |
| def sanitize_code(code: str) -> str: | |
| """Replace Unicode math symbols with ASCII equivalents and strip non-ASCII | |
| bytes from comments. | |
| Also removes lines that start with a shell/jupyter escape (``!pip install…``) | |
| — those get fed to the Python interpreter, which rejects them with a | |
| ``SyntaxError``. | |
| Code-line behaviour (non-comment): | |
| * any char present in the replacement table is mapped to its ASCII form | |
| * non-ASCII chars that have no replacement are left alone — they may | |
| be legitimate (e.g. a unicode string literal) | |
| Comment-line behaviour: | |
| * symbol replacement runs first | |
| * any remaining non-ASCII bytes are replaced by ``?`` (lossy) — comments | |
| never affect execution, so we drop noise eagerly | |
| """ | |
| if not code: | |
| return code | |
| out_lines: list[str] = [] | |
| for line in code.split("\n"): | |
| # Strip jupyter/shell escapes. | |
| if line.lstrip().startswith("!"): | |
| continue | |
| if any(ord(ch) > 127 for ch in line): | |
| stripped = line.lstrip() | |
| if stripped.startswith("#"): | |
| for sym, repl in _UNICODE_REPLACEMENTS.items(): | |
| line = line.replace(sym, repl) | |
| # Lossy: comments may be in any locale, we don't need them | |
| # round-tripped. | |
| line = line.encode("ascii", "replace").decode("ascii") | |
| else: | |
| for sym, repl in _UNICODE_REPLACEMENTS.items(): | |
| line = line.replace(sym, repl) | |
| out_lines.append(line) | |
| return "\n".join(out_lines) | |