Spaces:
Running
Running
File size: 3,111 Bytes
9d0fd45 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | """Unicode math-symbol sanitization for Python code blocks."""
from __future__ import annotations
_UNICODE_REPLACEMENTS: dict[str, str] = {
"\u222B": "integral", # Integral sign
"\u2211": "sum", # Summation
"\u221E": "inf", # Infinity
"\u00B2": "**2", # Superscript 2
"\u00B3": "**3", # Superscript 3
"\u2074": "**4", # Superscript 4
"\u207B": "-", # Superscript minus
"\u00B9": "**1", # Superscript 1
"\u27E8": "<", # Left angle bracket
"\u27E9": ">", # Right angle bracket
"\u2329": "<", # Left-pointing angle bracket
"\u232A": ">", # Right-pointing angle bracket
"\u2014": "-", # Em dash
"\u2013": "-", # En dash
"\u2018": "'", # Left single quote
"\u2019": "'", # Right single quote
"\u201C": '"', # Left double quote
"\u201D": '"', # Right double quote
"\u2264": "<=", # Less than or equal
"\u2265": ">=", # Greater than or equal
"\u2260": "!=", # Not equal
"\u00D7": "*", # Multiplication sign
"\u00F7": "/", # Division sign
"\u2248": "==", # Almost equal
"\u2245": "==", # Approximately equal
"\u2261": "==", # Identical to
"\u2192": "->", # Right arrow
"\u2190": "<-", # Left arrow
"\u221A": "sqrt", # Square root
"\u03C0": "pi", # Pi
}
def sanitize_code(code: str) -> str:
"""Replace Unicode math symbols with ASCII equivalents and strip non-ASCII
bytes from comments.
Also removes lines that start with a shell/jupyter escape (``!pip install…``)
— those get fed to the Python interpreter, which rejects them with a
``SyntaxError``.
Code-line behaviour (non-comment):
* any char present in the replacement table is mapped to its ASCII form
* non-ASCII chars that have no replacement are left alone — they may
be legitimate (e.g. a unicode string literal)
Comment-line behaviour:
* symbol replacement runs first
* any remaining non-ASCII bytes are replaced by ``?`` (lossy) — comments
never affect execution, so we drop noise eagerly
"""
if not code:
return code
out_lines: list[str] = []
for line in code.split("\n"):
# Strip jupyter/shell escapes.
if line.lstrip().startswith("!"):
continue
if any(ord(ch) > 127 for ch in line):
stripped = line.lstrip()
if stripped.startswith("#"):
for sym, repl in _UNICODE_REPLACEMENTS.items():
line = line.replace(sym, repl)
# Lossy: comments may be in any locale, we don't need them
# round-tripped.
line = line.encode("ascii", "replace").decode("ascii")
else:
for sym, repl in _UNICODE_REPLACEMENTS.items():
line = line.replace(sym, repl)
out_lines.append(line)
return "\n".join(out_lines)
|