| | """ |
| | Tools to open .py files as Unicode, using the encoding specified within the file, |
| | as per PEP 263. |
| | |
| | Much of the code is taken from the tokenize module in Python 3.2. |
| | """ |
| |
|
| | import io |
| | from io import TextIOWrapper, BytesIO |
| | from pathlib import Path |
| | import re |
| | from tokenize import open, detect_encoding |
| |
|
| | cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)", re.UNICODE) |
| | cookie_comment_re = re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE) |
| |
|
| | def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True): |
| | """Converts a bytes string with python source code to unicode. |
| | |
| | Unicode strings are passed through unchanged. Byte strings are checked |
| | for the python source file encoding cookie to determine encoding. |
| | txt can be either a bytes buffer or a string containing the source |
| | code. |
| | """ |
| | if isinstance(txt, str): |
| | return txt |
| | if isinstance(txt, bytes): |
| | buffer = BytesIO(txt) |
| | else: |
| | buffer = txt |
| | try: |
| | encoding, _ = detect_encoding(buffer.readline) |
| | except SyntaxError: |
| | encoding = "ascii" |
| | buffer.seek(0) |
| | with TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) as text: |
| | text.mode = 'r' |
| | if skip_encoding_cookie: |
| | return u"".join(strip_encoding_cookie(text)) |
| | else: |
| | return text.read() |
| |
|
| | def strip_encoding_cookie(filelike): |
| | """Generator to pull lines from a text-mode file, skipping the encoding |
| | cookie if it is found in the first two lines. |
| | """ |
| | it = iter(filelike) |
| | try: |
| | first = next(it) |
| | if not cookie_comment_re.match(first): |
| | yield first |
| | second = next(it) |
| | if not cookie_comment_re.match(second): |
| | yield second |
| | except StopIteration: |
| | return |
| | |
| | for line in it: |
| | yield line |
| |
|
| | def read_py_file(filename, skip_encoding_cookie=True): |
| | """Read a Python file, using the encoding declared inside the file. |
| | |
| | Parameters |
| | ---------- |
| | filename : str |
| | The path to the file to read. |
| | skip_encoding_cookie : bool |
| | If True (the default), and the encoding declaration is found in the first |
| | two lines, that line will be excluded from the output. |
| | |
| | Returns |
| | ------- |
| | A unicode string containing the contents of the file. |
| | """ |
| | filepath = Path(filename) |
| | with open(filepath) as f: |
| | if skip_encoding_cookie: |
| | return "".join(strip_encoding_cookie(f)) |
| | else: |
| | return f.read() |
| |
|
| | def read_py_url(url, errors='replace', skip_encoding_cookie=True): |
| | """Read a Python file from a URL, using the encoding declared inside the file. |
| | |
| | Parameters |
| | ---------- |
| | url : str |
| | The URL from which to fetch the file. |
| | errors : str |
| | How to handle decoding errors in the file. Options are the same as for |
| | bytes.decode(), but here 'replace' is the default. |
| | skip_encoding_cookie : bool |
| | If True (the default), and the encoding declaration is found in the first |
| | two lines, that line will be excluded from the output. |
| | |
| | Returns |
| | ------- |
| | A unicode string containing the contents of the file. |
| | """ |
| | |
| | from urllib.request import urlopen |
| | response = urlopen(url) |
| | buffer = io.BytesIO(response.read()) |
| | return source_to_unicode(buffer, errors, skip_encoding_cookie) |
| |
|
