| | |
| | |
| | """ |
| | Part of the astor library for Python AST manipulation. |
| | |
| | License: 3-clause BSD |
| | |
| | Copyright (c) 2015 Patrick Maupin |
| | """ |
| |
|
| | import sys |
| | import os |
| | import ast |
| | import shutil |
| | import logging |
| |
|
| | from astor.code_gen import to_source |
| | from astor.file_util import code_to_ast |
| | from astor.node_util import (allow_ast_comparison, dump_tree, |
| | strip_tree, fast_compare) |
| |
|
| |
|
| | dsttree = 'tmp_rtrip' |
| |
|
| | |
| |
|
| |
|
| | def out_prep(s, pre_encoded=(sys.version_info[0] == 2)): |
| | return s if pre_encoded else s.encode('utf-8') |
| |
|
| |
|
| | def convert(srctree, dsttree=dsttree, readonly=False, dumpall=False, |
| | ignore_exceptions=False, fullcomp=False): |
| | """Walk the srctree, and convert/copy all python files |
| | into the dsttree |
| | |
| | """ |
| |
|
| | if fullcomp: |
| | allow_ast_comparison() |
| |
|
| | parse_file = code_to_ast.parse_file |
| | find_py_files = code_to_ast.find_py_files |
| | srctree = os.path.normpath(srctree) |
| |
|
| | if not readonly: |
| | dsttree = os.path.normpath(dsttree) |
| | logging.info('') |
| | logging.info('Trashing ' + dsttree) |
| | shutil.rmtree(dsttree, True) |
| |
|
| | unknown_src_nodes = set() |
| | unknown_dst_nodes = set() |
| | badfiles = set() |
| | broken = [] |
| |
|
| | oldpath = None |
| |
|
| | allfiles = find_py_files(srctree, None if readonly else dsttree) |
| | for srcpath, fname in allfiles: |
| | |
| | if not readonly and srcpath != oldpath: |
| | oldpath = srcpath |
| | if srcpath >= srctree: |
| | dstpath = srcpath.replace(srctree, dsttree, 1) |
| | if not dstpath.startswith(dsttree): |
| | raise ValueError("%s not a subdirectory of %s" % |
| | (dstpath, dsttree)) |
| | else: |
| | assert srctree.startswith(srcpath) |
| | dstpath = dsttree |
| | os.makedirs(dstpath) |
| |
|
| | srcfname = os.path.join(srcpath, fname) |
| | logging.info('Converting %s' % srcfname) |
| | try: |
| | srcast = parse_file(srcfname) |
| | except SyntaxError: |
| | badfiles.add(srcfname) |
| | continue |
| |
|
| | try: |
| | dsttxt = to_source(srcast) |
| | except Exception: |
| | if not ignore_exceptions: |
| | raise |
| | dsttxt = '' |
| |
|
| | if not readonly: |
| | dstfname = os.path.join(dstpath, fname) |
| | try: |
| | with open(dstfname, 'wb') as f: |
| | f.write(out_prep(dsttxt)) |
| | except UnicodeEncodeError: |
| | badfiles.add(dstfname) |
| |
|
| | |
| | |
| | try: |
| | dstast = ast.parse(dsttxt) if readonly else parse_file(dstfname) |
| | except SyntaxError: |
| | dstast = [] |
| | if fullcomp: |
| | unknown_src_nodes.update(strip_tree(srcast)) |
| | unknown_dst_nodes.update(strip_tree(dstast)) |
| | bad = srcast != dstast |
| | else: |
| | bad = not fast_compare(srcast, dstast) |
| | if dumpall or bad: |
| | srcdump = dump_tree(srcast) |
| | dstdump = dump_tree(dstast) |
| | logging.warning(' calculating dump -- %s' % |
| | ('bad' if bad else 'OK')) |
| | if bad: |
| | broken.append(srcfname) |
| | if dumpall or bad: |
| | if not readonly: |
| | try: |
| | with open(dstfname[:-3] + '.srcdmp', 'wb') as f: |
| | f.write(out_prep(srcdump)) |
| | except UnicodeEncodeError: |
| | badfiles.add(dstfname[:-3] + '.srcdmp') |
| | try: |
| | with open(dstfname[:-3] + '.dstdmp', 'wb') as f: |
| | f.write(out_prep(dstdump)) |
| | except UnicodeEncodeError: |
| | badfiles.add(dstfname[:-3] + '.dstdmp') |
| | elif dumpall: |
| | sys.stdout.write('\n\nAST:\n\n ') |
| | sys.stdout.write(srcdump.replace('\n', '\n ')) |
| | sys.stdout.write('\n\nDecompile:\n\n ') |
| | sys.stdout.write(dsttxt.replace('\n', '\n ')) |
| | sys.stdout.write('\n\nNew AST:\n\n ') |
| | sys.stdout.write('(same as old)' if dstdump == srcdump |
| | else dstdump.replace('\n', '\n ')) |
| | sys.stdout.write('\n') |
| |
|
| | if badfiles: |
| | logging.warning('\nFiles not processed due to syntax errors:') |
| | for fname in sorted(badfiles): |
| | logging.warning(' %s' % fname) |
| | if broken: |
| | logging.warning('\nFiles failed to round-trip to AST:') |
| | for srcfname in broken: |
| | logging.warning(' %s' % srcfname) |
| |
|
| | ok_to_strip = 'col_offset _precedence _use_parens lineno _p_op _pp' |
| | ok_to_strip = set(ok_to_strip.split()) |
| | bad_nodes = (unknown_dst_nodes | unknown_src_nodes) - ok_to_strip |
| | if bad_nodes: |
| | logging.error('\nERROR -- UNKNOWN NODES STRIPPED: %s' % bad_nodes) |
| | logging.info('\n') |
| | return broken |
| |
|
| |
|
| | def usage(msg): |
| | raise SystemExit(textwrap.dedent(""" |
| | |
| | Error: %s |
| | |
| | Usage: |
| | |
| | python -m astor.rtrip [readonly] [<source>] |
| | |
| | |
| | This utility tests round-tripping of Python source to AST |
| | and back to source. |
| | |
| | If readonly is specified, then the source will be tested, |
| | but no files will be written. |
| | |
| | if the source is specified to be "stdin" (without quotes) |
| | then any source entered at the command line will be compiled |
| | into an AST, converted back to text, and then compiled to |
| | an AST again, and the results will be displayed to stdout. |
| | |
| | If neither readonly nor stdin is specified, then rtrip |
| | will create a mirror directory named tmp_rtrip and will |
| | recursively round-trip all the Python source from the source |
| | into the tmp_rtrip dir, after compiling it and then reconstituting |
| | it through code_gen.to_source. |
| | |
| | If the source is not specified, the entire Python library will be used. |
| | |
| | """) % msg) |
| |
|
| |
|
| | if __name__ == '__main__': |
| | import textwrap |
| |
|
| | args = sys.argv[1:] |
| |
|
| | readonly = 'readonly' in args |
| | if readonly: |
| | args.remove('readonly') |
| |
|
| | if not args: |
| | args = [os.path.dirname(textwrap.__file__)] |
| |
|
| | if len(args) > 1: |
| | usage("Too many arguments") |
| |
|
| | fname, = args |
| | dumpall = False |
| | if not os.path.exists(fname): |
| | dumpall = fname == 'stdin' or usage("Cannot find directory %s" % fname) |
| |
|
| | logging.basicConfig(format='%(msg)s', level=logging.INFO) |
| | convert(fname, readonly=readonly or dumpall, dumpall=dumpall) |
| |
|
