#!/usr/bin/env python """ Program to test that all methods/functions have at least one example doctest. Also checks if docstrings are imported into Sphinx. For this to work, the Sphinx docs need to be built first. Use "cd doc; make html" to build the Sphinx docs. Usage: ./bin/coverage_doctest.py sympy/core or ./bin/coverage_doctest.py sympy/core/basic.py If no arguments are given, all files in sympy/ are checked. """ from __future__ import print_function import os import sys import inspect from argparse import ArgumentParser, RawDescriptionHelpFormatter from pathlib import Path try: from HTMLParser import HTMLParser except ImportError: # It's html.parser in Python 3 from html.parser import HTMLParser from sympy.utilities.misc import filldedent # Load color templates, duplicated from sympy/testing/runtests.py color_templates = ( ("Black", "0;30"), ("Red", "0;31"), ("Green", "0;32"), ("Brown", "0;33"), ("Blue", "0;34"), ("Purple", "0;35"), ("Cyan", "0;36"), ("LightGray", "0;37"), ("DarkGray", "1;30"), ("LightRed", "1;31"), ("LightGreen", "1;32"), ("Yellow", "1;33"), ("LightBlue", "1;34"), ("LightPurple", "1;35"), ("LightCyan", "1;36"), ("White", "1;37"), ) colors = {} for name, value in color_templates: colors[name] = value c_normal = '\033[0m' c_color = '\033[%sm' def print_header(name, underline=None, color=None): print() if color: print("%s%s%s" % (c_color % colors[color], name, c_normal)) else: print(name) if underline and not color: print(underline*len(name)) def print_coverage(module_path, c, c_missing_doc, c_missing_doctest, c_indirect_doctest, c_sph, f, f_missing_doc, f_missing_doctest, f_indirect_doctest, f_sph, score, total_doctests, total_members, sphinx_score, total_sphinx, verbose=False, no_color=False, sphinx=True): """ Prints details (depending on verbose) of a module """ doctest_color = "Brown" sphinx_color = "DarkGray" less_100_color = "Red" less_50_color = "LightRed" equal_100_color = "Green" big_header_color = "LightPurple" small_header_color = "Purple" if no_color: score_string = "Doctests: %s%% (%s of %s)" % (score, total_doctests, total_members) elif score < 100: if score < 50: score_string = "%sDoctests:%s %s%s%% (%s of %s)%s" % \ (c_color % colors[doctest_color], c_normal, c_color % colors[less_50_color], score, total_doctests, total_members, c_normal) else: score_string = "%sDoctests:%s %s%s%% (%s of %s)%s" % \ (c_color % colors[doctest_color], c_normal, c_color % colors[less_100_color], score, total_doctests, total_members, c_normal) else: score_string = "%sDoctests:%s %s%s%% (%s of %s)%s" % \ (c_color % colors[doctest_color], c_normal, c_color % colors[equal_100_color], score, total_doctests, total_members, c_normal) if sphinx: if no_color: sphinx_score_string = "Sphinx: %s%% (%s of %s)" % (sphinx_score, total_members - total_sphinx, total_members) elif sphinx_score < 100: if sphinx_score < 50: sphinx_score_string = "%sSphinx:%s %s%s%% (%s of %s)%s" % \ (c_color % colors[sphinx_color], c_normal, c_color % colors[less_50_color], sphinx_score, total_members - total_sphinx, total_members, c_normal) else: sphinx_score_string = "%sSphinx:%s %s%s%% (%s of %s)%s" % \ (c_color % colors[sphinx_color], c_normal, c_color % colors[less_100_color], sphinx_score, total_members - total_sphinx, total_members, c_normal) else: sphinx_score_string = "%sSphinx:%s %s%s%% (%s of %s)%s" % \ (c_color % colors[sphinx_color], c_normal, c_color % colors[equal_100_color], sphinx_score, total_members - total_sphinx, total_members, c_normal) if verbose: print('\n' + '-'*70) print(module_path) print('-'*70) else: if sphinx: print("%s: %s %s" % (module_path, score_string, sphinx_score_string)) else: print("%s: %s" % (module_path, score_string)) if verbose: print_header('CLASSES', '*', not no_color and big_header_color) if not c: print_header('No classes found!') else: if c_missing_doc: print_header('Missing docstrings', '-', not no_color and small_header_color) for md in c_missing_doc: print(' * ' + md) if c_missing_doctest: print_header('Missing doctests', '-', not no_color and small_header_color) for md in c_missing_doctest: print(' * ' + md) if c_indirect_doctest: # Use "# indirect doctest" in the docstring to # suppress this warning. print_header('Indirect doctests', '-', not no_color and small_header_color) for md in c_indirect_doctest: print(' * ' + md) print('\n Use \"# indirect doctest\" in the docstring to suppress this warning') if c_sph: print_header('Not imported into Sphinx', '-', not no_color and small_header_color) for md in c_sph: print(' * ' + md) print_header('FUNCTIONS', '*', not no_color and big_header_color) if not f: print_header('No functions found!') else: if f_missing_doc: print_header('Missing docstrings', '-', not no_color and small_header_color) for md in f_missing_doc: print(' * ' + md) if f_missing_doctest: print_header('Missing doctests', '-', not no_color and small_header_color) for md in f_missing_doctest: print(' * ' + md) if f_indirect_doctest: print_header('Indirect doctests', '-', not no_color and small_header_color) for md in f_indirect_doctest: print(' * ' + md) print('\n Use \"# indirect doctest\" in the docstring to suppress this warning') if f_sph: print_header('Not imported into Sphinx', '-', not no_color and small_header_color) for md in f_sph: print(' * ' + md) if verbose: print('\n' + '-'*70) print(score_string) if sphinx: print(sphinx_score_string) print('-'*70) def _is_indirect(member, doc): """ Given string repr of doc and member checks if the member contains indirect documentation """ d = member in doc e = 'indirect doctest' in doc if not d and not e: return True else: return False def _get_arg_list(name, fobj): """ Given a function object, constructs a list of arguments and their defaults. Takes care of varargs and kwargs """ trunc = 20 # Sometimes argument length can be huge argspec = inspect.getfullargspec(fobj) arg_list = [] if argspec.args: for arg in argspec.args: arg_list.append(str(arg)) arg_list.reverse() # Now add the defaults if argspec.defaults: for i in range(len(argspec.defaults)): arg_list[i] = str(arg_list[i]) + '=' + str(argspec.defaults[-i]) # Get the list in right order arg_list.reverse() # Add var args if argspec.varargs: arg_list.append(argspec.varargs) if argspec.varkw: arg_list.append(argspec.varkw) # Truncate long arguments arg_list = [x[:trunc] for x in arg_list] # Construct the parameter string (enclosed in brackets) str_param = "%s(%s)" % (name, ', '.join(arg_list)) return str_param def get_mod_name(path, base): """ Gets a module name, given the path of file/dir and base dir of sympy """ rel_path = os.path.relpath(path, base) # Remove the file extension rel_path, ign = os.path.splitext(rel_path) # Replace separators by . for module path file_module = "" h, t = os.path.split(rel_path) while h or t: if t: file_module = t + '.' + file_module h, t = os.path.split(h) return file_module[:-1] class FindInSphinx(HTMLParser): is_imported = [] def handle_starttag(self, tag, attr): a = dict(attr) if tag == "div" and a.get('class', None) == "viewcode-block": self.is_imported.append(a['id']) def find_sphinx(name, mod_path, found={}): if mod_path in found: # Cache results return name in found[mod_path] doc_path = mod_path.split('.') doc_path[-1] += '.html' sphinx_path = os.path.join(sympy_top, 'doc', '_build', 'html', '_modules', *doc_path) if not os.path.exists(sphinx_path): return False html_txt = Path(sphinx_path).read_text() p = FindInSphinx() p.feed(html_txt) found[mod_path] = p.is_imported return name in p.is_imported def process_function(name, c_name, b_obj, mod_path, f_skip, f_missing_doc, f_missing_doctest, f_indirect_doctest, f_has_doctest, skip_list, sph, sphinx=True): """ Processes a function to get information regarding documentation. It is assume that the function calling this subrouting has already verified that it is a valid module function. """ if name in skip_list: return False, False # We add in the end, as inspect.getsourcelines is slow add_missing_doc = False add_missing_doctest = False add_indirect_doctest = False in_sphinx = True f_doctest = False function = False if inspect.isclass(b_obj): obj = getattr(b_obj, name) obj_name = c_name + '.' + name else: obj = b_obj obj_name = name full_name = _get_arg_list(name, obj) if name.startswith('_'): f_skip.append(full_name) else: doc = obj.__doc__ if isinstance(doc, str): if not doc: add_missing_doc = True elif not '>>>' in doc: add_missing_doctest = True elif _is_indirect(name, doc): add_indirect_doctest = True else: f_doctest = True elif doc is None: # this was a function defined in the docstring f_doctest = True else: raise TypeError('Current doc type for ', print(obj), ' is ', type(doc), '. Docstring must be a string, property, or none') function = True if sphinx: in_sphinx = find_sphinx(obj_name, mod_path) if add_missing_doc or add_missing_doctest or add_indirect_doctest or not in_sphinx: try: line_no = inspect.getsourcelines(obj)[1] except IOError: # Raised when source does not exist # which means the function is not there. return False, False full_name = "LINE %d: %s" % (line_no, full_name) if add_missing_doc: f_missing_doc.append(full_name) elif add_missing_doctest: f_missing_doctest.append(full_name) elif add_indirect_doctest: f_indirect_doctest.append(full_name) if not in_sphinx: sph.append(full_name) return f_doctest, function def process_class(c_name, obj, c_skip, c_missing_doc, c_missing_doctest, c_indirect_doctest, c_has_doctest, mod_path, sph, sphinx=True): """ Extracts information about the class regarding documentation. It is assumed that the function calling this subroutine has already checked that the class is valid. """ # Skip class case if c_name.startswith('_'): c_skip.append(c_name) return False, False, None c = False c_dt = False # Get the line number of class try: source, line_no = inspect.getsourcelines(obj) except IOError: # Raised when source does not exist # which means the class is not there. return False, False, None c = True full_name = "LINE %d: %s" % (line_no, c_name) doc = obj.__doc__ if isinstance(doc, str): if not doc: c_missing_doc.append(full_name) elif not '>>>' in doc: c_missing_doctest.append(full_name) elif _is_indirect(c_name, doc): c_indirect_doctest.append(full_name) else: c_dt = True c_has_doctest.append(full_name) elif doc is None: # this was a class defined in the docstring c_dt = True c_has_doctest.append(full_name) elif isinstance(doc, property): # skip class with dynamic doc c_skip.append(c_name) return False, False, None else: raise TypeError('Current doc type of ', print(obj), ' is ', type(doc), '. Docstring must be a string, property , or none') in_sphinx = False if sphinx: in_sphinx = find_sphinx(c_name, mod_path) if not in_sphinx: sph.append(full_name) return c_dt, c, source def coverage(module_path, verbose=False, no_color=False, sphinx=True): """ Given a module path, builds an index of all classes and functions contained. It then goes through each of the classes/functions to get the docstring and doctest coverage of the module. """ # Import the package and find members m = None try: __import__(module_path) m = sys.modules[module_path] except Exception as a: # Most likely cause, absence of __init__ print("%s could not be loaded due to %s." % (module_path, repr(a))) return 0, 0, 0 c_skipped = [] c_missing_doc = [] c_missing_doctest = [] c_has_doctest = [] c_indirect_doctest = [] classes = 0 c_doctests = 0 c_sph = [] f_skipped = [] f_missing_doc = [] f_missing_doctest = [] f_has_doctest = [] f_indirect_doctest = [] functions = 0 f_doctests = 0 f_sph = [] skip_members = ['__abstractmethods__'] # Get the list of members m_members = dir(m) for member in m_members: # Check for skipped functions first, they throw nasty errors # when combined with getattr if member in skip_members: continue # Identify if the member (class/def) is a part of this module obj = getattr(m, member) obj_mod = inspect.getmodule(obj) # Function not a part of this module if not obj_mod or not obj_mod.__name__ == module_path: continue # If it's a function if inspect.isfunction(obj) or inspect.ismethod(obj): f_dt, f = process_function(member, '', obj, module_path, f_skipped, f_missing_doc, f_missing_doctest, f_indirect_doctest, f_has_doctest, skip_members, f_sph, sphinx=sphinx) if f: functions += 1 if f_dt: f_doctests += 1 # If it's a class, look at it's methods too elif inspect.isclass(obj): # Process the class first c_dt, c, source = process_class(member, obj, c_skipped, c_missing_doc, c_missing_doctest, c_indirect_doctest, c_has_doctest, module_path, c_sph, sphinx=sphinx) if not c: continue else: classes += 1 if c_dt: c_doctests += 1 # Iterate through it's members for f_name in obj.__dict__: if f_name in skip_members or f_name.startswith('_'): continue # Check if def funcname appears in source if not ("def " + f_name) in ' '.join(source): continue # Identify the module of the current class member f_obj = getattr(obj, f_name) obj_mod = inspect.getmodule(f_obj) # Function not a part of this module if not obj_mod or not obj_mod.__name__ == module_path: continue # If it's a function if inspect.isfunction(f_obj) or inspect.ismethod(f_obj): f_dt, f = process_function(f_name, member, obj, module_path, f_skipped, f_missing_doc, f_missing_doctest, f_indirect_doctest, f_has_doctest, skip_members, f_sph, sphinx=sphinx) if f: functions += 1 if f_dt: f_doctests += 1 # Evaluate the percent coverage total_doctests = c_doctests + f_doctests total_members = classes + functions if total_members: score = 100 * float(total_doctests) / (total_members) else: score = 100 score = int(score) if sphinx: total_sphinx = len(c_sph) + len(f_sph) if total_members: sphinx_score = 100 - 100 * float(total_sphinx) / total_members else: sphinx_score = 100 sphinx_score = int(sphinx_score) else: total_sphinx = 0 sphinx_score = 0 # Sort functions/classes by line number c_missing_doc = sorted(c_missing_doc, key=lambda x: int(x.split()[1][:-1])) c_missing_doctest = sorted(c_missing_doctest, key=lambda x: int(x.split()[1][:-1])) c_indirect_doctest = sorted(c_indirect_doctest, key=lambda x: int(x.split()[1][:-1])) f_missing_doc = sorted(f_missing_doc, key=lambda x: int(x.split()[1][:-1])) f_missing_doctest = sorted(f_missing_doctest, key=lambda x: int(x.split()[1][:-1])) f_indirect_doctest = sorted(f_indirect_doctest, key=lambda x: int(x.split()[1][:-1])) print_coverage(module_path, classes, c_missing_doc, c_missing_doctest, c_indirect_doctest, c_sph, functions, f_missing_doc, f_missing_doctest, f_indirect_doctest, f_sph, score, total_doctests, total_members, sphinx_score, total_sphinx, verbose=verbose, no_color=no_color, sphinx=sphinx) return total_doctests, total_sphinx, total_members def go(sympy_top, file, verbose=False, no_color=False, exact=True, sphinx=True): # file names containing any string in skip_paths will be skipped, skip_paths = [] if os.path.isdir(file): doctests, total_sphinx, num_functions = 0, 0, 0 for F in os.listdir(file): _doctests, _total_sphinx, _num_functions = go(sympy_top, '%s/%s' % (file, F), verbose=verbose, no_color=no_color, exact=False, sphinx=sphinx) doctests += _doctests total_sphinx += _total_sphinx num_functions += _num_functions return doctests, total_sphinx, num_functions if (not (file.endswith((".py", ".pyx"))) or file.endswith('__init__.py') or not exact and ('test_' in file or 'bench_' in file or any(name in file for name in skip_paths))): return 0, 0, 0 if not os.path.exists(file): print("File(%s does not exist." % file) sys.exit(1) # Relpath for constructing the module name return coverage(get_mod_name(file, sympy_top), verbose=verbose, no_color=no_color, sphinx=sphinx) if __name__ == "__main__": bintest_dir = os.path.abspath(os.path.dirname(__file__)) # bin/cover... sympy_top = os.path.split(bintest_dir)[0] # ../ sympy_dir = os.path.join(sympy_top, 'sympy') # ../sympy/ if os.path.isdir(sympy_dir): sys.path.insert(0, sympy_top) usage = "usage: ./bin/doctest_coverage.py PATHS" parser = ArgumentParser( description=__doc__, usage=usage, formatter_class=RawDescriptionHelpFormatter, ) parser.add_argument("path", nargs='*', default=[os.path.join(sympy_top, 'sympy')]) parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False) parser.add_argument("--no-colors", action="store_true", dest="no_color", help="use no colors", default=False) parser.add_argument("--no-sphinx", action="store_false", dest="sphinx", help="don't report Sphinx coverage", default=True) args = parser.parse_args() if args.sphinx and not os.path.exists(os.path.join(sympy_top, 'doc', '_build', 'html')): print(filldedent(""" Cannot check Sphinx coverage without a documentation build. To build the docs, run "cd doc; make html". To skip checking Sphinx coverage, pass --no-sphinx. """)) sys.exit(1) full_coverage = True for file in args.path: file = os.path.normpath(file) print('DOCTEST COVERAGE for %s' % (file)) print('='*70) print() doctests, total_sphinx, num_functions = go(sympy_top, file, verbose=args.verbose, no_color=args.no_color, sphinx=args.sphinx) if num_functions == 0: score = 100 sphinx_score = 100 else: score = 100 * float(doctests) / num_functions score = int(score) if doctests < num_functions: full_coverage = False if args.sphinx: sphinx_score = 100 - 100 * float(total_sphinx) / num_functions sphinx_score = int(sphinx_score) if total_sphinx > 0: full_coverage = False print() print('='*70) if args.no_color: print("TOTAL DOCTEST SCORE for %s: %s%% (%s of %s)" % \ (get_mod_name(file, sympy_top), score, doctests, num_functions)) elif score < 100: print("TOTAL DOCTEST SCORE for %s: %s%s%% (%s of %s)%s" % \ (get_mod_name(file, sympy_top), c_color % (colors["Red"]), score, doctests, num_functions, c_normal)) else: print("TOTAL DOCTEST SCORE for %s: %s%s%% (%s of %s)%s" % \ (get_mod_name(file, sympy_top), c_color % (colors["Green"]), score, doctests, num_functions, c_normal)) if args.sphinx: if args.no_color: print("TOTAL SPHINX SCORE for %s: %s%% (%s of %s)" % \ (get_mod_name(file, sympy_top), sphinx_score, num_functions - total_sphinx, num_functions)) elif sphinx_score < 100: print("TOTAL SPHINX SCORE for %s: %s%s%% (%s of %s)%s" % \ (get_mod_name(file, sympy_top), c_color % (colors["Red"]), sphinx_score, num_functions - total_sphinx, num_functions, c_normal)) else: print("TOTAL SPHINX SCORE for %s: %s%s%% (%s of %s)%s" % \ (get_mod_name(file, sympy_top), c_color % (colors["Green"]), sphinx_score, num_functions - total_sphinx, num_functions, c_normal)) print() sys.exit(not full_coverage)