From 5e962f0449491afd765ad3f9e3e42bcc64ab77dc Mon Sep 17 00:00:00 2001 From: Florent Xicluna Date: Tue, 29 Jan 2013 18:41:18 +0100 Subject: [PATCH] Add support for doctests, contribution of Marius Gedminas --- NEWS.txt | 3 + pyflakes/checker.py | 49 ++++++++- pyflakes/messages.py | 6 ++ pyflakes/test/test_doctests.py | 175 +++++++++++++++++++++++++++++++++ 4 files changed, 230 insertions(+), 3 deletions(-) create mode 100644 pyflakes/test/test_doctests.py diff --git a/NEWS.txt b/NEWS.txt index 2d2e78ec..b9479ac6 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,3 +1,6 @@ +0.6.x (unreleased): + - Support checking doctests. + 0.6.1 (2013-01-29): - Fix detection of variables in augmented assignments. diff --git a/pyflakes/checker.py b/pyflakes/checker.py index fd38d189..3edb3ec7 100644 --- a/pyflakes/checker.py +++ b/pyflakes/checker.py @@ -3,6 +3,7 @@ # See LICENSE file for details import os.path +import sys try: import builtins PY2 = False @@ -194,6 +195,7 @@ class Checker(object): """ nodeDepth = 0 + linenoOffset = 0 traceTree = False builtIns = set(dir(builtins)) | set(_MAGIC_GLOBALS) @@ -230,21 +232,24 @@ def deferFunction(self, callable): `callable` is called, the scope at the time this is called will be restored, however it will contain any new bindings added to it. """ - self._deferredFunctions.append((callable, self.scopeStack[:])) + self._deferredFunctions.append((callable, self.scopeStack[:], + self.linenoOffset)) def deferAssignment(self, callable): """ Schedule an assignment handler to be called just after deferred function handlers. """ - self._deferredAssignments.append((callable, self.scopeStack[:])) + self._deferredAssignments.append((callable, self.scopeStack[:], + self.linenoOffset)) def runDeferred(self, deferred): """ Run the callables in C{deferred} using their associated scope stack. """ - for handler, scope in deferred: + for handler, scope, linenoOffset in deferred: self.scopeStack = scope + self.linenoOffset = linenoOffset handler() @property @@ -469,10 +474,18 @@ def isDocstring(self, node): return isinstance(node, ast.Str) or (isinstance(node, ast.Expr) and isinstance(node.value, ast.Str)) + def getDocstring(self, node): + if isinstance(node, ast.Expr): + node = node.value + assert isinstance(node, ast.Str) + return node.s + def handleNode(self, node, parent): if node is None: return node.parent = parent + if getattr(node, 'lineno', None) is not None: + node.lineno += self.linenoOffset if self.traceTree: print(' ' * self.nodeDepth + node.__class__.__name__) self.nodeDepth += 1 @@ -489,6 +502,34 @@ def handleNode(self, node, parent): if self.traceTree: print(' ' * self.nodeDepth + 'end ' + node.__class__.__name__) + def handleDoctests(self, node): + if node.body and self.isDocstring(node.body[0]): + docstring = self.getDocstring(node.body[0]) + else: + return + if not docstring: + return + import doctest + dtparser = doctest.DocTestParser() + try: + examples = dtparser.get_examples(docstring) + except ValueError: + # e.g. ValueError: line 6 of the docstring for has inconsistent leading whitespace: ... + return + self.pushFunctionScope() + for example in examples: + try: + tree = compile(example.source, "", "exec", ast.PyCF_ONLY_AST) + except SyntaxError: + e = sys.exc_info()[1] + self.report(messages.DoctestSyntaxError, + node.lineno + example.lineno + e.lineno) + else: + self.linenoOffset += node.lineno + example.lineno + self.handleChildren(tree) + self.linenoOffset -= node.lineno + example.lineno + self.popScope() + def ignore(self, node): pass @@ -593,6 +634,7 @@ def FUNCTIONDEF(self, node): self.handleNode(deco, node) self.addBinding(node, FunctionDefinition(node.name, node)) self.LAMBDA(node) + self.deferFunction(lambda: self.handleDoctests(node)) def LAMBDA(self, node): args = [] @@ -675,6 +717,7 @@ def CLASSDEF(self, node): for keywordNode in node.keywords: self.handleNode(keywordNode, node) self.pushClassScope() + self.deferFunction(lambda: self.handleDoctests(node)) for stmt in node.body: self.handleNode(stmt, node) self.popScope() diff --git a/pyflakes/messages.py b/pyflakes/messages.py index e3abab65..07f43728 100644 --- a/pyflakes/messages.py +++ b/pyflakes/messages.py @@ -60,6 +60,12 @@ def __init__(self, filename, lineno, name): Message.__init__(self, filename, lineno) self.message_args = (name,) +class DoctestSyntaxError(Message): + message = 'syntax error in doctest' + def __init__(self, filename, lineno): + Message.__init__(self, filename, lineno) + self.message_args = () + class UndefinedExport(Message): message = 'undefined name %r in __all__' diff --git a/pyflakes/test/test_doctests.py b/pyflakes/test/test_doctests.py new file mode 100644 index 00000000..7feade2a --- /dev/null +++ b/pyflakes/test/test_doctests.py @@ -0,0 +1,175 @@ +import textwrap +from unittest2 import skip + +from pyflakes.test.test_other import Test as TestOther +from pyflakes.test.test_imports import Test as TestImports +from pyflakes.test.test_undefined_names import Test as TestUndefinedNames + +import pyflakes.messages as m + +class Test(TestOther, TestImports, TestUndefinedNames): + + def doctestify(self, input): + lines = [] + for line in textwrap.dedent(input).splitlines(): + if line.strip() == '': + pass + elif (line.startswith(' ') or + line.startswith('except:') or + line.startswith('except ') or + line.startswith('finally:') or + line.startswith('else:') or + line.startswith('elif ')): + line = "... %s" % line + else: + line = ">>> %s" % line + lines.append(line) + doctestificator = textwrap.dedent('''\ + def doctest_something(): + """ + %s + """ + ''') + return doctestificator % "\n ".join(lines) + + def flakes(self, input, *args, **kw): + return super(Test, self).flakes(self.doctestify(input), + *args, **kw) + + def test_doubleNestingReportsClosestName(self): + """ + Lines in doctest are a bit different so we can't use the test + from TestUndefinedNames + """ + exc = super(Test, self).flakes(''' + def doctest_stuff(): + """ + >>> def a(): + ... x = 1 + ... def b(): + ... x = 2 # line 7 in the file + ... def c(): + ... x + ... x = 3 + ... return x + ... return x + ... return x + + """ + ''', m.UndefinedLocal).messages[0] + self.assertEqual(exc.message_args, ('x', 7)) + + def test_futureImport(self): + """XXX This test can't work in a doctest""" + + def test_importBeforeDoctest(self): + super(Test, self).flakes(""" + import foo + + def doctest_stuff(): + ''' + >>> foo + ''' + """) + + @skip("todo") + def test_importBeforeAndInDoctest(self): + super(Test, self).flakes(''' + import foo + + def doctest_stuff(): + """ + >>> import foo + >>> foo + """ + + foo + ''', m.Redefined) + + def test_importInDoctestAndAfter(self): + super(Test, self).flakes(''' + def doctest_stuff(): + """ + >>> import foo + >>> foo + """ + + import foo + foo() + ''') + + def test_lineNumbersInDoctests(self): + exc = super(Test, self).flakes(''' + + def doctest_stuff(): + """ + >>> x # line 5 + """ + + ''', m.UndefinedName).messages[0] + self.assertEqual(exc.lineno, 5) + + def test_lineNumbersInLambdasInDoctests(self): + exc = super(Test, self).flakes(''' + + def doctest_stuff(): + """ + >>> lambda: x # line 5 + """ + + ''', m.UndefinedName).messages[0] + self.assertEqual(exc.lineno, 5) + + + def test_lineNumbersAfterDoctests(self): + exc = super(Test, self).flakes(''' + + def doctest_stuff(): + """ + >>> x = 5 + """ + + x + + ''', m.UndefinedName).messages[0] + self.assertEqual(exc.lineno, 8) + + def test_syntaxErrorInDoctest(self): + exc = super(Test, self).flakes(''' + def doctest_stuff(): + """ + >>> from # line 4 + """ + ''', m.DoctestSyntaxError).messages[0] + self.assertEqual(exc.lineno, 4) + + def test_indentationErrorInDoctest(self): + exc = super(Test, self).flakes(''' + def doctest_stuff(): + """ + >>> if True: + ... pass + """ + ''', m.DoctestSyntaxError).messages[0] + self.assertEqual(exc.lineno, 5) + + def test_doctestCanReferToFunction(self): + super(Test, self).flakes(""" + def foo(): + ''' + >>> foo + ''' + """) + + def test_doctestCanReferToClass(self): + super(Test, self).flakes(""" + class Foo(): + ''' + >>> Foo + ''' + def bar(self): + ''' + >>> Foo + ''' + """) +