refactor: Layout a docstring parser base

src/pytkdocs/cli.py

@@ -21,9 +21,9 @@
 import traceback
 from typing import Dict, List, Optional, Sequence
 
-from .loader import Loader
-from .objects import Object
-from .serializer import serialize_object
+from pytkdocs.loader import Loader
+from pytkdocs.objects import Object
+from pytkdocs.serializer import serialize_object
 
 
 def process_config(config: dict) -> dict:

src/pytkdocs/loader.py

@@ -14,9 +14,10 @@
 from pathlib import Path
 from typing import Any, List, Optional, Set, Union
 
-from .objects import Attribute, Class, Function, Method, Module, Object, Source
-from .parsers.attributes import get_attributes
-from .properties import RE_SPECIAL
+from pytkdocs.objects import Attribute, Class, Function, Method, Module, Object, Source
+from pytkdocs.parsers.attributes import get_attributes
+from pytkdocs.parsers.docstrings import PARSERS
+from pytkdocs.properties import RE_SPECIAL
 
 
 class ObjectNode:
@@ -173,7 +174,12 @@ class Loader:
     Any error that occurred during collection of the objects and their documentation is stored in the `errors` list.
     """
 
-    def __init__(self, filters: Optional[List[str]] = None):
+    def __init__(
+        self,
+        filters: Optional[List[str]] = None,
+        docstring_style: str = "google",
+        docstring_options: Optional[dict] = None,
+    ) -> None:
         """
         Arguments:
             filters: A list of regular expressions to fine-grain select members. It is applied recursively.
@@ -182,6 +188,7 @@ def __init__(self, filters: Optional[List[str]] = None):
             filters = []
 
         self.filters = [(f, re.compile(f.lstrip("!"))) for f in filters]
+        self.docstring_parser = PARSERS[docstring_style](**(docstring_options or {}))  # type: ignore
         self.errors: List[str] = []
 
     def get_object_documentation(self, dotted_path: str, members: Optional[Union[Set[str], bool]] = None) -> Object:
@@ -236,7 +243,7 @@ def get_object_documentation(self, dotted_path: str, members: Optional[Union[Set
                     filtered.append(attribute)
             root_object.dispatch_attributes(filtered)
 
-        root_object.parse_all_docstring()
+        root_object.parse_all_docstring(self.docstring_parser)
 
         return root_object
 

src/pytkdocs/objects.py

@@ -22,9 +22,8 @@
 from pathlib import Path
 from typing import List, Optional, Union
 
-from pytkdocs.parsers.docstrings import parse
-
-from .properties import NAME_CLASS_PRIVATE, NAME_PRIVATE, NAME_SPECIAL, ApplicableNameProperty
+from pytkdocs.parsers.docstrings.base import Parser, Section
+from pytkdocs.properties import NAME_CLASS_PRIVATE, NAME_PRIVATE, NAME_SPECIAL, ApplicableNameProperty
 
 
 class Source:
@@ -92,6 +91,10 @@ def __init__(
         """The file path of the object's direct parent module."""
         self.docstring = docstring
         """The object's docstring."""
+        self.docstring_sections: List[Section] = []
+        """The object's docstring parsed into sections."""
+        self.docstring_errors: List[str] = []
+        """The errors detected while parsing the docstring."""
         self.properties = properties or []
         """The object's properties."""
         self.parent: Optional[Object] = None
@@ -283,23 +286,20 @@ def dispatch_attributes(self, attributes: List["Attribute"]) -> None:
                 attach_to.children.append(attribute)
                 attribute.parent = attach_to
 
-    def parse_all_docstring(self) -> None:
+    def parse_all_docstring(self, parser: Parser) -> None:
         """
         Recursively parse the docstring of this object and its children.
 
         I hope we can get rid of this code at some point as parsing docstring is not really our purpose.
         """
-        signature = None
-        if hasattr(self, "signature"):
-            signature = self.signature  # type: ignore
-        attr_type = None
-        if hasattr(self, "type"):
-            attr_type = self.type  # type: ignore
-        sections, errors = parse(self.path, self.docstring, signature, attr_type)
-        self.docstring_sections = sections
-        self.docstring_errors = errors
+        self.docstring_sections, self.docstring_errors = parser.parse(
+            self.docstring,
+            object_path=self.path,
+            object_signature=getattr(self, "signature", None),
+            object_type=getattr(self, "type", None),
+        )
         for child in self.children:
-            child.parse_all_docstring()
+            child.parse_all_docstring(parser)
 
     @lru_cache()
     def has_contents(self) -> bool:

src/pytkdocs/parsers/attributes.py

@@ -5,7 +5,7 @@
 from types import ModuleType
 from typing import Any, Iterable, List, Optional, Union
 
-from ..objects import Attribute
+from pytkdocs.objects import Attribute
 
 RECURSIVE_NODES = (ast.If, ast.IfExp, ast.Try, ast.With, ast.ExceptHandler)
 

src/pytkdocs/parsers/docstrings/__init__.py

@@ -0,0 +1,6 @@
+from typing import Dict, Type
+
+from pytkdocs.parsers.docstrings.base import Parser
+from pytkdocs.parsers.docstrings.google import Google
+
+PARSERS: Dict[str, Type[Parser]] = {"google": Google}

src/pytkdocs/parsers/docstrings/base.py

@@ -0,0 +1,127 @@
+import inspect
+from abc import ABCMeta, abstractmethod
+from typing import Any, List, Optional, Tuple
+
+empty = inspect.Signature.empty
+
+
+class AnnotatedObject:
+    """A helper class to store information about an annotated object."""
+
+    def __init__(self, annotation, description):
+        self.annotation = annotation
+        self.description = description
+
+
+class Parameter(AnnotatedObject):
+    """A helper class to store information about a signature parameter."""
+
+    def __init__(self, name, annotation, description, kind, default=empty):
+        super().__init__(annotation, description)
+        self.name = name
+        self.kind = kind
+        self.default = default
+
+    def __str__(self):
+        return self.name
+
+    def __repr__(self):
+        return f"<Parameter({self.name}, {self.annotation}, {self.description}, {self.kind}, {self.default})>"
+
+    @property
+    def is_optional(self):
+        return self.default is not empty
+
+    @property
+    def is_required(self):
+        return not self.is_optional
+
+    @property
+    def is_args(self):
+        return self.kind is inspect.Parameter.VAR_POSITIONAL
+
+    @property
+    def is_kwargs(self):
+        return self.kind is inspect.Parameter.VAR_KEYWORD
+
+    @property
+    def default_string(self):
+        if self.is_kwargs:
+            return "{}"
+        elif self.is_args:
+            return "()"
+        elif self.is_required:
+            return ""
+        return repr(self.default)
+
+
+class Section:
+    """A helper class to store a docstring section."""
+
+    class Type:
+        MARKDOWN = "markdown"
+        PARAMETERS = "parameters"
+        EXCEPTIONS = "exceptions"
+        RETURN = "return"
+
+    def __init__(self, section_type, value):
+        self.type = section_type
+        self.value = value
+
+    def __str__(self):
+        return self.type
+
+    def __repr__(self):
+        return f"<Section(type={self.type!r})>"
+
+
+class Parser(metaclass=ABCMeta):
+    """
+    A class to parse docstrings.
+
+    It is instantiated with an object's path, docstring, signature and return type.
+
+    The `parse` method then returns structured data,
+    in the form of a list of [`Section`][pytkdocs.parsers.docstrings.Section]s.
+    It also return the list of errors that occurred during parsing.
+    """
+
+    def __init__(self) -> None:
+        """Initialization method."""
+        self.object_path = ""
+        self.object_signature: Optional[inspect.Signature] = None
+        self.object_type = None
+        self.errors: List[str] = []
+
+    def set_state(
+        self, object_path: str, object_signature: Optional[inspect.Signature], object_type: Optional[Any],
+    ):
+        self.errors = []
+        self.object_path = object_path
+        self.object_signature = object_signature
+        self.object_type = object_type
+
+    def reset_state(self):
+        self.object_path = ""
+        self.object_signature = None
+        self.object_type = None
+
+    def parse(
+        self,
+        docstring: str,
+        object_path: str,
+        object_signature: Optional[inspect.Signature] = None,
+        object_type: Optional[Any] = None,
+    ) -> Tuple[List[Section], List[str]]:
+        self.set_state(object_path, object_signature, object_type)
+        sections = self.parse_sections(docstring)
+        errors = self.errors
+        self.reset_state()
+        return sections, errors
+
+    def error(self, message):
+        self.errors.append(f"{self.object_path}: {message}")
+
+    @abstractmethod
+    def parse_sections(self, docstring: str) -> List[Section]:
+        raise NotImplementedError