From d91cbf36fb907c26382afdee01826cf6e70a8a5b Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 19 Dec 2018 05:38:27 +1000 Subject: [PATCH 01/48] Add flag member details to stubs --- pgidocgen/stubs.py | 52 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 0fdaec07..14e2c5d5 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -39,6 +39,52 @@ def _main_many(target, namespaces): pass +class StubClass: + def __init__(self, classname): + self.classname = classname + self.parents = [] + self.members = [] + + def add_member(self, member): + self.members.append(member) + + @property + def class_line(self): + if self.parents: + parents = "({})".format(', '.join(self.parents)) + else: + parents = "" + return "class {}{}:".format(self.classname, parents) + + @property + def member_lines(self): + return [ + " {}".format(member) + for member in sorted(self.members) + ] + + def __str__(self): + return '\n'.join( + [self.class_line] + + self.member_lines + + [''] + ) + + +def stub_flag(flag) -> str: + stub = StubClass(flag.name) + for v in flag.values: + stub.add_member(f"{v.name} = ... # type: {flag.name}") + + if flag.methods or flag.vfuncs: + # This is unsupported simply because I can't find any GIR that + # has methods or vfuncs on its flag types. + raise NotImplementedError( + "Flag support doesn't annotate methods or vfuncs") + + return str(stub) + + def main(args): if not args.namespace: print("No namespace given") @@ -81,7 +127,7 @@ def get_to_write(dir_, namespace, version): for namespace, version in get_to_write(args.target, namespace, version): mod = Repository(namespace, version).parse() module_path = os.path.join(args.target, namespace + ".pyi") - types = mod.classes + mod.flags + mod.enums + \ + types = mod.classes + mod.enums + \ mod.structures + mod.unions with open(module_path, "w", encoding="utf-8") as h: for cls in types: @@ -89,6 +135,10 @@ def get_to_write(dir_, namespace, version): class {}: ... """.format(cls.name)) + for cls in mod.flags: + h.write(stub_flag(cls)) + h.write("\n\n") + for func in mod.functions: h.write("""\ def {}(*args, **kwargs): ... From 182b6f18c55022b44626801d53d04501e6ae76ab Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 19 Dec 2018 06:46:38 +1000 Subject: [PATCH 02/48] Add constant typing to stubs This also retains the original constant value on the Constant class so that the stubbing can introspect its type. We could instead do the typing-annotation style formatting on the constant class, but this same logic is going to be needed for e.g., formatting function parameter types. --- pgidocgen/docobj.py | 5 +++-- pgidocgen/stubs.py | 43 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/pgidocgen/docobj.py b/pgidocgen/docobj.py index cb099ff3..b20b08cd 100644 --- a/pgidocgen/docobj.py +++ b/pgidocgen/docobj.py @@ -881,12 +881,13 @@ def from_object(cls, repo, obj): class Constant(BaseDocObject): - def __init__(self, parent_fullname, name, value): + def __init__(self, parent_fullname, name, value, raw_value): self.fullname = parent_fullname + "." + name self.name = name self.info = None self.value = value + self.raw_value = raw_value @classmethod def from_object(cls, repo, parent_fullname, name, obj): @@ -898,7 +899,7 @@ def from_object(cls, repo, parent_fullname, name, obj): else: value = repr(obj) - instance = Constant(parent_fullname, name, value) + instance = Constant(parent_fullname, name, value, obj) instance.info = DocInfo.from_object(repo, "all", instance) return instance diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 14e2c5d5..676ba6d0 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -9,6 +9,7 @@ import subprocess import tempfile import os +import typing from .repo import Repository from .util import get_gir_files @@ -71,6 +72,38 @@ def __str__(self): ) +def get_typing_name(type_: typing.Any) -> str: + """Gives a name for a type that is suitable for a typing annotation. + + This is the Python annotation counterpart to funcsig.get_type_name(). + + int -> "int" + Gtk.Window -> "Gtk.Window" + [int] -> "Sequence[int]" + {int: Gtk.Button} -> "Mapping[int, Gtk.Button]" + """ + + if type_ is None: + return "" + elif isinstance(type_, str): + return type_ + elif isinstance(type_, list): + assert len(type_) == 1 + return "Sequence[%s]" % get_typing_name(type_[0]) + elif isinstance(type_, dict): + assert len(type_) == 1 + key, value = type_.popitem() + return "Mapping[%s, %s]" % (get_typing_name(key), get_typing_name(value)) + elif type_.__module__ in ("__builtin__", "builtins"): + return type_.__name__ + else: + # FIXME: We need better module handling here. I think we need to strip + # the module if the type's module is the *current* module being + # annotated, and if not then we need to track imports and probably add + # a "gi.repository." prefix. + return "%s.%s" % (type_.__module__, type_.__name__) + + def stub_flag(flag) -> str: stub = StubClass(flag.name) for v in flag.values: @@ -85,6 +118,11 @@ def stub_flag(flag) -> str: return str(stub) +def stub_constant(constant) -> str: + type_ = get_typing_name(type(constant.raw_value)) + return f"{constant.name} = ... # type: {type_}" + + def main(args): if not args.namespace: print("No namespace given") @@ -145,6 +183,5 @@ def {}(*args, **kwargs): ... """.format(func.name)) for const in mod.constants: - h.write("""\ -{} = ... -""".format(const.name)) + h.write(stub_constant(const)) + h.write("\n") From 800869b5472c7bae472a3b68c069a5fbadfb9dea Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Fri, 21 Dec 2018 06:22:42 +1000 Subject: [PATCH 03/48] Add stubs of module-level functions This also retains the full function signature on functions so that we can get the Python typing annotations for args and return values correct. --- pgidocgen/docobj.py | 2 ++ pgidocgen/stubs.py | 75 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/pgidocgen/docobj.py b/pgidocgen/docobj.py index b20b08cd..e99be50a 100644 --- a/pgidocgen/docobj.py +++ b/pgidocgen/docobj.py @@ -336,6 +336,7 @@ def from_object(cls, repo, parent_fullname, sig): print("FIXME: signal: %s " % inst.fullname) signature_desc = "(FIXME pgi-docgen: arguments are missing here)" + inst.full_signature = fsig inst.signature_desc = signature_desc inst.info = DocInfo.from_object(repo, "signals", inst, current_type=parent_fullname) @@ -788,6 +789,7 @@ def render(docs): signature = get_signature_string(obj) assert signature + instance.full_signature = func_sig instance.signature_desc = signature_desc instance.signature = signature instance.info.desc = desc diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 676ba6d0..2eb47308 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -104,6 +104,75 @@ def get_typing_name(type_: typing.Any) -> str: return "%s.%s" % (type_.__module__, type_.__name__) +def arg_to_annotation(text): + """Convert a docstring argument to a Python annotation string + + This is the Python annotation counterpart to funcsig.arg_to_class_ref(). + """ + + if not text.startswith(("[", "{")) or not text.endswith(("}", "]")): + parts = text.split(" or ") + else: + parts = [text] + + out = [] + for p in parts: + if p.startswith("["): + out.append("Sequence[%s]" % arg_to_annotation(p[1:-1])) + elif p.startswith("{"): + p = p[1:-1] + k, v = p.split(":", 1) + k = arg_to_annotation(k.strip()) + v = arg_to_annotation(v.strip()) + out.append("Mapping[%s, %s]" % (k, v)) + elif p: + out.append(p) + + if len(out) == 1: + return out[0] + elif len(out) == 2 and 'None' in out: + # This is not strictly necessary, but it's easier to read than the Union + out.pop(out.index('None')) + return f"Optional[{out[0]}]" + else: + return f"Union[{', '.join(out)}]" + + +def stub_function(function) -> str: + # We require the full signature details for argument types, and fallback + # to the simplest possible function signature if it's not available. + signature = getattr(function, 'full_signature', None) + if not signature: + print(f"Missing full signature for {function}; falling back") + return f"def {function.name}(*args, **kwargs): ..." + + # Decorator handling + decorator = "@staticmethod\n" if function.is_static else "" + + # Format argument types + arg_specs = [] + for key, value in signature.args: + arg_specs.append(f'{key}: {arg_to_annotation(value)}') + args = f'({", ".join(arg_specs)})' + + # Format return values + return_values = [] + for r in signature.res: + # We have either a (name, return type) pair, or just the return type. + type_ = r[1] if len(r) > 1 else r[0] + return_values.append(arg_to_annotation(type_)) + + # Additional handling for structuring return values + if len(return_values) == 0: + returns = 'None' + elif len(return_values) == 1: + returns = return_values[0] + else: + returns = f'Tuple[{", ".join(return_values)}]' + + return f'{decorator}def {function.name}{args} -> {returns}: ...' + + def stub_flag(flag) -> str: stub = StubClass(flag.name) for v in flag.values: @@ -178,9 +247,9 @@ class {}: ... h.write("\n\n") for func in mod.functions: - h.write("""\ -def {}(*args, **kwargs): ... -""".format(func.name)) + h.write(stub_function(func)) + # Extra \n because the signature lacks one. + h.write("\n\n\n") for const in mod.constants: h.write(stub_constant(const)) From d8d3e7d65a88b4241b62bde0395c9d0a76082399 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Fri, 21 Dec 2018 06:24:59 +1000 Subject: [PATCH 04/48] Add typing details for enums and support functions on class stubs --- pgidocgen/stubs.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 2eb47308..d437e993 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -45,10 +45,14 @@ def __init__(self, classname): self.classname = classname self.parents = [] self.members = [] + self.functions = [] def add_member(self, member): self.members.append(member) + def add_function(self, function): + self.functions.append(function) + @property def class_line(self): if self.parents: @@ -64,10 +68,20 @@ def member_lines(self): for member in sorted(self.members) ] + @property + def function_lines(self): + lines = [] + for function in self.functions: + lines.append('') + for line in function.splitlines(): + lines.append(f' {line}') + return lines + def __str__(self): return '\n'.join( [self.class_line] + self.member_lines + + self.function_lines + [''] ) @@ -187,6 +201,23 @@ def stub_flag(flag) -> str: return str(stub) +def stub_enum(enum) -> str: + stub = StubClass(enum.name) + for v in enum.values: + stub.add_member(f"{v.name} = ... # type: {enum.name}") + + for v in enum.methods: + stub.add_function(stub_function(v)) + + if enum.vfuncs: + # This is unsupported simply because I can't find any GIR that + # has vfuncs on its enum types. + raise NotImplementedError( + "Enum support doesn't annotate vfuncs") + + return str(stub) + + def stub_constant(constant) -> str: type_ = get_typing_name(type(constant.raw_value)) return f"{constant.name} = ... # type: {type_}" @@ -234,8 +265,7 @@ def get_to_write(dir_, namespace, version): for namespace, version in get_to_write(args.target, namespace, version): mod = Repository(namespace, version).parse() module_path = os.path.join(args.target, namespace + ".pyi") - types = mod.classes + mod.enums + \ - mod.structures + mod.unions + types = mod.classes + mod.structures + mod.unions with open(module_path, "w", encoding="utf-8") as h: for cls in types: h.write("""\ @@ -246,6 +276,10 @@ class {}: ... h.write(stub_flag(cls)) h.write("\n\n") + for cls in mod.enums: + h.write(stub_enum(cls)) + h.write("\n\n") + for func in mod.functions: h.write(stub_function(func)) # Extra \n because the signature lacks one. From b5c8f07691872947c7fd657becd3ecc895bc6607 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 23 Dec 2018 07:54:20 +1000 Subject: [PATCH 05/48] Rework constant formatting so that we can reuse it for class fields --- pgidocgen/docobj.py | 6 +++--- pgidocgen/stubs.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pgidocgen/docobj.py b/pgidocgen/docobj.py index e99be50a..5beff578 100644 --- a/pgidocgen/docobj.py +++ b/pgidocgen/docobj.py @@ -883,13 +883,13 @@ def from_object(cls, repo, obj): class Constant(BaseDocObject): - def __init__(self, parent_fullname, name, value, raw_value): + def __init__(self, parent_fullname, name, value, py_type): self.fullname = parent_fullname + "." + name self.name = name self.info = None self.value = value - self.raw_value = raw_value + self.py_type = py_type @classmethod def from_object(cls, repo, parent_fullname, name, obj): @@ -901,7 +901,7 @@ def from_object(cls, repo, parent_fullname, name, obj): else: value = repr(obj) - instance = Constant(parent_fullname, name, value, obj) + instance = Constant(parent_fullname, name, value, type(obj)) instance.info = DocInfo.from_object(repo, "all", instance) return instance diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index d437e993..c8a3ef3b 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -218,9 +218,8 @@ def stub_enum(enum) -> str: return str(stub) -def stub_constant(constant) -> str: - type_ = get_typing_name(type(constant.raw_value)) - return f"{constant.name} = ... # type: {type_}" +def format_field(field) -> str: + return f"{field.name} = ... # type: {get_typing_name(field.py_type)}" def main(args): @@ -286,5 +285,5 @@ class {}: ... h.write("\n\n\n") for const in mod.constants: - h.write(stub_constant(const)) + h.write(format_field(const)) h.write("\n") From 1b14ae1321e36002de72fe4511bd86345d00e421 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 23 Dec 2018 07:59:21 +1000 Subject: [PATCH 06/48] Add typing details for classes --- pgidocgen/docobj.py | 1 + pgidocgen/stubs.py | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pgidocgen/docobj.py b/pgidocgen/docobj.py index 5beff578..6ac03bca 100644 --- a/pgidocgen/docobj.py +++ b/pgidocgen/docobj.py @@ -649,6 +649,7 @@ def from_object(cls, repo, parent_fullname, field_info): name = field_info.name field = cls(parent_fullname, name) + field.py_type = field_info.py_type field.type_desc = py_type_to_class_ref(field_info.py_type) field.readable = field_info.readable field.writable = field_info.writeable diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index c8a3ef3b..1c7f1846 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -218,6 +218,27 @@ def stub_enum(enum) -> str: return str(stub) +def stub_class(cls) -> str: + stub = StubClass(cls.name) + + # TODO: These parent classes may require namespace prefix sanitising. + stub.parents = [b.name for b in cls.bases] + + # TODO: We don't handle: + # * child_properties: It's not clear how to annotate these + # * gtype_struct: I'm not sure what we'd use this for. + # * properties: It's not clear how to annotate these + # * signals: It's not clear how to annotate these + + for f in cls.fields: + stub.add_member(format_field(f)) + + for v in cls.methods + cls.vfuncs: + stub.add_function(stub_function(v)) + + return str(stub) + + def format_field(field) -> str: return f"{field.name} = ... # type: {get_typing_name(field.py_type)}" @@ -264,13 +285,17 @@ def get_to_write(dir_, namespace, version): for namespace, version in get_to_write(args.target, namespace, version): mod = Repository(namespace, version).parse() module_path = os.path.join(args.target, namespace + ".pyi") - types = mod.classes + mod.structures + mod.unions + types = mod.structures + mod.unions with open(module_path, "w", encoding="utf-8") as h: for cls in types: h.write("""\ class {}: ... """.format(cls.name)) + for cls in mod.classes: + h.write(stub_class(cls)) + h.write("\n\n") + for cls in mod.flags: h.write(stub_flag(cls)) h.write("\n\n") From fe62d7721ac5f11d9f4abb2aaa06cfd6ffc06ce0 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 23 Dec 2018 08:14:39 +1000 Subject: [PATCH 07/48] Fix function annotations to insert self for non-static methods --- pgidocgen/stubs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 1c7f1846..fd3354ad 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -165,6 +165,10 @@ def stub_function(function) -> str: # Format argument types arg_specs = [] + + if (function.is_method or function.is_vfunc) and not function.is_static: + arg_specs.append('self') + for key, value in signature.args: arg_specs.append(f'{key}: {arg_to_annotation(value)}') args = f'({", ".join(arg_specs)})' From 49a99bd42160dd8c4851a7912b6a74c13f26007a Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 23 Dec 2018 08:15:30 +1000 Subject: [PATCH 08/48] Adapt class stubbing to work for structs as well --- pgidocgen/stubs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index fd3354ad..f1d25403 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -225,8 +225,9 @@ def stub_enum(enum) -> str: def stub_class(cls) -> str: stub = StubClass(cls.name) + bases = getattr(cls, 'bases', []) # TODO: These parent classes may require namespace prefix sanitising. - stub.parents = [b.name for b in cls.bases] + stub.parents = [b.name for b in bases] # TODO: We don't handle: # * child_properties: It's not clear how to annotate these @@ -289,7 +290,7 @@ def get_to_write(dir_, namespace, version): for namespace, version in get_to_write(args.target, namespace, version): mod = Repository(namespace, version).parse() module_path = os.path.join(args.target, namespace + ".pyi") - types = mod.structures + mod.unions + types = mod.unions with open(module_path, "w", encoding="utf-8") as h: for cls in types: h.write("""\ @@ -300,6 +301,10 @@ class {}: ... h.write(stub_class(cls)) h.write("\n\n") + for cls in mod.structures: + h.write(stub_class(cls)) + h.write("\n\n") + for cls in mod.flags: h.write(stub_flag(cls)) h.write("\n\n") From afdedb2923187cec9d93192333615fd676271e42 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 23 Dec 2018 08:32:40 +1000 Subject: [PATCH 09/48] Add stubbing of unions (as though they're classes) --- pgidocgen/stubs.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index f1d25403..11496f01 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -290,18 +290,25 @@ def get_to_write(dir_, namespace, version): for namespace, version in get_to_write(args.target, namespace, version): mod = Repository(namespace, version).parse() module_path = os.path.join(args.target, namespace + ".pyi") - types = mod.unions + with open(module_path, "w", encoding="utf-8") as h: - for cls in types: - h.write("""\ -class {}: ... -""".format(cls.name)) for cls in mod.classes: h.write(stub_class(cls)) h.write("\n\n") for cls in mod.structures: + # From a GI point of view, structures are really just classes + # that can't inherit from anything. + h.write(stub_class(cls)) + h.write("\n\n") + + for cls in mod.unions: + # The semantics of a GI-mapped union type don't really map + # nicely to typing structures. It *is* a typing.Union[], but + # you can't add e.g., function signatures to one of those. + # + # In practical terms, treating these as classes seems best. h.write(stub_class(cls)) h.write("\n\n") From f2c4599500245f1cfbee37c18bbfa0fbe461be92 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 24 Dec 2018 06:17:05 +1000 Subject: [PATCH 10/48] Avoid re-stubbing dependency modules multiple times --- pgidocgen/stubs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 11496f01..31775376 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -276,14 +276,14 @@ def get_to_write(dir_, namespace, version): build directory is found, skipping it and all its deps. """ - mods = [] + mods = set() if os.path.exists(os.path.join(dir_, namespace + ".pyi")): return mods - mods.append((namespace, version)) + mods.add((namespace, version)) ns = get_namespace(namespace, version) for dep in ns.dependencies: - mods.extend(get_to_write(dir_, *dep)) + mods |= get_to_write(dir_, *dep) return mods From 948e8eea5cd7b0d5fdad91d4b7f2dd09b424cf28 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 24 Dec 2018 06:17:31 +1000 Subject: [PATCH 11/48] Update the shebang to use env This makes the virtualenv-installed setup work. --- pgidocgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgidocgen.py b/pgidocgen.py index 2229f5de..98f04c75 100755 --- a/pgidocgen.py +++ b/pgidocgen.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Copyright 2013, 2014 Christoph Reiter # # This library is free software; you can redistribute it and/or From 30e9ca787c3ed19a41f524795e6491e879c74044 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 24 Dec 2018 07:00:44 +1000 Subject: [PATCH 12/48] Import order cleanup --- pgidocgen/stubs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 31775376..e7237ee6 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -5,15 +5,15 @@ # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. -import sys +import os import subprocess +import sys import tempfile -import os import typing +from .namespace import get_namespace from .repo import Repository from .util import get_gir_files -from .namespace import get_namespace def add_parser(subparsers): From ad9e737f1308d5dca1356be0f35b07c989431866 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 24 Dec 2018 07:00:56 +1000 Subject: [PATCH 13/48] Handle empty class bodies in stub generation --- pgidocgen/stubs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index e7237ee6..ab811baa 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -78,10 +78,13 @@ def function_lines(self): return lines def __str__(self): + body_lines = self.member_lines + self.function_lines + if not body_lines: + body_lines = [' ...'] + return '\n'.join( [self.class_line] + - self.member_lines + - self.function_lines + + body_lines + [''] ) From b6ff08f7df953b0237d8ac17c9471033aeb27158 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 24 Dec 2018 08:29:42 +1000 Subject: [PATCH 14/48] Handle function arguments with no typing information at all --- pgidocgen/stubs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index ab811baa..7108121f 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -145,7 +145,9 @@ def arg_to_annotation(text): elif p: out.append(p) - if len(out) == 1: + if len(out) == 0: + return "Any" + elif len(out) == 1: return out[0] elif len(out) == 2 and 'None' in out: # This is not strictly necessary, but it's easier to read than the Union From 82c4cc9d57e19df7b4de0cf445274d40d1330d4b Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 24 Dec 2018 08:41:37 +1000 Subject: [PATCH 15/48] Module prefix all typing annotation strings for import ease This makes it a lot easier because we can just add a `typing` import to the top of the `.pyi` file and forget about everything else. --- pgidocgen/stubs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 7108121f..1d8499d8 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -106,11 +106,11 @@ def get_typing_name(type_: typing.Any) -> str: return type_ elif isinstance(type_, list): assert len(type_) == 1 - return "Sequence[%s]" % get_typing_name(type_[0]) + return "typing.Sequence[%s]" % get_typing_name(type_[0]) elif isinstance(type_, dict): assert len(type_) == 1 key, value = type_.popitem() - return "Mapping[%s, %s]" % (get_typing_name(key), get_typing_name(value)) + return "typing.Mapping[%s, %s]" % (get_typing_name(key), get_typing_name(value)) elif type_.__module__ in ("__builtin__", "builtins"): return type_.__name__ else: @@ -135,26 +135,26 @@ def arg_to_annotation(text): out = [] for p in parts: if p.startswith("["): - out.append("Sequence[%s]" % arg_to_annotation(p[1:-1])) + out.append("typing.Sequence[%s]" % arg_to_annotation(p[1:-1])) elif p.startswith("{"): p = p[1:-1] k, v = p.split(":", 1) k = arg_to_annotation(k.strip()) v = arg_to_annotation(v.strip()) - out.append("Mapping[%s, %s]" % (k, v)) + out.append("typing.Mapping[%s, %s]" % (k, v)) elif p: out.append(p) if len(out) == 0: - return "Any" + return "typing.Any" elif len(out) == 1: return out[0] elif len(out) == 2 and 'None' in out: # This is not strictly necessary, but it's easier to read than the Union out.pop(out.index('None')) - return f"Optional[{out[0]}]" + return f"typing.Optional[{out[0]}]" else: - return f"Union[{', '.join(out)}]" + return f"typing.Union[{', '.join(out)}]" def stub_function(function) -> str: @@ -191,7 +191,7 @@ def stub_function(function) -> str: elif len(return_values) == 1: returns = return_values[0] else: - returns = f'Tuple[{", ".join(return_values)}]' + returns = f'typing.Tuple[{", ".join(return_values)}]' return f'{decorator}def {function.name}{args} -> {returns}: ...' From d967c62bc2cf55a3130dd3f5f6fe1d01797062db Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 24 Dec 2018 08:42:28 +1000 Subject: [PATCH 16/48] Handle typing and dependency imports for stub generation --- pgidocgen/stubs.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 1d8499d8..2c35e4d5 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -253,6 +253,17 @@ def format_field(field) -> str: return f"{field.name} = ... # type: {get_typing_name(field.py_type)}" +def format_imports(namespace, version): + ns = get_namespace(namespace, version) + import_lines = [ + "import typing", + "", + *(f"from gi.repository import {dep[0]}" for dep in ns.dependencies), + "" + ] + return "\n".join(import_lines) + + def main(args): if not args.namespace: print("No namespace given") @@ -297,6 +308,9 @@ def get_to_write(dir_, namespace, version): module_path = os.path.join(args.target, namespace + ".pyi") with open(module_path, "w", encoding="utf-8") as h: + # Start by handling all required imports for type annotations + h.write(format_imports(namespace, version)) + h.write("\n\n") for cls in mod.classes: h.write(stub_class(cls)) From 0cedbb1cffeb8bfcba811a9fd524fc74138344e2 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 24 Dec 2018 09:32:26 +1000 Subject: [PATCH 17/48] Handling current-module-prefix stripping for stub generation The problem here is that we need class/struct/etc. references within the module being annotated to not have the module prefix, because otherwise we just get unresolvable typing references. We also need all of these to be forward references (i.e., strings) unless we want to do some kind of dependency ordering, which I don't. I'm not a fan of doing this with a global, but the other solutions are much more complicated. Maybe we can make a stubber state class instead of this, but it's not obvious how much better that would be. --- pgidocgen/stubs.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 2c35e4d5..bbb31c3e 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -16,6 +16,16 @@ from .util import get_gir_files +current_module: str = '-' + + +def strip_current_module(clsname: str) -> str: + # Strip GI module prefix from names in the current module + if clsname.startswith(current_module + "."): + return '"%s"' % clsname[len(current_module + "."):] + return clsname + + def add_parser(subparsers): parser = subparsers.add_parser("stubs", help="Create a typing stubs") parser.add_argument('target', @@ -56,7 +66,8 @@ def add_function(self, function): @property def class_line(self): if self.parents: - parents = "({})".format(', '.join(self.parents)) + parents = "({})".format( + ', '.join(strip_current_module(p) for p in self.parents)) else: parents = "" return "class {}{}:".format(self.classname, parents) @@ -112,12 +123,11 @@ def get_typing_name(type_: typing.Any) -> str: key, value = type_.popitem() return "typing.Mapping[%s, %s]" % (get_typing_name(key), get_typing_name(value)) elif type_.__module__ in ("__builtin__", "builtins"): + return '"%s"' % type_.__name__ + elif type_.__module__ == current_module: + # Strip GI module prefix from current-module types return type_.__name__ else: - # FIXME: We need better module handling here. I think we need to strip - # the module if the type's module is the *current* module being - # annotated, and if not then we need to track imports and probably add - # a "gi.repository." prefix. return "%s.%s" % (type_.__module__, type_.__name__) @@ -143,7 +153,7 @@ def arg_to_annotation(text): v = arg_to_annotation(v.strip()) out.append("typing.Mapping[%s, %s]" % (k, v)) elif p: - out.append(p) + out.append(strip_current_module(p)) if len(out) == 0: return "typing.Any" @@ -303,10 +313,17 @@ def get_to_write(dir_, namespace, version): return mods + # We track the module currently being stubbed for naming reasons. e.g., + # within GObject stubs, referring to "GObject.Object" is incorrect; we + # need the typing reference to be simply "Object". + global current_module + for namespace, version in get_to_write(args.target, namespace, version): mod = Repository(namespace, version).parse() module_path = os.path.join(args.target, namespace + ".pyi") + current_module = namespace + with open(module_path, "w", encoding="utf-8") as h: # Start by handling all required imports for type annotations h.write(format_imports(namespace, version)) From c0d06f4b1b266f8cd56638e5f55737c0d4617afa Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 24 Dec 2018 09:40:29 +1000 Subject: [PATCH 18/48] Check whether fields are valid Python identifiers before annotating --- pgidocgen/stubs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index bbb31c3e..3eac24af 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -209,6 +209,8 @@ def stub_function(function) -> str: def stub_flag(flag) -> str: stub = StubClass(flag.name) for v in flag.values: + if not v.name.isidentifier(): + continue stub.add_member(f"{v.name} = ... # type: {flag.name}") if flag.methods or flag.vfuncs: @@ -223,6 +225,8 @@ def stub_flag(flag) -> str: def stub_enum(enum) -> str: stub = StubClass(enum.name) for v in enum.values: + if not v.name.isidentifier(): + continue stub.add_member(f"{v.name} = ... # type: {enum.name}") for v in enum.methods: @@ -251,6 +255,8 @@ def stub_class(cls) -> str: # * signals: It's not clear how to annotate these for f in cls.fields: + if not f.name.isidentifier(): + continue stub.add_member(format_field(f)) for v in cls.methods + cls.vfuncs: From b5e6591cfcb68282817902cc8a4c3a54cfe67c9a Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 26 Dec 2018 06:48:45 +1000 Subject: [PATCH 19/48] Handle indirect GI module dependencies for imports While there's dependency information in the namespace class, it's not reliable when we need to have all relevant modules in the current namespace for name resolution. The solution here is to constantly update a list of modules that have been referenced by current-module annotations, since that's what mypy, etc. are going to need. This change also means that we can't write import order until the end of the stubbing, so we're now keeping the stubs in a StringIO. --- pgidocgen/stubs.py | 103 +++++++++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 40 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 3eac24af..9a385467 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -5,6 +5,7 @@ # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. +import io import os import subprocess import sys @@ -16,7 +17,9 @@ from .util import get_gir_files +# Initialising the current module to an invalid name current_module: str = '-' +current_module_dependencies = set() def strip_current_module(clsname: str) -> str: @@ -26,6 +29,14 @@ def strip_current_module(clsname: str) -> str: return clsname +def add_dependent_module(module: str): + # FIXME: Find a better way to check this. This currently won't work for GI + # modules that aren't installed in the current venv. + import gi.repository + if hasattr(gi.repository, module): + current_module_dependencies.add(module) + + def add_parser(subparsers): parser = subparsers.add_parser("stubs", help="Create a typing stubs") parser.add_argument('target', @@ -128,6 +139,7 @@ def get_typing_name(type_: typing.Any) -> str: # Strip GI module prefix from current-module types return type_.__name__ else: + add_dependent_module(type_.__module__) return "%s.%s" % (type_.__module__, type_.__name__) @@ -153,7 +165,10 @@ def arg_to_annotation(text): v = arg_to_annotation(v.strip()) out.append("typing.Mapping[%s, %s]" % (k, v)) elif p: - out.append(strip_current_module(p)) + class_str = strip_current_module(p) + if '.' in class_str: + add_dependent_module(class_str.split('.', 1)[0]) + out.append(class_str) if len(out) == 0: return "typing.Any" @@ -271,10 +286,13 @@ def format_field(field) -> str: def format_imports(namespace, version): ns = get_namespace(namespace, version) + for dep in ns.dependencies: + current_module_dependencies.add(dep[0]) + import_lines = [ "import typing", "", - *(f"from gi.repository import {dep[0]}" for dep in ns.dependencies), + *sorted(f"from gi.repository import {dep}" for dep in current_module_dependencies), "" ] return "\n".join(import_lines) @@ -323,50 +341,55 @@ def get_to_write(dir_, namespace, version): # within GObject stubs, referring to "GObject.Object" is incorrect; we # need the typing reference to be simply "Object". global current_module + global current_module_dependencies for namespace, version in get_to_write(args.target, namespace, version): mod = Repository(namespace, version).parse() module_path = os.path.join(args.target, namespace + ".pyi") current_module = namespace + current_module_dependencies = set() - with open(module_path, "w", encoding="utf-8") as h: - # Start by handling all required imports for type annotations - h.write(format_imports(namespace, version)) + h = io.StringIO() + + for cls in mod.classes: + h.write(stub_class(cls)) + h.write("\n\n") + + for cls in mod.structures: + # From a GI point of view, structures are really just classes + # that can't inherit from anything. + h.write(stub_class(cls)) + h.write("\n\n") + + for cls in mod.unions: + # The semantics of a GI-mapped union type don't really map + # nicely to typing structures. It *is* a typing.Union[], but + # you can't add e.g., function signatures to one of those. + # + # In practical terms, treating these as classes seems best. + h.write(stub_class(cls)) + h.write("\n\n") + + for cls in mod.flags: + h.write(stub_flag(cls)) + h.write("\n\n") + + for cls in mod.enums: + h.write(stub_enum(cls)) h.write("\n\n") - for cls in mod.classes: - h.write(stub_class(cls)) - h.write("\n\n") - - for cls in mod.structures: - # From a GI point of view, structures are really just classes - # that can't inherit from anything. - h.write(stub_class(cls)) - h.write("\n\n") - - for cls in mod.unions: - # The semantics of a GI-mapped union type don't really map - # nicely to typing structures. It *is* a typing.Union[], but - # you can't add e.g., function signatures to one of those. - # - # In practical terms, treating these as classes seems best. - h.write(stub_class(cls)) - h.write("\n\n") - - for cls in mod.flags: - h.write(stub_flag(cls)) - h.write("\n\n") - - for cls in mod.enums: - h.write(stub_enum(cls)) - h.write("\n\n") - - for func in mod.functions: - h.write(stub_function(func)) - # Extra \n because the signature lacks one. - h.write("\n\n\n") - - for const in mod.constants: - h.write(format_field(const)) - h.write("\n") + for func in mod.functions: + h.write(stub_function(func)) + # Extra \n because the signature lacks one. + h.write("\n\n\n") + + for const in mod.constants: + h.write(format_field(const)) + h.write("\n") + + with open(module_path, "w", encoding="utf-8") as f: + # Start by handling all required imports for type annotations + f.write(format_imports(namespace, version)) + f.write("\n\n") + f.write(h.getvalue()) From 61b98c6058ba8360de3e7b878208a7622db2756d Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 26 Dec 2018 06:52:57 +1000 Subject: [PATCH 20/48] Generate stubs for pyclasses This doesn't cover everything, but now at least we get GType, which was causing a lot of issues. --- pgidocgen/stubs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 9a385467..edad6a6c 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -268,8 +268,10 @@ def stub_class(cls) -> str: # * gtype_struct: I'm not sure what we'd use this for. # * properties: It's not clear how to annotate these # * signals: It's not clear how to annotate these + # * pyprops: These have no type information, and I'm not certain + # what they cover, etc. - for f in cls.fields: + for f in getattr(cls, 'fields', []): if not f.name.isidentifier(): continue stub.add_member(format_field(f)) @@ -352,6 +354,10 @@ def get_to_write(dir_, namespace, version): h = io.StringIO() + for cls in mod.pyclasses: + h.write(stub_class(cls)) + h.write("\n\n") + for cls in mod.classes: h.write(stub_class(cls)) h.write("\n\n") From 1c5ea23878867a1aa5a07c53878d8398c3fc2330 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 26 Dec 2018 06:53:39 +1000 Subject: [PATCH 21/48] Fix incorrect class name quoting --- pgidocgen/stubs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index edad6a6c..0771a6b5 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -134,10 +134,10 @@ def get_typing_name(type_: typing.Any) -> str: key, value = type_.popitem() return "typing.Mapping[%s, %s]" % (get_typing_name(key), get_typing_name(value)) elif type_.__module__ in ("__builtin__", "builtins"): - return '"%s"' % type_.__name__ + return type_.__name__ elif type_.__module__ == current_module: # Strip GI module prefix from current-module types - return type_.__name__ + return '"%s"' % type_.__name__ else: add_dependent_module(type_.__module__) return "%s.%s" % (type_.__module__, type_.__name__) From f365caf608b7b15e9eb08f17d23d4a52e83d4854 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 26 Dec 2018 07:16:11 +1000 Subject: [PATCH 22/48] Break out function return value handling for reuse --- pgidocgen/stubs.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 0771a6b5..95421807 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -182,6 +182,25 @@ def arg_to_annotation(text): return f"typing.Union[{', '.join(out)}]" +def format_function_returns(signature_result) -> str: + # Format return values + return_values = [] + for r in signature_result: + # We have either a (name, return type) pair, or just the return type. + type_ = r[1] if len(r) > 1 else r[0] + return_values.append(arg_to_annotation(type_)) + + # Additional handling for structuring return values + if len(return_values) == 0: + returns = 'None' + elif len(return_values) == 1: + returns = return_values[0] + else: + returns = f'typing.Tuple[{", ".join(return_values)}]' + + return returns + + def stub_function(function) -> str: # We require the full signature details for argument types, and fallback # to the simplest possible function signature if it's not available. @@ -203,20 +222,7 @@ def stub_function(function) -> str: arg_specs.append(f'{key}: {arg_to_annotation(value)}') args = f'({", ".join(arg_specs)})' - # Format return values - return_values = [] - for r in signature.res: - # We have either a (name, return type) pair, or just the return type. - type_ = r[1] if len(r) > 1 else r[0] - return_values.append(arg_to_annotation(type_)) - - # Additional handling for structuring return values - if len(return_values) == 0: - returns = 'None' - elif len(return_values) == 1: - returns = return_values[0] - else: - returns = f'typing.Tuple[{", ".join(return_values)}]' + returns = format_function_returns(signature.res) return f'{decorator}def {function.name}{args} -> {returns}: ...' From c1896b19eb87b37955f58ef431cc95c6bd3c2839 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 26 Dec 2018 07:37:35 +1000 Subject: [PATCH 23/48] Generate stubs for callback signatures --- pgidocgen/stubs.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 95421807..8fecf9f5 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -292,6 +292,13 @@ def format_field(field) -> str: return f"{field.name} = ... # type: {get_typing_name(field.py_type)}" +def format_callback(fn) -> str: + # We're formatting a callback signature here, not an actual function. + args = ", ".join(arg_to_annotation(v) for k, v in fn.full_signature.args) + returns = format_function_returns(fn.full_signature.res) + return f"{fn.name} = typing.Callable[[{args}], {returns}]" + + def format_imports(namespace, version): ns = get_namespace(namespace, version) for dep in ns.dependencies: @@ -391,6 +398,13 @@ def get_to_write(dir_, namespace, version): h.write(stub_enum(cls)) h.write("\n\n") + for fn in mod.callbacks: + h.write(format_callback(fn)) + h.write("\n") + + if mod.callbacks: + h.write("\n\n") + for func in mod.functions: h.write(stub_function(func)) # Extra \n because the signature lacks one. From 656afda18690368cce8b8c2be2e485f2facc4b90 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 26 Dec 2018 08:38:42 +1000 Subject: [PATCH 24/48] Ignore typing errors for constructors Firstly, it's worth noting that this only ignores the typing error. Things that call these APIs will still be checked! The problem is that the checking may be incorrect because of the API itself. There's a long discussion about this and related Liskov issues at: https://github.com/python/mypy/issues/1237 The short version is that this API is wrong-ish, but there's no mechanism within mypy to correctly annotate what's going on here. There *is* some of this around, because mypy treats __init__ and __new__ differently itself, but we can't apply that treatment to our constructors. This would be better if we actually knew which methods were constructors, instead of the name-based guessing here... but that's way too much complexity for me right now. --- pgidocgen/stubs.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 8fecf9f5..bc867a23 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -71,7 +71,9 @@ def __init__(self, classname): def add_member(self, member): self.members.append(member) - def add_function(self, function): + def add_function(self, function, *, ignore_type_error=False): + if ignore_type_error: + function += " # type: ignore" self.functions.append(function) @property @@ -283,7 +285,17 @@ def stub_class(cls) -> str: stub.add_member(format_field(f)) for v in cls.methods + cls.vfuncs: - stub.add_function(stub_function(v)) + # GObject-based constructors often violate Liskov substitution, + # leading to typing errors such as: + # Signature of "new" incompatible with supertype "Object" + # While we're waiting for a more general solution (see + # https://github.com/python/mypy/issues/1237) we'll just ignore + # the typing errors. + + # TODO: Extract constructor information from GIR and add it to + # docobj.Function to use here. + ignore = v.name == "new" + stub.add_function(stub_function(v), ignore_type_error=ignore) return str(stub) From 140dcf546844d297bbbc60930c756b9d6c242bc8 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Thu, 27 Dec 2018 07:53:13 +1000 Subject: [PATCH 25/48] Consolidate all the class-like stubbing --- pgidocgen/stubs.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index bc867a23..e2ceefaf 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -377,28 +377,23 @@ def get_to_write(dir_, namespace, version): current_module = namespace current_module_dependencies = set() - h = io.StringIO() - - for cls in mod.pyclasses: - h.write(stub_class(cls)) - h.write("\n\n") - - for cls in mod.classes: - h.write(stub_class(cls)) - h.write("\n\n") - - for cls in mod.structures: + class_likes = ( + mod.pyclasses + + mod.classes + # From a GI point of view, structures are really just classes # that can't inherit from anything. - h.write(stub_class(cls)) - h.write("\n\n") - - for cls in mod.unions: + mod.structures + # The semantics of a GI-mapped union type don't really map # nicely to typing structures. It *is* a typing.Union[], but # you can't add e.g., function signatures to one of those. # # In practical terms, treating these as classes seems best. + mod.unions + ) + + h = io.StringIO() + + for cls in class_likes: h.write(stub_class(cls)) h.write("\n\n") From d915b899c34e074f1033f1998228016709337c2a Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Thu, 27 Dec 2018 08:05:29 +1000 Subject: [PATCH 26/48] Reuse enum stub generation code for flags These are functionally identical anyway, and are *almost* the same as the generic class stubber. --- pgidocgen/stubs.py | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index e2ceefaf..cb44aa1c 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -229,33 +229,17 @@ def stub_function(function) -> str: return f'{decorator}def {function.name}{args} -> {returns}: ...' -def stub_flag(flag) -> str: - stub = StubClass(flag.name) - for v in flag.values: - if not v.name.isidentifier(): - continue - stub.add_member(f"{v.name} = ... # type: {flag.name}") - - if flag.methods or flag.vfuncs: - # This is unsupported simply because I can't find any GIR that - # has methods or vfuncs on its flag types. - raise NotImplementedError( - "Flag support doesn't annotate methods or vfuncs") - - return str(stub) - - -def stub_enum(enum) -> str: - stub = StubClass(enum.name) - for v in enum.values: +def stub_enum(cls) -> str: + stub = StubClass(cls.name) + for v in cls.values: if not v.name.isidentifier(): continue - stub.add_member(f"{v.name} = ... # type: {enum.name}") + stub.add_member(f"{v.name} = ... # type: {cls.name}") - for v in enum.methods: + for v in cls.methods: stub.add_function(stub_function(v)) - if enum.vfuncs: + if cls.vfuncs: # This is unsupported simply because I can't find any GIR that # has vfuncs on its enum types. raise NotImplementedError( @@ -397,11 +381,7 @@ def get_to_write(dir_, namespace, version): h.write(stub_class(cls)) h.write("\n\n") - for cls in mod.flags: - h.write(stub_flag(cls)) - h.write("\n\n") - - for cls in mod.enums: + for cls in mod.flags + mod.enums: h.write(stub_enum(cls)) h.write("\n\n") From eb086d99a536961740df38b564726fc1e1d95783 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Thu, 27 Dec 2018 08:39:35 +1000 Subject: [PATCH 27/48] Unify GEnum and GFlag code under the generic class stubber --- pgidocgen/stubs.py | 43 ++++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index cb44aa1c..2db5c545 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -229,31 +229,13 @@ def stub_function(function) -> str: return f'{decorator}def {function.name}{args} -> {returns}: ...' -def stub_enum(cls) -> str: - stub = StubClass(cls.name) - for v in cls.values: - if not v.name.isidentifier(): - continue - stub.add_member(f"{v.name} = ... # type: {cls.name}") - - for v in cls.methods: - stub.add_function(stub_function(v)) - - if cls.vfuncs: - # This is unsupported simply because I can't find any GIR that - # has vfuncs on its enum types. - raise NotImplementedError( - "Enum support doesn't annotate vfuncs") - - return str(stub) - - def stub_class(cls) -> str: stub = StubClass(cls.name) - bases = getattr(cls, 'bases', []) - # TODO: These parent classes may require namespace prefix sanitising. - stub.parents = [b.name for b in bases] + if hasattr(cls, 'bases'): + stub.parents = [b.name for b in cls.bases] + elif hasattr(cls, 'base'): + stub.parents = [cls.base] if cls.base else [] # TODO: We don't handle: # * child_properties: It's not clear how to annotate these @@ -268,6 +250,13 @@ def stub_class(cls) -> str: continue stub.add_member(format_field(f)) + # The `values` attribute is available on enums and flags, and its + # type will always be the current class. + for v in getattr(cls, 'values', []): + if not v.name.isidentifier(): + continue + stub.add_member(f"{v.name} = ... # type: {cls.name}") + for v in cls.methods + cls.vfuncs: # GObject-based constructors often violate Liskov substitution, # leading to typing errors such as: @@ -372,7 +361,11 @@ def get_to_write(dir_, namespace, version): # you can't add e.g., function signatures to one of those. # # In practical terms, treating these as classes seems best. - mod.unions + mod.unions + + # `GFlag`s and `GEnum`s are slightly different to classes, but + # easily covered by the same code. + mod.flags + + mod.enums ) h = io.StringIO() @@ -381,10 +374,6 @@ def get_to_write(dir_, namespace, version): h.write(stub_class(cls)) h.write("\n\n") - for cls in mod.flags + mod.enums: - h.write(stub_enum(cls)) - h.write("\n\n") - for fn in mod.callbacks: h.write(format_callback(fn)) h.write("\n") From 51e5bd33b7229b0479dc699ac0554b41354a9a5b Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Thu, 27 Dec 2018 09:16:02 +1000 Subject: [PATCH 28/48] Refactor function stub formatters to be more consistent --- pgidocgen/stubs.py | 67 +++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 2db5c545..9ba01c98 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -184,10 +184,26 @@ def arg_to_annotation(text): return f"typing.Union[{', '.join(out)}]" -def format_function_returns(signature_result) -> str: - # Format return values +def format_function_args(function, *, include_arg_names=True) -> str: + """Format function arguments as a type annotation fragment""" + + arg_specs = [] + + if (function.is_method or function.is_vfunc) and not function.is_static: + arg_specs.append('self') + + for key, value in function.full_signature.args: + spec = "{key}: {type}" if include_arg_names else "{type}" + arg_specs.append(spec.format(key=key, type=arg_to_annotation(value))) + + return ", ".join(arg_specs) + + +def format_function_returns(function) -> str: + """Format function return values as a type annotation fragment""" + return_values = [] - for r in signature_result: + for r in function.full_signature.res: # We have either a (name, return type) pair, or just the return type. type_ = r[1] if len(r) > 1 else r[0] return_values.append(arg_to_annotation(type_)) @@ -203,30 +219,19 @@ def format_function_returns(signature_result) -> str: return returns -def stub_function(function) -> str: +def format_function(function) -> str: # We require the full signature details for argument types, and fallback # to the simplest possible function signature if it's not available. - signature = getattr(function, 'full_signature', None) - if not signature: - print(f"Missing full signature for {function}; falling back") - return f"def {function.name}(*args, **kwargs): ..." - - # Decorator handling - decorator = "@staticmethod\n" if function.is_static else "" - - # Format argument types - arg_specs = [] - - if (function.is_method or function.is_vfunc) and not function.is_static: - arg_specs.append('self') - - for key, value in signature.args: - arg_specs.append(f'{key}: {arg_to_annotation(value)}') - args = f'({", ".join(arg_specs)})' - - returns = format_function_returns(signature.res) + if not getattr(function, 'full_signature', None): + print("Missing full signature for {}".format(function)) + return "def {}(*args, **kwargs): ...".format(function.name) - return f'{decorator}def {function.name}{args} -> {returns}: ...' + return '{decorator}def {name}({args}) -> {returns}: ...'.format( + decorator="@staticmethod\n" if function.is_static else "", + name=function.name, + args=format_function_args(function), + returns=format_function_returns(function), + ) def stub_class(cls) -> str: @@ -268,7 +273,7 @@ def stub_class(cls) -> str: # TODO: Extract constructor information from GIR and add it to # docobj.Function to use here. ignore = v.name == "new" - stub.add_function(stub_function(v), ignore_type_error=ignore) + stub.add_function(format_function(v), ignore_type_error=ignore) return str(stub) @@ -277,11 +282,13 @@ def format_field(field) -> str: return f"{field.name} = ... # type: {get_typing_name(field.py_type)}" -def format_callback(fn) -> str: +def format_callback(function) -> str: # We're formatting a callback signature here, not an actual function. - args = ", ".join(arg_to_annotation(v) for k, v in fn.full_signature.args) - returns = format_function_returns(fn.full_signature.res) - return f"{fn.name} = typing.Callable[[{args}], {returns}]" + return "{name} = typing.Callable[[{args}], {returns}]".format( + name=function.name, + args=format_function_args(function, include_arg_names=False), + returns=format_function_returns(function), + ) def format_imports(namespace, version): @@ -382,7 +389,7 @@ def get_to_write(dir_, namespace, version): h.write("\n\n") for func in mod.functions: - h.write(stub_function(func)) + h.write(format_function(func)) # Extra \n because the signature lacks one. h.write("\n\n\n") From 05fb13778c28f9185864ac208460280c96ad14e6 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Thu, 27 Dec 2018 09:16:50 +1000 Subject: [PATCH 29/48] Simplify the class-stub-container behaviour --- pgidocgen/stubs.py | 41 ++++++++++++----------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 9ba01c98..a9253181 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -62,6 +62,9 @@ def _main_many(target, namespaces): class StubClass: + + indent = " " * 4 + def __init__(self, classname): self.classname = classname self.parents = [] @@ -76,39 +79,19 @@ def add_function(self, function, *, ignore_type_error=False): function += " # type: ignore" self.functions.append(function) - @property - def class_line(self): - if self.parents: - parents = "({})".format( - ', '.join(strip_current_module(p) for p in self.parents)) - else: - parents = "" - return "class {}{}:".format(self.classname, parents) - - @property - def member_lines(self): - return [ - " {}".format(member) - for member in sorted(self.members) - ] - - @property - def function_lines(self): - lines = [] - for function in self.functions: - lines.append('') - for line in function.splitlines(): - lines.append(f' {line}') - return lines - def __str__(self): - body_lines = self.member_lines + self.function_lines + parents = ', '.join(strip_current_module(p) for p in self.parents) + class_line = "class {}({}):".format(self.classname, parents) + + body_lines = sorted(self.members) + for function in self.functions: + body_lines.extend([''] + function.splitlines()) if not body_lines: - body_lines = [' ...'] + body_lines = ['...'] return '\n'.join( - [self.class_line] + - body_lines + + [class_line] + + [(self.indent + line).rstrip() for line in body_lines] + [''] ) From afa7c5d367922ca5c31520eb5c0a1ac8060c6fa1 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 31 Dec 2018 07:09:10 +1000 Subject: [PATCH 30/48] Handle explicit NoneType types differently These are special cases for some base types lacking introspection information. Since nothing else seems to have this issue other than low-level GObject types, we'll treat this as a weird case. This gets rid of mypy errors like: GObject.pyi:1416: error: Name 'NoneType' is not defined in module-level definitions for e.g., GBoxed, GPointer, etc. --- pgidocgen/stubs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index a9253181..5ddbccde 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -109,6 +109,10 @@ def get_typing_name(type_: typing.Any) -> str: if type_ is None: return "" + elif type_ is type(None): + # As a weird corner-case, some non-introspectable base types + # actually give NoneType here. We treat them as very special. + return "typing.Any" elif isinstance(type_, str): return type_ elif isinstance(type_, list): From 0cdb17eee8a5659d6fffed3c163de03d6b0afae2 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Mon, 31 Dec 2018 13:53:59 +1000 Subject: [PATCH 31/48] Use the builtins module to explicitly namespace built-in types This isn't a generic solution for shadowing... that would require actual parsing and tricks. However, in practice the risky shadowing here happens when e.g., a class implements def int(self) -> int: ... and everything breaks. This works around that in a somewhat-inelegant but basically-correct kind of way. --- pgidocgen/stubs.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 5ddbccde..32c737e1 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -5,6 +5,7 @@ # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. +import builtins import io import os import subprocess @@ -21,6 +22,13 @@ current_module: str = '-' current_module_dependencies = set() +# Set of type names that can be unwittingly shadowed and will cause +# trouble with the type checker. +shadowed_builtins = { + builtin for builtin in builtins.__dict__ + if isinstance(builtins.__dict__[builtin], type) +} + def strip_current_module(clsname: str) -> str: # Strip GI module prefix from names in the current module @@ -123,7 +131,7 @@ def get_typing_name(type_: typing.Any) -> str: key, value = type_.popitem() return "typing.Mapping[%s, %s]" % (get_typing_name(key), get_typing_name(value)) elif type_.__module__ in ("__builtin__", "builtins"): - return type_.__name__ + return "builtins.%s" % type_.__name__ elif type_.__module__ == current_module: # Strip GI module prefix from current-module types return '"%s"' % type_.__name__ @@ -153,6 +161,8 @@ def arg_to_annotation(text): k = arg_to_annotation(k.strip()) v = arg_to_annotation(v.strip()) out.append("typing.Mapping[%s, %s]" % (k, v)) + elif p in shadowed_builtins: + out.append("builtins.%s" % p) elif p: class_str = strip_current_module(p) if '.' in class_str: @@ -284,6 +294,7 @@ def format_imports(namespace, version): current_module_dependencies.add(dep[0]) import_lines = [ + "import builtins", "import typing", "", *sorted(f"from gi.repository import {dep}" for dep in current_module_dependencies), From 5342f7577da20a89beffc4c3990ef95a547c1de2 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sat, 12 Jan 2019 08:54:46 +1000 Subject: [PATCH 32/48] Remove old-style variable annotations and string-ified class names According to mypy, these aren't necessary in stub files. --- pgidocgen/stubs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 32c737e1..5cdd532c 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -33,7 +33,7 @@ def strip_current_module(clsname: str) -> str: # Strip GI module prefix from names in the current module if clsname.startswith(current_module + "."): - return '"%s"' % clsname[len(current_module + "."):] + return clsname[len(current_module + "."):] return clsname @@ -134,7 +134,7 @@ def get_typing_name(type_: typing.Any) -> str: return "builtins.%s" % type_.__name__ elif type_.__module__ == current_module: # Strip GI module prefix from current-module types - return '"%s"' % type_.__name__ + return type_.__name__ else: add_dependent_module(type_.__module__) return "%s.%s" % (type_.__module__, type_.__name__) @@ -276,7 +276,7 @@ def stub_class(cls) -> str: def format_field(field) -> str: - return f"{field.name} = ... # type: {get_typing_name(field.py_type)}" + return f"{field.name}: {get_typing_name(field.py_type)}" def format_callback(function) -> str: From 462a2a815d95ed672b449d4410a67c2e7d59286d Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sat, 12 Jan 2019 08:55:24 +1000 Subject: [PATCH 33/48] Topologically sort GObject classes by MRO to avoid mypy errors See python/mypy#6119 for details. --- pgidocgen/stubs.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 5cdd532c..e108148a 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -104,6 +104,46 @@ def __str__(self): ) +def topological_sort(class_nodes): + """ + Topologically sort a list of class nodes according to inheritance + + We use this as a workaround for typing stub order dependencies + (see python/mypy#6119). + + Code adapted from https://stackoverflow.com/a/43702281/2963 + """ + # Map class name to actual class node being sorted + name_node_map = {node.fullname: node for node in class_nodes} + # Map class name to parent classes + name_parents_map = {} + # Map class name to child classes + name_children_map = {name: [] for name in name_node_map} + + for name, node in name_node_map.items(): + in_module_bases = [b.name for b in node.bases if b.name in name_node_map] + name_parents_map[name] = in_module_bases + for base_name in in_module_bases: + name_children_map[base_name].append(name) + + # Establish bases + sorted_names = [n for n, preds in name_parents_map.items() if not preds] + + for name in sorted_names: + for child in name_children_map[name]: + name_parents_map[child].remove(name) + if not name_parents_map[child]: + # Mutating list that we're iterating over, so that this + # class gets removed from subsequent pending parents + # lists. + sorted_names.append(child) + + if len(sorted_names) < len(name_node_map): + raise RuntimeError("Couldn't establish a topological ordering") + + return [name_node_map[name] for name in sorted_names] + + def get_typing_name(type_: typing.Any) -> str: """Gives a name for a type that is suitable for a typing annotation. @@ -357,7 +397,7 @@ def get_to_write(dir_, namespace, version): class_likes = ( mod.pyclasses + - mod.classes + + topological_sort(mod.classes) + # From a GI point of view, structures are really just classes # that can't inherit from anything. mod.structures + From cd18e22fa2fe8a405ae6aff814a494ebcc9dc56b Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 13 Jan 2019 12:42:39 +1000 Subject: [PATCH 34/48] Specifically handle an odd `_Value__data__union` annotation in GObject --- pgidocgen/stubs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index e108148a..f134ba35 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -290,6 +290,11 @@ def stub_class(cls) -> str: for f in getattr(cls, 'fields', []): if not f.name.isidentifier(): continue + + # Special case handling for weird annotations + if cls.fullname == 'GObject.Value' and f.name == 'data': + continue + stub.add_member(format_field(f)) # The `values` attribute is available on enums and flags, and its From 8246f8657e93921f08b329c23d7b3317d876de95 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 13 Jan 2019 12:43:29 +1000 Subject: [PATCH 35/48] Break out class-like access for easier testing --- pgidocgen/stubs.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index f134ba35..70195e00 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -348,6 +348,26 @@ def format_imports(namespace, version): return "\n".join(import_lines) +def get_module_classlikes(module): + return ( + module.pyclasses + + topological_sort(module.classes) + + # From a GI point of view, structures are really just classes + # that can't inherit from anything. + module.structures + + # The semantics of a GI-mapped union type don't really map + # nicely to typing structures. It *is* a typing.Union[], but + # you can't add e.g., function signatures to one of those. + # + # In practical terms, treating these as classes seems best. + module.unions + + # `GFlag`s and `GEnum`s are slightly different to classes, but + # easily covered by the same code. + module.flags + + module.enums + ) + + def main(args): if not args.namespace: print("No namespace given") @@ -400,27 +420,9 @@ def get_to_write(dir_, namespace, version): current_module = namespace current_module_dependencies = set() - class_likes = ( - mod.pyclasses + - topological_sort(mod.classes) + - # From a GI point of view, structures are really just classes - # that can't inherit from anything. - mod.structures + - # The semantics of a GI-mapped union type don't really map - # nicely to typing structures. It *is* a typing.Union[], but - # you can't add e.g., function signatures to one of those. - # - # In practical terms, treating these as classes seems best. - mod.unions + - # `GFlag`s and `GEnum`s are slightly different to classes, but - # easily covered by the same code. - mod.flags + - mod.enums - ) - h = io.StringIO() - for cls in class_likes: + for cls in get_module_classlikes(mod): h.write(stub_class(cls)) h.write("\n\n") From 8ae025e29ed719e8272ae6a82d2bc0e7701b4b1d Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 13 Jan 2019 12:43:54 +1000 Subject: [PATCH 36/48] Annotate GObject.Enum and GObject.Flags as also being integers They support common integer methods, so this is definitely reasonable. --- pgidocgen/stubs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 70195e00..e4c48033 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -13,6 +13,7 @@ import tempfile import typing +from .docobj import Flags from .namespace import get_namespace from .repo import Repository from .util import get_gir_files @@ -278,6 +279,9 @@ def stub_class(cls) -> str: stub.parents = [b.name for b in cls.bases] elif hasattr(cls, 'base'): stub.parents = [cls.base] if cls.base else [] + # GObject.Flags and GObject.Enum types support Python integer methods + if isinstance(cls, Flags): + stub.parents.append('builtins.int') # TODO: We don't handle: # * child_properties: It's not clear how to annotate these From 17042a5efa2f5b16502d84cc0059afbd52463fb7 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 13 Jan 2019 13:25:13 +1000 Subject: [PATCH 37/48] noqa a flake8 error flake8 is trying to do the right thing here, but in this instance we're dealing with getting an actual type, so the "use isinstance instead" error really isn't helpful or correct. --- pgidocgen/stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index e4c48033..a598f4ca 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -158,7 +158,7 @@ def get_typing_name(type_: typing.Any) -> str: if type_ is None: return "" - elif type_ is type(None): + elif type_ is type(None): # noqa: E721 # As a weird corner-case, some non-introspectable base types # actually give NoneType here. We treat them as very special. return "typing.Any" From 364110712b3a95355946960354ce0f7491bbc4b4 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 11 Aug 2019 08:08:29 +1000 Subject: [PATCH 38/48] Fix bad function call in debug path --- pgidocgen/debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgidocgen/debug.py b/pgidocgen/debug.py index 6145a28b..ca13654a 100644 --- a/pgidocgen/debug.py +++ b/pgidocgen/debug.py @@ -272,7 +272,7 @@ def get_abs_library_path(library_name): if "LD_LIBRARY_PATH" in os.environ: path = os.path.join(os.environ["LD_LIBRARY_PATH"], library_name) - path = os.path.abs(path) + path = os.path.abspath(path) if not os.path.exists(path): raise LookupError(library_name) return path From 7db61ad22cce4bc536d40feaf104ef4d1aff7fa0 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 11 Aug 2019 09:08:26 +1000 Subject: [PATCH 39/48] main: Don't hard-require apt libraries These aren't in the module requirements, and aren't installed by default on non-Debian-based systems. --- pgidocgen/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pgidocgen/main.py b/pgidocgen/main.py index 1542c77b..7d178901 100644 --- a/pgidocgen/main.py +++ b/pgidocgen/main.py @@ -9,7 +9,12 @@ import argparse -from . import create, build, stubs, create_debian, update +from . import create, build, stubs, update + +try: + from . import create_debian +except ImportError: + create_debian = None def main(argv): @@ -19,7 +24,8 @@ def main(argv): create.add_parser(subparser) build.add_parser(subparser) stubs.add_parser(subparser) - create_debian.add_parser(subparser) + if create_debian: + create_debian.add_parser(subparser) update.add_parser(subparser) args = parser.parse_args(argv[1:]) From 14855eb686251e649868542a84ed4fd09e17d141 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sun, 11 Aug 2019 09:10:21 +1000 Subject: [PATCH 40/48] Ignore spurious nodes in private member check The check for whether a record is private assumes a fixed child node structure that isn't the case in some environments. This patch tries to keep the spirit of the check by filtering out all known-unimportant child nodes and simply checking whether there's anything left. The expected unimportant nodes are: * empty text nodes containing formatting white-space * source-position elements that map the member to the original source file neither of which indicate that the member actually needs to be annotated. --- pgidocgen/namespace.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/pgidocgen/namespace.py b/pgidocgen/namespace.py index 2dbfa709..62209385 100644 --- a/pgidocgen/namespace.py +++ b/pgidocgen/namespace.py @@ -557,6 +557,20 @@ def is_available(mod, name): return types, type_structs, shadow_map, instance_params +def _is_empty_text_node(child) -> bool: + return ( + child.nodeType == child.TEXT_NODE and + not child.data.strip() + ) + + +def _is_source_position_node(child) -> bool: + return ( + child.nodeType == child.ELEMENT_NODE and + child.tagName == 'source-position' + ) + + def _parse_private(dom, namespace): private = set() @@ -566,9 +580,13 @@ def _parse_private(dom, namespace): disguised = bool(int(record.getAttribute("disguised") or "0")) is_gtype_struct = bool(record.getAttribute("glib:is-gtype-struct-for")) if disguised and not is_gtype_struct: - children = record.childNodes - if len(children) == 1 and \ - children[0].nodeType == children[0].TEXT_NODE: + + meaningful_children = [ + c for c in record.childNodes + if not _is_empty_text_node(c) and not _is_source_position_node(c) + ] + + if not meaningful_children: name = namespace + "." + record.getAttribute("name") private.add(name) From dde6e34113230128987b38f9674d1d3acb55f540 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 14 Aug 2019 08:11:21 +1000 Subject: [PATCH 41/48] Add support for ignoring type errors in class inheritance This is needed for e.g., classes that inherit from both Atk.Object and Atk.Action, which both define methods like `get_description` with very different signatures. --- pgidocgen/stubs.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index af57b80e..43665013 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -44,6 +44,8 @@ def add_dependent_module(module: str): import gi.repository if hasattr(gi.repository, module): current_module_dependencies.add(module) + + def add_parser(subparsers): parser = subparsers.add_parser("stubs", help="Create a typing stubs") parser.add_argument('target', @@ -68,6 +70,7 @@ def __init__(self, classname): self.parents = [] self.members = [] self.functions = [] + self.ignore_type_errors = False def add_member(self, member): self.members.append(member) @@ -77,9 +80,15 @@ def add_function(self, function, *, ignore_type_error=False): function += " # type: ignore" self.functions.append(function) - def __str__(self): + @property + def class_line(self): parents = ', '.join(strip_current_module(p) for p in self.parents) - class_line = "class {}({}):".format(self.classname, parents) + line = 'class {}({}):'.format(self.classname, parents) + if self.ignore_type_errors: + line += ' # type: ignore' + return line + + def __str__(self): body_lines = sorted(self.members) for function in self.functions: @@ -88,7 +97,7 @@ def __str__(self): body_lines = ['...'] return '\n'.join( - [class_line] + + [self.class_line] + [(self.indent + line).rstrip() for line in body_lines] + [''] ) From 9ef2d3fb8cef52861d5e72f8f107c0bff3508318 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 14 Aug 2019 08:12:44 +1000 Subject: [PATCH 42/48] Allow kwargs for GObject constructors GObject-derived widgets always accept keyword args for setting GObject properties. Ideally we'd check that the kwargs matched the object's properties, but for now we'll just accept all kwargs to avoid the type errors. --- pgidocgen/stubs.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 43665013..c0125cd2 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -220,7 +220,8 @@ def arg_to_annotation(text): return f"typing.Union[{', '.join(out)}]" -def format_function_args(function, *, include_arg_names=True) -> str: +def format_function_args( + function, *, accepts_kwargs: bool = False, include_arg_names=True) -> str: """Format function arguments as a type annotation fragment""" arg_specs = [] @@ -232,6 +233,9 @@ def format_function_args(function, *, include_arg_names=True) -> str: spec = "{key}: {type}" if include_arg_names else "{type}" arg_specs.append(spec.format(key=key, type=arg_to_annotation(value))) + if accepts_kwargs: + arg_specs.append('**kwargs') + return ", ".join(arg_specs) @@ -255,7 +259,7 @@ def format_function_returns(function) -> str: return returns -def format_function(function) -> str: +def format_function(function, *, accepts_kwargs=False) -> str: # We require the full signature details for argument types, and fallback # to the simplest possible function signature if it's not available. if not getattr(function, 'full_signature', None): @@ -265,7 +269,7 @@ def format_function(function) -> str: return '{decorator}def {name}({args}) -> {returns}: ...'.format( decorator="@staticmethod\n" if function.is_static else "", name=function.name, - args=format_function_args(function), + args=format_function_args(function, accepts_kwargs=accepts_kwargs), returns=format_function_returns(function), ) @@ -316,8 +320,18 @@ def stub_class(cls) -> str: # TODO: Extract constructor information from GIR and add it to # docobj.Function to use here. - ignore = v.name == "new" - stub.add_function(format_function(v), ignore_type_error=ignore) + is_constructor = v.name == "new" + + # All GObject-inheriting classes should accept keyword + # arguments to set GObject properties in their default + # constructor. + inherits_gobject = any(p == 'GObject.Object' for p in stub.parents) + accepts_kwargs = is_constructor and inherits_gobject + + stub.add_function( + format_function(v, accepts_kwargs=accepts_kwargs), + ignore_type_error=is_constructor + ) return str(stub) From cd92bba4f15a7b3ad664e797d5cc388613eddb78 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 14 Aug 2019 08:54:06 +1000 Subject: [PATCH 43/48] stubs: Add a method-specific ignore list for bad/weird/unused methods --- pgidocgen/stubs.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index c0125cd2..e3ca3a5e 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -19,6 +19,15 @@ from .util import get_gir_files +#: Methods that are left out of the type stub *entirely*. In general, +#: these are methods that have Liskov violations and are not in common +#: use or are deprecated. +OMITTED_METHODS = { + ('Gio.Initable', 'newv'), + ('GObject.Object', 'newv'), +} + + # Initialising the current module to an invalid name current_module: str = '-' current_module_dependencies = set() @@ -311,6 +320,10 @@ def stub_class(cls) -> str: stub.add_member(f"{v.name} = ... # type: {cls.name}") for v in cls.methods + cls.vfuncs: + # Check our ignored methods list before doing anything else + if (cls.fullname, v.name) in OMITTED_METHODS: + continue + # GObject-based constructors often violate Liskov substitution, # leading to typing errors such as: # Signature of "new" incompatible with supertype "Object" From a6dffe86bb1d08ea3417a77ee26a8fde22b3d24a Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 14 Aug 2019 08:54:38 +1000 Subject: [PATCH 44/48] stubs: Add a way to provide module-level aliases This is currently only for GObject, but there's several cases where backwards-compatible aliases have been provided. --- pgidocgen/stubs.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index e3ca3a5e..61e97e66 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -19,6 +19,13 @@ from .util import get_gir_files +#: Mapping from namespace to a list of tuples of (class name, alias) +MODULE_ALIASES = { + 'GObject': [ + ('Object', 'GObject'), + ], +} + #: Methods that are left out of the type stub *entirely*. In general, #: these are methods that have Liskov violations and are not in common #: use or are deprecated. @@ -474,6 +481,12 @@ def get_to_write(dir_, namespace, version): h.write(format_field(const)) h.write("\n") + aliases = MODULE_ALIASES.get(namespace, []) + if aliases: + h.write("\n\n") + for (real, alias) in aliases: + h.write('{} = {}\n'.format(alias, real)) + with open(module_path, "w", encoding="utf-8") as f: # Start by handling all required imports for type annotations f.write(format_imports(namespace, version)) From ccf067a787f9c10faf452b22cb795235ec0a98df Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 14 Aug 2019 17:27:10 +1000 Subject: [PATCH 45/48] stubs: Add the start of a list of Liskov issues that aren't fixable --- pgidocgen/stubs.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 61e97e66..66f26752 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -34,6 +34,32 @@ ('GObject.Object', 'newv'), } +#: Methods that have explicit `type: ignore` annotations added. These +#: are methods that violate Liskov by overriding superclass methods +#: with an incompatible signature. +LISKOV_VIOLATING_METHODS = { + ('Gio.Cancellable', 'connect'), + ('Gio.Cancellable', 'disconnect'), + ('Gio.MemoryOutputStream', 'get_data'), + ('Gio.MemoryOutputStream', 'steal_data'), + ('Gio.Socket', 'condition_wait'), + ('Gio.Socket', 'connect'), + ('Gio.Socket', 'receive_messages'), + ('Gio.Socket', 'send_messages'), + ('Gio.SocketClient', 'connect'), + ('Gio.SocketConnection', 'connect'), + ('GObject.TypeModule', 'use'), + ('Gtk.AccelGroup', 'connect'), + ('Gtk.AccelGroup', 'disconnect'), + ('Gtk.FileFilter', 'get_name'), + ('Gtk.RecentFilter', 'get_name'), + ('Gtk.Settings', 'install_property'), + ('Gtk.StyleContext', 'get_property'), + ('Gtk.StyleProperties', 'get_property'), + ('Gtk.StyleProperties', 'set_property'), + ('Gtk.ThemingEngine', 'get_property'), +} + # Initialising the current module to an invalid name current_module: str = '-' @@ -341,6 +367,10 @@ def stub_class(cls) -> str: # TODO: Extract constructor information from GIR and add it to # docobj.Function to use here. is_constructor = v.name == "new" + ignore_type_error = ( + is_constructor or + (cls.fullname, v.name) in LISKOV_VIOLATING_METHODS + ) # All GObject-inheriting classes should accept keyword # arguments to set GObject properties in their default @@ -350,7 +380,7 @@ def stub_class(cls) -> str: stub.add_function( format_function(v, accepts_kwargs=accepts_kwargs), - ignore_type_error=is_constructor + ignore_type_error=ignore_type_error, ) return str(stub) From f3ed01310505013102305084ade564f0cf654904 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Wed, 14 Aug 2019 17:27:42 +1000 Subject: [PATCH 46/48] stubs: Add a range of constructor-type names Ideally this would come from the GI constructor information, but we don't maintain that through to the point of use here yet. This list should be able to be removed with some work in the future though. --- pgidocgen/stubs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 66f26752..a23ce6b1 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -366,7 +366,14 @@ def stub_class(cls) -> str: # TODO: Extract constructor information from GIR and add it to # docobj.Function to use here. - is_constructor = v.name == "new" + # FIXME: This list is overly broad and very fragile. + is_constructor = v.name in ( + 'new', + 'new_from_stock', + 'new_with_label', + 'new_with_mnemonic', + 'new_with_range', + ) ignore_type_error = ( is_constructor or (cls.fullname, v.name) in LISKOV_VIOLATING_METHODS From eca1c0ebbe69b434cf59fb9f0e724844aefce4fa Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sat, 17 Aug 2019 08:36:28 +1000 Subject: [PATCH 47/48] stubs: Add new module for manual stub additions and handle in generation Object.Property and Object.Signal are the main two missing parts for general use, and both require hand-written stub information. The Property stub here is probably wrong in various ways and could definitely be a lot better, but it's better than nothing. --- pgidocgen/stuboverrides.py | 68 ++++++++++++++++++++++++++++++++++++++ pgidocgen/stubs.py | 10 +++++- 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 pgidocgen/stuboverrides.py diff --git a/pgidocgen/stuboverrides.py b/pgidocgen/stuboverrides.py new file mode 100644 index 00000000..65474909 --- /dev/null +++ b/pgidocgen/stuboverrides.py @@ -0,0 +1,68 @@ + +PROPERTY_STUB = """# TODO: Constrain T if possible. Builtins + GTypes might be sufficient? +T = typing.TypeVar('T') + +PropertyGetterFn = typing.Callable[[typing.Any], T] +PropertySetterFn = typing.Callable[[typing.Any, T], None] + + +class Property(typing.Generic[T]): + + name: typing.Optional[str] + type: typing.Type[T] + default: typing.Optional[T] + nick: str + blurb: str + flags: ParamFlags + minimum: typing.Optional[T] + maximum: typing.Optional[T] + + def __init__( + self, + getter: typing.Optional[PropertyGetterFn[T]] = None, + setter: typing.Optional[PropertySetterFn[T]] = None, + type: typing.Optional[typing.Type[T]] = None, + default: typing.Optional[T] = None, + nick: str = '', + blurb: str = '', + flags: ParamFlags = ParamFlags.READWRITE, + minimum: typing.Optional[T] = None, + maximum: typing.Optional[T] = None, + ) -> None: + ... + + def __get__(self, instance: typing.Any, klass: typing.Type) -> T: + ... + + def __set__(self, instance: typing.Any, value: T) -> None: + ... + + def __call__(self, PropertyGetterFn) -> Property[T]: + ... + + def getter(self: Property[T], fget: PropertyGetterFn) -> Property[T]: + ... + + def setter(self: Property[T], fset: PropertySetterFn) -> Property[T]: + ... + + # TODO: There's three Tuple variant structures that could be + # returned here, and they're all unpleasantly complicated. + def get_pspec_args(self) -> typing.Sequence[typing.Any]: + ... +""" + +#: Map of namespace to additional manually-written stub classes that +#: should be added to the top of the generated stub. +NAMESPACE_OVERRIDES = { + 'GObject': [ + PROPERTY_STUB, + ], +} + +#: Map of class full name to attributes to be added to the class. +OBJECT_OVERRIDES = { + 'GObject.Object': { + 'Property': 'Property', + }, +} diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index a23ce6b1..55dddcff 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -10,12 +10,12 @@ import os import subprocess import sys -import tempfile import typing from .docobj import Flags from .namespace import get_namespace, set_cache_prefix_path from .repo import Repository +from .stuboverrides import NAMESPACE_OVERRIDES, OBJECT_OVERRIDES from .util import get_gir_files @@ -327,6 +327,10 @@ def stub_class(cls) -> str: if isinstance(cls, Flags): stub.parents.append('builtins.int') + overrides = OBJECT_OVERRIDES.get(cls.fullname, {}) + for name, value in overrides.items(): + stub.add_member(f'{name} = {value}') + # TODO: We don't handle: # * child_properties: It's not clear how to annotate these # * gtype_struct: I'm not sure what we'd use this for. @@ -498,6 +502,10 @@ def get_to_write(dir_, namespace, version): h = io.StringIO() + for override in NAMESPACE_OVERRIDES.get(namespace, []): + h.write(override) + h.write("\n\n") + for cls in get_module_classlikes(mod): h.write(stub_class(cls)) h.write("\n\n") From 992976e34c5b3846ed487cbeaac916ab55dfba25 Mon Sep 17 00:00:00 2001 From: Kai Willadsen Date: Sat, 17 Aug 2019 09:38:03 +1000 Subject: [PATCH 48/48] stubs: Make GLib.Flags boolean operators retain flag type --- pgidocgen/stuboverrides.py | 18 +++++++++++++++--- pgidocgen/stubs.py | 6 +++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pgidocgen/stuboverrides.py b/pgidocgen/stuboverrides.py index 65474909..0b8986e1 100644 --- a/pgidocgen/stuboverrides.py +++ b/pgidocgen/stuboverrides.py @@ -52,9 +52,16 @@ def get_pspec_args(self) -> typing.Sequence[typing.Any]: ... """ +FLAGS_TYPEVAR = """FlagsT = typing.TypeVar('FlagsT') +""" + + #: Map of namespace to additional manually-written stub classes that #: should be added to the top of the generated stub. NAMESPACE_OVERRIDES = { + 'GLib': [ + FLAGS_TYPEVAR, + ], 'GObject': [ PROPERTY_STUB, ], @@ -62,7 +69,12 @@ def get_pspec_args(self) -> typing.Sequence[typing.Any]: #: Map of class full name to attributes to be added to the class. OBJECT_OVERRIDES = { - 'GObject.Object': { - 'Property': 'Property', - }, + 'GLib.Flags': [ + 'def __or__(self: FlagsT, other: typing.Union[int, FlagsT]) -> FlagsT: ...', + 'def __and__(self: FlagsT, other: typing.Union[int, FlagsT]) -> FlagsT: ...', + 'def __xor__(self: FlagsT, other: typing.Union[int, FlagsT]) -> FlagsT: ...', + ], + 'GObject.Object': [ + 'Property = Property', + ], } diff --git a/pgidocgen/stubs.py b/pgidocgen/stubs.py index 55dddcff..ebbad43e 100644 --- a/pgidocgen/stubs.py +++ b/pgidocgen/stubs.py @@ -327,9 +327,9 @@ def stub_class(cls) -> str: if isinstance(cls, Flags): stub.parents.append('builtins.int') - overrides = OBJECT_OVERRIDES.get(cls.fullname, {}) - for name, value in overrides.items(): - stub.add_member(f'{name} = {value}') + overrides = OBJECT_OVERRIDES.get(cls.fullname, []) + for member in overrides: + stub.add_member(member) # TODO: We don't handle: # * child_properties: It's not clear how to annotate these