Ensure that unit tests can be executed under version of Windows diffe…

…rent than the one used for building NVDA (#12835)

Fixes regression from #12617. Discovered when working on #12753 as that PR required executing unit tests under various versions of Windows

Summary of the issue:
It is sometimes useful to execute unit tests under various versions of Windows where it is impractical, or even impossible to have full build environment for NVDA. With current master some tests are failing because when importing COM interfaces comtypes complains about them being created on different version of Windows. Before PR #12617 this worked by chance since our monkey patches for compypes were applied when importing core and it just so happened that core was imported before any COM interface was.

Description of how this pull request fixes the issue:
Since we have monkey patch which stops comtypes from complaining about typelib being different than COM interface it has been applied to comtypes before tests starts. To avoid copying code of this monkey patch it was necessary to refactor our monkey patches to make each of them into a separate function - that allows us to apply them selectively rather than just all of them during import.

Testing strategy:
Ensured that unit tests pass under a never version of Windows than the one under which virtual environment for NVDA has been created
Ensured that comtypes monkey patches are properly applied for NVDA in particular that core.CallCancelled is raised when appropriate.

source/core.py

@@ -14,14 +14,7 @@ class CallCancelled(Exception):
 	"""Raised when a call is cancelled.
 	"""
 
-# Initialise comtypes.client.gen_dir and the comtypes.gen search path 
-# and Append our comInterfaces directory to the comtypes.gen search path.
 import comtypes
-import comtypes.client
-import comtypes.gen
-import comInterfaces
-comtypes.gen.__path__.append(comInterfaces.__path__[0])
-
 import sys
 import winVersion
 import threading

source/monkeyPatches/__init__.py

@@ -11,8 +11,8 @@
 
 def applyMonkeyPatches():
 	# Apply several monkey patches to comtypes
-	# F401 - imported but unused: Patches are applied during import
-	from . import comtypesMonkeyPatches  # noqa: F401
+	from . import comtypesMonkeyPatches
+	comtypesMonkeyPatches.applyMonkeyPatches()
 
 	# Apply patches to Enum, prevent cyclic references on ValueError during construction
 	from . import enumPatches

source/monkeyPatches/comtypesMonkeyPatches.py

@@ -3,19 +3,22 @@
 # This file is covered by the GNU General Public License.
 # See the file COPYING for more details.
 
-# Warning: no comtypes modules can be imported until ctypes.WINFUNCTYPE has been replaced further down.
+# Warning: no comtypes modules can be imported at the module level
+# since we need to replace ctypes.WINFUNCTYPE with our custom version.
 
 import ctypes
 import _ctypes
+from ctypes import cast, c_void_p
+from _ctypes import _Pointer
 import importlib
 import sys
 
 
-# A version of ctypes.WINFUNCTYPE 
-# that produces a WinFunctionType class whose instance will convert COMError into a CallCancelled exception when called as a function.
-old_WINFUNCTYPE=ctypes.WINFUNCTYPE
 def new_WINFUNCTYPE(restype,*argtypes,**kwargs):
-	cls=old_WINFUNCTYPE(restype,*argtypes,**kwargs)
+	"""A version of ctypes.WINFUNCTYPE
+	that produces a WinFunctionType class
+	whose instance will convert COMError into a CallCancelled exception when called as a function."""
+	cls = ctypes.WINFUNCTYPE_orig(restype, *argtypes, **kwargs)
 	class WinFunctionType(cls):
 		# We must manually pull the mandatory class variables from the super class,
 		# as the metaclass of _ctypes.CFuncPtr seems to expect these on the outermost subclass.
@@ -37,72 +40,75 @@ def __call__(self,*args,**kwargs):
 				raise
 	return WinFunctionType
 
-# While importing comtypes,
-# Replace WINFUNCTYPE in ctypes with our own version,
-# So that comtypes will use this in all its COM method calls. 
-# As comtypes imports WINFUNCTYPE from ctypes by name,
-# We only need to replace it for the duration of importing comtypes, 
-# as it will then have it for ever.
-ctypes.WINFUNCTYPE=new_WINFUNCTYPE
-try:
-	import comtypes
-finally:
-	ctypes.WINFUNCTYPE=old_WINFUNCTYPE
-
-# It is safe to import any comtypes modules from here on down.
-
-from logHandler import log
-import garbageHandler  # noqa: E402
 
+def replace_WINFUNCTYPE() -> None:
+	# While importing comtypes,
+	# Replace WINFUNCTYPE in ctypes with our own version,
+	# So that comtypes will use this in all its COM method calls.
+	# As comtypes imports WINFUNCTYPE from ctypes by name,
+	# We only need to replace it for the duration of importing comtypes,
+	# as it will then have it for ever.
+	ctypes.WINFUNCTYPE_orig = ctypes.WINFUNCTYPE
+	ctypes.WINFUNCTYPE = new_WINFUNCTYPE
+	try:
+		import comtypes
+		if comtypes.WINFUNCTYPE != new_WINFUNCTYPE:
+			raise RuntimeError("Failed to replace WINFUNCTYPE with the custom version")
+	finally:
+		ctypes.WINFUNCTYPE = ctypes.WINFUNCTYPE_orig
 
-from comtypes import COMError
-from comtypes.hresult import *
 
-#Monkey patch comtypes to support byref in variants
-from comtypes.automation import (  # noqa: E402
-	VARIANT,
-	_vartype_to_ctype,
-	VT_BYREF,
-	VT_R8,
-	IDispatch
-)
-from ctypes import cast, c_void_p
-from _ctypes import _Pointer
-oldVARIANT_value_fset=VARIANT.value.fset
 def newVARIANT_value_fset(self,value):
+	from comtypes.automation import VARIANT
 	realValue=value
 	if isinstance(value,_Pointer):
 		try:
 			value=value.contents
 		except (NameError,AttributeError):
 			pass
-	oldVARIANT_value_fset(self,value)
+	VARIANT.VALUE_FSEWT_ORIG(self, value)
 	if realValue is not value:
+		from comtypes.automation import VT_BYREF
 		self.vt|=VT_BYREF
 		self._.c_void_p=cast(realValue,c_void_p)
-VARIANT.value=property(VARIANT.value.fget,newVARIANT_value_fset,VARIANT.value.fdel)
 
-#Monkeypatch comtypes lazybind dynamic IDispatch support to fallback to the more basic dynamic IDispatch support if the former does not work
-#Example: ITypeComp.bind gives back a vardesc, which comtypes does not yet support
-import comtypes.client.lazybind
-old__getattr__=comtypes.client.lazybind.Dispatch.__getattr__
+
+def support_byref_in_variants() -> None:
+	# Monkey patch comtypes to support byref in variants
+	from comtypes.automation import VARIANT
+	VARIANT.VALUE_FSEWT_ORIG = VARIANT.value.fset
+	VARIANT.value = property(VARIANT.value.fget, newVARIANT_value_fset, VARIANT.value.fdel)
+
+
 def new__getattr__(self,name):
+	import comtypes.client.lazybind
 	try:
-		return old__getattr__(self,name)
+		return comtypes.client.lazybind.Dispatch.__getattr__orig(self, name)
 	except (NameError, AttributeError):
 		return getattr(comtypes.client.dynamic._Dispatch(self._comobj),name)
-comtypes.client.lazybind.Dispatch.__getattr__=new__getattr__
 
-#Monkeypatch comtypes to allow its basic dynamic Dispatch support to  support invoke 0 (calling the actual IDispatch object itself)
+
+def lazybind_dynamic_to_basic() -> None:
+	# Monkeypatch comtypes lazybind dynamic IDispatch support
+	# to fallback to the more basic dynamic IDispatch support if the former does not work
+	# Example: ITypeComp.bind gives back a vardesc, which comtypes does not yet support
+	import comtypes.client.lazybind
+	comtypes.client.lazybind.Dispatch.__getattr__orig = comtypes.client.lazybind.Dispatch.__getattr__
+	comtypes.client.lazybind.Dispatch.__getattr__ = new__getattr__
+
+
 def new__call__(self,*args,**kwargs):
+	import comtypes.client
 	return comtypes.client.dynamic.MethodCaller(0,self)(*args,**kwargs)
-comtypes.client.dynamic._Dispatch.__call__=new__call__
 
-# Work around an issue with comtypes where __del__ seems to be called twice on COM pointers.
-# This causes Release() to be called more than it should, which is very nasty and will eventually cause us to access pointers which have been freed.
-from comtypes import _compointer_base
 
-_compointer_base._oldCpbDel = _compointer_base.__del__
+def support_invoke_zero() -> None:
+	# Monkeypatch comtypes to allow its basic dynamic Dispatch support
+	# to support invoke 0 (calling the actual IDispatch object itself)
+	import comtypes.client
+	comtypes.client.dynamic._Dispatch.__call__ = new__call__
+
+
 def newCpbDel(self):
 	# __del__ may be called while Python is exiting.
 	# In this state, global symbols may be set to None
@@ -114,66 +120,118 @@ def newCpbDel(self):
 	if hasattr(self, "_deleted"):
 		# Don't allow this to be called more than once.
 		if not isFinalizing:
+			from logHandler import log
 			log.debugWarning("COM pointer %r already deleted" % self)
 		return
 	if not isFinalizing:
+		import garbageHandler
 		garbageHandler.notifyObjectDeletion(self)
 	self._oldCpbDel()
 	self._deleted = True
 newCpbDel.__name__ = "__del__"
-_compointer_base.__del__ = newCpbDel
-del _compointer_base
 
-#Monkey patch to force dynamic Dispatch on all vt_dispatch variant values.
-#Certainly needed for comtypes COM servers, but currently very fiddly to do just for that case 
-oldVARIANT_value_fget=VARIANT.value.fget
+
+def replace_cpb_del() -> None:
+	# Work around an issue with comtypes where __del__ seems to be called twice on COM pointers.
+	# This causes Release() to be called more than it should,
+	# which is very nasty and will eventually cause us to access pointers which have been freed.
+	from comtypes import _compointer_base
+	_compointer_base._oldCpbDel = _compointer_base.__del__
+	_compointer_base.__del__ = newCpbDel
+	del _compointer_base
+
+
 def newVARIANT_value_fget(self):
 	return self._get_value(dynamic=True)
-VARIANT.value=property(newVARIANT_value_fget,VARIANT.value.fset,VARIANT.value.fdel)
 
-# #4258: monkeypatch to better handle error where IDispatch's GetTypeInfo can return a NULL pointer. Affects QT5
-oldGetTypeInfo=IDispatch._GetTypeInfo
+
+def replace_VARIAN_value_fget() -> None:
+	# Monkey patch to force dynamic Dispatch on all vt_dispatch variant values.
+	# Certainly needed for comtypes COM servers, but currently very fiddly to do just for that case
+	from comtypes.automation import VARIANT
+	VARIANT.value = property(newVARIANT_value_fget, VARIANT.value.fset, VARIANT.value.fdel)
+
+
 def newGetTypeInfo(self,index,lcid=0):
-	res=oldGetTypeInfo(self,index,lcid)
+	from comtypes.automation import IDispatch
+	res = IDispatch._GetTypeInfo_orig(self, index, lcid)
 	if not res:
+		from comtypes import COMError
+		from comtypes.hresult import E_NOTIMPL
 		raise COMError(E_NOTIMPL,None,None)
 	return res
-IDispatch._GetTypeInfo=newGetTypeInfo
 
-# Windows updates often include newer versions of dlls/typelibs we use.
-# The typelib being newer than the comtypes generated module doesn't hurt us,
-# so kill the "Typelib newer than module" ImportError.
-# comtypes doesn't let us disable this when running from source, so we need to monkey patch.
-# This is just the code from the original comtypes._check_version excluding the time check.
-import comtypes
+
+def replace_idispatch_getTypeInfo() -> None:
+	# #4258: monkeypatch to better handle error where IDispatch's GetTypeInfo can return a NULL pointer.
+	# Affects QT5
+	from comtypes.automation import IDispatch
+	IDispatch._GetTypeInfo_orig = IDispatch._GetTypeInfo
+	IDispatch._GetTypeInfo = newGetTypeInfo
 
 
 def _check_version(actual, tlib_cached_mtime=None):
 	from comtypes.tools.codegenerator import version as required
 	if actual != required:
 		raise ImportError("Wrong version")
-comtypes._check_version = _check_version
-
 
-# Monkeypatch comtypes to clear the importlib cache when importing a new module
 
-# We must import comtypes.client._generate here as it must be done after other monkeypatching
-import comtypes.client._generate  # noqa: E402
-
-old_my_import = comtypes.client._generate._my_import
+def replace_check_version() -> None:
+	# Windows updates often include newer versions of dlls/typelibs we use.
+	# The typelib being newer than the comtypes generated module doesn't hurt us,
+	# so kill the "Typelib newer than module" ImportError.
+	# comtypes doesn't let us disable this when running from source, so we need to monkey patch.
+	# This is just the code from the original comtypes._check_version excluding the time check.
+	import comtypes
+	comtypes._check_version = _check_version
 
 
 def new_my_import(fullname):
+	import comtypes.client._generate
 	importlib.invalidate_caches()
-	return old_my_import(fullname)
-
-
-comtypes.client._generate._my_import = new_my_import
-
-# Correctly map VT_R8 to c_double.
-# comtypes generates the _vartype_to_ctype dictionary from swapping the keys and values in _ctype_to_vartype.
-# Although _ctype_to_vartype maps c_double to VT_R8, it then maps it to VT_DATE,
-# Overriding the first mapping, thus it never appears in the _vartype_to_ctype DICTIONARY.
-# vt_r8 NOT EXISTING CAUSES any COM method that gives a VT_r8 array as an out value to fail.
-# For example, the cellSize UIA custom property in Excel.
-_vartype_to_ctype[VT_R8] = ctypes.c_double
+	return comtypes.client._generate._my_import_orig(fullname)
+
+
+def replace_my_import() -> None:
+	# Monkeypatch comtypes to clear the importlib cache when importing a new module
+	import comtypes.client._generate
+	comtypes.client._generate._my_import_orig = comtypes.client._generate._my_import
+	comtypes.client._generate._my_import = new_my_import
+
+
+def vt_R8_to_c_double() -> None:
+	# Correctly map VT_R8 to c_double.
+	# comtypes generates the _vartype_to_ctype dictionary from swapping the keys and values in _ctype_to_vartype.
+	# Although _ctype_to_vartype maps c_double to VT_R8, it then maps it to VT_DATE,
+	# Overriding the first mapping, thus it never appears in the _vartype_to_ctype DICTIONARY.
+	# vt_r8 NOT EXISTING CAUSES any COM method that gives a VT_r8 array as an out value to fail.
+	# For example, the cellSize UIA custom property in Excel.
+	from comtypes.automation import _vartype_to_ctype, VT_R8
+	_vartype_to_ctype[VT_R8] = ctypes.c_double
+
+
+def appendComInterfacesToGenSearchPath() -> None:
+	# Initialise comtypes.client.gen_dir and the comtypes.gen search path
+	# and append our comInterfaces directory to the comtypes.gen search path.
+	import comtypes.client
+	import comtypes.gen
+	import comInterfaces
+	comtypes.gen.__path__.append(comInterfaces.__path__[0])
+
+
+def applyMonkeyPatches() -> None:
+	# Ensure no comtypes modules were imported
+	# before we had a chance to replace `ctypes.WINFUNCTYPE` with our custom version.
+	if any(filter(lambda modName: modName.startswith("comtypes"), sys.modules.keys())):
+		raise RuntimeError("Comtypes module imported before `ctypes.WINFUNCTYPE` has been replaced")
+	replace_WINFUNCTYPE()
+	support_byref_in_variants()
+	lazybind_dynamic_to_basic()
+	support_invoke_zero()
+	replace_cpb_del()
+	replace_VARIAN_value_fget()
+	replace_idispatch_getTypeInfo()
+	replace_check_version()
+	replace_my_import()
+	vt_R8_to_c_double()
+	appendComInterfacesToGenSearchPath()

source/nvda_slave.pyw

@@ -12,15 +12,10 @@ import sys
 import os
 import globalVars
 import winKernel
+import monkeyPatches.comtypesMonkeyPatches
 
-
-# Initialise comtypes.client.gen_dir and the comtypes.gen search path 
-# and Append our comInterfaces directory to the comtypes.gen search path.
-import comtypes
-import comtypes.client
-import comtypes.gen
-import comInterfaces
-comtypes.gen.__path__.append(comInterfaces.__path__[0])
+# Ensure that slave uses generated comInterfaces by adding our comInterfaces to `comtypes.gen` search path.
+monkeyPatches.comtypesMonkeyPatches.appendComInterfacesToGenSearchPath()
 
 
 if hasattr(sys, "frozen"):

tests/unit/__init__.py

@@ -35,7 +35,14 @@
 # Suppress Flake8 warning F401 (module imported but unused)
 # as this module is imported to expand the system path.
 import sourceEnv  # noqa: F401
-
+# Apply several monkey patches to comtypes to make sure that it would search for generated interfaces
+# rather than creating them on the fly. Also stop module being never than typelib error, seen when
+# virtual environment has been created under different version of Windows than the one used for unit tests.
+# Suppress Flake8 warning E402 (module import not at top of file) as this cannot be imported until source
+# directory is appended to python path.
+import monkeyPatches.comtypesMonkeyPatches  # noqa: E402
+monkeyPatches.comtypesMonkeyPatches.replace_check_version()
+monkeyPatches.comtypesMonkeyPatches.appendComInterfacesToGenSearchPath()
 import globalVars