Patch wx overriding the correct python locale

source/core.py

@@ -449,6 +449,8 @@ def handlePowerStatusChange(self):
 			locale.Init(wxLang.Language)
 		except:
 			log.error("Failed to initialize wx locale",exc_info=True)
+		# Revert wx's changes to the python locale
+		languageHandler.setLocale(languageHandler.curLang)
 
 	log.debug("Initializing garbageHandler")
 	garbageHandler.initialize()

source/languageHandler.py

@@ -13,6 +13,7 @@
 import locale
 import gettext
 import globalVars
+from logHandler import log
 
 #a few Windows locale constants
 LOCALE_SLANGUAGE=0x2
@@ -146,37 +147,119 @@ def getWindowsLanguage():
 			localeName="en"
 	return localeName
 
-def setLanguage(lang):
+
+def setLanguage(lang: str) -> None:
+	'''
+	Sets the following using `lang` such as "en", "ru_RU", or "es-ES". Use "Windows" to use the system locale
+	 - the windows locale for the thread (fallback to system locale)
+	 - the translation service (fallback to English)
+	 - languageHandler.curLang (match the translation service)
+	 - the python locale for the thread (match the translation service, fallback to system default)
+	'''
 	global curLang
-	try:
-		if lang=="Windows":
-			localeName=getWindowsLanguage()
-			trans=gettext.translation('nvda',localedir='locale',languages=[localeName])
-			curLang=localeName
-		else:
-			trans=gettext.translation("nvda", localedir="locale", languages=[lang])
-			curLang=lang
-			localeChanged=False
-			#Try setting Python's locale to lang
-			try:
-				locale.setlocale(locale.LC_ALL,lang)
-				localeChanged=True
-			except:
-				pass
-			if not localeChanged and '_' in lang:
-				#Python couldn'tsupport the language_country locale, just try language.
-				try:
-					locale.setlocale(locale.LC_ALL,lang.split('_')[0])
-				except:
-					pass
-			#Set the windows locale for this thread (NVDA core) to this locale.
-			LCID=localeNameToWindowsLCID(lang)
+	if lang == "Windows":
+		localeName = getWindowsLanguage()
+	else:
+		localeName = lang
+		# Set the windows locale for this thread (NVDA core) to this locale.
+		try:
+			LCID = localeNameToWindowsLCID(lang)
 			ctypes.windll.kernel32.SetThreadLocale(LCID)
+		except IOError:
+			log.debugWarning(f"couldn't set windows thread locale to {lang}")
+
+	try:
+		trans = gettext.translation("nvda", localedir="locale", languages=[localeName])
+		curLang = localeName
 	except IOError:
-		trans=gettext.translation("nvda",fallback=True)
-		curLang="en"
+		log.debugWarning(f"couldn't set the translation service locale to {localeName}")
+		trans = gettext.translation("nvda", fallback=True)
+		curLang = "en"
+
 	# #9207: Python 3.8 adds gettext.pgettext, so add it to the built-in namespace.
 	trans.install(names=["pgettext"])
+	setLocale(curLang)
+
+
+def setLocale(localeName: str) -> None:
+	'''
+	Set python's locale using a `localeName` such as "en", "ru_RU", or "es-ES".
+	Will fallback on `curLang` if it cannot be set and finally fallback to the system locale.
+	'''
+
+	r'''
+	Python 3.8's locale system allows you to set locales that you cannot get
+	so we must test for both ValueErrors and locale.Errors
+
+	>>> import locale
+	>>> locale.setlocale(locale.LC_ALL, 'foobar')
+	Traceback (most recent call last):
+	File "<stdin>", line 1, in <module>
+	File "Python38-32\lib\locale.py", line 608, in setlocale
+		return _setlocale(category, locale)
+	locale.Error: unsupported locale setting
+	>>> locale.setlocale(locale.LC_ALL, 'en-GB')
+	'en-GB'
+	>>> locale.getlocale()
+	Traceback (most recent call last):
+	File "<stdin>", line 1, in <module>
+	File "Python38-32\lib\locale.py", line 591, in getlocale
+		return _parse_localename(localename)
+	File "Python38-32\lib\locale.py", line 499, in _parse_localename
+		raise ValueError('unknown locale: %s' % localename)
+	ValueError: unknown locale: en-GB
+	'''
+	originalLocaleName = localeName
+	# Try setting Python's locale to localeName
+	try:
+		locale.setlocale(locale.LC_ALL, localeName)
+		locale.getlocale()
+		log.debug(f"setting python locale to {localeName}")
+		return
+	except locale.Error:
+		log.debugWarning(f"python locale {localeName} could not be set")
+	except ValueError:
+		log.debugWarning(f"python locale {localeName} could not be retrieved with getlocale")
+
+	if '-' in localeName:
+		# Python couldn't support the language-country locale, try language_country.
+		try:
+			localeName = localeName.replace('-', '_')
+			locale.setlocale(locale.LC_ALL, localeName)
+			locale.getlocale()
+			log.debug(f"setting python locale to {localeName}")
+			return
+		except locale.Error:
+			log.debugWarning(f"python locale {localeName} could not be set")
+		except ValueError:
+			log.debugWarning(f"python locale {localeName} could not be retrieved with getlocale")
+
+	if '_' in localeName:
+		# Python couldn't support the language_country locale, just try language.
+		try:
+			localeName = localeName.split('_')[0]
+			locale.setlocale(locale.LC_ALL, localeName)
+			locale.getlocale()
+			log.debug(f"setting python locale to {localeName}")
+			return
+		except locale.Error:
+			log.debugWarning(f"python locale {localeName} could not be set")
+		except ValueError:
+			log.debugWarning(f"python locale {localeName} could not be retrieved with getlocale")
+
+	try:
+		locale.getlocale()
+	except ValueError:
+		# as the locale may have been changed to something that getlocale() couldn't retrieve
+		# reset to default locale
+		if originalLocaleName == curLang:
+			log.debugWarning(f"setting python locale to system default")
+			# reset to system locale default if we can't set the current lang's locale
+			locale.setlocale(locale.LC_ALL, "")
+		else:
+			log.debugWarning(f"setting python locale to the current language {curLang}")
+			# fallback and try to reset the locale to the current lang
+			setLocale(curLang)
 
 
 def getLanguage() -> str:
@@ -314,5 +397,9 @@ def normalizeLanguage(lang):
 	134:'qut',
 	135:'rw',
 	136:'wo',
-	140:'gbz'
+	140: 'gbz',
+	1170: 'ckb',
+	1109: 'my',
+	1143: 'so',
+	9242: 'sr',
 }

tests/unit/test_languageHandler.py

@@ -1,20 +1,32 @@
-#tests/unit/test_languageHandler.py
-#A part of NonVisual Desktop Access (NVDA)
-#This file is covered by the GNU General Public License.
-#See the file COPYING for more details.
-#Copyright (C) 2017 NV Access Limited
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2017-2021 NV Access Limited
 
 """Unit tests for the languageHandler module.
 """
 
 import unittest
 import languageHandler
-from languageHandler import LCID_NONE
+from languageHandler import LCID_NONE, windowsPrimaryLCIDsToLocaleNames
+import locale
+import ctypes
 
 LCID_ENGLISH_US = 0x0409
+UNSUPPORTED_PYTHON_LOCALES = {
+	"an",
+	"ckb",
+	"kmr",
+	"mn",
+	"my",
+	"ne",
+	"so",
+}
+TRANSLATABLE_LANGS = set(l[0] for l in languageHandler.getAvailableLanguages()) - {"Windows"}
+WINDOWS_LANGS = set(locale.windows_locale.values()).union(windowsPrimaryLCIDsToLocaleNames.values())
 
-class TestLocaleNameToWindowsLCID(unittest.TestCase):
 
+class TestLocaleNameToWindowsLCID(unittest.TestCase):
 	def test_knownLocale(self):
 		lcid = languageHandler.localeNameToWindowsLCID("en")
 		self.assertEqual(lcid, LCID_ENGLISH_US)
@@ -31,3 +43,157 @@ def test_nonStandardLocale(self):
 	def test_invalidLocale(self):
 		lcid = languageHandler.localeNameToWindowsLCID("zzzz")
 		self.assertEqual(lcid, LCID_NONE)
+
+
+class TestSetLocale(unittest.TestCase):
+	"""
+	Tests setting the python locale for possible locales set by NVDA user preferences or the System.
+	"""
+
+	SUPPORTED_LOCALES = [("en", "en_US"), ("fa-IR", "fa_IR"), ("an-ES", "an_ES")]
+
+	def setUp(self):
+		"""
+		`setLocale` doesn't change `languageHandler.curLang`, so reset the locale using `setLanguage` to
+		the current language for each test.
+		"""
+		languageHandler.setLanguage(languageHandler.curLang)
+
+	@classmethod
+	def tearDownClass(cls):
+		"""
+		`setLocale` doesn't change `languageHandler.curLang`, so reset the locale using `setLanguage` to
+		the current language so the tests can continue normally.
+		"""
+		languageHandler.setLanguage(languageHandler.curLang)
+
+	def test_SetLocale_SupportedLocale_LocaleIsSet(self):
+		"""
+		Tests several locale formats that should result in an expected python locale being set.
+		"""
+		for localeName in self.SUPPORTED_LOCALES:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLocale(localeName[0])
+				self.assertEqual(locale.getlocale()[0], localeName[1])
+
+	def test_SetLocale_PythonUnsupportedLocale_LocaleUnchanged(self):
+		"""
+		Tests several locale formats that python doesn't support which will result in a return to the
+		current locale
+		"""
+		original_locale = locale.getlocale()
+		for localeName in UNSUPPORTED_PYTHON_LOCALES:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLocale(localeName)
+				self.assertEqual(locale.getlocale(), original_locale)
+
+	def test_SetLocale_NVDASupportedAndPythonSupportedLocale_LanguageCodeMatches(self):
+		"""
+	 	Tests all the translatable languages that NVDA shows in the user preferences
+		excludes the locales that python doesn't support, as the expected behaviour is different.
+		"""
+		for localeName in TRANSLATABLE_LANGS - UNSUPPORTED_PYTHON_LOCALES:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLocale(localeName)
+				current_locale = locale.getlocale()
+
+				if localeName == "uk":
+					self.assertEqual(current_locale[0], "English_United Kingdom")
+				else:
+					pythonLang = current_locale[0].split("_")[0]
+					langOnly = localeName.split("_")[0]
+					self.assertEqual(
+						langOnly,
+						pythonLang,
+						f"full values: {localeName} {current_locale[0]}",
+					)
+
+	def test_SetLocale_WindowsLang_LocaleCanBeRetrieved(self):
+		"""
+		We don't know whether python supports a specific windows locale so just ensure locale isn't
+		broken after testing these values.
+		"""
+		for localeName in WINDOWS_LANGS:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLocale(localeName)
+				locale.getlocale()
+
+
+class TestSetLanguage(unittest.TestCase):
+	"""
+	Tests setting the NVDA language set by NVDA user preferences or the System.
+	"""
+	UNSUPPORTED_WIN_LANGUAGES = ["an", "kmr"]
+
+	def tearDown(self):
+		"""
+		Resets the language to whatever it was before the testing suite begun.
+		"""
+		languageHandler.setLanguage(self._prevLang)
+
+	def __init__(self, *args, **kwargs):
+		self._prevLang = languageHandler.getLanguage()
+
+		ctypes.windll.kernel32.SetThreadLocale(0)
+		defaultThreadLocale = ctypes.windll.kernel32.GetThreadLocale()
+		self._defaultThreadLocaleName = languageHandler.windowsLCIDToLocaleName(
+			defaultThreadLocale
+		)
+
+		locale.setlocale(locale.LC_ALL, "")
+		self._defaultPythonLocale = locale.getlocale()
+
+		languageHandler.setLanguage(self._prevLang)
+		super().__init__(*args, **kwargs)
+
+	def test_SetLanguage_NVDASupportedLanguages_LanguageIsSetCorrectly(self):
+		"""
+		Tests languageHandler.setLanguage, using all NVDA supported languages, which should do the following:
+		- set the translation service and languageHandler.curLang
+		- set the windows locale for the thread (fallback to system default)
+		- set the python locale for the thread (match the translation service, fallback to system default)
+		"""
+		for localeName in TRANSLATABLE_LANGS:
+			with self.subTest(localeName=localeName):
+				langOnly = localeName.split("_")[0]
+				languageHandler.setLanguage(localeName)
+				# check curLang/translation service is set
+				self.assertEqual(languageHandler.curLang, localeName)
+
+				# check Windows thread is set
+				threadLocale = ctypes.windll.kernel32.GetThreadLocale()
+				threadLocaleName = languageHandler.windowsLCIDToLocaleName(threadLocale)
+				threadLocaleLang = threadLocaleName.split("_")[0]
+				if localeName in self.UNSUPPORTED_WIN_LANGUAGES:
+					# our translatable locale isn't supported by windows
+					# check that the system locale is unchanged
+					self.assertEqual(self._defaultThreadLocaleName, threadLocaleName)
+				else:
+					# check that the language codes are correctly set for the thread
+					self.assertEqual(
+						langOnly,
+						threadLocaleLang,
+						f"full values: {localeName} {threadLocaleName}",
+					)
+
+				# check that the python locale is set
+				python_locale = locale.getlocale()
+				if localeName in UNSUPPORTED_PYTHON_LOCALES:
+					self.assertEqual(self._defaultPythonLocale, python_locale)
+				elif localeName == "uk":
+					self.assertEqual(python_locale[0], "English_United Kingdom")
+				else:
+					# checks that the language codes part of the locales are correct
+					pythonLang = python_locale[0].split("_")[0]
+					self.assertEqual(
+						langOnly, pythonLang, f"full values: {localeName} {python_locale}"
+					)
+
+	def test_SetLanguage_WindowsLanguages_NoErrorsThrown(self):
+		"""
+		We don't know whether python or our translator system supports a specific windows locale
+		so just ensure the setLanguage process doesn't fail.
+		"""
+		for localeName in WINDOWS_LANGS:
+			with self.subTest(localeName=localeName):
+				languageHandler.setLanguage(localeName)