Patch wx overriding the correct python locale

source/core.py

@@ -447,7 +447,7 @@ def handlePowerStatusChange(self):
 	if wxLang:
 		try:
 			locale.Init(wxLang.Language)
-			# Fix whatever wx locale did to python's locale
+			# Revert wx's changes to the python locale
 			languageHandler.setLocale(languageHandler.curLang)
 		except:
 			log.error("Failed to initialize wx locale",exc_info=True)

source/languageHandler.py

@@ -147,7 +147,8 @@ def getWindowsLanguage():
 			localeName="en"
 	return localeName
 
-def setLanguage(lang):
+
+def setLanguage(lang: str) -> None:
 	'''
 	Sets the following using `lang`
 	 - languageHandler.curLang
@@ -156,28 +157,35 @@ def setLanguage(lang):
 	 - the python locale for the thread (match the translation service, fallback to python default)
 	'''
 	global curLang
+	if lang == "Windows":
+		localeName = getWindowsLanguage()
+	else:
+		localeName = lang
 	try:
-		if lang=="Windows":
-			localeName=getWindowsLanguage()
-			trans=gettext.translation('nvda',localedir='locale',languages=[localeName])
-			curLang=localeName
-		else:
-			trans=gettext.translation("nvda", localedir="locale", languages=[lang])
-			curLang=lang
-			#Set the windows locale for this thread (NVDA core) to this locale.
-			LCID=localeNameToWindowsLCID(lang)
-			ctypes.windll.kernel32.SetThreadLocale(LCID)
+		trans = gettext.translation("nvda", localedir="locale", languages=[localeName])
+		curLang = localeName
 	except IOError:
-		trans=gettext.translation("nvda",fallback=True)
-		curLang="en"
+		log.debugWarning(f"couldn't set the translation service locale to {localeName}")
+		trans = gettext.translation("nvda", fallback=True)
+		curLang = "en"
+
+	# Set the windows locale for this thread (NVDA core) to this locale.
+	if lang != "Windows":
+		try:
+			LCID = localeNameToWindowsLCID(localeName)
+			ctypes.windll.kernel32.SetThreadLocale(LCID)
+		except IOError:
+			log.debugWarning(f"couldn't set windows thread locale to {lang}")
+
 	# #9207: Python 3.8 adds gettext.pgettext, so add it to the built-in namespace.
 	trans.install(names=["pgettext"])
 	setLocale(curLang)
 
 
-def setLocale(localeName):
+def setLocale(localeName: str) -> None:
 	'''
 	Set python's locale using a `localeName` set by `setLanguage`.
+	Will fallback on `curLang` if it cannot be set and finally fallback to the system locale.
 	Python 3.8's locale system allows you to set locales that you cannot get
 	so we must test for both ValueErrors and locale.Errors
 	'''
@@ -201,37 +209,47 @@ def setLocale(localeName):
 		raise ValueError('unknown locale: %s' % localename)
 	ValueError: unknown locale: en-GB
 	'''
-	# Try setting Python's locale to lang
-	localeChanged = False
+	originalLocaleName = localeName
+	# Try setting Python's locale to localeName
 	try:
 		locale.setlocale(locale.LC_ALL, localeName)
 		locale.getlocale()
-		localeChanged = True
+		return
 	except (locale.Error, ValueError):
 		pass
-	if not localeChanged and '-' in localeName:
+	if '-' in localeName:
 		# Python couldn't support the language-country locale, try language_country.
 		try:
 			localeName = localeName.replace('-', '_')
 			locale.setlocale(locale.LC_ALL, localeName)
 			locale.getlocale()
-			localeChanged = True
+			return
 		except (locale.Error, ValueError):
 			pass
-	if not localeChanged and '_' in localeName:
+	if '_' in localeName:
 		# Python couldn't support the language_country locale, just try language.
 		try:
 			localeName = localeName.split('_')[0]
 			locale.setlocale(locale.LC_ALL, localeName)
 			locale.getlocale()
-			localeChanged = True
+			return
 		except (locale.Error, ValueError):
 			pass
-	if not localeChanged:
-		log.debugWarning(f"python locale {localeName} could not be set")
-		# as the locale may have been set to something that getlocale() couldn't retrieve
+
+	log.debugWarning(f"python locale {localeName} could not be set")
+	try:
+		locale.getlocale()
+	except ValueError:
+		# as the locale may have been changed to something that getlocale() couldn't retrieve
 		# reset to default locale
-		locale.setlocale(locale.LC_ALL, "")
+		if originalLocaleName == curLang:
+			log.debugWarning(f"setting python locale to system default")
+			# reset to system locale default if we can't set the current lang's locale
+			locale.setlocale(locale.LC_ALL, "")
+		else:
+			log.debugWarning(f"setting python locale to {curLang}")
+			# fallback and try to reset the locale to the current lang
+			setLocale(curLang)
 
 
 def getLanguage() -> str:
@@ -369,5 +387,9 @@ def normalizeLanguage(lang):
 	134:'qut',
 	135:'rw',
 	136:'wo',
-	140:'gbz'
+	140: 'gbz',
+	1170: 'ckb',
+	1109: 'my',
+	1143: 'so',
+	9242: 'sr',
 }

tests/unit/test_languageHandler.py

@@ -1,20 +1,87 @@
-#tests/unit/test_languageHandler.py
-#A part of NonVisual Desktop Access (NVDA)
-#This file is covered by the GNU General Public License.
-#See the file COPYING for more details.
-#Copyright (C) 2017 NV Access Limited
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2017-2021 NV Access Limited
 
 """Unit tests for the languageHandler module.
 """
 
 import unittest
 import languageHandler
-from languageHandler import LCID_NONE
+from languageHandler import LCID_NONE, windowsPrimaryLCIDsToLocaleNames
+import locale
+import ctypes
 
 LCID_ENGLISH_US = 0x0409
+UNSUPPORTED_PYTHON_LOCALES = {
+	"an",
+	"arn",
+	"arn_CL",
+	"ba",
+	"ba_RU",
+	"bn",
+	"bo",
+	"bo_BT",
+	"ckb",
+	"co",
+	"co_FR",
+	"en_BZ",
+	"en_CB",
+	"en_JA",
+	"en_MY",
+	"en_TT",
+	"en_US",
+	"fil",
+	"fy",
+	"gbz",
+	"gbz_AF",
+	"gu",
+	"ha",
+	"hy",
+	"ii",
+	"ii_CN",
+	"kh",
+	"kh_KH",
+	"kk",
+	"kmr",
+	"kok",
+	"lb",
+	"mn",
+	"mn_CN",
+	"moh",
+	"moh_CA",
+	"my",
+	"ne",
+	"ns",
+	"ns_ZA",
+	"ps",
+	"qut",
+	"qut_GT",
+	"quz",
+	"quz_BO",
+	"quz_EC",
+	"rm",
+	"rm_CH",
+	"sa",
+	"se_FI",
+	"se_SE",
+	"so",
+	"sw",
+	"tk",
+	"tmz",
+	"tmz_DZ",
+	"ug",
+	"wen",
+	"wen_DE",
+	"wo",
+	"yo",
+}
+TRANSLATABLE_LANGS = set(l[0] for l in languageHandler.getAvailableLanguages()) - {"Windows"}
+WINDOWS_LANGS = set(locale.windows_locale.values())
+WINDOWS_LANGS.update(windowsPrimaryLCIDsToLocaleNames.values())
 
-class TestLocaleNameToWindowsLCID(unittest.TestCase):
 
+class TestLocaleNameToWindowsLCID(unittest.TestCase):
 	def test_knownLocale(self):
 		lcid = languageHandler.localeNameToWindowsLCID("en")
 		self.assertEqual(lcid, LCID_ENGLISH_US)
@@ -31,3 +98,119 @@ def test_nonStandardLocale(self):
 	def test_invalidLocale(self):
 		lcid = languageHandler.localeNameToWindowsLCID("zzzz")
 		self.assertEqual(lcid, LCID_NONE)
+
+
+class TestSetLocale(unittest.TestCase):
+	"""
+	tests setting the python locale for all possible locales set by NVDA or the System
+	"""
+
+	SUPPORTED_LOCALES = [("en", "en_US"), ("fa-IR", "fa_IR"), ("an-ES", "an_ES")]
+
+	@classmethod
+	def tearDownClass(cls):
+		languageHandler.setLanguage(languageHandler.curLang)
+
+	def setUp(self):
+		languageHandler.setLanguage(languageHandler.curLang)
+
+	def testSupportedLocales(self):
+		for localeName in self.SUPPORTED_LOCALES:
+			languageHandler.setLocale(localeName[0])
+			self.assertEqual(locale.getlocale()[0], localeName[1])
+
+	def testUnsupportedLocales(self):
+		original_locale = locale.getlocale()
+		for localeName in UNSUPPORTED_PYTHON_LOCALES:
+			with self.subTest():
+				languageHandler.setLocale(localeName)
+				self.assertEqual(locale.getlocale(), original_locale)
+
+	def testAllSupportedLangs(self):
+		for localeName in TRANSLATABLE_LANGS - UNSUPPORTED_PYTHON_LOCALES:
+			with self.subTest():
+				languageHandler.setLocale(localeName)
+				current_locale = locale.getlocale()
+
+				if localeName == "uk":
+					self.assertEqual(current_locale[0], "English_United Kingdom")
+				else:
+					pythonLang = current_locale[0].split("_")[0]
+					langOnly = localeName.split("_")[0]
+					self.assertEqual(
+						langOnly,
+						pythonLang,
+						f"full values: {localeName} {current_locale[0]}",
+					)
+
+	def testAllWindowsLangs(self):
+		prev_locale = locale.getlocale()
+		for localeName in WINDOWS_LANGS:
+			with self.subTest():
+				languageHandler.setLocale(localeName)
+				current_locale = locale.getlocale()
+				if localeName == languageHandler.curLang:
+					self.assertEqual(current_locale, prev_locale)
+				elif localeName in UNSUPPORTED_PYTHON_LOCALES:
+					self.assertEqual(current_locale, prev_locale)
+				else:
+					self.assertNotEqual(current_locale, prev_locale, localeName)
+
+
+class TestSetLanguage(unittest.TestCase):
+	"""
+	tests setting the NVDA language for all possible locales set by NVDA
+	"""
+	UNSUPPORTED_WIN_LANGUAGES = ["an", "kmr"]
+
+	def tearDown(self):
+		languageHandler.setLanguage(self._prevLang)
+
+	def __init__(self, *args, **kwargs):
+		self._prevLang = languageHandler.getLanguage()
+
+		ctypes.windll.kernel32.SetThreadLocale(0)
+		defaultThreadLocale = ctypes.windll.kernel32.GetThreadLocale()
+		self._defaultThreadLocaleName = languageHandler.windowsLCIDToLocaleName(
+			defaultThreadLocale
+		)
+
+		locale.setlocale(locale.LC_ALL, "")
+		self._defaultPythonLocale = locale.getlocale()
+
+		languageHandler.setLanguage(self._prevLang)
+		super().__init__(*args, **kwargs)
+
+	def testTranslatableLanguages(self):
+		for lang in TRANSLATABLE_LANGS:
+			langOnly = lang.split("_")[0]
+			with self.subTest():
+				languageHandler.setLanguage(lang)
+				# check curLang/translation service is set
+				self.assertEqual(languageHandler.curLang, lang)
+
+				# check Windows thread is set
+				threadLocale = ctypes.windll.kernel32.GetThreadLocale()
+				threadLocaleName = languageHandler.windowsLCIDToLocaleName(threadLocale)
+				threadLocaleLang = threadLocaleName.split("_")[0]
+				if lang in self.UNSUPPORTED_WIN_LANGUAGES:
+					self.assertEqual(self._defaultThreadLocaleName, threadLocaleName)
+				else:
+					# check that the language codes are correctly set
+					self.assertEqual(
+						langOnly,
+						threadLocaleLang,
+						f"full values: {lang} {threadLocaleName}",
+					)
+
+				# check that the python locale is set
+				python_locale = locale.getlocale()
+				if lang in UNSUPPORTED_PYTHON_LOCALES:
+					self.assertEqual(self._defaultPythonLocale, python_locale)
+				elif lang == "uk":
+					self.assertEqual(python_locale[0], "English_United Kingdom")
+				else:
+					pythonLang = python_locale[0].split("_")[0]
+					self.assertEqual(
+						langOnly, pythonLang, f"full values: {lang} {python_locale}"
+					)