Py3.8: Transparently use a Python virtual environment under the hood

.gitignore

@@ -42,3 +42,4 @@ tests/unit/nvda.ini
 source/locale/*/cldr.dic
 .vscode
 .vs
+.venv

.gitmodules

@@ -15,28 +15,12 @@
 	path = include/sonic
 	url = https://github.com/waywardgeek/sonic.git
 	ignore = untracked
-[submodule "include/pyserial"]
-	path = include/pyserial
-	url = https://github.com/pyserial/pyserial.git
-	ignore = untracked
-[submodule "include/comtypes"]
-	path = include/comtypes
-	url = https://github.com/nvaccess/comtypes-bin
-[submodule "include/scons"]
-	path = include/scons
-	url = https://github.com/SCons/scons
 [submodule "include/ia2"]
 	path = include/ia2
 	url = https://github.com/LinuxA11y/IAccessible2.git
-[submodule "include/wxPython"]
-	path = include/wxPython
-	url = https://github.com/nvaccess/wxPython-bin.git
 [submodule "include/configobj"]
 	path = include/configobj
 	url = https://github.com/DiffSK/configobj.git
-[submodule "include/py2exe"]
-	path = include/py2exe
-	url = https://github.com/nvaccess/py2exe-bin
 [submodule "include/javaAccessBridge32"]
 	path = include/javaAccessBridge32
 	url = https://github.com/nvaccess/javaAccessBridge32-bin.git

appveyor.yml

@@ -11,7 +11,7 @@ branches:
   - /release-.*/
 
 environment:
- PY_PYTHON: 3.7-32
+ PY_PYTHON: 3.8-32
  encFileKey:
   secure: ekOvuyywHuDdGZmRmoj+b3jfrq39A2xlx4RD5ZUGd/8=
  mozillaSymsAuthToken:
@@ -59,6 +59,12 @@ init:
 clone_depth: 1
 
 install:
+ # Ensure we have the latest version of pip
+ - py -m pip install --upgrade pip
+ # Activate the NvDA build system Python virtual environment, creating / updating it if necessary.
+ - .\venvUtils\ensureAndActivate.bat
+ # Save an exact list of the actual Python packages and their versions that got installed, to aide in reproducing a build
+ - py -m pip freeze >output\pip_packages.txt
  - cd appveyor
  # Decrypt files.
  - ps: |
@@ -95,12 +101,12 @@ build_script:
      Set-AppveyorBuildVariable "sconsOutTargets" $sconsOutTargets
      Set-AppveyorBuildVariable "sconsArgs" $sconsArgs
  - 'echo scons args: %sconsArgs%'
- - py scons.py source %sconsArgs%
+ - scons source %sconsArgs%
  # We don't need launcher to run checkPot, so run the checkPot before launcher.
- - py scons.py checkPot %sconsArgs%
+ - scons checkPot %sconsArgs%
  # The pot gets built by tests, but we don't actually need it as a build artifact.
  - 'echo scons output targets: %sconsOutTargets%'
- - py scons.py %sconsOutTargets% %sconsArgs%
+ - scons %sconsOutTargets% %sconsArgs%
  # Build symbol store.
  - ps: |
      foreach ($syms in
@@ -118,9 +124,6 @@ build_script:
  - cd ..
 
 before_test:
- # install required packages
- - py -m pip install --upgrade pip
- - py -m pip install -r tests/system/requirements.txt -r tests/lint/lintInstall/requirements.txt
  - mkdir testOutput
  - mkdir testOutput\unit
  - mkdir testOutput\system
@@ -153,7 +156,7 @@ test_script:
      $errorCode=0
      $outDir = (Resolve-Path .\testOutput\unit\)
      $unitTestsXml = "$outDir\unitTests.xml"
-     py -m nose -sv --with-xunit --xunit-file="$unitTestsXml" ./tests/unit
+     py -m nose -sv --with-xunit --xunit-file="$unitTestsXml" --traverse-namespace -w ./tests/unit
      if($LastExitCode -ne 0) {
       $errorCode=$LastExitCode
       Add-AppveyorMessage "Unit test failure"

devDocs/sconscript

@@ -19,10 +19,6 @@ Import("env", "outputDir", "sourceDir", "t2tBuildConf")
 
 env = env.Clone()
 
-devDocsInstall = env.SConscript("devDocsInstall/sconscript", exports=["env"])
-env.Depends(devDocsInstall, env.Dir("devDocs/devDocsInstall/requirements.txt"))
-env.Alias("devDocsInstall", devDocsInstall)
-
 devDocsOutputDir=outputDir.Dir('devDocs')
 
 #Build the developer guide and move it to the output directory
@@ -69,7 +65,6 @@ sphinxAPIDocs = env.Command(
 		] + [f"{sourceDir}\\{f}" for f in ignorePaths]
 	]
 )
-env.Depends(sphinxAPIDocs, devDocsInstall)
 sphinxHtml = env.Command(
 	"_build",
 	sphinxAPIDocs,

nvdaHelper/espeak/sconscript

@@ -1,4 +1,4 @@
-from include.scons.SCons.Util import CLVar
+from SCons.Util import CLVar
 
 Import([
 	'thirdPartyEnv',

nvdaHelper/liblouis/sconscript

@@ -17,8 +17,8 @@ import re
 import glob
 from SCons.Tool.MSCommon.vc import find_vc_pdir
 import typing
-from include.scons.SCons.Environment import Environment
-from include.scons.SCons.Environment import Base
+from SCons.Environment import Environment
+from SCons.Environment import Base
 
 Import([
 	"thirdPartyEnv",

readme.md

@@ -55,9 +55,8 @@ The NVDA source depends on several other packages to run correctly.
 ### Installed Dependencies
 The following dependencies need to be installed on your system:
 
-* [Python](https://www.python.org/), version 3.7, 32 bit
+* [Python](https://www.python.org/), version 3.8, 32 bit
 	* Use latest minor version if possible.
-	* Don't use `3.7.6` it causes an error while building, see #10696.
 * Microsoft Visual Studio 2019 Community, Version 16.3 or later:
 	* Download from https://visualstudio.microsoft.com/vs/
 	* When installing Visual Studio, you need to enable the following:
@@ -75,20 +74,17 @@ The following dependencies need to be installed on your system:
 
 
 ### Git Submodules
-Most of the dependencies are contained in Git submodules.
+Some of the dependencies are contained in Git submodules.
 If you didn't pass the `--recursive` option to git clone, you will need to run `git submodule update --init`.
 Whenever a required submodule commit changes (e.g. after git pull), you will need to run `git submodule update`.
 If you aren't sure, run `git submodule update` after every git pull, merge or checkout.
 
 For reference, the following run time dependencies are included in Git submodules:
 
-* [comtypes](https://github.com/enthought/comtypes), version 1.1.7
-* [wxPython](https://www.wxpython.org/), version 4.0.3
 * [eSpeak NG](https://github.com/espeak-ng/espeak-ng), version 1.51-dev commit 82d5b7b04
 * [Sonic](https://github.com/waywardgeek/sonic), commit 4f8c1d11
 * [IAccessible2](https://wiki.linuxfoundation.org/accessibility/iaccessible2/start), commit cbc1f29631780
 * [ConfigObj](https://github.com/DiffSK/configobj), commit f9a265c
-* [Six](https://pypi.python.org/pypi/six), version 1.12.0, required by wxPython and ConfigObj
 * [liblouis](http://www.liblouis.org/), version 3.16.1
 * [Unicode Common Locale Data Repository (CLDR)](http://cldr.unicode.org/), version 38.1
 * NVDA images and sounds
@@ -97,37 +93,27 @@ For reference, the following run time dependencies are included in Git submodule
 * [MinHook](https://github.com/RaMMicHaeL/minhook), tagged version 1.2.2
 * brlapi Python bindings, version 0.8 or later, distributed with [BRLTTY for Windows](https://brltty.app/download.html), version 6.1
 * lilli.dll, version 2.1.0.0
-* [pySerial](https://pypi.python.org/pypi/pyserial), version 3.4
 * [Python interface to FTDI driver/chip](http://fluidmotion.dyndns.org/zenphoto/index.php?p=news&title=Python-interface-to-FTDI-driver-chip)
 * Java Access Bridge 32 bit, from Zulu Community OpenJDK build 13.0.1+10Zulu (13.28.11)
 
-Additionally, the following build time dependencies are included in Git submodules:
+Additionally, the following build time dependencies are included in the miscDeps git submodule: 
 
-* [Py2Exe](https://github.com/albertosottile/py2exe/), version 0.9.3.2 commit b372a8e
-* [Python Windows Extensions](https://sourceforge.net/projects/pywin32/ ), build 224, required by py2exe
 * [txt2tags](https://txt2tags.org/), version 2.5
-* [SCons](https://www.scons.org/), version 4.0.1
 * [Nulsoft Install System](https://nsis.sourceforge.io/Main_Page/), version 2.51
 * [NSIS UAC plug-in](https://nsis.sourceforge.io/UAC_plug-in), version 0.2.4, ansi
 * xgettext and msgfmt from [GNU gettext](https://sourceforge.net/projects/cppcms/files/boost_locale/gettext_for_windows/)
 * [Boost Optional (stand-alone header)](https://github.com/akrzemi1/Optional), from commit [3922965](https://github.com/akrzemi1/Optional/commit/3922965396fc455c6b1770374b9b4111799588a9)
 
-### Other Dependencies
-To lint using Flake 8 locally using our SCons integration, some dependencies are installed (automatically) via pip.
-Although this [must be run manually](#linting-your-changes), developers may wish to first configure a Python Virtual Environment to ensure their general install is not affected.
-* Flake8
-* Flake8-tabs
-
-
 The following dependencies aren't needed by most people, and are not included in Git submodules:
-
-* To generate developer documentation: [Sphinx](http://sphinx-doc.org/), version 3.4.1
 * To generate developer documentation for nvdaHelper: [Doxygen Windows installer](http://www.doxygen.nl/download.html), version 1.8.15:
 * When you are using Visual Studio Code as your integrated development environment of preference, you can make use of our [prepopulated workspace configuration](https://github.com/nvaccess/vscode-nvda/) for [Visual Studio Code](https://code.visualstudio.com/).
 	While this VSCode project is not included as a submodule in the NVDA repository, you can easily check out the workspace configuration in your repository by executing the following from the root of the repository.
 
 	```git clone https://github.com/nvaccess/vscode-nvda.git .vscode```
 
+### Python dependencies
+NVDA and its build system also depend on an extensive list of Python packages. They are all listed with their specific versions in a requirements.txt file in the root of this repository. However, the build system takes care of fetching these itself when needed. these packages will be installed into an isolated Python virtual environment within this repository, and will not affect your system-wide set of packages.
+
 ## Preparing the Source Tree
 Before you can run the NVDA source code, you must prepare the source tree.
 You do this by opening a command prompt, changing to the root of the NVDA source distribution and typing:
@@ -173,13 +159,14 @@ By default, builds also do not use any compiler optimizations.
 Please see the `release` keyword argument for what compiler optimizations it will enable.
 
 ## Running the Source Code
-Most developers run directly from source by:
+Most developers run directly from source by executing:
+```
+runnvda.bat
+```
+This command will block until NVDA is exited. However, you can run it in the background with:
 ```
-cd source
-pythonw.exe nvda.pyw
+start /b runnvda.bat
 ```
-Note: Since NVDA is a Windows application (rather than command line), it is best to run it with `pythonw.exe`.
-However, if during development you encounter an error early in the startup of NVDA, you can use `python.exe` which is likely to give more information about the error.
 
 To view help on the arguments that NVDA will accept, use the `-h` or `--help` option.
 These arguments are also documented in the user guide.

requirements.txt

@@ -0,0 +1,26 @@
+# NVDA's build system is SCons
+SCons==4.1.0.post1
+
+# NVDA's runtime dependencies
+comtypes==1.1.7
+pyserial==3.5
+wxPython==4.1.1
+
+# Packaging NVDA
+py2exe==0.10.1.0
+
+# For building developer documentation
+sphinx==3.4.1
+sphinx_rtd_theme
+
+# Requirements for automated linting
+flake8 ~= 3.7.7
+flake8-tabs == 2.1.0
+
+# Requirements for unit tests
+nose==1.3.7
+
+# Requirements for system tests
+robotframework==3.2.2
+robotremoteserver==1.1
+robotframework-screencaplibrary==1.5.0

runnvda.bat

@@ -0,0 +1,2 @@
+@echo off
+call "%~dp0\venvUtils\venvCmd.bat" py "%~dp0\source\nvda.pyw" %*

scons.bat

@@ -1,13 +1,3 @@
 @echo off
-rem We need this script because .py probably isn't in pathext.
-rem We can't just call python -c because it may not be in the path.
-rem Instead, find the python launcher (installed by python 3)
-where py 1>nul 2>&1
-if "%ERRORLEVEL%" == "0" (
-	rem Python launcher is present in the PATH
-	rem Call python 3.7 for 32 bits
-	py -3.7-32 "%~dp0\scons.py" %*
-) else (
-	rem Python registers itself with the .py extension, so call scons.py.
-	"%~dp0\scons.py" %*
-)
+rem Executes SScons within the NVDA build system's Python virtual environment.
+"%~dp0\venvUtils\venvCmd.bat" py -m SCons %*

sconstruct

@@ -7,9 +7,19 @@ import sys
 import os
 import platform
 
+# Ensure we are inside the NVDA build system's Python virtual environment.
+nvdaVenv = os.getenv("NVDA_VENV")
+virtualEnv = os.getenv("VIRTUAL_ENV")
+if not nvdaVenv or nvdaVenv != virtualEnv:
+	print(
+		"Error: SCons cannot detect  the NVDA build system Python virtual environment.\n"
+		"SCons must be executed using scons.bat in the root of this repository."
+	)
+	sys.exit(1)
+
 # Variables for storing required version of Python, and the version which is used to run this script.
 requiredPythonMajor ="3"
-requiredPythonMinor = "7"
+requiredPythonMinor = "8"
 requiredPythonArchitecture = "32bit"
 installedPythonMajor = str(sys.version_info.major)
 installedPythonMinor = str(sys.version_info.minor)
@@ -34,10 +44,6 @@ if (
 		requiredPythonArchitecture
 	)
 	)
-if sys.version_info.micro == 6:
-	# #10696: Building with Python 3.7.6 fails. Innform user and exit.
-	Py376FailMsg = "Building with Python 3.7.6 is not possible.\nPlease use more  recent version of Python 3."
-	raise RuntimeError(Py376FailMsg)
 sourceEnvPath = os.path.abspath(os.path.join(Dir('.').srcnode().path, "source"))
 sys.path.append(sourceEnvPath)
 import sourceEnv

source/core.py

@@ -560,6 +560,15 @@ def run(self):
 	else:
 		log.debug("initializing updateCheck")
 		updateCheck.initialize()
+	# If running from source, try to disconnect from the console we may have been executed in.
+	# NVDA may reconnect to read it later,
+	# but it is better to assume we are not connected to anything at the start.
+	try:
+		ctypes.windll.kernel32.FreeConsole()
+	except WindowsError:
+		pass
+	else:
+		log.debug("Freed from parent console process.")
 	log.info("NVDA initialized")
 	postNvdaStartup.notify()