talipot-python: Create a virtualenv for the embedded interpreter (WIP)

.github/workflows/code-coverage.yml

@@ -66,7 +66,7 @@ jobs:
           -DTALIPOT_CODE_COVERAGE=ON
       - name: Talipot build
         working-directory: ./build
-        run: ninja -j4
+        run: ninja -j4 install
       - name: Run Talipot unit tests
         working-directory: ./build
         run: xvfb-run ninja tests

.github/workflows/macos-homebrew-build.yml

@@ -114,6 +114,7 @@ jobs:
           hdiutil attach Talipot*.dmg
           cp -r /Volumes/Talipot*/Talipot*.app /Applications/
           sudo xattr -r -d com.apple.quarantine /Applications/Talipot*.app
+          rm -rf ~/.Talipot*
           /Applications/Talipot*.app/Contents/MacOS/Talipot \
             --check-application-starts \
             --debug-plugins-load

.github/workflows/macos-macports-build.yml

@@ -125,6 +125,7 @@ jobs:
           hdiutil attach Talipot*.dmg
           cp -r /Volumes/Talipot*/Talipot*.app /Applications/
           sudo xattr -r -d com.apple.quarantine /Applications/Talipot*.app
+          rm -rf ~/.Talipot*
           /Applications/Talipot*.app/Contents/MacOS/Talipot \
             --check-application-starts \
             --debug-plugins-load

.github/workflows/windows-mingw64-build.yml

@@ -7,6 +7,8 @@ jobs:
   mingw64:
     name: ${{ matrix.config.name }}
     runs-on: windows-latest
+    env:
+      PYTHON_VERSION: "3.13"
     defaults:
       run:
         shell: msys2 {0}
@@ -40,16 +42,22 @@ jobs:
             mingw-w64-${{ matrix.config.arch }}-qhull
             mingw-w64-${{ matrix.config.arch }}-graphviz
             mingw-w64-${{ matrix.config.arch }}-libgit2
-            mingw-w64-${{ matrix.config.arch }}-python
             mingw-w64-${{ matrix.config.arch }}-cppunit
             mingw-w64-${{ matrix.config.arch }}-fontconfig
             mingw-w64-${{ matrix.config.arch }}-freetype
             mingw-w64-${{ matrix.config.arch }}-fribidi
             mingw-w64-${{ matrix.config.arch }}-glew
             mingw-w64-${{ matrix.config.arch }}-qt5
             mingw-w64-${{ matrix.config.arch }}-quazip
-            mingw-w64-${{ matrix.config.arch }}-python-sphinx
-            mingw-w64-${{ matrix.config.arch }}-sip
+      - name: Install Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        id: python-install
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+      - name: Install sip and sphinx
+        run: |
+          export PATH=$Python3_ROOT_DIR\Scripts:$PATH
+          pip install sip sphinx
       - name: Prepare ccache timestamp
         id: get-current-date
         run: |
@@ -75,7 +83,7 @@ jobs:
           -DCMAKE_BUILD_TYPE=Release
           -DCMAKE_NEED_RESPONSE=ON
           -DCMAKE_INSTALL_PREFIX=$PWD/install
-          -DPython3_EXECUTABLE=/${{ matrix.config.msystem }}/bin/python3
+          -DPython3_EXECUTABLE=$Python3_ROOT_DIR/python.exe
           -DTALIPOT_BUILD_TESTS=ON
           -DTALIPOT_USE_CCACHE=ON ..
       - name: Talipot build

bundlers/linux/make_appimage_bundle.sh.in

@@ -103,6 +103,7 @@ export LD_LIBRARY_PATH=${QT_INSTALL_LIBS_DIR}:${LD_LIBRARY_PATH}
 export LD_LIBRARY_PATH=$(dirname @PYTHON_LIBRARY@):$LD_LIBRARY_PATH
 
 # add Python environment
+cp -v -p $TALIPOT_INSTALL_DIR/bin/python3* $BUNDLE_BIN_DIR
 rm -f $BUNDLE_LIB_DIR/libpython*
 PYTHON_LIB=$(ldd $(ls $BUNDLE_LIB_DIR/libtalipot-python*) | \
   grep libpython | awk '{print $3}')
@@ -125,9 +126,16 @@ if [ "$PYTHON_LIB" != "" ]; then
   fi
   echo "copying $(dirname $PYTHON_LIB)/$PYTHON_PACKAGE_DIR files into \
   $PYTHON_PACKAGE_BUNDLE_DIR"
-  find . \( -type f \) \( ! -name "*.pyc" \) \( ! -name "*.pyo" \) -exec \
+  find . \( -type f \) \( ! -name "*.pyo" \) -exec \
     cp --parents --preserve=mode {} $PYTHON_PACKAGE_BUNDLE_DIR \;
   popd > /dev/null 2>&1
+  mkdir -p $PYTHON_PACKAGE_BUNDLE_DIR/ensurepip/_bundled
+  pushd $PYTHON_PACKAGE_BUNDLE_DIR/ensurepip/_bundled > /dev/null 2>&1
+  setuptools_version=$(python3.11 -c "import ensurepip;print(ensurepip._SETUPTOOLS_VERSION)")
+  pip_version=$(python3.11 -c "import ensurepip;print(ensurepip._PIP_VERSION)")
+  wget https://pypi.debian.net/setuptools/setuptools-${setuptools_version}-py3-none-any.whl
+  wget https://pypi.debian.net/pip/pip-${pip_version}-py3-none-any.whl
+  popd > /dev/null 2>&1
 fi
 
 # copy required shared libs using linuxdeployqt tool

library/talipot-python/include/talipot/PythonInterpreter.h

@@ -1,6 +1,6 @@
 /**
  *
- * Copyright (C) 2019-2021  The Talipot developers
+ * Copyright (C) 2019-2024  The Talipot developers
  *
  * Talipot is a fork of Tulip, created by David Auber
  * and the Tulip development Team from LaBRI, University of Bordeaux
@@ -51,6 +51,7 @@ class TLP_PYTHON_SCOPE PythonInterpreter : public QObject, public Singleton<Pyth
 
   void setDefaultConsoleWidget(QAbstractScrollArea *consoleWidget);
   void setConsoleWidget(QAbstractScrollArea *consoleWidget);
+  void setupVirtualEnv();
 
   bool _wasInit;
   bool _runningScript;

library/talipot-python/src/PythonInterpreter.cpp

@@ -1,6 +1,6 @@
 /**
  *
- * Copyright (C) 2019-2024  The Talipot developers
+ * Copyright (C) 2019-2025  The Talipot developers
  *
  * Talipot is a fork of Tulip, created by David Auber
  * and the Tulip development Team from LaBRI, University of Bordeaux
@@ -27,6 +27,7 @@
 #if defined(__MINGW32__)
 #include <QSslSocket>
 #endif
+#include <QFileInfo>
 
 #include <talipot/Release.h>
 #include <talipot/PythonVersionChecker.h>
@@ -133,8 +134,10 @@ int tracefunc(PyObject *, PyFrameObject *, int what, PyObject *) {
 const QString PythonInterpreter::pythonPluginsPath(tlpStringToQString(tlp::TalipotLibDir) +
                                                    "talipot/python/");
 
-const QString PythonInterpreter::pythonPluginsPathHome(QDir::homePath() + "/.Talipot-" +
-                                                       TALIPOT_MM_VERSION + "/plugins/python");
+const QString talipotUserDirectory = QDir::homePath() + "/.Talipot-" + TALIPOT_MM_VERSION;
+const QString talipotVenvDirectory =
+    talipotUserDirectory + "/venv" + PythonVersionChecker::compiledVersion();
+const QString PythonInterpreter::pythonPluginsPathHome(talipotUserDirectory + "/plugins/python");
 
 const char PythonInterpreter::pythonReservedCharacters[] = {
     '#', '%',  '/',  '+', '-', '&', '*', '<', '>', '|', '~', '^', '=',
@@ -211,6 +214,10 @@ PythonInterpreter::PythonInterpreter()
 #endif
   }
 
+#ifndef MSYS2_PYTHON
+  setupVirtualEnv();
+#endif
+
   holdGIL();
 
   importModule("sys");
@@ -305,6 +312,59 @@ PythonInterpreter::PythonInterpreter()
   }
 
   releaseGIL();
+
+#ifndef MSYS2_PYTHON
+  setupVirtualEnv();
+#endif
+}
+
+void PythonInterpreter::setupVirtualEnv() {
+#ifdef Q_OS_WIN
+  if (!QFileInfo(talipotVenvDirectory + "/Scripts/pip.exe").exists()) {
+#else
+  if (!QFileInfo(talipotVenvDirectory + "/bin/pip").exists()) {
+#endif
+    runString(QString(R"(
+import os
+import platform
+import sys
+import venv
+python_command = 'python3'
+if platform.system() == 'Windows':
+    python_command = 'python.exe'
+sys._base_executable = os.path.join('%1', python_command)
+os.environ['LD_LIBRARY_PATH'] = '%2';
+venv.create('%3', with_pip=True)
+)")
+                  .arg(QApplication::applicationDirPath(), tlpStringToQString(tlp::TalipotLibDir),
+                       talipotVenvDirectory));
+  }
+
+  runString(QString(R"(
+import os
+import platform
+import sys
+
+base = '%1'
+if platform.system() == 'Windows':
+    site_packages = os.path.join(base, 'Lib', 'site-packages')
+else:
+    site_packages = os.path.join(
+        base, 'lib',
+        'python%s.%s' % (sys.version_info.major, sys.version_info.minor),
+        'site-packages')
+prev_sys_path = list(sys.path)
+import site
+site.addsitedir(site_packages)
+sys.real_prefix = sys.prefix
+sys.prefix = base
+new_sys_path = []
+for item in list(sys.path):
+    if item not in prev_sys_path:
+        new_sys_path.append(item)
+        sys.path.remove(item)
+sys.path[:0] = new_sys_path)")
+                .arg(talipotVenvDirectory));
 }
 
 PythonInterpreter::~PythonInterpreter() {

software/talipot/CMakeLists.txt

@@ -269,3 +269,23 @@ IF(LINUX)
           OUTPUT_QUIET ERROR_QUIET)")
   ENDIF(TALIPOT_LINUX_DESKTOP_REGISTRATION)
 ENDIF(LINUX)
+
+IF(NOT MSYS2_PYTHON)
+  STRING(REPLACE "\\" "/" PYTHON_EXE_PATH "${PYTHON_EXECUTABLE}")
+  GET_FILENAME_COMPONENT(PYTHON_EXE_NAME "${PYTHON_EXECUTABLE}" NAME)
+
+  INSTALL(
+    CODE "
+FILE(COPY \"${PYTHON_EXE_PATH}\"
+     DESTINATION \"\${CMAKE_INSTALL_PREFIX}/bin/\" FOLLOW_SYMLINK_CHAIN)
+IF(NOT WIN32
+  AND NOT ${PYTHON_EXE_NAME} STREQUAL python3
+  AND NOT EXISTS ${CMAKE_INSTALL_PREFIX}/bin/python3)
+  FILE(CREATE_LINK ${CMAKE_INSTALL_PREFIX}/bin/${PYTHON_EXE_NAME}
+       ${CMAKE_INSTALL_PREFIX}/bin/python3 SYMBOLIC)
+ENDIF(
+  NOT WIN32
+  AND NOT ${PYTHON_EXE_NAME} STREQUAL python3
+  AND NOT EXISTS ${CMAKE_INSTALL_PREFIX}/bin/python3)
+")
+ENDIF(NOT MSYS2_PYTHON)