Create OpenBB Platform Installer with conda constructor

.codespell.skip

@@ -1,5 +1,6 @@
 ./tests
 .git
+*.bat
 *.css
 *.csv
 *.html
@@ -8,6 +9,7 @@
 *.js
 *.json
 *.lock
+*.nsi
 *.scss
 *.txt
 *.yaml

.github/workflows/README.md

@@ -71,7 +71,7 @@ It consists of two jobs: `code-linting` and `markdown-link-check`.
 The first job, `code-linting`, runs on an Ubuntu machine and performs several linting tasks on the code in the repository, including:
 
 - Checking out the code from the repository
-- Setting up Python 3.9
+- Setting up Python 3.10
 - Installing a number of Python packages necessary for the linting tasks
 - Running `bandit` to check for security vulnerabilities
 - Running `black` to check the code formatting

.github/workflows/general-linting.yml

@@ -91,3 +91,6 @@ jobs:
       - name: json-yaml-validate
         id: json-yaml-validate
         uses: GrantBirki/[email protected]
+        with:
+          yaml_exclude_regex: "construct.yaml"
+          use_gitignore: false

.github/workflows/macos-build.yml

@@ -0,0 +1,218 @@
+name: MacOS Build
+
+env:
+  OPENBB_LOG_COLLECT: false
+  OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
+  PIP_DEFAULT_TIMEOUT: 100
+  PYTHONNOUSERSITE: 1
+
+on: workflow_dispatch
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  M1-MacOs-Build:
+    name: M1 MacOS Build
+    runs-on: [self-hosted, macos, ARM64]
+    steps:
+      # Checkout repository main branch. this allows for the commit hashes to line up
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Git Log
+        run: git log
+      # The following commands to clear previous PATHS and restore to defaults since we have to maintain the instance ourselves
+      - name: Clean Previous Path
+        run: |
+          export PATH=""
+          export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+          echo $PATH
+      # Set up caching for conda env so that the workflow runs quickly after the first time
+      - name: Setup Conda Caching
+        uses: actions/cache@v3
+        with:
+          path: ~/conda_pkgs_dir
+          key: conda-macos-3-10-${{ hashFiles('build/conda/environments/constructor.yml') }}
+      # Set up miniconda using the environment yaml file within the repo
+      - name: Setup Miniconda
+        uses: conda-incubator/[email protected]
+        with:
+          miniconda-version: "latest"
+          auto-update-conda: true
+          channels: conda-forge,defaults
+          show-channel-urls: true
+          channel-priority: flexible
+          environment-file: build/conda/environments/constructor.yml
+          activate-environment: constructor
+          auto-activate-base: false
+
+      - name: Creating Application Keychain
+        env:
+          MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }}
+          MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
+          MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
+          MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
+        run:
+          | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity
+          echo "Ensuring Keychain with same name does not exist"
+          rm -rf /Users/openbb/Library/Keychains/build.keychain-db
+          echo "Decoding certificate"
+          echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
+          echo "Creating Keychain"
+          security create-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
+          echo "Setting Default Keychain"
+          security default-keychain -s build.keychain
+          echo "Unlocking Keychain"
+          security unlock-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
+          echo "Importing Keychain"
+          security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign
+          echo "Setting Partition List"
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PWD build.keychain
+
+      - name: Create Signed Package
+        env:
+          MACOS_INSTALLER_KEYCHAIN_PWD: ${{ secrets.MACOS_INSTALLER_KEYCHAIN_PWD }}
+          MACOS_INSTALLER_SIGNING_IDENTITY_NAME: ${{ secrets.MACOS_CODESIGN_INSTALLER_IDENTITY_NAME }}
+          MACOS_APPLICATION_SIGNING_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
+        run: |
+          security unlock-keychain -p $MACOS_INSTALLER_KEYCHAIN_PWD install.keychain
+          echo "signing_identity_name: $MACOS_INSTALLER_SIGNING_IDENTITY_NAME" >> build/conda/installer/construct.yaml
+          echo "notarization_identity_name: $MACOS_APPLICATION_SIGNING_IDENTITY" >> build/conda/installer/construct.yaml
+          cd build/conda && constructor installer/. && cd ../../
+          mv build/conda/OpenBB-Platform-MacOSX-arm64.pkg OpenBB-Platform-MacOSX-arm64.pkg
+        shell: bash -l {0}
+
+      - name: Deleting Previous Keychain
+        run: |
+          echo "Deleting Previous Keychain to Clean Instance"
+          rm -rf /Users/openbb/Library/Keychains/build.keychain-db
+
+      - name: Clean up Build Artifacts
+        run: |
+          rm -rf build/conda/tmp
+
+      - name: Notarize DMG
+        env:
+          NOTARIZE_APPLE_ID: ${{ secrets.NOTARIZE_APPLE_ID }}
+          NOTARIZE_APPLE_PWD: ${{ secrets.NOTARIZE_APPLE_PWD }}
+          NOTARIZE_APPLE_TEAM_ID: ${{ secrets.NOTARIZE_APPLE_TEAM_ID }}
+        run: |
+          xcrun notarytool submit OpenBB-Platform-MacOSX-arm64.pkg --apple-id "$NOTARIZE_APPLE_ID" --password "$NOTARIZE_APPLE_PWD" --team-id "$NOTARIZE_APPLE_TEAM_ID" --wait
+
+      - name: Staple
+        run: |
+          xcrun stapler staple OpenBB-Platform-MacOSX-arm64.pkg
+
+      - name: Save Build Artifact PKG
+        uses: actions/upload-artifact@v3
+        with:
+          name: OpenBBM1.pkg
+          path: OpenBB-Platform-MacOSX-arm64.pkg
+
+      - name: Clean up Build Artifacts
+        run: |
+          rm OpenBB-Platform-MacOSX-arm64.pkg
+
+  # Job to build the MacOS Intel version of the Terminal===================================
+  Intel-MacOs-Build:
+    name: Intel MacOS Build
+    runs-on: [self-hosted, macos, x64]
+    steps:
+      # Checkout repository main branch. this allows for the commit hashes to line up
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Git Log
+        run: git log
+      # The following commands to clear previous PATHS and restore to defaults since we have to maintain the instance ourselves
+      - name: Clean Previous Path
+        run: |
+          export PATH=""
+          export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+          echo $PATH
+      # Set up caching for conda env so that the workflow runs quickly after the first time
+      - name: Setup Conda Caching
+        uses: actions/cache@v3
+        with:
+          path: ~/conda_pkgs_dir
+          key: conda-macos-3-10-${{ hashFiles('build/conda/environments/constructor.yml') }}
+      # Set up miniconda using the environment yaml file within the repo
+      - name: Setup Miniconda
+        uses: conda-incubator/[email protected]
+        with:
+          miniconda-version: "latest"
+          auto-update-conda: true
+          channels: conda-forge,defaults
+          show-channel-urls: true
+          channel-priority: flexible
+          environment-file: build/conda/environments/constructor.yml
+          activate-environment: constructor
+          auto-activate-base: false
+
+      - name: Creating Application Keychain
+        env:
+          MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }}
+          MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
+          MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
+          MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
+        run:
+          | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity
+          echo "Ensuring Keychain with same name does not exist"
+          rm -rf /Users/openbb/Library/Keychains/build.keychain-db
+          echo "Decoding certificate"
+          echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
+          echo "Creating Keychain"
+          security create-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
+          echo "Setting Default Keychain"
+          security default-keychain -s build.keychain
+          echo "Unlocking Keychain"
+          security unlock-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
+          echo "Importing Keychain"
+          security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign
+          echo "Setting Partition List"
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PWD build.keychain
+
+      - name: Create Signed Package
+        env:
+          MACOS_INSTALLER_KEYCHAIN_PWD: ${{ secrets.MACOS_INSTALLER_KEYCHAIN_PWD }}
+          MACOS_INSTALLER_SIGNING_IDENTITY_NAME: ${{ secrets.MACOS_CODESIGN_INSTALLER_IDENTITY_NAME }}
+          MACOS_APPLICATION_SIGNING_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
+        run: |
+          security unlock-keychain -p $MACOS_INSTALLER_KEYCHAIN_PWD install.keychain
+          echo "signing_identity_name: $MACOS_INSTALLER_SIGNING_IDENTITY_NAME" >> build/conda/installer/construct.yaml
+          echo "notarization_identity_name: $MACOS_APPLICATION_SIGNING_IDENTITY" >> build/conda/installer/construct.yaml
+          cd build/conda && constructor installer/. && cd ../../
+          mv build/conda/OpenBB-Platform-MacOSX-x86_64.pkg OpenBB-Platform-MacOSX-x86_64.pkg
+        shell: bash -l {0}
+
+      - name: Deleting Previous Keychain
+        run: |
+          echo "Deleting Previous Keychain to Clean Instance"
+          rm -rf /Users/openbb/Library/Keychains/build.keychain-db
+
+      - name: Clean up Build Artifacts
+        run: |
+          rm -rf build/conda/tmp
+
+      - name: Notarize DMG
+        env:
+          NOTARIZE_APPLE_ID: ${{ secrets.NOTARIZE_APPLE_ID }}
+          NOTARIZE_APPLE_PWD: ${{ secrets.NOTARIZE_APPLE_PWD }}
+          NOTARIZE_APPLE_TEAM_ID: ${{ secrets.NOTARIZE_APPLE_TEAM_ID }}
+        run: |
+          xcrun notarytool submit OpenBB-Platform-MacOSX-x86_64.pkg --apple-id "$NOTARIZE_APPLE_ID" --password "$NOTARIZE_APPLE_PWD" --team-id "$NOTARIZE_APPLE_TEAM_ID" --wait
+
+      - name: Staple
+        run: |
+          xcrun stapler staple OpenBB-Platform-MacOSX-x86_64.pkg
+
+      - name: Save Build Artifact PKG
+        uses: actions/upload-artifact@v3
+        with:
+          name: OpenBBIntel.pkg
+          path: OpenBB-Platform-MacOSX-x86_64.pkg
+
+      - name: Clean up Build Artifacts
+        run: |
+          rm OpenBB-Platform-MacOSX-x86_64.pkg

.gitignore

@@ -55,3 +55,9 @@ openbb_platform/openbb/package/*
 
 # Dev Container env
 obb/*
+
+# OpenBB Distribution
+!build/conda/installer/*.sh
+*.pkg
+*.exe
+build/conda/tmp

.pre-commit-config.yaml

@@ -3,6 +3,7 @@ repos:
     rev: v4.4.0
     hooks:
       - id: check-yaml
+        exclude: 'construct.yaml'
       - id: end-of-file-fixer
         exclude_types: [css, markdown, text, svg]
       - id: trailing-whitespace

build/conda/README.md

@@ -0,0 +1,38 @@
+# OpenBB Platform Installer
+
+The code in this folder creates a packaged installer for the OpenBB Platform, using Conda Constructor.
+
+The installers can be created for Windows (.exe), MacOS (.pkg).
+
+The installer is distributed with a Conda environment that contains Python and package management tools.
+
+The dependencies required to run the application are installed from PyPI by the constructor's post_install script. They are installed by poetry and are defined in the `installer/assets/openbb_platform_installer/pyproject.toml` file.
+
+The version of Python is defined in the `construct.yaml` file. It is set for 3.12.
+
+## Build Instructions
+
+**In order to build the installer, conda should be already installed and initialized your current shell profile.**
+
+Navigate into the `build/conda` folder and then begin.
+
+1. Create the Conda environment.
+   - `conda env create --file environments/constructor.yml`
+2. Activate the environment.
+   - `conda activate constructor`
+3. Build the installer.
+   - `constructor installer/.`
+
+## For when we have a full NSIS
+
+To check nsis file - run
+
+```batch
+miniconda3\envs\constructor\NSIS\makensis.exe /V2 assets\installer.nsi
+```
+
+gives better output for errors
+
+```batch
+miniconda3\envs\constructor\NSIS\makensis.exe /V4 assets\installer.nsi
+```

build/conda/environments/constructor.yml

@@ -0,0 +1,10 @@
+name: constructor
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - conda-build
+  - constructor>=3.9.3
+  - six
+  - urllib3
+  - requests