Validate ExecuTorch path for more models in CI

pytorch · Apr 9, 2024 · a72e61a · a72e61a
1 parent 67cad1f
commit a72e61a
Show file tree

Hide file tree

Showing 4 changed files with 285 additions and 26 deletions.
diff --git a/.ci/scripts/download_checkpoints.sh b/.ci/scripts/download_checkpoints.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+CHECKPOINT_NAME="$1"
+RESOURCES_STRING="$2"
+
+pushd "${LLAMA_FAST_ROOT}" || exit
+
+# Create the directory for the checkpoint
+mkdir -p "checkpoints/${CHECKPOINT_NAME}"
+cd "checkpoints/${CHECKPOINT_NAME}" || exit
+
+# Download all resources
+IFS=',' read -r -a resources <<< "$RESOURCES_STRING"
+for resource in "${resources[@]}"
+do
+  echo "Downloading: $resource"
+  wget "$resource"
+done
+
+popd || exit
diff --git a/.github/workflows/executorch.yml b/.github/workflows/executorch.yml
@@ -0,0 +1,110 @@
+name: executorch
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  gather-models:
+    runs-on: ubuntu-22.04
+    outputs:
+      models: ${{ steps.gather-models.outputs.models }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'false'
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Extract the list of models to test
+        id: gather-models
+        run: |
+          set -eux
+          PYTHONPATH="${PWD}" python ./scripts/gather_test_models.py
+  test-models-mobile-linux:
+    name: test-models-mobile-linux
+    needs: gather-models
+    strategy:
+      matrix: ${{ fromJSON(needs.gather-models.outputs.models) }}
+    runs-on: "32-core-ubuntu"
+    env:
+      LLAMA_FAST_ROOT: ${{ github.workspace }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Print machine info
+        run: |
+          echo "Platform: $(uname -s)"
+          echo "Processor type: $(uname -p)"
+          echo "Number of processors: $(nproc)"
+          echo "RAM: $(free -h | awk '/Mem/ {print $2}')"
+          echo "Disk space: $(df -h --total | awk '/total/ {print $2}')"
+      - name: Install ExecuTorch
+        run: |
+          bash ${LLAMA_FAST_ROOT}/scripts/install_et.sh
+      - name: Download checkpoints
+        run: |
+          bash ${LLAMA_FAST_ROOT}/.ci/scripts/download_checkpoints.sh ${{ matrix.checkpoint_name }} "${{ matrix.resources }}"
+      - name: Run inferencec
+        run: |
+          pushd ${LLAMA_FAST_ROOT}
+          export CHECKPOINT_PATH=${LLAMA_FAST_ROOT}/checkpoints/${CHECKPOINT_NAME}/${CHECKPOINT_NAME}.pt
+          export MODEL_NAME=${CHECKPOINT_NAME}
+
+          python generate.py --checkpoint-path ${CHECKPOINT_PATH} --temperature 0 > ${PWD}/output_eager
+          cat ${PWD}/output_eager
+
+          python export.py --checkpoint-path ${CHECKPOINT_PATH} --output-pte-path ${PWD}/${MODEL_NAME}.pte
+          python generate.py --checkpoint-path ${CHECKPOINT_PATH} --temperature 0 --pte-path ${PWD}/${MODEL_NAME}.pte  > ${PWD}/output_et
+          cat ${PWD}/output_et
+
+          echo "Tests complete."
+  test-models-mobile-macos:
+    name: test-models-mobile-macos
+    needs: gather-models
+    strategy:
+      matrix: ${{ fromJSON(needs.gather-models.outputs.models) }}
+    runs-on: "macos-12"
+    env:
+      LLAMA_FAST_ROOT: ${{ github.workspace }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Print machine info
+        run: |
+          echo "Platform: $(uname -s)"
+          echo "Processor type: $(uname -p)"
+          echo "Number of processors: $(sysctl -n hw.ncpu)"
+          echo "RAM: $(sysctl -n hw.memsize | awk '{print $0/1024/1024/1024"g"}')"
+          echo "Disk space: $(df -h / | awk 'NR==2 {print $2}')"
+      - name: Install ExecuTorch
+        run: |
+          bash ${LLAMA_FAST_ROOT}/scripts/install_et.sh
+      - name: Download checkpoints
+        run: |
+          bash ${LLAMA_FAST_ROOT}/.ci/scripts/download_checkpoints.sh ${{ matrix.checkpoint_name }} "${{ matrix.resources }}"
+      - name: Run inference
+        run: |
+          pushd ${LLAMA_FAST_ROOT}
+          export CHECKPOINT_PATH=${LLAMA_FAST_ROOT}/checkpoints/${CHECKPOINT_NAME}/${CHECKPOINT_NAME}.pt
+          export MODEL_NAME=${CHECKPOINT_NAME}
+
+          python generate.py --checkpoint-path ${CHECKPOINT_PATH} --temperature 0 > ${PWD}/output_eager
+          cat ${PWD}/output_eager
+
+          python export.py --checkpoint-path ${CHECKPOINT_PATH} --output-pte-path ${PWD}/${MODEL_NAME}.pte
+          python generate.py --checkpoint-path ${CHECKPOINT_PATH} --temperature 0 --pte-path ${PWD}/${MODEL_NAME}.pte  > ${PWD}/output_et
+          cat ${PWD}/output_et
+
+          echo "Tests complete."
diff --git a/scripts/gather_test_models.py b/scripts/gather_test_models.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import itertools
+import json
+import os
+from typing import Any
+
+
+MODEL_CHECKPOINT_NAMES = {
+    "stories15M": "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt,https://github.com/karpathy/llama2.c/raw/master/tokenizer.model,https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin",
+    "stories42M": "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories42M.pt,https://github.com/karpathy/llama2.c/raw/master/tokenizer.model,https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin",
+    "stories100M": "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories100M.pt,https://github.com/karpathy/llama2.c/raw/master/tokenizer.model,https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin",
+    # "open_llama_7b",
+    # "Llama-2-7b-chat-hf",
+    # "Llama-2-13b-chat-hf",
+    # "Llama-2-70b-chat-hf",
+    # "CodeLlama-7b-Python-hf",
+    # "CodeLlama-34b-Python-hf",
+    # "Mistral-7B-v0.1",
+    # "Mistral-7B-Instruct-v0.1",
+    # "Mistral-7B-Instruct-v0.2",
+}
+
+
+DEFAULT_RUNNERS = {
+    "linux": "32-core-ubuntu",
+    "macos": "macos-m1-12",
+}
+
+
+def parse_args() -> Any:
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser("Gather all models to test on CI for the target OS")
+    parser.add_argument(
+        "--target-os",
+        type=str,
+        default="linux",
+        help="the target OS",
+    )
+
+    return parser.parse_args()
+
+
+def set_output(name: str, val: Any) -> None:
+    """
+    Set the GitHb output so that it can be accessed by other jobs
+    """
+    print(f"Setting {val} to GitHub output")
+
+    if os.getenv("GITHUB_OUTPUT"):
+        with open(str(os.getenv("GITHUB_OUTPUT")), "a") as env:
+            print(f"{name}={val}", file=env)
+    else:
+        print(f"::set-output name={name}::{val}")
+
+
+def export_models_for_ci() -> dict[str, dict]:
+    """
+    This gathers all the models that we want to test on GitHub OSS CI
+    """
+    args = parse_args()
+    target_os = args.target_os
+
+    # This is the JSON syntax for configuration matrix used by GitHub
+    # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
+    models = {"include": []}
+
+    for checkpoint_name, export_config in itertools.product(
+        MODEL_CHECKPOINT_NAMES,
+        [
+            "xnnpack",
+        ]
+    ):
+        record = {
+            "checkpoint_name": checkpoint_name,
+            "resources": MODEL_CHECKPOINT_NAMES[checkpoint_name],
+            "export_config": export_config,
+            "runner": DEFAULT_RUNNERS.get(target_os),
+            "timeout": 90,
+        }
+
+        models["include"].append(record)
+
+    print(json.dumps(models))
+    set_output("models", json.dumps(models))
+
+
+if __name__ == "__main__":
+    export_models_for_ci()
diff --git a/scripts/install_et.sh b/scripts/install_et.sh
@@ -1,26 +1,53 @@
-cd ${LLAMA_FAST_ROOT}
-echo "Inside: $LLAMA_FAST_ROOT"
-
-echo "Cloning executorch to ${LLAMA_FAST_ROOT}/build/src"
-rm -rf ${LLAMA_FAST_ROOT}/build
-mkdir -p ${LLAMA_FAST_ROOT}/build/src
-cd ${LLAMA_FAST_ROOT}/build/src
-git clone https://github.com/pytorch/executorch.git
-cd executorch
-echo "Install executorch: submodule update"
-git submodule sync
-git submodule update --init
-
-echo "Applying fixes"
-cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/module.cpp ${LLAMA_FAST_ROOT}/build/src/executorch/extension/module/module.cpp # ET uses non-standard C++ that does not compile in GCC
-cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/managed_tensor.h ${LLAMA_FAST_ROOT}/build/src/executorch/extension/runner_util/managed_tensor.h # ET is missing headers for vector/memory.  This causes downstream issues when building runner-et.
-
-echo "Building and installing python libraries"
-./install_requirements.sh --pybind xnnpack
-
-echo "Building and installing C++ libraries"
-echo "Inside: ${PWD}"
-mkdir cmake-out
-cmake -DCMAKE_BUILD_TYPE=Release -DEXECUTORCH_BUILD_OPTIMIZED=ON -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON -DEXECUTORCH_BUILD_XNNPACK=ON -S . -B cmake-out -G Ninja
-cmake --build cmake-out
-cmake --install cmake-out --prefix ${LLAMA_FAST_ROOT}/build/install
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+install_pip_dependencies() {
+  echo "Intalling common pip packages"
+
+  pip install wheel
+  pip install cmake
+  pip install ninja
+  pip install zstd
+  pushd ${LLAMA_FAST_ROOT}
+  pip install -r ./requirements.txt
+  popd
+}
+
+install_executorch() {
+  echo "Cloning executorch to ${LLAMA_FAST_ROOT}/build/src"
+  rm -rf ${LLAMA_FAST_ROOT}/build
+  mkdir -p ${LLAMA_FAST_ROOT}/build/src
+  pushd ${LLAMA_FAST_ROOT}/build/src
+  git clone https://github.com/pytorch/executorch.git
+  cd executorch
+  echo "Install executorch: submodule update"
+  git submodule sync
+  git submodule update --init
+
+  echo "Applying fixes"
+  cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/module.cpp ${LLAMA_FAST_ROOT}/build/src/executorch/extension/module/module.cpp # ET uses non-standard C++ that does not compile in GCC
+  cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/managed_tensor.h ${LLAMA_FAST_ROOT}/build/src/executorch/extension/runner_util/managed_tensor.h # ET is missing headers for vector/memory.  This causes downstream issues when building runner-et.
+
+  echo "Building and installing python libraries"
+  bash ./install_requirements.sh --pybind xnnpack
+  pip list
+
+  echo "Building and installing C++ libraries"
+  echo "Inside: ${PWD}"
+  mkdir cmake-out
+  cmake -DCMAKE_BUILD_TYPE=Release -DEXECUTORCH_BUILD_OPTIMIZED=ON -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON -DEXECUTORCH_BUILD_XNNPACK=ON -S . -B cmake-out -G Ninja
+  cmake --build cmake-out
+  cmake --install cmake-out --prefix ${LLAMA_FAST_ROOT}/build/install
+  popd
+}
+
+pushd ${LLAMA_FAST_ROOT}
+install_pip_dependencies
+install_executorch
+popd