Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GHA] CPU sanitizers for functional tests. #26028

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 107 additions & 12 deletions .github/workflows/linux_sanitizers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,35 @@ on:
# run daily at 00:00
- cron: '0 0 * * *'
workflow_dispatch:
inputs:
# to be defined for specified test scopes (e.g. CPU func. tests)
sanitizerMode:
description: 'Sanitizer mode for CPU func. tests (All || AddressAndLeak || UndefinedBehavior || Thread).'
required: false
default: 'All'
gtestFilter:
description: 'Filter for CPU func. tests.'
required: false
default: '*smoke*'
pull_request:
paths:
- '.github/workflows/linux_sanitizers.yml'

concurrency:
# github.ref is not unique in post-commit
group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux-sanitizers
group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux-sanitizers-tests
cancel-in-progress: true

permissions: read-all

env:
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}
BUILD_DIR: /__w/openvino/openvino/openvino_build
OPENVINO_REPO: /__w/openvino/openvino/openvino
TBB_ENABLE_SANITIZERS: 1
ASAN_OPTIONS: halt_on_error=0:suppressions=/__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp
LSAN_OPTIONS: suppressions=/__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt:NEOReadDebugKeys=1:DisableDeepBind=1
TSAN_OPTIONS: suppressions=/__w/openvino/openvino/openvino/tests/sanitizers/tsan/suppressions.txt

jobs:
Smart_CI:
Expand Down Expand Up @@ -93,11 +112,9 @@ jobs:
CMAKE_BUILD_TYPE: 'RelWithDebInfo'
CMAKE_GENERATOR: 'Ninja'
GITHUB_WORKSPACE: '/__w/openvino/openvino'
OPENVINO_REPO: /__w/openvino/openvino/openvino
OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib
INSTALL_DIR: /__w/openvino/openvino/openvino_install
INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install
BUILD_DIR: /__w/openvino/openvino/openvino_build
CMAKE_CXX_COMPILER_LAUNCHER: sccache
CMAKE_C_COMPILER_LAUNCHER: sccache
SCCACHE_IGNORE_SERVER_IO_ERROR: 1
Expand Down Expand Up @@ -252,14 +269,8 @@ jobs:
- SANITIZER: 'UndefinedBehavior'
- SANITIZER: 'Thread'
env:
OPENVINO_REPO: /__w/openvino/openvino/openvino
INSTALL_DIR: /__w/openvino/openvino/install
INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests
BUILD_DIR: /__w/openvino/openvino/openvino_build
TBB_ENABLE_SANITIZERS: 1
ASAN_OPTIONS: halt_on_error=0:suppressions=/__w/openvino/openvino/openvino/tests/sanitizers/asan/suppressions.supp
LSAN_OPTIONS: suppressions=/__w/openvino/openvino/openvino/tests/sanitizers/lsan/suppressions.txt:NEOReadDebugKeys=1:DisableDeepBind=1
TSAN_OPTIONS: suppressions=/__w/openvino/openvino/openvino/tests/sanitizers/tsan/suppressions.txt

steps:
- name: Download OpenVINO package
Expand Down Expand Up @@ -405,7 +416,7 @@ jobs:
source ${INSTALL_DIR}/setupvars.sh
${INSTALL_TEST_DIR}/ov_cpu_unit_tests --gtest_print_time=1 \
--gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml

- name: ov_subgraphs_dumper_tests tests
if: ${{ !cancelled() }}
run: |
Expand Down Expand Up @@ -441,7 +452,7 @@ jobs:
${INSTALL_TEST_DIR}/ov_template_func_tests --gtest_print_time=1 \
--gtest_filter=*smoke* \
--gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TemplateFuncTests.xml

- name: OpenVINO C API tests
if: ${{ !cancelled() }}
run: |
Expand Down Expand Up @@ -483,6 +494,90 @@ jobs:
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
if: ${{ !cancelled() }}
with:
name: test-results-cpp_${{ matrix.SANITIZER }}
name: unit-test-results-cpp_${{ matrix.SANITIZER }}
path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml
if-no-files-found: 'error'

CXX_Func_Tests:
# todo: move another plugin's func tests here
name: C++ func tests
if: ${{ github.repository_owner == 'openvinotoolkit' }}
needs: [Docker, Build]
timeout-minutes: 120
runs-on: aks-linux-16-cores-32gb
container:
image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}
defaults:
run:
shell: bash
strategy:
max-parallel: 3
fail-fast: false
matrix:
include:
- SANITIZER: 'AddressAndLeak'
- SANITIZER: 'UndefinedBehavior'
- SANITIZER: 'Thread'
env:
INSTALL_DIR: /__w/openvino/openvino/install
INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests

steps:
- name: Download OpenVINO package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: ${{ format('openvino_package_{0}', matrix.SANITIZER) }}
path: ${{ env.INSTALL_DIR }}

- name: Download OpenVINO tests package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: ${{ format('openvino_tests_{0}', matrix.SANITIZER) }}
path: ${{ env.INSTALL_TEST_DIR }}

- name: Extract OpenVINO packages
run: |
pushd $INSTALL_DIR
pigz -dc openvino_package.tar.gz | tar -xf - -C ${INSTALL_DIR}
popd
pushd $INSTALL_TEST_DIR
pigz -dc openvino_tests.tar.gz | tar -xf - -C ${INSTALL_DIR}
popd

- name: Fetch Sanitizer Suppression Lists
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
timeout-minutes: 15
with:
sparse-checkout: |
tests/sanitizers/lsan/suppressions.txt
tests/sanitizers/asan/suppressions.supp
tests/sanitizers/tsan/suppressions.txt
sparse-checkout-cone-mode: false
path: ${{ env.OPENVINO_REPO }}

#
# Tests
#

- name: CPU plugin func tests
id: cpu_func
# by default in scheduled launch we run only AddressAndLeak with *smoke*Conv_* pattern
# to prevent false-positive alerts and time exceeded error,
# in workflow_dispatch case all smoke tests will be launched with all types of sanitizers
# if gtestFilter and sanMode inputs are not specified
if: ${{ !cancelled() && (github.event_name == 'workflow_dispatch' && (inputs.sanitizerMode == 'All' || inputs.sanitizerMode == matrix.SANITIZER) || matrix.SANITIZER == 'AddressAndLeak')}}
env:
GTEST_FILTER: ${{ github.event_name == 'workflow_dispatch' && inputs.gtestFilter || '*smoke*Conv_*' }}
run: |
source ${INSTALL_DIR}/setupvars.sh
${INSTALL_TEST_DIR}/ov_cpu_func_tests --gtest_print_time=1 \
--gtest_filter=${{ env.GTEST_FILTER }} \
--gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUFuncTests.xml

- name: Upload Test Results
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
if: ${{ !cancelled() && steps.cpu_func.conclusion != 'skipped'}}
with:
name: func-test-results-cpp_${{ matrix.SANITIZER }}
path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml
if-no-files-found: 'error'
Loading