From 1a6a45b5c54c2fc88f8a8fa5d3a27d66f681f9fb Mon Sep 17 00:00:00 2001 From: Huy Do <huydhn@gmail.com> Date: Thu, 14 Dec 2023 14:53:04 -0800 Subject: [PATCH 1/6] Debug aarch64 clone --- test/smoke_test/smoke_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index f4c06150e..0bebed58f 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -258,7 +258,11 @@ def smoke_test_modules(): if module["repo"]: if not os.path.exists(f"{cwd}/{module['repo_name']}"): print(f"Path does not exist: {cwd}/{module['repo_name']}") - subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True) + try: + output = subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as exc: + print(f"==== {output}") + raise RuntimeError(f"CLONE FAIL: {exc.returncode} {exc.output}") from exc try: smoke_test_command = f"python3 {module['smoke_test']}" if target_os == 'windows': From f329da542149f22a3d019a54a4b0b1111d892fbd Mon Sep 17 00:00:00 2001 From: Huy Do <huydhn@gmail.com> Date: Thu, 14 Dec 2023 16:00:13 -0800 Subject: [PATCH 2/6] Debug --- test/smoke_test/smoke_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index 0bebed58f..f3dd95763 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -261,7 +261,6 @@ def smoke_test_modules(): try: output = subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as exc: - print(f"==== {output}") raise RuntimeError(f"CLONE FAIL: {exc.returncode} {exc.output}") from exc try: smoke_test_command = f"python3 {module['smoke_test']}" From 0241cf6100bb3fb36c1040fe199cea7b03b24e2f Mon Sep 17 00:00:00 2001 From: Huy Do <huydhn@gmail.com> Date: Fri, 15 Dec 2023 12:47:12 -0800 Subject: [PATCH 3/6] Fix validation workflow with conda 23.11.0 and GLIBC_2.25 --- .github/scripts/validate_binaries.sh | 6 ++++++ test/smoke_test/smoke_test.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 22fc9d109..a21732860 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -39,6 +39,12 @@ else ${PWD}/check_binary.sh fi + # NB: The latest conda 23.11.0 pulls in some dependencies of conda-libmamba-solver that + # require GLIBC_2.25, which is not available in the current aarch64 image causing the + # suqsequence git command to fail. Basically, they don't work with CentOS 7 which AML 2 + # is based on https://github.com/ContinuumIO/anaconda-issues/issues/12822 + unset LD_LIBRARY_PATH + if [[ ${TARGET_OS} == 'windows' ]]; then python ./test/smoke_test/smoke_test.py ${TEST_SUFFIX} else diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index f3dd95763..8a2ca2ac1 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -259,9 +259,9 @@ def smoke_test_modules(): if not os.path.exists(f"{cwd}/{module['repo_name']}"): print(f"Path does not exist: {cwd}/{module['repo_name']}") try: - output = subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True) + subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as exc: - raise RuntimeError(f"CLONE FAIL: {exc.returncode} {exc.output}") from exc + raise RuntimeError(f"Cloning {module['repo']} FAIL: {exc.returncode} Output: {exc.output}") from exc try: smoke_test_command = f"python3 {module['smoke_test']}" if target_os == 'windows': From d6ad9cb88681bb0f0001d1b5d96b9d52f4eb3801 Mon Sep 17 00:00:00 2001 From: Huy Do <huydhn@gmail.com> Date: Fri, 15 Dec 2023 13:13:48 -0800 Subject: [PATCH 4/6] Gate the change on linux-aarch64 and keep the old LD_LIBRARY_PATH --- .github/scripts/validate_binaries.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index a21732860..aee132b6f 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -39,11 +39,14 @@ else ${PWD}/check_binary.sh fi - # NB: The latest conda 23.11.0 pulls in some dependencies of conda-libmamba-solver that - # require GLIBC_2.25, which is not available in the current aarch64 image causing the - # suqsequence git command to fail. Basically, they don't work with CentOS 7 which AML 2 - # is based on https://github.com/ContinuumIO/anaconda-issues/issues/12822 - unset LD_LIBRARY_PATH + if [[ ${TARGET_OS} == 'linux-aarch64' ]]; then + OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} + # NB: The latest conda 23.11.0 pulls in some dependencies of conda-libmamba-solver that + # require GLIBC_2.25, which is not available in the current aarch64 image causing the + # suqsequence git command to fail. Basically, they don't work with CentOS 7 which AML 2 + # is based on https://github.com/ContinuumIO/anaconda-issues/issues/12822 + unset LD_LIBRARY_PATH + fi if [[ ${TARGET_OS} == 'windows' ]]; then python ./test/smoke_test/smoke_test.py ${TEST_SUFFIX} @@ -57,4 +60,8 @@ else conda deactivate conda env remove -n ${ENV_NAME} + + if [[ ${TARGET_OS} == 'linux-aarch64' ]]; then + LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH} + fi fi From fa7e4c388e92e618add72080caf0a4806cd336cf Mon Sep 17 00:00:00 2001 From: Huy Do <huydhn@gmail.com> Date: Fri, 15 Dec 2023 13:28:51 -0800 Subject: [PATCH 5/6] Try to unset LD_LIBRARY_PATH in the workflow instead --- .github/scripts/validate_binaries.sh | 13 ------------- .../workflows/validate-aarch64-linux-binaries.yml | 8 +++++++- test/smoke_test/smoke_test.py | 10 ++++++++++ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index aee132b6f..22fc9d109 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -39,15 +39,6 @@ else ${PWD}/check_binary.sh fi - if [[ ${TARGET_OS} == 'linux-aarch64' ]]; then - OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} - # NB: The latest conda 23.11.0 pulls in some dependencies of conda-libmamba-solver that - # require GLIBC_2.25, which is not available in the current aarch64 image causing the - # suqsequence git command to fail. Basically, they don't work with CentOS 7 which AML 2 - # is based on https://github.com/ContinuumIO/anaconda-issues/issues/12822 - unset LD_LIBRARY_PATH - fi - if [[ ${TARGET_OS} == 'windows' ]]; then python ./test/smoke_test/smoke_test.py ${TEST_SUFFIX} else @@ -60,8 +51,4 @@ else conda deactivate conda env remove -n ${ENV_NAME} - - if [[ ${TARGET_OS} == 'linux-aarch64' ]]; then - LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH} - fi fi diff --git a/.github/workflows/validate-aarch64-linux-binaries.yml b/.github/workflows/validate-aarch64-linux-binaries.yml index 14b7b6395..6b1a60d7c 100644 --- a/.github/workflows/validate-aarch64-linux-binaries.yml +++ b/.github/workflows/validate-aarch64-linux-binaries.yml @@ -95,5 +95,11 @@ jobs: printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json eval "$(conda shell.bash hook)" - # Standart case: Validate binaries + # NB: The latest conda 23.11.0 pulls in some dependencies of conda-libmamba-solver that + # require GLIBC_2.25, which is not available in the current aarch64 image causing the + # subsequence git command to fail. Basically, they don't work with CentOS 7 which AML 2 + # is based on https://github.com/ContinuumIO/anaconda-issues/issues/12822 + unset LD_LIBRARY_PATH + + # Standard case: Validate binaries source ./.github/scripts/validate_binaries.sh diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index 8a2ca2ac1..3b476ee24 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -258,6 +258,16 @@ def smoke_test_modules(): if module["repo"]: if not os.path.exists(f"{cwd}/{module['repo_name']}"): print(f"Path does not exist: {cwd}/{module['repo_name']}") + try: + subprocess.check_output( + f"git clone --depth 1 {module['repo']}", + stderr=subprocess.STDOUT, + shell=True, + ) + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f"Cloning {module['repo']} FAIL: {exc.returncode} Output: {exc.output}" + ) from exc try: subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as exc: From 07e5cd981ea73f80f48a099b28ad387413838ec9 Mon Sep 17 00:00:00 2001 From: Huy Do <huydhn@gmail.com> Date: Fri, 15 Dec 2023 13:42:48 -0800 Subject: [PATCH 6/6] Fix copy/paste typo --- test/smoke_test/smoke_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index 3b476ee24..14e04d366 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -268,10 +268,6 @@ def smoke_test_modules(): raise RuntimeError( f"Cloning {module['repo']} FAIL: {exc.returncode} Output: {exc.output}" ) from exc - try: - subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True) - except subprocess.CalledProcessError as exc: - raise RuntimeError(f"Cloning {module['repo']} FAIL: {exc.returncode} Output: {exc.output}") from exc try: smoke_test_command = f"python3 {module['smoke_test']}" if target_os == 'windows':