Skip to content

Commit

Permalink
cmake: twister: Using common script for toolchain setting verification
Browse files Browse the repository at this point in the history
Fixes: zephyrproject-rtos#30713

Now using the common script `verify-toolchain.cmake` for verifying
toolchain settings.

Signed-off-by: Torsten Rasmussen <[email protected]>
  • Loading branch information
tejlmand committed Feb 4, 2021
1 parent ca1ae3d commit 8739d53
Show file tree
Hide file tree
Showing 5 changed files with 182 additions and 108 deletions.
1 change: 1 addition & 0 deletions cmake/app/boilerplate.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,7 @@ message(STATUS "Cache files will be written to: ${USER_CACHE_DIR}")
set(CMAKE_C_COMPILER_FORCED 1)
set(CMAKE_CXX_COMPILER_FORCED 1)

include(${ZEPHYR_BASE}/cmake/verify-toolchain.cmake)
include(${ZEPHYR_BASE}/cmake/host-tools.cmake)

# Include board specific device-tree flags before parsing.
Expand Down
15 changes: 0 additions & 15 deletions cmake/generic_toolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,6 @@ but Zephyr ignores flags from the environment. Use 'cmake -DEXTRA_${var}=$ENV{${
endif()
endforeach()

if(NOT ZEPHYR_TOOLCHAIN_VARIANT)
if(DEFINED ENV{ZEPHYR_TOOLCHAIN_VARIANT})
set(ZEPHYR_TOOLCHAIN_VARIANT $ENV{ZEPHYR_TOOLCHAIN_VARIANT})
elseif(CROSS_COMPILE OR (DEFINED ENV{CROSS_COMPILE}))
set(ZEPHYR_TOOLCHAIN_VARIANT cross-compile)
endif()
endif()

# Host-tools don't unconditionally set TOOLCHAIN_HOME anymore,
# but in case Zephyr's SDK toolchain is used, set TOOLCHAIN_HOME
if("${ZEPHYR_TOOLCHAIN_VARIANT}" STREQUAL "zephyr")
Expand All @@ -40,13 +32,6 @@ assert(TOOLCHAIN_ROOT "Zephyr toolchain root path invalid: please set the TOOLCH
# Set cached ZEPHYR_TOOLCHAIN_VARIANT.
set(ZEPHYR_TOOLCHAIN_VARIANT ${ZEPHYR_TOOLCHAIN_VARIANT} CACHE STRING "Zephyr toolchain variant")

# Pick host system's toolchain if we are targeting posix
if(${ARCH} STREQUAL "posix")
if(NOT "${ZEPHYR_TOOLCHAIN_VARIANT}" STREQUAL "llvm")
set(ZEPHYR_TOOLCHAIN_VARIANT "host")
endif()
endif()

# Configure the toolchain based on what SDK/toolchain is in use.
include(${TOOLCHAIN_ROOT}/cmake/toolchain/${ZEPHYR_TOOLCHAIN_VARIANT}/generic.cmake)

Expand Down
95 changes: 4 additions & 91 deletions cmake/toolchain/zephyr/host-tools.cmake
Original file line number Diff line number Diff line change
@@ -1,77 +1,20 @@
# SPDX-License-Identifier: Apache-2.0

# Lots of duplications here.
# FIXME: maintain this only in one place.

# We need to separate actual toolchain from the host-tools required by Zephyr
# and currently provided by the Zephyr SDK. Those tools will need to be
# provided for different OSes and sepearately from the toolchain.

# This is the minimum required version which supports CMake package
set(MINIMUM_REQUIRED_SDK_VERSION 0.11.3)

# This is the minimum required version for Zephyr to work (Old style)
set(REQUIRED_SDK_VER 0.11.1)
cmake_host_system_information(RESULT TOOLCHAIN_ARCH QUERY OS_PLATFORM)

set_ifndef(ZEPHYR_TOOLCHAIN_VARIANT $ENV{ZEPHYR_TOOLCHAIN_VARIANT} "")
set_ifndef(ZEPHYR_SDK_INSTALL_DIR $ENV{ZEPHYR_SDK_INSTALL_DIR} "")

# There are three scenarios where Zephyr SDK should be looked up:
# 1) Zephyr specified as toolchain (ZEPHYR_SDK_INSTALL_DIR still used if defined)
# 2) No toolchain specified == Default to Zephyr toolchain (Linux only)
# 3) Other toolchain specified, but ZEPHYR_SDK_INSTALL_DIR also given.
# This means Zephyr SDK toolchain will not be used for compilation,
# but other supplementary host tools will be used.
if(("zephyr" STREQUAL ${ZEPHYR_TOOLCHAIN_VARIANT}) OR
((NOT DEFINED ZEPHYR_TOOLCHAIN_VARIANT) AND (${CMAKE_HOST_SYSTEM_NAME} STREQUAL Linux)) OR
(DEFINED ZEPHYR_SDK_INSTALL_DIR))

# No toolchain was specified, so inform user that we will be searching.
if (NOT DEFINED ZEPHYR_SDK_INSTALL_DIR AND NOT DEFINED ZEPHYR_TOOLCHAIN_VARIANT)
message("ZEPHYR_TOOLCHAIN_VARIANT not set, trying to locate Zephyr SDK")
endif()

# This ensure packages are sorted in descending order.
SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION_CURRENT ${CMAKE_FIND_PACKAGE_SORT_DIRECTION})
SET(CMAKE_FIND_PACKAGE_SORT_ORDER_CURRENT ${CMAKE_FIND_PACKAGE_SORT_ORDER})
SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC)
SET(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL)

if(DEFINED ZEPHYR_SDK_INSTALL_DIR)
# The Zephyr SDK will automatically set the toolchain variant.
# To support Zephyr SDK tools (DTC, and other tools) with 3rd party toolchains
# then we keep track of current toolchain variant.
set(ZEPHYR_CURRENT_TOOLCHAIN_VARIANT ${ZEPHYR_TOOLCHAIN_VARIANT})
find_package(Zephyr-sdk ${MINIMUM_REQUIRED_SDK_VERSION} QUIET HINTS $ENV{ZEPHYR_SDK_INSTALL_DIR})
if(ZEPHYR_CURRENT_TOOLCHAIN_VARIANT)
if(NOT "zephyr" STREQUAL ${ZEPHYR_CURRENT_TOOLCHAIN_VARIANT})
set(ZEPHYR_TOOLCHAIN_VARIANT ${ZEPHYR_CURRENT_TOOLCHAIN_VARIANT})
endif()
endif()
else()
find_package(Zephyr-sdk ${MINIMUM_REQUIRED_SDK_VERSION} QUIET PATHS
/usr
/usr/local
/opt
$ENV{HOME}
$ENV{HOME}/.local
$ENV{HOME}/.local/opt
$ENV{HOME}/bin)
endif()

SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION ${CMAKE_FIND_PACKAGE_SORT_DIRECTION_CURRENT})
SET(CMAKE_FIND_PACKAGE_SORT_ORDER ${CMAKE_FIND_PACKAGE_SORT_ORDER_CURRENT})
else ()
if(NOT DEFINED ZEPHYR_SDK_INSTALL_DIR)
# Until https://github.com/zephyrproject-rtos/zephyr/issues/4912 is
# resolved we use ZEPHYR_SDK_INSTALL_DIR to determine whether the user
# wants to use the Zephyr SDK or not.
return()
endif()

# Cache the Zephyr SDK install dir.
set(ZEPHYR_SDK_INSTALL_DIR ${ZEPHYR_SDK_INSTALL_DIR} CACHE PATH "Zephyr SDK install directory")

if(NOT ${Zephyr-sdk_FOUND})
if(NOT DEFINED SDK_VERSION)
if(ZEPHYR_TOOLCHAIN_VARIANT AND ZEPHYR_SDK_INSTALL_DIR)
# Manual detection for Zephyr SDK 0.11.1 and 0.11.2 for backward compatibility.
set(sdk_version_path ${ZEPHYR_SDK_INSTALL_DIR}/sdk_version)
Expand All @@ -97,41 +40,11 @@ if(NOT ${Zephyr-sdk_FOUND})
Expected format: x.y.z
Check whether the Zephyr SDK was installed correctly.
")
elseif(${SDK_VERSION} VERSION_GREATER_EQUAL ${REQUIRED_SDK_VER})
set(Zephyr-sdk_FOUND TRUE)
endif()
endif()
endif()

if(NOT ${Zephyr-sdk_FOUND})
# Note: When CMake mimimun version becomes >= 3.17, change this loop into:
# foreach(version config IN ZIP_LISTS Zephyr-sdk_CONSIDERED_VERSIONS Zephyr-sdk_CONSIDERED_CONFIGS)
set(missing_version "You need SDK version ${REQUIRED_SDK_VER} or newer.")
foreach (version ${Zephyr-sdk_CONSIDERED_VERSIONS})
if(${version} VERSION_GREATER ${MINIMUM_REQUIRED_SDK_VERSION})
set(missing_version "You need SDK version ${MINIMUM_REQUIRED_SDK_VERSION} or compatible version.")
endif()
list(GET Zephyr-sdk_CONSIDERED_CONFIGS 0 zephyr-sdk-candidate)
list(REMOVE_AT Zephyr-sdk_CONSIDERED_CONFIGS 0)
get_filename_component(zephyr-sdk-path ${zephyr-sdk-candidate}/../.. ABSOLUTE)
string(APPEND version_path " ${version} (${zephyr-sdk-path})\n")
endforeach()

if(NOT ZEPHYR_TOOLCHAIN_VARIANT AND NOT ZEPHYR_SDK_INSTALL_DIR)
set(error_note "Note: If you are using SDK 0.11.1 or 0.11.2, remember to set ZEPHYR_SDK_INSTALL_DIR and ZEPHYR_TOOLCHAIN_VARIANT")
endif()

message(FATAL_ERROR "The SDK version you are using is not supported, please update your SDK.
${missing_version}
You have version(s):
${version_path}
The SDK can be downloaded from:
https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${MINIMUM_REQUIRED_SDK_VERSION}/zephyr-sdk-${MINIMUM_REQUIRED_SDK_VERSION}-setup.run
${error_note}
")
endif()

message(STATUS "Found toolchain: zephyr (${ZEPHYR_SDK_INSTALL_DIR})")
message(STATUS "Using toolchain: zephyr ${SDK_VERSION} (${ZEPHYR_SDK_INSTALL_DIR})")

if(${SDK_VERSION} VERSION_LESS_EQUAL 0.11.2)
# For backward compatibility with 0.11.1 and 0.11.2
Expand Down
140 changes: 140 additions & 0 deletions cmake/verify-toolchain.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
# The purpose of this file is to verify that required variables has been
# defined for proper toolchain use.
#
# It also offers the possibility to verify that the selected toolchain matches
# a specific version.
# Currently only when using the Zephyr SDK the version is verified, but other
# other version verification for other toolchains can be added as needed.
#
# This file can also be executed in script mode so that it can be used in other
# places, such as python scripts.
#
# When invoked as a script with -P:
# cmake [options] -P verify-toolchain.cmake
#
# it takes the following arguments:
# FORMAT=json: Print the output as a json formatted string, useful for Python

# This is the minimum required Zephyr-SDK version which supports CMake package
set(TOOLCHAIN_ZEPHYR_MINIMUM_REQUIRED_VERSION 0.11.3)

# Set internal variables if set in environment.
if(NOT DEFINED ZEPHYR_TOOLCHAIN_VARIANT)
set(ZEPHYR_TOOLCHAIN_VARIANT $ENV{ZEPHYR_TOOLCHAIN_VARIANT})
endif()

if(NOT DEFINED ZEPHYR_SDK_INSTALL_DIR)
set(ZEPHYR_SDK_INSTALL_DIR $ENV{ZEPHYR_SDK_INSTALL_DIR})
endif()

# Pick host system's toolchain if we are targeting posix
if("${ARCH}" STREQUAL "posix")
if(NOT "${ZEPHYR_TOOLCHAIN_VARIANT}" STREQUAL "llvm")
set(ZEPHYR_TOOLCHAIN_VARIANT "host")
endif()
return()
endif()

if(NOT ZEPHYR_TOOLCHAIN_VARIANT AND
(CROSS_COMPILE OR (DEFINED ENV{CROSS_COMPILE})))
set(ZEPHYR_TOOLCHAIN_VARIANT cross-compile)
endif()

# Verify Zephyr SDK Toolchain.
# There are three scenarios where Zephyr SDK should be looked up:
# 1) Zephyr specified as toolchain (ZEPHYR_SDK_INSTALL_DIR still used if defined)
# 2) No toolchain specified == Default to Zephyr toolchain (Linux only)
# Until we completely deprecate it
if(("zephyr" STREQUAL ${ZEPHYR_TOOLCHAIN_VARIANT}) OR
((NOT DEFINED ZEPHYR_TOOLCHAIN_VARIANT) AND (${CMAKE_HOST_SYSTEM_NAME} STREQUAL Linux)) OR
(DEFINED ZEPHYR_SDK_INSTALL_DIR))

# No toolchain was specified, so inform user that we will be searching.
if (NOT DEFINED ZEPHYR_SDK_INSTALL_DIR AND
NOT DEFINED ZEPHYR_TOOLCHAIN_VARIANT
AND NOT CMAKE_SCRIPT_MODE_FILE)
message(STATUS "ZEPHYR_TOOLCHAIN_VARIANT not set, trying to locate Zephyr SDK")
endif()

# This ensure packages are sorted in descending order.
SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION_CURRENT ${CMAKE_FIND_PACKAGE_SORT_DIRECTION})
SET(CMAKE_FIND_PACKAGE_SORT_ORDER_CURRENT ${CMAKE_FIND_PACKAGE_SORT_ORDER})
SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC)
SET(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL)

if(DEFINED ZEPHYR_SDK_INSTALL_DIR)
# The Zephyr SDK will automatically set the toolchain variant.
# To support Zephyr SDK tools (DTC, and other tools) with 3rd party toolchains
# then we keep track of current toolchain variant.
set(ZEPHYR_CURRENT_TOOLCHAIN_VARIANT ${ZEPHYR_TOOLCHAIN_VARIANT})
find_package(Zephyr-sdk ${TOOLCHAIN_ZEPHYR_MINIMUM_REQUIRED_VERSION} QUIET HINTS $ENV{ZEPHYR_SDK_INSTALL_DIR})
if(DEFINED ZEPHYR_CURRENT_TOOLCHAIN_VARIANT)
set(ZEPHYR_TOOLCHAIN_VARIANT ${ZEPHYR_CURRENT_TOOLCHAIN_VARIANT})
endif()
else()
find_package(Zephyr-sdk ${TOOLCHAIN_ZEPHYR_MINIMUM_REQUIRED_VERSION} QUIET PATHS
/usr
/usr/local
/opt
$ENV{HOME}
$ENV{HOME}/.local
$ENV{HOME}/.local/opt
$ENV{HOME}/bin)
endif()

SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION ${CMAKE_FIND_PACKAGE_SORT_DIRECTION_CURRENT})
SET(CMAKE_FIND_PACKAGE_SORT_ORDER ${CMAKE_FIND_PACKAGE_SORT_ORDER_CURRENT})
endif()

if(NOT DEFINED ZEPHYR_TOOLCHAIN_VARIANT)
if (NOT Zephyr-sdk_CONSIDERED_VERSIONS)
set(error_msg "ZEPHYR_TOOLCHAIN_VARIANT not specified and no Zephyr SDK is installed.\n")
string(APPEND error_msg "Please set ZEPHYR_TOOLCHAIN_VARIANT to the toolchain to use or install the Zephyr SDK.")

if(NOT ZEPHYR_TOOLCHAIN_VARIANT AND NOT ZEPHYR_SDK_INSTALL_DIR)
set(error_note "Note: If you are using Zephyr SDK 0.11.1 or 0.11.2, remember to set ZEPHYR_SDK_INSTALL_DIR and ZEPHYR_TOOLCHAIN_VARIANT")
endif()
else()
# Note: When CMake mimimun version becomes >= 3.17, change this loop into:
# foreach(version config IN ZIP_LISTS Zephyr-sdk_CONSIDERED_VERSIONS Zephyr-sdk_CONSIDERED_CONFIGS)
set(error_msg "The Zephyr SDK version you are using is not supported, please update your SDK.\n")
set(missing_version "You need SDK version ${TOOLCHAIN_ZEPHYR_MINIMUM_REQUIRED_VERSION} or newer.")
foreach (version ${Zephyr-sdk_CONSIDERED_VERSIONS})
if(${version} VERSION_GREATER ${TOOLCHAIN_ZEPHYR_MINIMUM_REQUIRED_VERSION})
set(missing_version "You need SDK version ${TOOLCHAIN_ZEPHYR_MINIMUM_REQUIRED_VERSION} or compatible version.")
endif()
list(GET Zephyr-sdk_CONSIDERED_CONFIGS 0 zephyr-sdk-candidate)
list(REMOVE_AT Zephyr-sdk_CONSIDERED_CONFIGS 0)
get_filename_component(zephyr-sdk-path ${zephyr-sdk-candidate}/../.. ABSOLUTE)
string(APPEND version_path " ${version} (${zephyr-sdk-path})")
endforeach()
string(APPEND error_msg "${missing_version}")
string(APPEND error_msg "You have version(s):")
string(APPEND error_msg "${version_path}")
endif()

message(FATAL_ERROR "${error_msg}
The Zephyr SDK can be downloaded from:
https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${TOOLCHAIN_ZEPHYR_MINIMUM_REQUIRED_VERSION}/zephyr-sdk-${TOOLCHAIN_ZEPHYR_MINIMUM_REQUIRED_VERSION}-setup.run
${error_note}
")

endif()

if(CMAKE_SCRIPT_MODE_FILE)
if("${FORMAT}" STREQUAL "json")
set(json "{\"ZEPHYR_TOOLCHAIN_VARIANT\" : \"${ZEPHYR_TOOLCHAIN_VARIANT}\", ")
string(APPEND json "\"SDK_VERSION\": \"${SDK_VERSION}\", ")
string(APPEND json "\"ZEPHYR_SDK_INSTALL_DIR\" : \"${ZEPHYR_SDK_INSTALL_DIR}\"}")
message("${json}")
else()
message(STATUS "ZEPHYR_TOOLCHAIN_VARIANT: ${ZEPHYR_TOOLCHAIN_VARIANT}")
if(DEFINED SDK_VERSION)
message(STATUS "SDK_VERSION: ${SDK_VERSION}")
endif()

if(DEFINED ZEPHYR_SDK_INSTALL_DIR)
message(STATUS "ZEPHYR_SDK_INSTALL_DIR : ${ZEPHYR_SDK_INSTALL_DIR}")
endif()
endif()
endif()
39 changes: 37 additions & 2 deletions scripts/pylib/twister/twisterlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -2037,6 +2037,37 @@ def run_cmake(self, args=[]):

return results

@staticmethod
def run_cmake_script(args=[]):

logger.debug("Running cmake script %s" % (args[0]))

cmake_args = ["-D{}".format(a.replace('"', '')) for a in args[1:]]
cmake_args.extend(['-P', args[0]])

logger.debug("Calling cmake with arguments: {}".format(cmake_args))
cmake = shutil.which('cmake')
cmd = [cmake] + cmake_args

kwargs = dict()
kwargs['stdout'] = subprocess.PIPE
# CMake sends the output of message() to stderr unless it's STATUS
kwargs['stderr'] = subprocess.STDOUT

p = subprocess.Popen(cmd, **kwargs)
out, _ = p.communicate()

if p.returncode == 0:
msg = "Finished running %s" % (args[0])
logger.debug(msg)
results = {"returncode": p.returncode, "msg": msg, "stdout": out}

else:
logger.error("Cmake script failure: %s" % (args[0]))
results = {"returncode": p.returncode}

return results


class FilterBuilder(CMake):

Expand Down Expand Up @@ -2809,13 +2840,17 @@ def get_all_tests(self):

@staticmethod
def get_toolchain():
toolchain = os.environ.get("ZEPHYR_TOOLCHAIN_VARIANT", None)
toolchain_script = ZEPHYR_BASE + "/cmake/verify-toolchain.cmake"
result = CMake.run_cmake_script([toolchain_script, "FORMAT=json"])

try:
if not toolchain:
if result['returncode']:
raise TwisterRuntimeError("E: Variable ZEPHYR_TOOLCHAIN_VARIANT is not defined")
except Exception as e:
print(str(e))
sys.exit(2)
toolchain = json.loads(result['stdout'])['ZEPHYR_TOOLCHAIN_VARIANT']
logger.info(f"Using '{toolchain}' toolchain.")

return toolchain

Expand Down

0 comments on commit 8739d53

Please sign in to comment.