-
Notifications
You must be signed in to change notification settings - Fork 12.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
PR for llvm/llvm-project#80565 #80566
Conversation
@aaupov What do you think about merging this PR to the release branch? |
@llvm/pr-subscribers-clang Author: None (llvmbot) Changesresolves llvm/llvm-project#80565 Full diff: https://github.com/llvm/llvm-project/pull/80566.diff 3 Files Affected:
diff --git a/clang/utils/perf-training/CMakeLists.txt b/clang/utils/perf-training/CMakeLists.txt
index c6d51863fb1b5..93744f4606023 100644
--- a/clang/utils/perf-training/CMakeLists.txt
+++ b/clang/utils/perf-training/CMakeLists.txt
@@ -1,6 +1,10 @@
+include(LLVMExternalProjectUtils)
+
set(CLANG_PGO_TRAINING_DATA "${CMAKE_CURRENT_SOURCE_DIR}" CACHE PATH
"The path to a lit testsuite containing samples for PGO and order file generation"
)
+set(CLANG_PGO_TRAINING_DATA_SOURCE_DIR OFF CACHE STRING "Path to source directory containing cmake project with source files to use for generating pgo data")
+set(CLANG_PGO_TRAINING_DEPS "" CACHE STRING "Extra dependencies needed to build the PGO training data.")
if(LLVM_BUILD_INSTRUMENTED)
configure_lit_site_cfg(
@@ -11,11 +15,11 @@ if(LLVM_BUILD_INSTRUMENTED)
add_lit_testsuite(generate-profraw "Generating clang PGO data"
${CMAKE_CURRENT_BINARY_DIR}/pgo-data/
EXCLUDE_FROM_CHECK_ALL
- DEPENDS clang clear-profraw ${CLANG_PERF_TRAINING_DEPS}
+ DEPENDS clang clear-profraw ${CLANG_PGO_TRAINING_DEPS}
)
add_custom_target(clear-profraw
- COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py clean ${CMAKE_CURRENT_BINARY_DIR} profraw
+ COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py clean ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/profiles/ profraw
COMMENT "Clearing old profraw data")
if(NOT LLVM_PROFDATA)
@@ -26,9 +30,14 @@ if(LLVM_BUILD_INSTRUMENTED)
message(STATUS "To enable merging PGO data LLVM_PROFDATA has to point to llvm-profdata")
else()
add_custom_target(generate-profdata
- COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} ${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR}
+ COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} ${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/profiles/
COMMENT "Merging profdata"
DEPENDS generate-profraw)
+ if (CLANG_PGO_TRAINING_DATA_SOURCE_DIR)
+ llvm_ExternalProject_Add(generate-profraw-external ${CLANG_PGO_TRAINING_DATA_SOURCE_DIR}
+ USE_TOOLCHAIN EXLUDE_FROM_ALL NO_INSTALL DEPENDS generate-profraw)
+ add_dependencies(generate-profdata generate-profraw-external)
+ endif()
endif()
endif()
diff --git a/clang/utils/perf-training/perf-helper.py b/clang/utils/perf-training/perf-helper.py
index 99d6a3333b6ef..3e92cd38a7145 100644
--- a/clang/utils/perf-training/perf-helper.py
+++ b/clang/utils/perf-training/perf-helper.py
@@ -30,26 +30,28 @@ def findFilesWithExtension(path, extension):
def clean(args):
- if len(args) != 2:
+ if len(args) < 2:
print(
- "Usage: %s clean <path> <extension>\n" % __file__
+ "Usage: %s clean <paths> <extension>\n" % __file__
+ "\tRemoves all files with extension from <path>."
)
return 1
- for filename in findFilesWithExtension(args[0], args[1]):
- os.remove(filename)
+ for path in args[1:-1]:
+ for filename in findFilesWithExtension(path, args[-1]):
+ os.remove(filename)
return 0
def merge(args):
- if len(args) != 3:
+ if len(args) < 3:
print(
- "Usage: %s merge <llvm-profdata> <output> <path>\n" % __file__
+ "Usage: %s merge <llvm-profdata> <output> <paths>\n" % __file__
+ "\tMerges all profraw files from path into output."
)
return 1
cmd = [args[0], "merge", "-o", args[1]]
- cmd.extend(findFilesWithExtension(args[2], "profraw"))
+ for path in args[2:]:
+ cmd.extend(findFilesWithExtension(path, "profraw"))
subprocess.check_call(cmd)
return 0
diff --git a/llvm/docs/AdvancedBuilds.rst b/llvm/docs/AdvancedBuilds.rst
index 960b19fa5317f..ee178dd3772c4 100644
--- a/llvm/docs/AdvancedBuilds.rst
+++ b/llvm/docs/AdvancedBuilds.rst
@@ -145,6 +145,29 @@ that also enables ThinTLO, use the following command:
-DPGO_INSTRUMENT_LTO=Thin \
<path to source>/llvm
+By default, clang will generate profile data by compiling a simple
+hello world program. You can also tell clang use an external
+project for generating profile data that may be a better fit for your
+use case. The project you specify must either be a lit test suite
+(use the CLANG_PGO_TRAINING_DATA option) or a CMake project (use the
+CLANG_PERF_TRAINING_DATA_SOURCE_DIR option).
+
+For example, If you wanted to use the
+`LLVM Test Suite <https://github.com/llvm/llvm-test-suite/>`_ to generate
+profile data you would use the following command:
+
+.. code-block:: console
+
+ $ cmake -G Ninja -C <path to source>/clang/cmake/caches/PGO.cmake \
+ -DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=<path to llvm-test-suite> \
+ -DBOOTSTRAP_CLANG_PGO_TRAINING_DEPS=runtimes
+
+The BOOTSTRAP\_ prefixes tells CMake to pass the variables on to the instrumented
+stage two build. And the CLANG_PGO_TRAINING_DEPS option let's you specify
+additional build targets to build before building the external project. The
+LLVM Test Suite requires compiler-rt to build, so we need to add the
+`runtimes` target as a dependency.
+
After configuration, building the stage2-instrumented-generate-profdata target
will automatically build the stage1 compiler, build the instrumented compiler
with the stage1 compiler, and then run the instrumented compiler against the
@@ -172,12 +195,12 @@ You can feed that file into the LLVM_PROFDATA_FILE option when you build your
optimized compiler.
It may be necessary to build additional targets before running perf training, such as
-builtins and runtime libraries. You can use the :code:`CLANG_PERF_TRAINING_DEPS` CMake
+builtins and runtime libraries. You can use the :code:`CLANG_PGO_TRAINING_DEPS` CMake
variable for that purpose:
.. code-block:: cmake
- set(CLANG_PERF_TRAINING_DEPS builtins runtimes CACHE STRING "")
+ set(CLANG_PGO_TRAINING_DEPS builtins runtimes CACHE STRING "")
The PGO cache has a slightly different stage naming scheme than other
multi-stage builds. It generates three stages: stage1, stage2-instrumented, and
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
LGTM! Will be nice to have this available in the release branch for the release builds and the CI container.
…file data (llvm#78879) The new CLANG_PGO_TRAINING_DATA_SOURCE_DIR allows users to specify a CMake project to use for generating the profile data. For example, to use the llvm-test-suite to generate profile data you would do: $ cmake -G Ninja -B build -S llvm -C <path to source>/clang/cmake/caches/PGO.cmake \ -DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=<path to llvm-test-suite> \ -DBOOTSTRAP_CLANG_PGO_TRAINING_DEPS=runtimes Note that the CLANG_PERF_TRAINING_DEPS has been renamed to CLANG_PGO_TRAINING_DEPS. --------- Co-authored-by: Petr Hosek <[email protected]> (cherry picked from commit dd0356d)
resolves #80565