From 84035f5d2be628d9cb40b9812154b64d3ea53357 Mon Sep 17 00:00:00 2001 From: "Damien Buhl (alias daminetreg)" Date: Mon, 17 Nov 2025 09:53:04 +0000 Subject: [PATCH] :gear: cmake-re v0.0.81 configuration to build and cache on RBE This provides configuration + documentation to run a build on a remote caching and execution cluster. This was tested on a 500 cores EngFlow cluster, bringing the 1min clean build to 10sec and mere seconds when the cache is loaded. --- .gitmodules | 3 +++ README.md | 20 +++++++++++++++++++ .../how_to_build_and_run_examples.dox | 20 ++++++++++++++----- environments | 1 + 4 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 .gitmodules create mode 160000 environments diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..1e36d3630a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "environments"] + path = environments + url = https://github.com/tipi-build/environments.git diff --git a/README.md b/README.md index 03936fc02c..05d6fa8f51 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,26 @@ Pre-build binaries are generated with the following security / good coding pract [docs-howtobuild]: https://artificial-intelligence.sites.arm.com/computelibrary/latest/how_to_build.xhtml +### Run a CMake `--distributed` build with caching and remote execution on an RBE cluster +Fast cached and distributed builds can be made through the use of `cmake-re` : + +```bash +# Authenticate +export RBE_service=:443 +export RBE_tls_client_auth_key=$HOME/engflow-mTLS/engflow.key +export RBE_tls_client_auth_cert=$HOME/engflow-mTLS/engflow.crt + +export TIPI_CACHE_CONSUME_ONLY=ON # Disable TIPI additional cache layers + +cmake-re -S . -B build/aarch64-re -DCMAKE_TOOLCHAIN_FILE=environments/linux-ubuntu-2404-aarch64-linux-gnu.cmake +cmake-re --distributed --build build/aarch64-re -j500 +``` + +When remote execution is not wished, and only remote caching is wanted one can set the environment variable : + * `RBE_exec_strategy=local` + +Usage & Details on the [`cmake-re` documentation](https://tipi.build/documentation/0352-distributed-builds) +
## How to contribute diff --git a/docs/user_guide/how_to_build_and_run_examples.dox b/docs/user_guide/how_to_build_and_run_examples.dox index 28c8c459a4..7cf6913f12 100644 --- a/docs/user_guide/how_to_build_and_run_examples.dox +++ b/docs/user_guide/how_to_build_and_run_examples.dox @@ -493,19 +493,20 @@ Build validation and benchmarking: bazel build //tests:all -@subsection S1_8_2_cmake_build CMake build +@subsection S1_8_2_cmake_build CMake RE build @subsubsection S1_8_2_1_file_structure File structure -File structure for all files included in the CMake build: +File structure for all files included in the CMake RE build: . ├── CMakeLists.txt ├── cmake │ ├── Options.cmake - │ ├── Version.cmake - │  └── toolchains - │  └── aarch64_linux_toolchain.cmake + │ └── Version.cmake + ├── environments + │ ├── linux-ubuntu-2404-aarch64-linux-gnu.pkr.js + │  └── linux-ubuntu-2404-aarch64-linux-gnu.cmake ├── examples │  └── CMakeLists.txt ├── src @@ -539,6 +540,15 @@ To build libraries, examples and tests: cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DARM_COMPUTE_ENABLE_OPENMP=1 -DARM_COMPUTE_BUILD_EXAMPLES=1 -DARM_COMPUTE_BUILD_TESTING=1 cmake --build build -j32 +If you want to speed-up the build you can run it accelerated with remote +execution and caching with a RE-API build cluster (CMAKE_TOOLCHAIN_FILE +is mandatory to specify the environments in which the builds will be +distributed, read more in the [cmake-re environments documentation](https://tipi.build/documentation/0400-environments)): + + export RBE_service=:443 + cmake-re -S . -B build -DCMAKE_TOOLCHAIN_FILE=environments/linux-ubuntu-2404-aarch64-linux-gnu.cmake + cmake-re --build build -j500 --distributed + @section S1_9_fixed_format Building with support for fixed format kernels @subsection S1_9_1_intro_to_fixed_format_kernels What are fixed format kernels? diff --git a/environments b/environments new file mode 160000 index 0000000000..e57875e4f5 --- /dev/null +++ b/environments @@ -0,0 +1 @@ +Subproject commit e57875e4f5b891a331725355a07a0454760b8068