From 298f0e1c991ace81cefa4b49bcbc4e6c4466fe1f Mon Sep 17 00:00:00 2001 From: Mikyx-1 Date: Sun, 16 Aug 2026 17:28:17 +0700 Subject: [PATCH] Report dependency download progress during CMake configure FetchContent is quiet by default, so a cold configure sits silent for minutes while it clones dependencies and gives no sign it is alive. nlohmann/json alone is ~105k objects and took 639s of the 669s configure measured on a slow link, all of it without output. Add cmake/GemmaFetch.cmake, whose gemma_fetch() wraps FetchContent_Declare and FetchContent_MakeAvailable with a progress bar naming each dependency and its elapsed time, and turns FETCHCONTENT_QUIET off so git's own "Receiving objects" counter shows the fine-grained progress CMake cannot know about. Configure with -DGEMMA_QUIET_FETCH=ON for the old behavior. Used by the top-level build and by both examples, which are standalone projects with their own fetches. The examples fall back to a plain gemma_fetch() if the folder is copied out of the tree. The module is deliberately not include_guard()ed so a nested build (an example fetching gemma.cpp as a subproject) restarts its own counter instead of continuing the outer project's numbering. --- CMakeLists.txt | 29 ++++---- cmake/GemmaFetch.cmake | 88 ++++++++++++++++++++++++ examples/hello_world/CMakeLists.txt | 28 ++++++-- examples/simplified_gemma/CMakeLists.txt | 28 ++++++-- 4 files changed, 145 insertions(+), 28 deletions(-) create mode 100644 cmake/GemmaFetch.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index f199a323..b55f9bf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,8 +60,15 @@ if(EMSCRIPTEN) add_link_options("-sEXIT_RUNTIME=1") endif() -FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa EXCLUDE_FROM_ALL) -FetchContent_MakeAvailable(highway) +# Dependency fetching. gemma_fetch() reports download progress so the configure +# step does not sit silent for minutes; see cmake/GemmaFetch.cmake. +set(GEMMA_DEP_TOTAL 5) +if(GEMMA_ONEDNN_BRGEMM OR GEMMA_ONEDNN_MATMUL) + math(EXPR GEMMA_DEP_TOTAL "${GEMMA_DEP_TOTAL} + 1") +endif() +include(${CMAKE_CURRENT_LIST_DIR}/cmake/GemmaFetch.cmake) + +gemma_fetch(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa EXCLUDE_FROM_ALL) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15) @@ -78,11 +85,9 @@ endif() ## Note: absl needs to be installed by sentencepiece. This will only happen if ## cmake is invoked with -DSPM_ENABLE_SHARED=OFF and -DSPM_ABSL_PROVIDER=module -FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9 EXCLUDE_FROM_ALL) -FetchContent_MakeAvailable(sentencepiece) +gemma_fetch(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9 EXCLUDE_FROM_ALL) -FetchContent_Declare(json GIT_REPOSITORY https://github.com/nlohmann/json.git GIT_TAG 9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03 EXCLUDE_FROM_ALL) -FetchContent_MakeAvailable(json) +gemma_fetch(json GIT_REPOSITORY https://github.com/nlohmann/json.git GIT_TAG 9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03 EXCLUDE_FROM_ALL) # Find OpenSSL for HTTPS support find_package(OpenSSL) @@ -95,8 +100,7 @@ else() endif() # HTTP library for API server -FetchContent_Declare(httplib GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git GIT_TAG v0.18.1 EXCLUDE_FROM_ALL) -FetchContent_MakeAvailable(httplib) +gemma_fetch(httplib GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git GIT_TAG v0.18.1 EXCLUDE_FROM_ALL) # Create interface target for httplib (header-only library) add_library(httplib_interface INTERFACE) @@ -109,8 +113,7 @@ endif() set(BENCHMARK_ENABLE_TESTING OFF) set(BENCHMARK_ENABLE_GTEST_TESTS OFF) -FetchContent_Declare(benchmark GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v1.8.2 EXCLUDE_FROM_ALL) -FetchContent_MakeAvailable(benchmark) +gemma_fetch(benchmark GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v1.8.2 EXCLUDE_FROM_ALL) if(EMSCRIPTEN) target_compile_options(benchmark PRIVATE -Wno-c2y-extensions) endif() @@ -123,12 +126,11 @@ if(GEMMA_ONEDNN_BRGEMM) set(DNNL_GPU_RUNTIME "NONE" CACHE STRING "" FORCE) set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "" FORCE) set(DNNL_EXPERIMENTAL_UKERNEL ON CACHE BOOL "" FORCE) - FetchContent_Declare(onednn + gemma_fetch(onednn GIT_REPOSITORY https://github.com/uxlfoundation/oneDNN.git GIT_TAG v3.11 EXCLUDE_FROM_ALL ) - FetchContent_MakeAvailable(onednn) message(STATUS "OneDNN BRGeMM micro-kernel support enabled") endif() @@ -141,12 +143,11 @@ if(GEMMA_ONEDNN_MATMUL) set(DNNL_CPU_RUNTIME "THREADPOOL" CACHE STRING "" FORCE) set(DNNL_GPU_RUNTIME "NONE" CACHE STRING "" FORCE) set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "" FORCE) - FetchContent_Declare(onednn + gemma_fetch(onednn GIT_REPOSITORY https://github.com/uxlfoundation/oneDNN.git GIT_TAG v3.11 EXCLUDE_FROM_ALL ) - FetchContent_MakeAvailable(onednn) message(STATUS "OneDNN matmul primitive (threadpool runtime) support enabled") endif() diff --git a/cmake/GemmaFetch.cmake b/cmake/GemmaFetch.cmake new file mode 100644 index 00000000..970adadc --- /dev/null +++ b/cmake/GemmaFetch.cmake @@ -0,0 +1,88 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Progress reporting for FetchContent dependencies. +# +# A cold configure clones several repositories from GitHub -- roughly 200 MB for +# the top-level build, of which nlohmann/json alone is ~105k objects and can +# take over ten minutes on a slow link. FetchContent hides download output by +# default, so the configure step sits silent and looks like it has stalled. +# +# gemma_fetch() wraps FetchContent_Declare + FetchContent_MakeAvailable with a +# progress bar naming the dependency and its elapsed time, and turns +# FETCHCONTENT_QUIET off so git's own "Receiving objects: ..." counter shows the +# fine-grained progress that CMake cannot know about. Configure with +# -DGEMMA_QUIET_FETCH=ON to restore the old silent behavior. +# +# Set GEMMA_DEP_TOTAL to the number of gemma_fetch() calls before the first one +# so the bar knows its denominator. +# +# Deliberately not include_guard()ed: the examples build gemma.cpp as a +# subproject, and each project that includes this module gets its own counter +# reset in its own directory scope, so the inner build counts 1/5..5/5 rather +# than continuing the outer project's numbering. + +include(FetchContent) + +option(GEMMA_QUIET_FETCH "Hide dependency download progress during configure" OFF) +set(FETCHCONTENT_QUIET ${GEMMA_QUIET_FETCH}) + +if(NOT DEFINED GEMMA_DEP_TOTAL OR GEMMA_DEP_TOTAL LESS 1) + set(GEMMA_DEP_TOTAL 1) +endif() +set(GEMMA_DEP_INDEX 0) + +# gemma_fetch( [...]) +# Extra arguments are forwarded verbatim to FetchContent_Declare. +function(gemma_fetch name) + math(EXPR _index "${GEMMA_DEP_INDEX} + 1") + set(GEMMA_DEP_INDEX ${_index} PARENT_SCOPE) + + # 20-cell bar counting dependencies already available. It steps once per + # dependency; git prints the intra-clone progress itself. + math(EXPR _done "(${_index} - 1) * 20 / ${GEMMA_DEP_TOTAL}") + math(EXPR _pct "(${_index} - 1) * 100 / ${GEMMA_DEP_TOTAL}") + set(_bar "") + set(_cell 0) + while(_cell LESS 20) + if(_cell LESS _done) + string(APPEND _bar "=") + else() + string(APPEND _bar "-") + endif() + math(EXPR _cell "${_cell} + 1") + endwhile() + message(STATUS + "[${_bar}] ${_pct}% | dependency ${_index}/${GEMMA_DEP_TOTAL}: ${name} -- fetching and configuring, please wait") + + string(TIMESTAMP _start "%s") + # GIT_PROGRESS makes git report transfer progress even though its stderr is + # not a terminal here. Only meaningful for git downloads, so guard it. + if("GIT_REPOSITORY" IN_LIST ARGN) + FetchContent_Declare(${name} ${ARGN} GIT_PROGRESS TRUE) + else() + FetchContent_Declare(${name} ${ARGN}) + endif() + FetchContent_MakeAvailable(${name}) + string(TIMESTAMP _end "%s") + math(EXPR _elapsed "${_end} - ${_start}") + message(STATUS " ${name} ready (${_elapsed}s)") + + # FetchContent sets these in the caller's scope, which here is this function. + # Hoist them so callers still see e.g. sentencepiece_SOURCE_DIR. + string(TOLOWER "${name}" _lc) + set(${_lc}_SOURCE_DIR "${${_lc}_SOURCE_DIR}" PARENT_SCOPE) + set(${_lc}_BINARY_DIR "${${_lc}_BINARY_DIR}" PARENT_SCOPE) + set(${_lc}_POPULATED "${${_lc}_POPULATED}" PARENT_SCOPE) +endfunction() diff --git a/examples/hello_world/CMakeLists.txt b/examples/hello_world/CMakeLists.txt index c253b768..a933ac90 100644 --- a/examples/hello_world/CMakeLists.txt +++ b/examples/hello_world/CMakeLists.txt @@ -18,14 +18,29 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) include(FetchContent) -FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa) -FetchContent_MakeAvailable(highway) + +# Download progress reporting, shared with the top-level build. This example is +# a standalone project, so fall back to plain FetchContent if it was copied out +# of the gemma.cpp tree and the module is not next to it. +set(GEMMA_DEP_TOTAL 3) +set(_gemma_fetch_module "${CMAKE_CURRENT_LIST_DIR}/../../cmake/GemmaFetch.cmake") +if(EXISTS "${_gemma_fetch_module}") + include("${_gemma_fetch_module}") +else() + function(gemma_fetch name) + FetchContent_Declare(${name} ${ARGN}) + FetchContent_MakeAvailable(${name}) + string(TOLOWER "${name}" _lc) + set(${_lc}_SOURCE_DIR "${${_lc}_SOURCE_DIR}" PARENT_SCOPE) + endfunction() +endif() + +gemma_fetch(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15) target_compile_definitions(hwy PUBLIC HWY_DISABLED_TARGETS=HWY_AVX10_2) endif() -FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9) -FetchContent_MakeAvailable(sentencepiece) +gemma_fetch(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9) @@ -36,11 +51,10 @@ if (NOT BUILD_MODE) endif() if (BUILD_MODE STREQUAL "local") # Relative path to gemma.cpp from examples/hello_world/build/ - FetchContent_Declare(gemma SOURCE_DIR ../../..) + gemma_fetch(gemma SOURCE_DIR ../../..) else() - FetchContent_Declare(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG 4a924f179448dc83e46a2af9520c61b4ef56174c) + gemma_fetch(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG 4a924f179448dc83e46a2af9520c61b4ef56174c) endif() -FetchContent_MakeAvailable(gemma) if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release") diff --git a/examples/simplified_gemma/CMakeLists.txt b/examples/simplified_gemma/CMakeLists.txt index 471cdfa3..b7f9e248 100644 --- a/examples/simplified_gemma/CMakeLists.txt +++ b/examples/simplified_gemma/CMakeLists.txt @@ -18,14 +18,29 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) include(FetchContent) -FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa) -FetchContent_MakeAvailable(highway) + +# Download progress reporting, shared with the top-level build. This example is +# a standalone project, so fall back to plain FetchContent if it was copied out +# of the gemma.cpp tree and the module is not next to it. +set(GEMMA_DEP_TOTAL 3) +set(_gemma_fetch_module "${CMAKE_CURRENT_LIST_DIR}/../../cmake/GemmaFetch.cmake") +if(EXISTS "${_gemma_fetch_module}") + include("${_gemma_fetch_module}") +else() + function(gemma_fetch name) + FetchContent_Declare(${name} ${ARGN}) + FetchContent_MakeAvailable(${name}) + string(TOLOWER "${name}" _lc) + set(${_lc}_SOURCE_DIR "${${_lc}_SOURCE_DIR}" PARENT_SCOPE) + endfunction() +endif() + +gemma_fetch(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15) target_compile_definitions(hwy PUBLIC HWY_DISABLED_TARGETS=HWY_AVX10_2) endif() -FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9) -FetchContent_MakeAvailable(sentencepiece) +gemma_fetch(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9) @@ -36,11 +51,10 @@ if (NOT BUILD_MODE) endif() if (BUILD_MODE STREQUAL "local") # Relative path to gemma.cpp from examples/simplified_gemma/build/ - FetchContent_Declare(gemma SOURCE_DIR ../../..) + gemma_fetch(gemma SOURCE_DIR ../../..) else() - FetchContent_Declare(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG a9aa63fd2ea6b786ed0706d619588bfe2d43370e) + gemma_fetch(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG a9aa63fd2ea6b786ed0706d619588bfe2d43370e) endif() -FetchContent_MakeAvailable(gemma) if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release")