Report dependency download progress during CMake configure

CMakeLists.txt

@@ -60,8 +60,15 @@ if(EMSCRIPTEN)
   add_link_options("-sEXIT_RUNTIME=1")
 endif()
 
-FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa EXCLUDE_FROM_ALL)
-FetchContent_MakeAvailable(highway)
+# Dependency fetching. gemma_fetch() reports download progress so the configure
+# step does not sit silent for minutes; see cmake/GemmaFetch.cmake.
+set(GEMMA_DEP_TOTAL 5)
+if(GEMMA_ONEDNN_BRGEMM OR GEMMA_ONEDNN_MATMUL)
+  math(EXPR GEMMA_DEP_TOTAL "${GEMMA_DEP_TOTAL} + 1")
+endif()
+include(${CMAKE_CURRENT_LIST_DIR}/cmake/GemmaFetch.cmake)
+
+gemma_fetch(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa EXCLUDE_FROM_ALL)
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
    CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15)
@@ -78,11 +85,9 @@ endif()
 
 ## Note: absl needs to be installed by sentencepiece. This will only happen if
 ## cmake is invoked with -DSPM_ENABLE_SHARED=OFF and -DSPM_ABSL_PROVIDER=module
-FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9 EXCLUDE_FROM_ALL)
-FetchContent_MakeAvailable(sentencepiece)
+gemma_fetch(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9 EXCLUDE_FROM_ALL)
 
-FetchContent_Declare(json GIT_REPOSITORY https://github.com/nlohmann/json.git GIT_TAG 9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03 EXCLUDE_FROM_ALL)
-FetchContent_MakeAvailable(json)
+gemma_fetch(json GIT_REPOSITORY https://github.com/nlohmann/json.git GIT_TAG 9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03 EXCLUDE_FROM_ALL)
 
 # Find OpenSSL for HTTPS support
 find_package(OpenSSL)
@@ -95,8 +100,7 @@ else()
 endif()
 
 # HTTP library for API server
-FetchContent_Declare(httplib GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git GIT_TAG v0.18.1 EXCLUDE_FROM_ALL)
-FetchContent_MakeAvailable(httplib)
+gemma_fetch(httplib GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git GIT_TAG v0.18.1 EXCLUDE_FROM_ALL)
 
 # Create interface target for httplib (header-only library)
 add_library(httplib_interface INTERFACE)
@@ -109,8 +113,7 @@ endif()
 set(BENCHMARK_ENABLE_TESTING OFF)
 set(BENCHMARK_ENABLE_GTEST_TESTS OFF)
 
-FetchContent_Declare(benchmark GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v1.8.2 EXCLUDE_FROM_ALL)
-FetchContent_MakeAvailable(benchmark)
+gemma_fetch(benchmark GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v1.8.2 EXCLUDE_FROM_ALL)
 if(EMSCRIPTEN)
   target_compile_options(benchmark PRIVATE -Wno-c2y-extensions)
 endif()
@@ -123,12 +126,11 @@ if(GEMMA_ONEDNN_BRGEMM)
   set(DNNL_GPU_RUNTIME "NONE" CACHE STRING "" FORCE)
   set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "" FORCE)
   set(DNNL_EXPERIMENTAL_UKERNEL ON CACHE BOOL "" FORCE)
-  FetchContent_Declare(onednn
+  gemma_fetch(onednn
     GIT_REPOSITORY https://github.com/uxlfoundation/oneDNN.git
     GIT_TAG v3.11
     EXCLUDE_FROM_ALL
   )
-  FetchContent_MakeAvailable(onednn)
   message(STATUS "OneDNN BRGeMM micro-kernel support enabled")
 endif()
 
@@ -141,12 +143,11 @@ if(GEMMA_ONEDNN_MATMUL)
   set(DNNL_CPU_RUNTIME "THREADPOOL" CACHE STRING "" FORCE)
   set(DNNL_GPU_RUNTIME "NONE" CACHE STRING "" FORCE)
   set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "" FORCE)
-  FetchContent_Declare(onednn
+  gemma_fetch(onednn
     GIT_REPOSITORY https://github.com/uxlfoundation/oneDNN.git
     GIT_TAG v3.11
     EXCLUDE_FROM_ALL
   )
-  FetchContent_MakeAvailable(onednn)
   message(STATUS "OneDNN matmul primitive (threadpool runtime) support enabled")
 endif()
 

cmake/GemmaFetch.cmake

@@ -0,0 +1,88 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Progress reporting for FetchContent dependencies.
+#
+# A cold configure clones several repositories from GitHub -- roughly 200 MB for
+# the top-level build, of which nlohmann/json alone is ~105k objects and can
+# take over ten minutes on a slow link. FetchContent hides download output by
+# default, so the configure step sits silent and looks like it has stalled.
+#
+# gemma_fetch() wraps FetchContent_Declare + FetchContent_MakeAvailable with a
+# progress bar naming the dependency and its elapsed time, and turns
+# FETCHCONTENT_QUIET off so git's own "Receiving objects: ..." counter shows the
+# fine-grained progress that CMake cannot know about. Configure with
+# -DGEMMA_QUIET_FETCH=ON to restore the old silent behavior.
+#
+# Set GEMMA_DEP_TOTAL to the number of gemma_fetch() calls before the first one
+# so the bar knows its denominator.
+#
+# Deliberately not include_guard()ed: the examples build gemma.cpp as a
+# subproject, and each project that includes this module gets its own counter
+# reset in its own directory scope, so the inner build counts 1/5..5/5 rather
+# than continuing the outer project's numbering.
+
+include(FetchContent)
+
+option(GEMMA_QUIET_FETCH "Hide dependency download progress during configure" OFF)
+set(FETCHCONTENT_QUIET ${GEMMA_QUIET_FETCH})
+
+if(NOT DEFINED GEMMA_DEP_TOTAL OR GEMMA_DEP_TOTAL LESS 1)
+  set(GEMMA_DEP_TOTAL 1)
+endif()
+set(GEMMA_DEP_INDEX 0)
+
+# gemma_fetch(<name> [<FetchContent_Declare args>...])
+# Extra arguments are forwarded verbatim to FetchContent_Declare.
+function(gemma_fetch name)
+  math(EXPR _index "${GEMMA_DEP_INDEX} + 1")
+  set(GEMMA_DEP_INDEX ${_index} PARENT_SCOPE)
+
+  # 20-cell bar counting dependencies already available. It steps once per
+  # dependency; git prints the intra-clone progress itself.
+  math(EXPR _done "(${_index} - 1) * 20 / ${GEMMA_DEP_TOTAL}")
+  math(EXPR _pct "(${_index} - 1) * 100 / ${GEMMA_DEP_TOTAL}")
+  set(_bar "")
+  set(_cell 0)
+  while(_cell LESS 20)
+    if(_cell LESS _done)
+      string(APPEND _bar "=")
+    else()
+      string(APPEND _bar "-")
+    endif()
+    math(EXPR _cell "${_cell} + 1")
+  endwhile()
+  message(STATUS
+    "[${_bar}] ${_pct}% | dependency ${_index}/${GEMMA_DEP_TOTAL}: ${name} -- fetching and configuring, please wait")
+
+  string(TIMESTAMP _start "%s")
+  # GIT_PROGRESS makes git report transfer progress even though its stderr is
+  # not a terminal here. Only meaningful for git downloads, so guard it.
+  if("GIT_REPOSITORY" IN_LIST ARGN)
+    FetchContent_Declare(${name} ${ARGN} GIT_PROGRESS TRUE)
+  else()
+    FetchContent_Declare(${name} ${ARGN})
+  endif()
+  FetchContent_MakeAvailable(${name})
+  string(TIMESTAMP _end "%s")
+  math(EXPR _elapsed "${_end} - ${_start}")
+  message(STATUS "         ${name} ready (${_elapsed}s)")
+
+  # FetchContent sets these in the caller's scope, which here is this function.
+  # Hoist them so callers still see e.g. sentencepiece_SOURCE_DIR.
+  string(TOLOWER "${name}" _lc)
+  set(${_lc}_SOURCE_DIR "${${_lc}_SOURCE_DIR}" PARENT_SCOPE)
+  set(${_lc}_BINARY_DIR "${${_lc}_BINARY_DIR}" PARENT_SCOPE)
+  set(${_lc}_POPULATED "${${_lc}_POPULATED}" PARENT_SCOPE)
+endfunction()

examples/hello_world/CMakeLists.txt

@@ -18,14 +18,29 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 include(FetchContent)
-FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa)
-FetchContent_MakeAvailable(highway)
+
+# Download progress reporting, shared with the top-level build. This example is
+# a standalone project, so fall back to plain FetchContent if it was copied out
+# of the gemma.cpp tree and the module is not next to it.
+set(GEMMA_DEP_TOTAL 3)
+set(_gemma_fetch_module "${CMAKE_CURRENT_LIST_DIR}/../../cmake/GemmaFetch.cmake")
+if(EXISTS "${_gemma_fetch_module}")
+  include("${_gemma_fetch_module}")
+else()
+  function(gemma_fetch name)
+    FetchContent_Declare(${name} ${ARGN})
+    FetchContent_MakeAvailable(${name})
+    string(TOLOWER "${name}" _lc)
+    set(${_lc}_SOURCE_DIR "${${_lc}_SOURCE_DIR}" PARENT_SCOPE)
+  endfunction()
+endif()
+
+gemma_fetch(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa)
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
    CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15)
   target_compile_definitions(hwy PUBLIC HWY_DISABLED_TARGETS=HWY_AVX10_2)
 endif()
-FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9)
-FetchContent_MakeAvailable(sentencepiece)
+gemma_fetch(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9)
 
 
 
@@ -36,11 +51,10 @@ if (NOT BUILD_MODE)
 endif()
 if (BUILD_MODE STREQUAL "local")
   # Relative path to gemma.cpp from examples/hello_world/build/
-  FetchContent_Declare(gemma SOURCE_DIR ../../..)
+  gemma_fetch(gemma SOURCE_DIR ../../..)
 else()
-  FetchContent_Declare(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG 4a924f179448dc83e46a2af9520c61b4ef56174c)
+  gemma_fetch(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG 4a924f179448dc83e46a2af9520c61b4ef56174c)
 endif()
-FetchContent_MakeAvailable(gemma)
 
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Release")

examples/simplified_gemma/CMakeLists.txt

@@ -18,14 +18,29 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 include(FetchContent)
-FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa)
-FetchContent_MakeAvailable(highway)
+
+# Download progress reporting, shared with the top-level build. This example is
+# a standalone project, so fall back to plain FetchContent if it was copied out
+# of the gemma.cpp tree and the module is not next to it.
+set(GEMMA_DEP_TOTAL 3)
+set(_gemma_fetch_module "${CMAKE_CURRENT_LIST_DIR}/../../cmake/GemmaFetch.cmake")
+if(EXISTS "${_gemma_fetch_module}")
+  include("${_gemma_fetch_module}")
+else()
+  function(gemma_fetch name)
+    FetchContent_Declare(${name} ${ARGN})
+    FetchContent_MakeAvailable(${name})
+    string(TOLOWER "${name}" _lc)
+    set(${_lc}_SOURCE_DIR "${${_lc}_SOURCE_DIR}" PARENT_SCOPE)
+  endfunction()
+endif()
+
+gemma_fetch(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9d5b12611fcfe145f988771c45e7bae9f78cb7fa)
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
    CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15)
   target_compile_definitions(hwy PUBLIC HWY_DISABLED_TARGETS=HWY_AVX10_2)
 endif()
-FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9)
-FetchContent_MakeAvailable(sentencepiece)
+gemma_fetch(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9)
 
 
 
@@ -36,11 +51,10 @@ if (NOT BUILD_MODE)
 endif()
 if (BUILD_MODE STREQUAL "local")
   # Relative path to gemma.cpp from examples/simplified_gemma/build/
-  FetchContent_Declare(gemma SOURCE_DIR ../../..)
+  gemma_fetch(gemma SOURCE_DIR ../../..)
 else()
-  FetchContent_Declare(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG a9aa63fd2ea6b786ed0706d619588bfe2d43370e)
+  gemma_fetch(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG a9aa63fd2ea6b786ed0706d619588bfe2d43370e)
 endif()
-FetchContent_MakeAvailable(gemma)
 
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Release")