upgrade ggml to 0.20.2

README.md

@@ -213,7 +213,8 @@ tools/transcribe-quantize/ Quantization tool source
 bindings/                  Python, TypeScript, Rust, and Swift bindings
 docs/                      Porting and validation guidance
 scripts/                   Python converter + test tooling
-ggml/                      Vendored ggml (see ggml/UPSTREAM for pinned SHA)
+ggml/                      Vendored ggml (see ggml/UPSTREAM for its recipe)
+patches/ggml/              Downstream patches applied by scripts/sync-ggml.sh
 src/third_party/miniz/     Vendored miniz deflate codec (see its UPSTREAM file)
 samples/                   Test audio files
 tests/                     Unit and smoke tests

ggml/.pi/SYSTEM.md

@@ -0,0 +1 @@
+gg/SYSTEM.md

ggml/.pi/gg/SYSTEM.md

@@ -0,0 +1,27 @@
+You are a coding agent. Here are some very important rules that you must follow:
+
+General:
+- Be very precise and concise when writing code, comments, explanations, etc.
+- PR and commit titles format: `<module> : <title>`. Lookup recents for examples
+- Don't try to build or run the code unless you are explicitly asked to do so
+- Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
+
+Coding:
+- When in doubt, always refer to the CONTRIBUTING.md file of the project
+- When referencing issues or PRs in comments, use the format:
+  - C/C++ code: `// ref: <url>`
+  - Other (CMake, etc.): `# ref: <url>`
+
+Pull requests (PRs):
+- New branch names are prefixed with "gg/"
+- Before opening a pull request, ask the user to confirm the description
+- When creating a pull request, look for the repository's PR template and follow it
+- For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]"
+- Ask the user to tell you what model was used and write it in place of [MODEL]
+- Always create the pull requests in draft mode
+
+Commits:
+- On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag
+- Do not explicitly set the git author in commits - rely on the default git config
+- Always use `--no-gpg-sign` when committing
+- Never `git push` without explicit confirmation from the user

ggml/CMakeLists.txt

@@ -4,7 +4,7 @@ project("ggml" C CXX ASM)
 
 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 15)
+set(GGML_VERSION_MINOR 20)
 set(GGML_VERSION_PATCH 2)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
 
@@ -216,7 +216,6 @@ option(GGML_HIP                             "ggml: use HIP"
 option(GGML_HIP_GRAPHS                      "ggml: use HIP graph"                              ON)
 option(GGML_HIP_RCCL                        "ggml: use ROCm Collective Comm. Library"         OFF)
 option(GGML_HIP_NO_VMM                      "ggml: do not try to use HIP VMM"                 ON)
-option(GGML_HIP_ROCWMMA_FATTN               "ggml: enable rocWMMA for FlashAttention"         OFF)
 option(GGML_HIP_MMQ_MFMA                    "ggml: enable MFMA MMA for CDNA in MMQ"           ON)
 option(GGML_HIP_EXPORT_METRICS              "ggml: enable kernel perf metrics output"         OFF)
 option(GGML_MUSA_GRAPHS                     "ggml: use MUSA graph, experimental, unstable"    OFF)
@@ -257,6 +256,8 @@ set   (GGML_SYCL_DEVICE_ARCH "" CACHE STRING
                                             "ggml: sycl device architecture")
 
 option(GGML_OPENVINO                        "ggml: use OPENVINO"                              OFF)
+option(GGML_ET                              "ggml: use ET backend"                            OFF)
+option(GGML_ET_SYSEMU                       "ggml: use ET backend via sysemu"                 OFF)
 
 option(GGML_OPENCL                          "ggml: use OpenCL"                                OFF)
 option(GGML_OPENCL_PROFILING                "ggml: use OpenCL profiling (increases overhead)" OFF)
@@ -266,7 +267,6 @@ set   (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING
                                             "ggml: OpenCL API version to target")
 
 option(GGML_HEXAGON                         "ggml: enable Hexagon backend"                    OFF)
-set(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING "ggml: quantize group size (32, 64, or 128)")
 
 # toolchain for vulkan-shaders-gen
 set   (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen")
@@ -402,7 +402,7 @@ configure_package_config_file(
               GGML_BIN_INSTALL_DIR)
 
 write_basic_package_version_file(
-        ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
+        ${CMAKE_CURRENT_BINARY_DIR}/ggml-config-version.cmake
     VERSION ${GGML_INSTALL_VERSION}
     COMPATIBILITY SameMajorVersion)
 
@@ -414,7 +414,7 @@ message(STATUS "ggml version: ${GGML_INSTALL_VERSION}")
 message(STATUS "ggml commit:  ${GGML_BUILD_COMMIT}")
 
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
-              ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
+              ${CMAKE_CURRENT_BINARY_DIR}/ggml-config-version.cmake
         DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml)
 
 if (MSVC)

ggml/UPSTREAM

@@ -1,8 +1,9 @@
-repo: https://github.com/ggml-org/ggml
-sha:  707321c4cf6d21cb4bc831aa8b687dbf01a521ce
+repo: git@github.com:ggml-org/ggml.git
+sha:  8c63e70982c95ceb862e3a1073a2c1beef75d60a
+patches:
+  patches/ggml/0001-fix-threadpool-oversubscription.patch
 
-This directory is a vendored snapshot of ggml at the SHA above. Do not edit
-files in this directory by hand; local changes are overwritten on the next sync.
-To move the snapshot, run scripts/sync-ggml.sh <ref> from the repo root: it
-re-vendors this directory and rewrites this file. The snapshot is upstream's
-tracked tree at the SHA, minus .github/ (upstream CI, irrelevant to a vendor).
+This directory is generated from the upstream ggml tree at the SHA above, minus
+.github/, with the listed downstream patches applied in order. Do not edit it by
+hand. Run scripts/sync-ggml.sh <ref> from the repo root to reproduce or upgrade
+it; the script rewrites this file.

ggml/cmake/ggml-config.cmake.in

@@ -113,6 +113,7 @@ set_and_check(GGML_LIB_DIR "@PACKAGE_GGML_LIB_INSTALL_DIR@")
 if(NOT TARGET ggml::ggml)
     find_package(Threads REQUIRED)
 
+    unset(GGML_LIBRARY CACHE)
     find_library(GGML_LIBRARY ggml
         REQUIRED
         HINTS ${GGML_LIB_DIR}
@@ -121,8 +122,10 @@ if(NOT TARGET ggml::ggml)
     add_library(ggml::ggml UNKNOWN IMPORTED)
     set_target_properties(ggml::ggml
         PROPERTIES
-            IMPORTED_LOCATION "${GGML_LIBRARY}")
+            IMPORTED_LOCATION "${GGML_LIBRARY}"
+            INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}")
 
+    unset(GGML_BASE_LIBRARY CACHE)
     find_library(GGML_BASE_LIBRARY ggml-base
         REQUIRED
         HINTS ${GGML_LIB_DIR}
@@ -132,6 +135,7 @@ if(NOT TARGET ggml::ggml)
     set_target_properties(ggml::ggml-base
         PROPERTIES
             IMPORTED_LOCATION "${GGML_BASE_LIBRARY}"
+            INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}"
             INTERFACE_LINK_LIBRARIES "${GGML_BASE_INTERFACE_LINK_LIBRARIES}")
 
     set(_ggml_all_targets "")
@@ -140,6 +144,7 @@ if(NOT TARGET ggml::ggml)
             string(REPLACE "-" "_" _ggml_backend_pfx "${_ggml_backend}")
             string(TOUPPER "${_ggml_backend_pfx}" _ggml_backend_pfx)
 
+            unset(${_ggml_backend_pfx}_LIBRARY CACHE)
             find_library(${_ggml_backend_pfx}_LIBRARY ${_ggml_backend}
                 REQUIRED
                 HINTS ${GGML_LIB_DIR}

ggml/docs/gguf.md

@@ -44,9 +44,9 @@ The components are:
     - This can be derived from gguf metadata `general.version`
 1. **Encoding**: Indicates the weights encoding scheme that was applied to the model. Content, type mixture and arrangement however are determined by user code and can vary depending on project needs.
 1. **Type**: Indicates the kind of gguf file and the intended purpose for it
-  - If missing, then file is by default a typical gguf tensor model file
-  - `LoRA` : GGUF file is a LoRA adapter
-  - `vocab` : GGUF file with only vocab data and metadata
+    - If missing, then file is by default a typical gguf tensor model file
+    - `LoRA` : GGUF file is a LoRA adapter
+    - `vocab` : GGUF file with only vocab data and metadata
 1. **Shard**: (Optional) Indicates and denotes that the model has been split into multiple shards, formatted as `<ShardNum>-of-<ShardTotal>`.
     - *ShardNum* : Shard position in this model. Must be 5 digits padded by zeros.
       - Shard number always starts from `00001` onwards (e.g. First shard always starts at `00001-of-XXXXX` rather than `00000-of-XXXXX`).

ggml/examples/common-ggml.cpp

@@ -75,6 +75,7 @@ bool ggml_common_quantize_0(
         case GGML_FTYPE_MOSTLY_MXFP4:
         case GGML_FTYPE_MOSTLY_NVFP4:
         case GGML_FTYPE_MOSTLY_Q1_0:
+        case GGML_FTYPE_MOSTLY_Q2_0:
                 {
                     fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype);
                     return false;
@@ -217,6 +218,7 @@ bool ggml_common_quantize_0(
                 case GGML_TYPE_MXFP4:
                 case GGML_TYPE_NVFP4:
                 case GGML_TYPE_Q1_0:
+                case GGML_TYPE_Q2_0:
                 case GGML_TYPE_COUNT:
                     {
                         fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype));

ggml/examples/simple/simple-backend.cpp

@@ -51,7 +51,7 @@ float matrix_B[rows_B * cols_B] = {
 };
 
 
-// initialize the tensors of the model in this case two matrices 2x2
+// initialize the backends and scheduler
 void init_model(simple_model & model) {
     ggml_log_set(ggml_log_callback_default, nullptr);
 

ggml/examples/test-cmake/.gitignore

@@ -0,0 +1,8 @@
+ggml-build-install
+ggml-build-install-dl
+
+install
+install-dl
+
+build
+build-dl

ggml/examples/test-cmake/CMakeLists.txt

@@ -3,7 +3,7 @@ project(ggml-simple)
 
 set(CMAKE_CXX_STANDARD 17)
 
-find_package(ggml CONFIG REQUIRED)
+find_package(ggml 0.19.0 REQUIRED)
 
 set(TEST_TARGET test-cmake)
 add_executable(test-cmake test-cmake.cpp)

ggml/examples/test-cmake/README.md

@@ -1,3 +1,21 @@
 ## cmake-test
 
-This directory can be built as a separate project with an installed ggml.
+This directory can be built as a separate project to test/verify a ggml
+installation.
+
+### Usage
+The following will configure, build, and install ggml using two different
+configurations. One will use dynamically linked backends (GGML_BACKEND_DL=ON)
+and one without.
+
+Configuring, build, and install ggml:
+```console
+./build-install.sh
+```
+
+Build this project twice using the installations created above:
+```console
+./build.sh
+```
+
+_wip_

ggml/examples/test-cmake/build-install.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+# Remove installation target directory
+rm -rf install-dl install ggml-build-install-dl ggml-build-install
+
+### Build standard installation
+build_dir=ggml-build-install
+install_dir=install
+
+cmake --fresh -S ../../. -B $build_dir -DCMAKE_BUILD_TYPE=Release \
+  -DBUILD_SHARED_LIBS=ON \
+  -DCMAKE_INSTALL_PREFIX="${PWD}/${install_dir}"
+
+cmake --build $build_dir --parallel 12
+cmake --install $build_dir
+
+### Build dynamic backend
+build_dir=ggml-build-install-dl
+install_dir=install-dl
+
+# Build dynamic backend modules and install them in libexec, mirroring a package
+# manager's private plugin directory. GGML_BACKEND_DIR is compiled into ggml as
+# its plugin search path, and cmake --install copies the backend modules there.
+cmake --fresh -S ../../. -B $build_dir -DCMAKE_BUILD_TYPE=Release \
+  -DBUILD_SHARED_LIBS=ON \
+  -DGGML_BACKEND_DL=ON \
+  -DGGML_CPU_ALL_VARIANTS=ON \
+  -DCMAKE_INSTALL_PREFIX="${PWD}/${install_dir}" \
+  -DGGML_BACKEND_DIR="${PWD}/${install_dir}/libexec"
+
+cmake --build $build_dir --parallel 12
+cmake --install $build_dir

ggml/examples/test-cmake/test-cmake.cpp

@@ -1,6 +1,11 @@
 #include "ggml-backend.h"
+#include "ggml.h"
+#include <cstdio>
 
 int main(void) {
+    printf("[test-cmake] Using ggml version %s\n", ggml_version());
+    printf("[test-cmake] Loading all backends...\n");
     ggml_backend_load_all();
+    printf("[test-cmake] Succesfully loaded all backend.\n");
     return 0;
 }

ggml/include/ggml-backend.h

@@ -154,6 +154,8 @@ extern "C" {
         bool buffer_from_host_ptr;
         // event synchronization
         bool events;
+        // mmap is supported for loading
+        bool mmap_support;
     };
 
     // all the device properties

ggml/include/ggml-cpu.h

@@ -100,6 +100,7 @@ extern "C" {
     GGML_BACKEND_API int ggml_cpu_has_sve        (void);
     GGML_BACKEND_API int ggml_cpu_get_sve_cnt    (void);  // sve vector length in bytes
     GGML_BACKEND_API int ggml_cpu_has_sme        (void);
+    GGML_BACKEND_API int ggml_cpu_has_sme2       (void);
     // other
     GGML_BACKEND_API int ggml_cpu_has_riscv_v    (void);
     GGML_BACKEND_API int ggml_cpu_get_rvv_vlen   (void);  // risc-v vector length in bytes

ggml/include/ggml-cuda.h

@@ -30,9 +30,6 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int de
 // conduct allreduce operation between devices
 GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends);
 
-// split tensor buffer that splits matrices by rows across multiple devices
-GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split);
-
 // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
 GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);
 

ggml/include/ggml-et.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GGML_ET_NAME "ET"
+
+// backend API
+GGML_BACKEND_API ggml_guid_t     ggml_backend_et_guid(void);
+GGML_BACKEND_API ggml_backend_t ggml_backend_et_init(size_t devidx);
+
+GGML_BACKEND_API bool ggml_backend_is_et(ggml_backend_t backend);
+GGML_BACKEND_API int  ggml_backend_et_get_device_count(void);
+GGML_BACKEND_API void ggml_backend_et_get_device_description(int devidx, char * description, size_t description_size);
+GGML_BACKEND_API void ggml_backend_et_get_device_memory(int devidx, size_t * free, size_t * total);
+
+GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_et_buffer_type(size_t dev_num);
+GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_et_host_buffer_type(void);
+
+GGML_BACKEND_API ggml_backend_reg_t ggml_backend_et_reg(void);
+
+#ifdef  __cplusplus
+}
+#endif

ggml/include/ggml-rpc.h

@@ -6,12 +6,12 @@
 extern "C" {
 #endif
 
-#define RPC_PROTO_MAJOR_VERSION    4
+#define RPC_PROTO_MAJOR_VERSION    5
 #define RPC_PROTO_MINOR_VERSION    0
-#define RPC_PROTO_PATCH_VERSION    1
+#define RPC_PROTO_PATCH_VERSION    0
 
 #ifdef  __cplusplus
-static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
+static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
 #endif
 
 #define GGML_RPC_MAX_SERVERS       16

ggml/include/ggml-sycl.h

@@ -27,6 +27,14 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int de
 // split tensor buffer that splits matrices by rows across multiple devices
 GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split);
 
+// Tensor parallelism (--split-mode tensor): comm_init/free/allreduce_tensor
+// trio queried by the meta-backend via ggml_backend_reg_get_proc_address.
+// See typedefs in ggml/include/ggml-backend.h. Mirrors the CUDA backend's
+// pattern (ggml_backend_cuda_comm_*).
+GGML_BACKEND_API void * ggml_backend_sycl_comm_init(ggml_backend_t * backends, size_t n_backends);
+GGML_BACKEND_API void   ggml_backend_sycl_comm_free(void * comm_ctx);
+GGML_BACKEND_API bool   ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx, struct ggml_tensor ** tensors);
+
 // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
 GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type(void);