sync : llama.cpp

.github/workflows/ci.yml

@@ -7,38 +7,6 @@ on:
     branches: [ master ]
 
 jobs:
-  test-ubuntu-opencl:
-    if: false
-    runs-on: ubuntu-latest
-    env:
-      GGML_NLOOP: 3
-      GGML_NITER: 1
-      GGML_N_THREADS: 2
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Dependencies
-      run: |
-        wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
-        echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
-        sudo apt-get update
-        sudo apt-get install -y --no-install-recommends llvm intel-oneapi-runtime-opencl intel-oneapi-runtime-compilers libclblast-dev
-    - name: Create Build Environment
-      run: mkdir build
-
-    - name: Configure CMake
-      working-directory: ./build
-      run: cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DGGML_CLBLAST=ON ..
-
-    - name: Build
-      working-directory: ./build
-      run: make
-
-    - name: Test
-      working-directory: ./build
-      run: ctest --verbose --timeout 900
-
   test-macos-metal:
     runs-on: macos-13
     env:

.gitignore

@@ -26,6 +26,7 @@ zig-out/
 zig-cache/
 
 *.o
+*.d
 *.dot
 
 *.sw?

AUTHORS

@@ -1,14 +1,16 @@
-# date: Thu Sep 26 09:19:50 CDT 2024
+# date: Tue Dec  3 20:24:37 EET 2024
 # this file is auto-generated by scripts/gen-authors.sh
 
 0cc4m <[email protected]>
 65a <[email protected]>
 AT <[email protected]>
 Abhilash Majumder <[email protected]>
 Adam Tazi <[email protected]>
+Ahmad Tameem <[email protected]>
 AidanBeltonS <[email protected]>
 AidanBeltonS <[email protected]>
 Akarshan Biswas <[email protected]>
+Akarshan Biswas <[email protected]>
 Albert Jin <[email protected]>
 Alberto Cabrera Pérez <[email protected]>
 Alberto Cabrera Pérez <[email protected]>
@@ -20,6 +22,7 @@ AmirAli Mirian <[email protected]>
 Ananta Bastola <[email protected]>
 Andreas (Andi) Kunar <[email protected]>
 Andrei <[email protected]>
+Andrew Minh Nguyen <[email protected]>
 Arjun <[email protected]>
 Ashraful Islam <[email protected]>
 Astariul <[email protected]>
@@ -35,6 +38,9 @@ Bryan Lozano <[email protected]>
 Carolinabanana <[email protected]>
 CarterLi999 <[email protected]>
 Cebtenzzre <[email protected]>
+Changyeon Kim <[email protected]>
+Charles Xu <[email protected]>
+Charles Xu <[email protected]>
 Chen Xi <[email protected]>
 Chen Xi <[email protected]>
 Chris Elrod <[email protected]>
@@ -44,6 +50,8 @@ Cordeiro <[email protected]>
 Cristiano Calcagno <[email protected]>
 DAN™ <[email protected]>
 Dan Forbes <[email protected]>
+Dan Johansson <[email protected]>
+Dan Johansson <[email protected]>
 Daniel Bevenius <[email protected]>
 Daniel Ziegenberg <[email protected]>
 Daniele <[email protected]>
@@ -56,13 +64,17 @@ DavidKorczynski <[email protected]>
 Davidson Francis <[email protected]>
 Dibakar Gope <[email protected]>
 Didzis Gosko <[email protected]>
+Diego Devesa <[email protected]>
 Diogo <[email protected]>
 Djip007 <[email protected]>
+Dou Xinpeng <[email protected]>
+Dou Xinpeng <[email protected]>
 Dr. Tom Murphy VII Ph.D <[email protected]>
 Ebey Abraham <[email protected]>
 Eldar Yusupov <[email protected]>
 Emmanuel Durand <[email protected]>
 Engininja2 <[email protected]>
+Eric Zhang <[email protected]>
 Erik Scholz <[email protected]>
 Ettore Di Giacinto <[email protected]>
 Eve <[email protected]>
@@ -71,9 +83,12 @@ Faisal Zaghloul <[email protected]>
 FantasyGmm <[email protected]>
 Felix <[email protected]>
 Finn Voorhees <[email protected]>
+FirstTimeEZ <[email protected]>
+Frankie Robertson <[email protected]>
 GainLee <[email protected]>
 George Hindle <[email protected]>
 Georgi Gerganov <[email protected]>
+Gilad S <[email protected]>
 Gilad S <[email protected]>
 Guillaume Wenzek <[email protected]>
 Halalaluyafail3 <[email protected]>
@@ -85,6 +100,7 @@ Hyunsung Lee <[email protected]>
 IGUILIZ Salah-Eddine <[email protected]>
 Ian Bull <[email protected]>
 Ikko Eltociear Ashimine <[email protected]>
+Ivan <[email protected]>
 Ivan Filipov <[email protected]>
 Ivan Stepanov <[email protected]>
 Ivan Zdane <[email protected]>
@@ -106,6 +122,7 @@ Johannes Gäßler <[email protected]>
 John Balis <[email protected]>
 Josh Bleecher Snyder <[email protected]>
 Judd <[email protected]>
+Jun Hee Yoo <[email protected]>
 Justina Cho <[email protected]>
 Justine Tunney <[email protected]>
 Justine Tunney <[email protected]>
@@ -117,7 +134,9 @@ LoganDark <[email protected]>
 LoganDark <[email protected]>
 LostRuins <[email protected]>
 Lukas Möller <[email protected]>
+M Refi D.A <[email protected]>
 M. Yusuf Sarıgöz <[email protected]>
+Ma Mingfei <[email protected]>
 Mahesh Madhav <[email protected]>
 MaiHD <[email protected]>
 Mark Zhuang <[email protected]>
@@ -126,6 +145,7 @@ Masaya, Kato <[email protected]>
 Mathijs de Bruin <[email protected]>
 Matt Stephenson <[email protected]>
 Max Krasnyansky <[email protected]>
+Max Krasnyansky <[email protected]>
 Mayank Kumar Pal <[email protected]>
 Meng, Hengyu <[email protected]>
 Mengqing Cao <[email protected]>
@@ -150,7 +170,9 @@ PAB <[email protected]>
 Paul Tsochantaris <[email protected]>
 Philpax <[email protected]>
 Pierre Alexandre SCHEMBRI <[email protected]>
+Plamen Minev <[email protected]>
 Playdev <[email protected]>
+Prashant Vithule <[email protected]>
 Przemysław Pawełczyk <[email protected]>
 R0CKSTAR <[email protected]>
 R0CKSTAR <[email protected]>
@@ -162,15 +184,20 @@ Reinforce-II <[email protected]>
 Reza Rezvan <[email protected]>
 Rick G <[email protected]>
 RiverZhou <[email protected]>
+Romain Biessy <[email protected]>
 Ronsor <[email protected]>
 Rotem Dan <[email protected]>
 Ryan Hitchman <[email protected]>
+SRHMorris <[email protected]>
+SXX <[email protected]>
 Salvatore Mesoraca <[email protected]>
 Sam Spilsbury <[email protected]>
 Sanchit Gandhi <[email protected]>
 Santtu Keskinen <[email protected]>
+Sergio López <[email protected]>
 Sergio López <[email protected]>
 Shijie <[email protected]>
+Shupei Fan <[email protected]>
 Siddharth Ramakrishnan <[email protected]>
 Sigbjørn Skjæret <[email protected]>
 Skyler Celestinian-Sterling <[email protected]>
@@ -186,18 +213,24 @@ Timothy Cronin <[email protected]>
 Tom Bailey <[email protected]>
 Tom Jobbins <[email protected]>
 Tony Wasserka <[email protected]>
+Tristan Druyen <[email protected]>
 Tyé singwa <[email protected]>
 UEXTM.com <[email protected]>
 WillCorticesAI <[email protected]>
 William Tambellini <[email protected]>
 William Tambellini <[email protected]>
 XiaotaoChen <[email protected]>
+Xinpeng Dou <[email protected]>
 Xuan Son Nguyen <[email protected]>
 Yavor Ivanov <[email protected]>
 YavorGIvanov <[email protected]>
 Yilong Guo <[email protected]>
 Yilong Guo <[email protected]>
+Yuri Khrustalev <[email protected]>
+Zhenwei Jin <[email protected]>
+Zhiyuan Li <[email protected]>
 agray3 <[email protected]>
+amritahs-ibm <[email protected]>
 apcameron <[email protected]>
 appvoid <[email protected]>
 ariez-xyz <[email protected]>
@@ -232,9 +265,11 @@ l3utterfly <[email protected]>
 le.chang <[email protected]>
 leejet <[email protected]>
 leejet <[email protected]>
+leo-pony <[email protected]>
 liuwei-git <[email protected]>
 luoyu-intel <[email protected]>
 magicse <[email protected]>
+mahorozte <[email protected]>
 mashizora <[email protected]>
 matteo <[email protected]>
 ochafik <[email protected]>
@@ -254,6 +289,7 @@ ucag.li <[email protected]>
 ulatekh <[email protected]>
 wangshuai09 <[email protected]>
 woachk <[email protected]>
+xctan <[email protected]>
 yangyaofei <[email protected]>
 yuri@FreeBSD <yuri@FreeBSD>
 zhentaoyu <[email protected]>

CMakeLists.txt

@@ -33,6 +33,7 @@ else()
 endif()
 
 option(BUILD_SHARED_LIBS "ggml: build shared libraries" ${BUILD_SHARED_LIBS_DEFAULT})
+option(GGML_BACKEND_DL   "ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)" OFF)
 
 #
 # option list
@@ -95,6 +96,7 @@ option(GGML_CPU_HBM     "ggml: use memkind for CPU HBM" OFF)
 option(GGML_CPU_AARCH64 "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON)
 
 option(GGML_AVX         "ggml: enable AVX"              ${INS_ENB})
+option(GGML_AVX_VNNI    "ggml: enable AVX-VNNI"         OFF)
 option(GGML_AVX2        "ggml: enable AVX2"             ${INS_ENB})
 option(GGML_AVX512      "ggml: enable AVX512"           OFF)
 option(GGML_AVX512_VBMI "ggml: enable AVX512-VBMI"      OFF)
@@ -109,6 +111,7 @@ if (NOT MSVC)
 endif()
 option(GGML_LASX        "ggml: enable lasx"             ON)
 option(GGML_LSX         "ggml: enable lsx"              ON)
+option(GGML_RVV         "ggml: enable rvv"              ON)
 option(GGML_SVE         "ggml: enable SVE"              OFF)
 
 if (WIN32)
@@ -159,7 +162,6 @@ set   (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING
 set   (GGML_METAL_STD "" CACHE STRING       "ggml: metal standard version (-std flag)")
 option(GGML_OPENMP                          "ggml: use OpenMP"                                ON)
 option(GGML_RPC                             "ggml: use RPC"                                   OFF)
-option(GGML_AMX                             "ggml: use AMX"                                   OFF)
 option(GGML_SYCL                            "ggml: use SYCL"                                  OFF)
 option(GGML_SYCL_F16                        "ggml: use 16 bit floats for sycl calculations"   OFF)
 set   (GGML_SYCL_TARGET "INTEL" CACHE STRING

ci/run.sh

@@ -293,15 +293,23 @@ function gg_sum_yolo {
 
 ## main
 
-if [ -z $GG_BUILD_LOW_PERF ]; then
+if [ -z ${GG_BUILD_LOW_PERF} ]; then
+    # Create symlink: ./ggml/models-mnt -> $MNT/models/models-mnt
     rm -rf ${SRC}/models-mnt
-
     mnt_models=${MNT}/models
     mkdir -p ${mnt_models}
     ln -sfn ${mnt_models} ${SRC}/models-mnt
+
+    # Create a fresh python3 venv and enter it
+    if ! python3 -m venv "$MNT/venv"; then
+        echo "Error: Failed to create Python virtual environment at $MNT/venv."
+        exit 1
+    fi
+    source "$MNT/venv/bin/activate"
+
+    pip install -r ${SRC}/requirements.txt --disable-pip-version-check
 fi
 
-python3 -m pip install -r ${SRC}/requirements.txt
 
 ret=0
 

examples/common-ggml.cpp

@@ -217,6 +217,7 @@ bool ggml_common_quantize_0(
                 case GGML_TYPE_Q4_0_8_8:
                 case GGML_TYPE_TQ1_0:
                 case GGML_TYPE_TQ2_0:
+                case GGML_TYPE_IQ4_NL_4_4:
                 case GGML_TYPE_COUNT:
                     {
                         fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype));

include/ggml-backend.h

@@ -190,6 +190,14 @@ extern "C" {
     typedef void                         (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
     // Get additional buffer types provided by the device (returns a NULL-terminated array)
     typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
+    // Set the abort callback for the backend
+    typedef void                         (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
+    // Get a list of feature flags supported by the backend (returns a NULL-terminated array)
+    struct ggml_backend_feature {
+        const char * name;
+        const char * value;
+    };
+    typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);
 
     //
     // Backend registry
@@ -214,6 +222,13 @@ extern "C" {
     // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)
     GGML_API ggml_backend_t ggml_backend_init_best(void);
 
+    // Load a backend from a dynamic library and register it
+    GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);
+    // Unload a backend if loaded dynamically and unregister it
+    GGML_API void               ggml_backend_unload(ggml_backend_reg_t reg);
+    // Load all known backends from dynamic libraries
+    GGML_API void               ggml_backend_load_all(void);
+
     //
     // Backend scheduler
     //