Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7879491cf4 | |||
| 8466395353 | |||
| dd002b91f5 | |||
| 1d88cb05f1 | |||
| 3bb63cbcbf | |||
| f711c8f303 | |||
| 921257f500 | |||
| 3ffb62935b |
@@ -5,8 +5,8 @@ exclude =
|
||||
# Do not traverse examples and tools
|
||||
examples,
|
||||
tools,
|
||||
# Reference scripts for skills
|
||||
skills,
|
||||
# Do not traverse .venv directories
|
||||
.venv*,
|
||||
# Do not include package initializers
|
||||
__init__.py,
|
||||
# No need to traverse our git directory
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
android-ndk-snapdragon:
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
linux-iot-snapdragon:
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
name: CI (wasi)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/build-wasi.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.wgsl',
|
||||
'**/*.tmpl',
|
||||
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/build-wasi.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.wgsl',
|
||||
'**/*.tmpl',
|
||||
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
WASI_SDK_VERSION: 34
|
||||
DAWN_VERSION: v20260908.214631
|
||||
DAWN_COMMIT: 94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655
|
||||
WASI_WEBGPU_HEADERS_COMMIT: 917765edd7cb8f8cf95bda1217514c78f71b582b
|
||||
|
||||
jobs:
|
||||
ubuntu-webgpu:
|
||||
runs-on: ubuntu-24.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: Install wasi-sdk
|
||||
uses: bytecodealliance/setup-wasi-sdk-action@77d09f3df1e8da344d8403cf8631af8977938b4c # v1.0.0
|
||||
with:
|
||||
version: ${{ env.WASI_SDK_VERSION }}
|
||||
add-to-path: false
|
||||
|
||||
- name: wasi:webgpu Dependency
|
||||
run: |
|
||||
mkdir -p wasi-webgpu/dawn-headers
|
||||
curl -fL "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/dawn-headers-${DAWN_COMMIT}.tar.gz" |
|
||||
tar -xz -C wasi-webgpu/dawn-headers --strip-components=1
|
||||
|
||||
git init -q wasi-webgpu/wasi-webgpu-headers
|
||||
cd wasi-webgpu/wasi-webgpu-headers
|
||||
git fetch -q --depth 1 https://github.com/wasi-gfx/wasi-webgpu-headers.git ${WASI_WEBGPU_HEADERS_COMMIT}
|
||||
git checkout -q FETCH_HEAD
|
||||
git submodule update -q --init --depth 1
|
||||
cd ../..
|
||||
|
||||
# ggml-webgpu does find_package(Dawn) and links dawn::webgpu_dawn. There is no Dawn on WASI,
|
||||
# so this target instead compiles wasi-webgpu-headers (webgpu.h over wasi:webgpu imports)
|
||||
# and exposes Dawn's webgpu_cpp.h on top of it.
|
||||
cat > wasi-webgpu/DawnConfig.cmake << 'EOF'
|
||||
set(_wasi_webgpu "${CMAKE_CURRENT_LIST_DIR}/wasi-webgpu-headers")
|
||||
add_library(dawn::webgpu_dawn INTERFACE IMPORTED)
|
||||
set_target_properties(dawn::webgpu_dawn PROPERTIES
|
||||
INTERFACE_SOURCES "${_wasi_webgpu}/webgpu.c;${_wasi_webgpu}/async_futures.c;${_wasi_webgpu}/imports.c;${_wasi_webgpu}/imports_component_type.o"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/dawn-headers/include/webgpu_upstream;${_wasi_webgpu}")
|
||||
EOF
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-ubuntu-24.04-arm-wasi
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
# llama-common does not build for WASI, so the executables are linked by hand
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
WASI_FLAGS="-msimd128 -D_WASI_EMULATED_SIGNAL -fwasm-exceptions -mllvm -wasm-use-legacy-eh=false"
|
||||
cmake -B build-wasi \
|
||||
-G "Ninja" \
|
||||
-DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_PATH}/share/cmake/wasi-sdk-p3.cmake \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_FLAGS="${WASI_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${WASI_FLAGS}" \
|
||||
-DCMAKE_HAVE_LIBC_PTHREAD=1 \
|
||||
-DBUILD_SHARED_LIBS=OFF \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_OPENMP=OFF \
|
||||
-DGGML_WEBGPU=ON \
|
||||
-DDawn_DIR=$PWD/wasi-webgpu \
|
||||
-DLLAMA_BUILD_COMMON=OFF
|
||||
|
||||
time cmake --build build-wasi --config Release --target llama -j $(nproc)
|
||||
|
||||
LIBS="build-wasi/ggml/src/libggml.a build-wasi/ggml/src/ggml-webgpu/libggml-webgpu.a build-wasi/ggml/src/libggml-cpu.a build-wasi/ggml/src/libggml-base.a"
|
||||
LDFLAGS="-lunwind -ldl -Wl,-z,stack-size=8388608"
|
||||
CXX="${WASI_SDK_PATH}/bin/clang++ --target=wasm32-wasip3 -O3 -std=c++17 ${WASI_FLAGS} -I include -I ggml/include"
|
||||
|
||||
$CXX examples/simple/simple.cpp build-wasi/src/libllama.a $LIBS $LDFLAGS -o build-wasi/llama-simple.wasm
|
||||
$CXX tests/test-backend-ops.cpp $LIBS $LDFLAGS -o build-wasi/test-backend-ops.wasm
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: webgpu-ubuntu-24.04-arm-wasi
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
@@ -547,7 +547,7 @@ jobs:
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8'
|
||||
|
||||
defaults:
|
||||
run:
|
||||
@@ -600,7 +600,7 @@ jobs:
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
|
||||
container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8'
|
||||
|
||||
defaults:
|
||||
run:
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ include("ggml/cmake/common.cmake")
|
||||
|
||||
function(llama_add_compile_flags)
|
||||
if (LLAMA_FATAL_WARNINGS)
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM")
|
||||
list(APPEND C_FLAGS -Werror)
|
||||
list(APPEND CXX_FLAGS -Werror)
|
||||
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
|
||||
|
||||
@@ -6,8 +6,8 @@ The cross-compilation toolchain images are provided by the
|
||||
[Qualcomm Snapdragon Toolchain registry](https://github.com/snapdragon-toolchain).
|
||||
These Docker images include the Android NDK, OpenCL SDK, Hexagon SDK, CMake, and the necessary cross-compilers:
|
||||
|
||||
* **Android toolchain**: `ghcr.io/snapdragon-toolchain/arm64-android:v0.7`
|
||||
* **Linux toolchain**: `ghcr.io/snapdragon-toolchain/arm64-linux:v0.7`
|
||||
* **Android toolchain**: `ghcr.io/snapdragon-toolchain/arm64-android:v0.8`
|
||||
* **Linux toolchain**: `ghcr.io/snapdragon-toolchain/arm64-linux:v0.8`
|
||||
|
||||
The unified build utility (`scripts/snapdragon/build.py`) automatically pulls
|
||||
and orchestrates these containers to perform target compilation.
|
||||
@@ -40,7 +40,7 @@ Alternatively, you can build llama.cpp manually by entering the cross-compilatio
|
||||
|
||||
```bash
|
||||
# Start the cross-compilation container manually:
|
||||
~/src/llama.cpp$ docker run -it --rm -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-android:v0.7
|
||||
~/src/llama.cpp$ docker run -it --rm -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-android:v0.8
|
||||
|
||||
# Inside the container, build the project using presets:
|
||||
[d]/workspace> cp docs/backend/snapdragon/CMakeUserPresets.json .
|
||||
@@ -53,7 +53,7 @@ Preset CMake variables:
|
||||
GGML_HEXAGON="ON"
|
||||
GGML_OPENCL="ON"
|
||||
GGML_OPENMP="OFF"
|
||||
HEXAGON_SDK_ROOT="/opt/hexagon/6.6.0.0"
|
||||
HEXAGON_SDK_ROOT="/opt/hexagon/6.7.0.0"
|
||||
...
|
||||
-- Including OpenCL backend
|
||||
-- Including Hexagon backend
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
The cross-compilation is performed using the Snapdragon Linux Docker toolchain image (see
|
||||
[github.com/snapdragon-toolchain](https://github.com/snapdragon-toolchain)):
|
||||
|
||||
* **Linux toolchain**: `ghcr.io/snapdragon-toolchain/arm64-linux:v0.7`
|
||||
* **Linux toolchain**: `ghcr.io/snapdragon-toolchain/arm64-linux:v0.8`
|
||||
|
||||
The unified build utility (`scripts/snapdragon/build.py`) automatically pulls
|
||||
and orchestrates this container to perform target compilation. You only need to
|
||||
@@ -29,7 +29,7 @@ Alternatively, you can build llama.cpp manually by entering the cross-compilatio
|
||||
|
||||
```bash
|
||||
# Start the cross-compilation container manually:
|
||||
~/src/llama.cpp$ docker run -it --rm -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-linux:v0.7
|
||||
~/src/llama.cpp$ docker run -it --rm -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-linux:v0.8
|
||||
|
||||
# Inside the container, build the project using presets:
|
||||
[d]/workspace> cp docs/backend/snapdragon/CMakeUserPresets.json .
|
||||
|
||||
@@ -8,7 +8,7 @@ Native Windows 11 arm64 builds have the following tool dependencies:
|
||||
- UCRT and Driver Kit
|
||||
- LLVM core libraries and Clang compiler (winget)
|
||||
- CMake, Git, Python (winget)
|
||||
- Hexagon SDK Community Edition 6.6 or later (see below)
|
||||
- Hexagon SDK Community Edition 6.7 or later (see below)
|
||||
- OpenCL SDK 2.3 or later (see below)
|
||||
|
||||
Note: The rest of the **Windows** build process assumes that you're running natively in Powershell.
|
||||
@@ -54,15 +54,15 @@ c:\Qualcomm\OpenCL_SDK\2.3.2
|
||||
|
||||
Either use the trimmed down version (optimized for CI) from
|
||||
|
||||
https://github.com/snapdragon-toolchain/hexagon-sdk/releases/download/v6.6.0.0/hexagon-sdk-v6.6.0.0-arm64-wos.tar.xz
|
||||
https://github.com/snapdragon-toolchain/hexagon-sdk/releases/download/v6.7.0.0/hexagon-sdk-v6.7.0.0-arm64-wos.tar.xz
|
||||
|
||||
Or download the complete official version from
|
||||
|
||||
https://softwarecenter.qualcomm.com/catalog/item/Hexagon_SDK?version=6.6.0.0
|
||||
https://softwarecenter.qualcomm.com/catalog/item/Hexagon_SDK?version=6.7.0.0
|
||||
|
||||
Unzip/untar the archive into
|
||||
```
|
||||
c:\Qualcomm\Hexagon_SDK\6.6.0.0
|
||||
c:\Qualcomm\Hexagon_SDK\6.7.0.0
|
||||
```
|
||||
|
||||
## Install the latest Adreno GPU driver
|
||||
@@ -150,8 +150,8 @@ The overall Hexagon backend build procedure for Windows on Snapdragon is the sam
|
||||
However, additional settings are required for generating and signing HTP Ops libraries.
|
||||
```
|
||||
> $env:OPENCL_SDK_ROOT="C:\Qualcomm\OpenCL_SDK\2.3.2"
|
||||
> $env:HEXAGON_SDK_ROOT="C:\Qualcomm\Hexagon_SDK\6.6.0.0"
|
||||
> $env:HEXAGON_TOOLS_ROOT="C:\Qualcomm\Hexagon_SDK\6.6.0.0\tools\HEXAGON_Tools\19.0.07"
|
||||
> $env:HEXAGON_SDK_ROOT="C:\Qualcomm\Hexagon_SDK\6.7.0.0"
|
||||
> $env:HEXAGON_TOOLS_ROOT="C:\Qualcomm\Hexagon_SDK\6.7.0.0\tools\HEXAGON_Tools\21.0.04"
|
||||
> $env:HEXAGON_HTP_CERT="c:\Users\MyUsers\Certs\ggml-htp-v1.pfx"
|
||||
> $env:WINDOWS_SDK_BIN="C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0"
|
||||
|
||||
|
||||
@@ -852,6 +852,36 @@ WebGPU allows cross-platform access to the GPU from supported browsers. We utili
|
||||
|
||||
Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/src/emdawnwebgpu/) to download or build the emdawnwebgpu package (Note that it might be safer to build the emdawnwebgpu package locally, so that it stays in sync with the version of Dawn you have installed above). When building using CMake, the path to the emdawnwebgpu port file needs to be set with the flag `EMDAWNWEBGPU_DIR`.
|
||||
|
||||
## WASI
|
||||
|
||||
Requires [wasi-sdk](https://github.com/WebAssembly/wasi-sdk) 34+ with `WASI_SDK_PATH` set. Only the libraries build (`llama-common` does not support WASI yet), so link your program against them:
|
||||
|
||||
```bash
|
||||
FLAGS="-msimd128 -D_WASI_EMULATED_SIGNAL -fwasm-exceptions -mllvm -wasm-use-legacy-eh=false"
|
||||
cmake -B build-wasi -DCMAKE_TOOLCHAIN_FILE=$WASI_SDK_PATH/share/cmake/wasi-sdk-p2.cmake \
|
||||
-DCMAKE_C_FLAGS="$FLAGS" -DCMAKE_CXX_FLAGS="$FLAGS" -DCMAKE_HAVE_LIBC_PTHREAD=1 \
|
||||
-DBUILD_SHARED_LIBS=OFF -DGGML_NATIVE=OFF -DGGML_OPENMP=OFF -DLLAMA_BUILD_COMMON=OFF
|
||||
cmake --build build-wasi --target llama
|
||||
|
||||
$WASI_SDK_PATH/bin/clang++ --target=wasm32-wasip3 -O3 $FLAGS -I include -I ggml/include examples/simple/simple.cpp \
|
||||
build-wasi/src/libllama.a build-wasi/ggml/src/libggml*.a -lunwind -ldl -Wl,-z,stack-size=8388608 -o llama-simple.wasm
|
||||
wasmtime run -W exceptions=y --dir models llama-simple.wasm -m models/model.gguf "Hello"
|
||||
```
|
||||
|
||||
### wasi:webgpu
|
||||
|
||||
The WebGPU backend can target [wasi:webgpu](https://github.com/WebAssembly/wasi-webgpu) via [wasi-webgpu-headers](https://github.com/wasi-gfx/wasi-webgpu-headers) (with its submodule) and Dawn's `dawn-headers` release. Point `Dawn_DIR` at a directory containing this `DawnConfig.cmake` next to both:
|
||||
|
||||
```cmake
|
||||
set(_wasi_webgpu "${CMAKE_CURRENT_LIST_DIR}/wasi-webgpu-headers")
|
||||
add_library(dawn::webgpu_dawn INTERFACE IMPORTED)
|
||||
set_target_properties(dawn::webgpu_dawn PROPERTIES
|
||||
INTERFACE_SOURCES "${_wasi_webgpu}/webgpu.c;${_wasi_webgpu}/async_futures.c;${_wasi_webgpu}/imports.c;${_wasi_webgpu}/imports_component_type.o"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/dawn-headers/include/webgpu_upstream;${_wasi_webgpu}")
|
||||
```
|
||||
|
||||
Then build as above, but with `wasi-sdk-p3.cmake` / `--target=wasm32-wasip3`, `-DGGML_WEBGPU=ON -DDawn_DIR=...` at configure, and `build-wasi/ggml/src/ggml-webgpu/libggml-webgpu.a` added at link. See `.github/workflows/build-wasi.yml` for a more complete example.
|
||||
|
||||
## IBM Z & LinuxONE
|
||||
|
||||
To read documentation for how to build on IBM Z & LinuxONE, [click here](./build-s390x.md)
|
||||
|
||||
@@ -2,10 +2,16 @@ function(ggml_get_flags CCID CCVER)
|
||||
set(C_FLAGS "")
|
||||
set(CXX_FLAGS "")
|
||||
|
||||
if (CCID MATCHES "Clang")
|
||||
if (CCID MATCHES "Clang" OR CCID MATCHES "IntelLLVM")
|
||||
set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return)
|
||||
set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)
|
||||
|
||||
if (CCID STREQUAL "IntelLLVM")
|
||||
# for Intel DPC++ and LLVM SYCL, using debug does not disable optimization
|
||||
list(APPEND C_FLAGS -Rno-debug-disables-optimization)
|
||||
list(APPEND CXX_FLAGS -Rno-debug-disables-optimization)
|
||||
endif()
|
||||
|
||||
if (
|
||||
(CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
|
||||
(CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
|
||||
|
||||
@@ -21,6 +21,7 @@ message(STATUS "hexagon: using ${HEXAGON_SDK_ROOT} and ${HEXAGON_TOOLS_ROOT} for
|
||||
|
||||
include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake)
|
||||
include(ExternalProject)
|
||||
find_package(Python3 REQUIRED COMPONENTS Interpreter)
|
||||
|
||||
option(GGML_HEXAGON_HTP_DEBUG "ggml-hexagon: enable HTP debug output" OFF)
|
||||
set(GGML_HEXAGON_HTP_BUILD_TYPE "Release" CACHE STRING "ggml-hexagon: HTP skel build type (Release, RelWithDebInfo, Debug)")
|
||||
@@ -74,7 +75,8 @@ function(build_htp_skel V)
|
||||
-DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_ROOT}
|
||||
-DHEXAGON_HTP_DEBUG=${GGML_HEXAGON_HTP_DEBUG}
|
||||
-DDSP_VERSION=${V}
|
||||
-DPREBUILT_LIB_DIR="toolv19_${V}")
|
||||
-DPython3_EXECUTABLE=${Python3_EXECUTABLE}
|
||||
-DPREBUILT_LIB_DIR="toolv21_${V}")
|
||||
list(APPEND HTP_SKELS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so)
|
||||
list(APPEND HTP_PROJECTS htp-${V})
|
||||
set(HTP_SKELS ${HTP_SKELS} PARENT_SCOPE)
|
||||
@@ -85,6 +87,7 @@ build_htp_skel(v73)
|
||||
build_htp_skel(v75)
|
||||
build_htp_skel(v79)
|
||||
build_htp_skel(v81)
|
||||
build_htp_skel(v85)
|
||||
|
||||
# Install Hexagon skels required at runtime
|
||||
install(FILES ${HTP_SKELS} TYPE LIB)
|
||||
|
||||
@@ -8399,9 +8399,9 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
|
||||
if (opt_arch < 73) {
|
||||
GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is under supported range, capping at v73\n", opt_arch);
|
||||
opt_arch = 73;
|
||||
} else if (opt_arch > 81) {
|
||||
GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is over supported range, capping at v81\n", opt_arch);
|
||||
opt_arch = 81;
|
||||
} else if (opt_arch > 85) {
|
||||
GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is over supported range, capping at v85\n", opt_arch);
|
||||
opt_arch = 85;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -53,9 +53,17 @@ target_compile_definitions(${HTP_LIB} PRIVATE
|
||||
$<IF:$<BOOL:${HEXAGON_HTP_DEBUG}>,HTP_DEBUG=1,NDEBUG=1>
|
||||
$<IF:$<BOOL:${HEXAGON_HTP_DEBUG}>,FARF_HIGH=1,>)
|
||||
|
||||
|
||||
build_idl(htp_iface.idl ${HTP_LIB})
|
||||
|
||||
set_target_properties(${HTP_LIB} PROPERTIES EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
if (NOT Python3_EXECUTABLE)
|
||||
find_package(Python3 REQUIRED COMPONENTS Interpreter)
|
||||
endif()
|
||||
|
||||
add_custom_command(TARGET ${HTP_LIB} POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E echo "Adding SWIV section $<TARGET_FILE_NAME:${HTP_LIB}>"
|
||||
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/htp-swiv.py -i $<TARGET_FILE:${HTP_LIB}> -o $<TARGET_FILE:${HTP_LIB}>
|
||||
VERBATIM)
|
||||
|
||||
install(TARGETS ${HTP_LIB})
|
||||
|
||||
@@ -0,0 +1,512 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import zlib
|
||||
|
||||
ELF_MAGIC = b'\x7fELF'
|
||||
ELF_CLASS_INDEX = 4
|
||||
ELFCLASS32 = 1
|
||||
ELFCLASS64 = 2
|
||||
ELF_VERSION_INDEX = 6
|
||||
EV_CURRENT = 1
|
||||
MAX_PHDR_COUNT = 100
|
||||
|
||||
LOAD_TYPE = 0x1
|
||||
|
||||
SHT_NOBITS = 8
|
||||
|
||||
SWIV_MAGIC = b'SWIV'
|
||||
SWIV_SEG_SIZE_MIN = 16
|
||||
# SHT_LOUSER plus ASCII "SWIV".
|
||||
SWIV_SECTION_TYPE = 0xD3574956
|
||||
|
||||
logger = logging.getLogger("htp-swiv")
|
||||
|
||||
|
||||
class Elf_Ehdr_common:
|
||||
s = struct.Struct('16sHHI')
|
||||
|
||||
def __init__(self, data):
|
||||
unpacked_data = (Elf_Ehdr_common.s).unpack(data)
|
||||
self.e_ident = unpacked_data[0]
|
||||
self.e_type = unpacked_data[1]
|
||||
self.e_machine = unpacked_data[2]
|
||||
self.e_version = unpacked_data[3]
|
||||
|
||||
|
||||
class Elf32_Ehdr:
|
||||
s = struct.Struct('16sHHIIIIIHHHHHH')
|
||||
|
||||
def __init__(self, data):
|
||||
unpacked_data = (Elf32_Ehdr.s).unpack(data)
|
||||
self.e_ident = unpacked_data[0]
|
||||
self.e_type = unpacked_data[1]
|
||||
self.e_machine = unpacked_data[2]
|
||||
self.e_version = unpacked_data[3]
|
||||
self.e_entry = unpacked_data[4]
|
||||
self.e_phoff = unpacked_data[5]
|
||||
self.e_shoff = unpacked_data[6]
|
||||
self.e_flags = unpacked_data[7]
|
||||
self.e_ehsize = unpacked_data[8]
|
||||
self.e_phentsize = unpacked_data[9]
|
||||
self.e_phnum = unpacked_data[10]
|
||||
self.e_shentsize = unpacked_data[11]
|
||||
self.e_shnum = unpacked_data[12]
|
||||
self.e_shstrndx = unpacked_data[13]
|
||||
|
||||
def get_packed_data(self):
|
||||
values = [self.e_ident,
|
||||
self.e_type,
|
||||
self.e_machine,
|
||||
self.e_version,
|
||||
self.e_entry,
|
||||
self.e_phoff,
|
||||
self.e_shoff,
|
||||
self.e_flags,
|
||||
self.e_ehsize,
|
||||
self.e_phentsize,
|
||||
self.e_phnum,
|
||||
self.e_shentsize,
|
||||
self.e_shnum,
|
||||
self.e_shstrndx
|
||||
]
|
||||
|
||||
return (Elf32_Ehdr.s).pack(*values)
|
||||
|
||||
|
||||
class Elf32_Phdr:
|
||||
s = struct.Struct('I' * 8)
|
||||
|
||||
def __init__(self, data):
|
||||
unpacked_data = (Elf32_Phdr.s).unpack(data)
|
||||
self.p_type = unpacked_data[0]
|
||||
self.p_offset = unpacked_data[1]
|
||||
self.p_vaddr = unpacked_data[2]
|
||||
self.p_paddr = unpacked_data[3]
|
||||
self.p_filesz = unpacked_data[4]
|
||||
self.p_memsz = unpacked_data[5]
|
||||
self.p_flags = unpacked_data[6]
|
||||
self.p_align = unpacked_data[7]
|
||||
|
||||
def get_packed_data(self):
|
||||
values = [self.p_type,
|
||||
self.p_offset,
|
||||
self.p_vaddr,
|
||||
self.p_paddr,
|
||||
self.p_filesz,
|
||||
self.p_memsz,
|
||||
self.p_flags,
|
||||
self.p_align
|
||||
]
|
||||
|
||||
return (Elf32_Phdr.s).pack(*values)
|
||||
|
||||
|
||||
class Elf32_Shdr:
|
||||
s = struct.Struct('I' * 10)
|
||||
|
||||
def __init__(self, data):
|
||||
unpacked_data = (Elf32_Shdr.s).unpack(data)
|
||||
self.sh_name = unpacked_data[0]
|
||||
self.sh_type = unpacked_data[1]
|
||||
self.sh_flags = unpacked_data[2]
|
||||
self.sh_addr = unpacked_data[3]
|
||||
self.sh_offset = unpacked_data[4]
|
||||
self.sh_size = unpacked_data[5]
|
||||
self.sh_link = unpacked_data[6]
|
||||
self.sh_info = unpacked_data[7]
|
||||
self.sh_addralign = unpacked_data[8]
|
||||
self.sh_entsize = unpacked_data[9]
|
||||
self.sh_name_str = ""
|
||||
|
||||
def get_packed_data(self):
|
||||
values = [self.sh_name,
|
||||
self.sh_type,
|
||||
self.sh_flags,
|
||||
self.sh_addr,
|
||||
self.sh_offset,
|
||||
self.sh_size,
|
||||
self.sh_link,
|
||||
self.sh_info,
|
||||
self.sh_addralign,
|
||||
self.sh_entsize]
|
||||
|
||||
return (Elf32_Shdr.s).pack(*values)
|
||||
|
||||
|
||||
class Elf64_Ehdr:
|
||||
s = struct.Struct('16sHHIQQQIHHHHHH')
|
||||
|
||||
def __init__(self, data):
|
||||
unpacked_data = (Elf64_Ehdr.s).unpack(data)
|
||||
self.e_ident = unpacked_data[0]
|
||||
self.e_type = unpacked_data[1]
|
||||
self.e_machine = unpacked_data[2]
|
||||
self.e_version = unpacked_data[3]
|
||||
self.e_entry = unpacked_data[4]
|
||||
self.e_phoff = unpacked_data[5]
|
||||
self.e_shoff = unpacked_data[6]
|
||||
self.e_flags = unpacked_data[7]
|
||||
self.e_ehsize = unpacked_data[8]
|
||||
self.e_phentsize = unpacked_data[9]
|
||||
self.e_phnum = unpacked_data[10]
|
||||
self.e_shentsize = unpacked_data[11]
|
||||
self.e_shnum = unpacked_data[12]
|
||||
self.e_shstrndx = unpacked_data[13]
|
||||
|
||||
def get_packed_data(self):
|
||||
values = [self.e_ident,
|
||||
self.e_type,
|
||||
self.e_machine,
|
||||
self.e_version,
|
||||
self.e_entry,
|
||||
self.e_phoff,
|
||||
self.e_shoff,
|
||||
self.e_flags,
|
||||
self.e_ehsize,
|
||||
self.e_phentsize,
|
||||
self.e_phnum,
|
||||
self.e_shentsize,
|
||||
self.e_shnum,
|
||||
self.e_shstrndx
|
||||
]
|
||||
|
||||
return (Elf64_Ehdr.s).pack(*values)
|
||||
|
||||
|
||||
class Elf64_Phdr:
|
||||
s = struct.Struct('IIQQQQQQ')
|
||||
|
||||
def __init__(self, data):
|
||||
unpacked_data = (Elf64_Phdr.s).unpack(data)
|
||||
self.p_type = unpacked_data[0]
|
||||
self.p_flags = unpacked_data[1]
|
||||
self.p_offset = unpacked_data[2]
|
||||
self.p_vaddr = unpacked_data[3]
|
||||
self.p_paddr = unpacked_data[4]
|
||||
self.p_filesz = unpacked_data[5]
|
||||
self.p_memsz = unpacked_data[6]
|
||||
self.p_align = unpacked_data[7]
|
||||
|
||||
def get_packed_data(self):
|
||||
values = [self.p_type,
|
||||
self.p_flags,
|
||||
self.p_offset,
|
||||
self.p_vaddr,
|
||||
self.p_paddr,
|
||||
self.p_filesz,
|
||||
self.p_memsz,
|
||||
self.p_align
|
||||
]
|
||||
|
||||
return (Elf64_Phdr.s).pack(*values)
|
||||
|
||||
|
||||
class Elf64_Shdr:
|
||||
s = struct.Struct('IIQQQQIIQQ')
|
||||
|
||||
def __init__(self, data):
|
||||
unpacked_data = (Elf64_Shdr.s).unpack(data)
|
||||
self.sh_name = unpacked_data[0]
|
||||
self.sh_type = unpacked_data[1]
|
||||
self.sh_flags = unpacked_data[2]
|
||||
self.sh_addr = unpacked_data[3]
|
||||
self.sh_offset = unpacked_data[4]
|
||||
self.sh_size = unpacked_data[5]
|
||||
self.sh_link = unpacked_data[6]
|
||||
self.sh_info = unpacked_data[7]
|
||||
self.sh_addralign = unpacked_data[8]
|
||||
self.sh_entsize = unpacked_data[9]
|
||||
self.sh_name_str = ""
|
||||
|
||||
def get_packed_data(self):
|
||||
values = [self.sh_name,
|
||||
self.sh_type,
|
||||
self.sh_flags,
|
||||
self.sh_addr,
|
||||
self.sh_offset,
|
||||
self.sh_size,
|
||||
self.sh_link,
|
||||
self.sh_info,
|
||||
self.sh_addralign,
|
||||
self.sh_entsize]
|
||||
|
||||
return (Elf64_Shdr.s).pack(*values)
|
||||
|
||||
|
||||
class SWIV_Seg:
|
||||
s = struct.Struct('4sIQ')
|
||||
|
||||
def __init__(self):
|
||||
self.magic = SWIV_MAGIC
|
||||
self.crc = 0
|
||||
self.padding = 0
|
||||
|
||||
def get_packed_data(self):
|
||||
values = [self.magic,
|
||||
self.crc,
|
||||
self.padding
|
||||
]
|
||||
|
||||
return (SWIV_Seg.s).pack(*values)
|
||||
|
||||
|
||||
def is_elf64(elf_header):
|
||||
return elf_header.e_ident[ELF_CLASS_INDEX] == ELFCLASS64
|
||||
|
||||
|
||||
def verify_elf_header(elf_header):
|
||||
ident = elf_header.e_ident
|
||||
return (ident.startswith(ELF_MAGIC)
|
||||
and ident[ELF_CLASS_INDEX] in (ELFCLASS32, ELFCLASS64)
|
||||
and ident[ELF_VERSION_INDEX] == EV_CURRENT)
|
||||
|
||||
|
||||
def file_copy_offset(in_fp, in_off, out_fp, out_off, num_bytes):
|
||||
in_fp.seek(in_off)
|
||||
read_in = in_fp.read(num_bytes)
|
||||
out_fp.seek(out_off)
|
||||
out_fp.write(read_in)
|
||||
|
||||
return num_bytes
|
||||
|
||||
|
||||
def preprocess_elf_file(elf_file_name):
|
||||
with open(elf_file_name, 'rb') as elf_fp:
|
||||
elf_header = Elf_Ehdr_common(elf_fp.read(Elf_Ehdr_common.s.size))
|
||||
|
||||
if verify_elf_header(elf_header) is False:
|
||||
raise RuntimeError("Input ELF verification failed : " + elf_file_name)
|
||||
|
||||
elf_fp.seek(0)
|
||||
|
||||
if is_elf64(elf_header):
|
||||
elf_header = Elf64_Ehdr(elf_fp.read(Elf64_Ehdr.s.size))
|
||||
else:
|
||||
elf_header = Elf32_Ehdr(elf_fp.read(Elf32_Ehdr.s.size))
|
||||
|
||||
phdr_table = []
|
||||
|
||||
phdr_size = elf_header.e_phentsize
|
||||
|
||||
file_offset = elf_header.e_phoff
|
||||
elf_fp.seek(file_offset)
|
||||
|
||||
for _ in range(elf_header.e_phnum):
|
||||
if is_elf64(elf_header):
|
||||
phdr_table.append(Elf64_Phdr(elf_fp.read(phdr_size)))
|
||||
else:
|
||||
phdr_table.append(Elf32_Phdr(elf_fp.read(phdr_size)))
|
||||
|
||||
return [elf_header, phdr_table]
|
||||
|
||||
|
||||
def process_section_header(elf_file_name):
|
||||
with open(elf_file_name, 'rb') as elf_fp:
|
||||
elf_header = Elf_Ehdr_common(elf_fp.read(Elf_Ehdr_common.s.size))
|
||||
|
||||
if verify_elf_header(elf_header) is False:
|
||||
raise RuntimeError("Input ELF verification failed : " + elf_file_name)
|
||||
|
||||
elf_fp.seek(0)
|
||||
|
||||
if is_elf64(elf_header):
|
||||
elf_header = Elf64_Ehdr(elf_fp.read(Elf64_Ehdr.s.size))
|
||||
else:
|
||||
elf_header = Elf32_Ehdr(elf_fp.read(Elf32_Ehdr.s.size))
|
||||
|
||||
shdr_table = []
|
||||
|
||||
shdr_size = elf_header.e_shentsize
|
||||
elf_fp.seek(elf_header.e_shoff)
|
||||
|
||||
for _ in range(elf_header.e_shnum):
|
||||
if is_elf64(elf_header):
|
||||
shdr_table.append(Elf64_Shdr(elf_fp.read(shdr_size)))
|
||||
else:
|
||||
shdr_table.append(Elf32_Shdr(elf_fp.read(shdr_size)))
|
||||
|
||||
return shdr_table
|
||||
|
||||
|
||||
def crc32_update_file(elf_fp, offset, size, crc):
|
||||
elf_fp.seek(offset)
|
||||
remaining = size
|
||||
while remaining > 0:
|
||||
chunk = elf_fp.read(min(1024 * 1024, remaining))
|
||||
if not chunk:
|
||||
raise RuntimeError("Unexpected EOF while calculating CRC")
|
||||
crc = zlib.crc32(chunk, crc)
|
||||
remaining -= len(chunk)
|
||||
return crc
|
||||
|
||||
|
||||
def crc32_update_zeros(size, crc):
|
||||
zero_chunk = b'\x00' * min(1024 * 1024, size)
|
||||
remaining = size
|
||||
while remaining > 0:
|
||||
chunk = zero_chunk[:min(len(zero_chunk), remaining)]
|
||||
crc = zlib.crc32(chunk, crc)
|
||||
remaining -= len(chunk)
|
||||
return crc
|
||||
|
||||
|
||||
def crc32_calculate(elf_fp, phdr_table):
|
||||
crc = 0
|
||||
|
||||
# SWIV validates loadable segment contents, including BSS as zeros.
|
||||
for curr_phdr in phdr_table:
|
||||
if curr_phdr.p_type != LOAD_TYPE:
|
||||
continue
|
||||
|
||||
crc = crc32_update_file(elf_fp, curr_phdr.p_offset, curr_phdr.p_filesz, crc)
|
||||
|
||||
zero_size = curr_phdr.p_memsz - curr_phdr.p_filesz
|
||||
if zero_size > 0:
|
||||
crc = crc32_update_zeros(zero_size, crc)
|
||||
|
||||
crc &= 0xffffffff
|
||||
logger.info("calculated CRC: %#x", crc)
|
||||
|
||||
return crc
|
||||
|
||||
|
||||
def verify_swiv_seg(elf_in_fp, offset):
|
||||
|
||||
elf_in_fp.seek(offset)
|
||||
swiv_seg = SWIV_Seg()
|
||||
unpacked_data = (SWIV_Seg.s).unpack(elf_in_fp.read(SWIV_SEG_SIZE_MIN))
|
||||
swiv_seg.magic = unpacked_data[0]
|
||||
|
||||
return swiv_seg.magic == SWIV_MAGIC
|
||||
|
||||
|
||||
def same_output_file(input_elf, output_elf):
|
||||
try:
|
||||
return os.path.exists(output_elf) and os.path.samefile(input_elf, output_elf)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def make_output_target(input_elf, output_elf):
|
||||
if not same_output_file(input_elf, output_elf):
|
||||
return output_elf, None
|
||||
|
||||
output_dir = os.path.dirname(os.path.abspath(output_elf))
|
||||
output_base = os.path.basename(output_elf)
|
||||
fd, temp_output = tempfile.mkstemp(prefix=output_base + ".", suffix=".tmp", dir=output_dir)
|
||||
os.close(fd)
|
||||
shutil.copymode(input_elf, temp_output)
|
||||
return temp_output, temp_output
|
||||
|
||||
|
||||
def swiv_generate(input_elf, output_elf):
|
||||
output_target, temp_output = make_output_target(input_elf, output_elf)
|
||||
|
||||
try:
|
||||
with open(input_elf, "rb") as elf_in_fp, open(output_target, "wb+") as elf_out_fp:
|
||||
[elf_header, phdr_table] = preprocess_elf_file(input_elf)
|
||||
|
||||
num_phdrs = elf_header.e_phnum
|
||||
num_shdrs = elf_header.e_shnum
|
||||
phdr_size = elf_header.e_phentsize
|
||||
|
||||
if num_phdrs == 0 or num_phdrs > MAX_PHDR_COUNT:
|
||||
raise RuntimeError("Input ELF has invalid number of program headers.")
|
||||
|
||||
if elf_header.e_shoff == 0 or num_shdrs == 0:
|
||||
raise RuntimeError("Input ELF does not contain a section header table.")
|
||||
|
||||
shdr_table_input = process_section_header(input_elf)
|
||||
shdr_table_output = process_section_header(input_elf)
|
||||
|
||||
last_phdr = phdr_table[num_phdrs - 1]
|
||||
if last_phdr.p_memsz == SWIV_SEG_SIZE_MIN and verify_swiv_seg(elf_in_fp, last_phdr.p_offset):
|
||||
raise RuntimeError("A SWIV segment is detected in the input ELF.")
|
||||
|
||||
last_shdr = shdr_table_input[elf_header.e_shnum - 1]
|
||||
if last_shdr.sh_size == SWIV_SEG_SIZE_MIN and verify_swiv_seg(elf_in_fp, last_shdr.sh_offset):
|
||||
raise RuntimeError("A SWIV section is detected in the input ELF.")
|
||||
|
||||
if is_elf64(elf_header):
|
||||
swiv_shdr = Elf64_Shdr(b'\0' * Elf64_Shdr.s.size)
|
||||
else:
|
||||
swiv_shdr = Elf32_Shdr(b'\0' * Elf32_Shdr.s.size)
|
||||
|
||||
swiv_shdr.sh_name = 0
|
||||
swiv_shdr.sh_type = SWIV_SECTION_TYPE
|
||||
swiv_shdr.sh_flags = 0
|
||||
swiv_shdr.sh_addr = 0
|
||||
swiv_shdr.sh_size = SWIV_SEG_SIZE_MIN
|
||||
swiv_shdr.sh_offset = elf_header.e_shoff + (elf_header.e_shnum + 1) * elf_header.e_shentsize
|
||||
shdr_table_output.append(swiv_shdr)
|
||||
elf_header.e_shnum += 1
|
||||
|
||||
if elf_header.e_phoff != 0:
|
||||
for phdr in phdr_table:
|
||||
file_copy_offset(elf_in_fp, phdr.p_offset, elf_out_fp, phdr.p_offset, phdr.p_filesz)
|
||||
|
||||
elf_out_fp.seek(0)
|
||||
elf_out_fp.write(elf_header.get_packed_data())
|
||||
|
||||
phdr_start = elf_header.e_phoff
|
||||
for phdr in phdr_table:
|
||||
elf_out_fp.seek(phdr_start)
|
||||
elf_out_fp.write(phdr.get_packed_data())
|
||||
phdr_start += phdr_size
|
||||
|
||||
for shdr in shdr_table_input[:num_shdrs]:
|
||||
if shdr.sh_type == SHT_NOBITS or shdr.sh_size == 0:
|
||||
continue
|
||||
file_copy_offset(elf_in_fp, shdr.sh_offset, elf_out_fp, shdr.sh_offset, shdr.sh_size)
|
||||
|
||||
shdr_start = elf_header.e_shoff
|
||||
for shdr in shdr_table_output:
|
||||
elf_out_fp.seek(shdr_start)
|
||||
elf_out_fp.write(shdr.get_packed_data())
|
||||
shdr_start += elf_header.e_shentsize
|
||||
|
||||
elf_out_fp.flush()
|
||||
|
||||
swiv_seg = SWIV_Seg()
|
||||
swiv_seg.crc = crc32_calculate(elf_out_fp, phdr_table)
|
||||
elf_out_fp.seek(swiv_shdr.sh_offset)
|
||||
elf_out_fp.write(swiv_seg.get_packed_data())
|
||||
|
||||
if temp_output is not None:
|
||||
os.replace(temp_output, output_elf)
|
||||
temp_output = None
|
||||
|
||||
return 0
|
||||
finally:
|
||||
if temp_output is not None and os.path.exists(temp_output):
|
||||
os.unlink(temp_output)
|
||||
|
||||
|
||||
def main(argv):
|
||||
parser = argparse.ArgumentParser(description="Add a Software Image Validation (SWIV) section to an HTP ELF file.")
|
||||
parser.add_argument("-i", "--input", dest="input_file", metavar="INPUT", required=True)
|
||||
parser.add_argument("-o", "--output", dest="output_file", metavar="OUTPUT", required=True)
|
||||
parser.add_argument("-v", "--verbose", action="store_true", help="show calculated CRC")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO if args.verbose else logging.WARNING,
|
||||
format="%(message)s",
|
||||
)
|
||||
|
||||
try:
|
||||
return swiv_generate(args.input_file, args.output_file)
|
||||
except RuntimeError as exc:
|
||||
logger.error("ERROR: %s", exc)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -18,6 +18,7 @@ libggml-htp-v73.so = 1
|
||||
libggml-htp-v75.so = 1
|
||||
libggml-htp-v79.so = 1
|
||||
libggml-htp-v81.so = 1
|
||||
libggml-htp-v85.so = 1
|
||||
|
||||
[ControlFlags]
|
||||
ExcludeFromSelect = *
|
||||
@@ -30,6 +31,7 @@ libggml-htp-v73.so,,,0x10 ;COPYFLG_NO_OVERWRITE
|
||||
libggml-htp-v75.so,,,0x10 ;COPYFLG_NO_OVERWRITE
|
||||
libggml-htp-v79.so,,,0x10 ;COPYFLG_NO_OVERWRITE
|
||||
libggml-htp-v81.so,,,0x10 ;COPYFLG_NO_OVERWRITE
|
||||
libggml-htp-v85.so,,,0x10 ;COPYFLG_NO_OVERWRITE
|
||||
|
||||
[Strings]
|
||||
GGML = 'GGML'
|
||||
|
||||
@@ -44,7 +44,7 @@ if (WIN32)
|
||||
if(DEFINED ENV{LEVEL_ZERO_V1_SDK_PATH})
|
||||
set(LEVEL_ZERO_V1_SDK_PATH $ENV{LEVEL_ZERO_V1_SDK_PATH})
|
||||
if(EXISTS "${LEVEL_ZERO_V1_SDK_PATH}")
|
||||
target_include_directories(ggml-sycl PRIVATE "${LEVEL_ZERO_V1_SDK_PATH}/include")
|
||||
target_include_directories(ggml-sycl SYSTEM PRIVATE "${LEVEL_ZERO_V1_SDK_PATH}/include")
|
||||
set(LEVEL_ZERO_V1_SDK_LIB_PATH "${LEVEL_ZERO_V1_SDK_PATH}/lib")
|
||||
else()
|
||||
message(WARNING "LEVEL_ZERO_V1_SDK_PATH set but folder not found: ${LEVEL_ZERO_V1_SDK_PATH}")
|
||||
|
||||
@@ -504,6 +504,5 @@ void ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
|
||||
default:
|
||||
fprintf(stderr, "%s: unsupported types: dst: %s\n", __func__, ggml_type_name(dst->type));
|
||||
GGML_ASSERT(false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ static void dequantize_row_q2_K_sycl_reorder(const void *vx, dst_t *y, const int
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)),
|
||||
[=](sycl::nd_item<3> item_ct1) {
|
||||
dequantize_block_q2_K_reorder(vx, y, item_ct1, nb);
|
||||
dequantize_block_q2_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -129,7 +129,7 @@ static void dequantize_row_q3_K_sycl_reorder(const void *vx, dst_t *y, const int
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)),
|
||||
[=](sycl::nd_item<3> item_ct1) {
|
||||
dequantize_block_q3_K_reorder(vx, y, item_ct1, nb);
|
||||
dequantize_block_q3_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -231,7 +231,7 @@ static void dequantize_row_q4_K_sycl_reorder(const void * vx, dst_t * y, const i
|
||||
|
||||
cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(global_size), sycl::range<1>(local_size)),
|
||||
[=](sycl::nd_item<1> item_ct1) {
|
||||
dequantize_block_q4_K_reorder(vx, y, get_pointer(scale_local_acc), item_ct1, nb);
|
||||
dequantize_block_q4_K_reorder(vx, y + item_ct1.get_group(0) * QK_K, item_ct1.get_group(0), get_pointer(scale_local_acc), item_ct1, nb);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -258,7 +258,7 @@ static void dequantize_row_q5_K_sycl_reorder(const void * vx, dst_t * y, const i
|
||||
cgh.parallel_for(
|
||||
sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)),
|
||||
[=](sycl::nd_item<3> item_ct1) {
|
||||
dequantize_block_q5_K_reorder(vx, y, get_pointer(scale_local_acc), item_ct1, nb);
|
||||
dequantize_block_q5_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), get_pointer(scale_local_acc), item_ct1, nb);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -303,7 +303,7 @@ static void dequantize_row_q6_K_sycl_reorder(const void * vx, dst_t * y, const i
|
||||
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)),
|
||||
[=](sycl::nd_item<3> item_ct1) { dequantize_block_q6_K_reorder(vx, y, item_ct1, nb); });
|
||||
[=](sycl::nd_item<3> item_ct1) { dequantize_block_q6_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb); });
|
||||
}
|
||||
|
||||
template <typename dst_t>
|
||||
@@ -757,7 +757,6 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {
|
||||
#endif
|
||||
default:
|
||||
GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type));
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -861,7 +860,6 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) {
|
||||
#endif
|
||||
default:
|
||||
GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type));
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -877,7 +875,6 @@ to_bf16_sycl_t ggml_get_to_bf16_sycl(ggml_type type, ggml_tensor * /*dst*/) {
|
||||
return convert_unary_sycl<sycl::ext::oneapi::bfloat16>;
|
||||
default:
|
||||
GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type));
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -74,6 +74,5 @@ void ggml_sycl_count_equal(ggml_backend_sycl_context &ctx, ggml_tensor *dst) {
|
||||
} break;
|
||||
default:
|
||||
GGML_ASSERT(false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -926,10 +926,9 @@ static void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restri
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy,
|
||||
static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i,
|
||||
const sycl::nd_item<3> & item_ct1, int64_t n_blocks) {
|
||||
#if QK_K == 256
|
||||
const int64_t i = item_ct1.get_group(2);
|
||||
if (i >= n_blocks) {
|
||||
return;
|
||||
}
|
||||
@@ -949,7 +948,7 @@ static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t *
|
||||
const int64_t is = 8 * n + l / 16;
|
||||
|
||||
const uint8_t q = qs[32 * n + l];
|
||||
dst_t * y = yy + i * QK_K + 128 * n;
|
||||
dst_t * y = yy + 128 * n;
|
||||
|
||||
const float dall = (*dm)[0];
|
||||
const float dmin = (*dm)[1];
|
||||
@@ -960,6 +959,7 @@ static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t *
|
||||
#else
|
||||
GGML_UNUSED(vx);
|
||||
GGML_UNUSED(yy);
|
||||
GGML_UNUSED(i);
|
||||
GGML_UNUSED(item_ct1);
|
||||
GGML_UNUSED(n_blocks);
|
||||
GGML_ABORT("Q2_K reorder dequantize not supported for QK_K != 256");
|
||||
@@ -1022,10 +1022,9 @@ static void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restri
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_block_q3_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy,
|
||||
static void dequantize_block_q3_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i,
|
||||
const sycl::nd_item<3> & item_ct1, int64_t n_blocks) {
|
||||
#if QK_K == 256
|
||||
const int64_t i = item_ct1.get_group(2);
|
||||
if (i >= n_blocks) {
|
||||
return;
|
||||
}
|
||||
@@ -1062,7 +1061,7 @@ static void dequantize_block_q3_K_reorder(const void * __restrict__ vx, dst_t *
|
||||
|
||||
const float dl = d_all * (us - 32);
|
||||
|
||||
dst_t * y = yy + i * QK_K + 128 * n + 32 * j;
|
||||
dst_t * y = yy + 128 * n + 32 * j;
|
||||
const uint8_t * q = qs + 32 * n;
|
||||
const uint8_t * hm = hmask;
|
||||
|
||||
@@ -1072,6 +1071,7 @@ static void dequantize_block_q3_K_reorder(const void * __restrict__ vx, dst_t *
|
||||
#else
|
||||
GGML_UNUSED(vx);
|
||||
GGML_UNUSED(yy);
|
||||
GGML_UNUSED(i);
|
||||
GGML_UNUSED(item_ct1);
|
||||
GGML_UNUSED(n_blocks);
|
||||
GGML_ABORT("Q3_K reorder dequantize not supported for QK_K != 256");
|
||||
@@ -1160,14 +1160,13 @@ inline void dequantize_q4_K_common(dst_t * __restrict__ y, const uint8_t * __res
|
||||
}
|
||||
|
||||
template <typename dst_t>
|
||||
static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, uint8_t * scales_local,
|
||||
const sycl::nd_item<1> & item_ct1, int64_t nb) {
|
||||
const int64_t i = item_ct1.get_group(0); // block index
|
||||
static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i,
|
||||
uint8_t * scales_local, const sycl::nd_item<1> & item_ct1, int64_t nb) {
|
||||
const int64_t tid = item_ct1.get_local_id(0); // thread index within block
|
||||
const int64_t il = tid / 8;
|
||||
const int64_t ir = tid % 8;
|
||||
|
||||
dst_t * y = yy + i * QK_K + 64 * il + 4 * ir;
|
||||
dst_t * y = yy + 64 * il + 4 * ir;
|
||||
|
||||
const uint8_t * base = static_cast<const uint8_t *>(vx);
|
||||
const size_t qs_offset = i * (QK_K / 2);
|
||||
@@ -1237,9 +1236,8 @@ static void dequantize_block_q5_K_wide(const void * __restrict__ vx, dst_t * __r
|
||||
}
|
||||
|
||||
template <typename dst_t>
|
||||
static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy,
|
||||
static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t ib,
|
||||
uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t n_blocks) {
|
||||
const int64_t ib = item_ct1.get_group(2);
|
||||
|
||||
#if QK_K == 256
|
||||
// assume 64 threads
|
||||
@@ -1248,7 +1246,7 @@ static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t *
|
||||
const int64_t ir = tid % 16; // 0...15
|
||||
const int64_t is = 2 * il;
|
||||
|
||||
dst_t * y = yy + ib * QK_K + 64 * il + 2 * ir;
|
||||
dst_t * y = yy + 64 * il + 2 * ir;
|
||||
|
||||
const uint8_t * base = static_cast<const uint8_t *>(vx);
|
||||
|
||||
@@ -1340,9 +1338,8 @@ static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restri
|
||||
}
|
||||
|
||||
template <typename dst_t>
|
||||
static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy,
|
||||
static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t ib,
|
||||
const sycl::nd_item<3> & item_ct1, int64_t n_blocks) {
|
||||
const int64_t ib = item_ct1.get_group(2);
|
||||
|
||||
const int64_t tid = item_ct1.get_local_id(2);
|
||||
const int64_t ip = tid / 32; // ip is 0 or 1
|
||||
@@ -1359,7 +1356,7 @@ static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t *
|
||||
const uint8_t * scales_ptr = base_ptr + base_scales_offset;
|
||||
const ggml_half * d = (const ggml_half *) (base_ptr + base_d_offset) + ib;
|
||||
|
||||
dst_t * y = yy + ib * QK_K + 128 * ip + il;
|
||||
dst_t * y = yy + 128 * ip + il;
|
||||
|
||||
const uint8_t * ql = ql_ptr + 64 * ip + il;
|
||||
const uint8_t qh = *(qh_ptr + 32 * ip + il);
|
||||
|
||||
@@ -3159,7 +3159,7 @@ namespace dpct
|
||||
_local_mem_size = local_mem_size;
|
||||
|
||||
|
||||
};
|
||||
}
|
||||
static inline std::mutex kernel_function_ptr_map_mutex;
|
||||
|
||||
public:
|
||||
|
||||
@@ -1169,7 +1169,7 @@ static void swiglu_oai_sycl(const T * x,
|
||||
});
|
||||
}
|
||||
|
||||
void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
static void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
void * src0_d = src0->data;
|
||||
|
||||
@@ -832,6 +832,29 @@ static void flash_attn_combine_results(const float * __restrict__ VKQ_parts,
|
||||
dst[tid] = VKQ_numerator / VKQ_denominator;
|
||||
}
|
||||
|
||||
// Carries the sub-group size (and grf_size_automatic when use_large_grf) as
|
||||
// kernel properties via get(properties_tag), in place of the
|
||||
// reqd_sub_group_size attribute and the deprecated parallel_for overload.
|
||||
template <typename KernelFunc, uint32_t Size, bool UseGrf>
|
||||
struct props_kernel {
|
||||
KernelFunc kernel;
|
||||
|
||||
void operator()(sycl::nd_item<3> item) const {
|
||||
kernel(item);
|
||||
}
|
||||
|
||||
auto get(sycl::ext::oneapi::experimental::properties_tag) const {
|
||||
if constexpr (UseGrf) {
|
||||
return sycl::ext::oneapi::experimental::properties{
|
||||
sycl::ext::intel::experimental::grf_size_automatic,
|
||||
sycl::ext::oneapi::experimental::sub_group_size<Size> };
|
||||
} else {
|
||||
return sycl::ext::oneapi::experimental::properties{
|
||||
sycl::ext::oneapi::experimental::sub_group_size<Size> };
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <fattn_kernel_t fattn_kernel, int warp_size, bool use_large_grf = false>
|
||||
static void lauch_kernel(
|
||||
dpct::dim3 group_range,
|
||||
@@ -881,7 +904,7 @@ static void lauch_kernel(
|
||||
static_cast<sycl::range<3>>(group_range * local_range),
|
||||
static_cast<sycl::range<3>>(local_range));
|
||||
|
||||
const auto kernel = [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] {
|
||||
const auto kernel = [=](sycl::nd_item<3> item_ct1) {
|
||||
GGML_UNUSED(item_ct1);
|
||||
fattn_kernel(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,
|
||||
max_bias, m0, m1, n_head_log2, logit_softcap, ne00,
|
||||
@@ -892,14 +915,7 @@ static void lauch_kernel(
|
||||
|
||||
q->submit([&](sycl::handler &cgh) {
|
||||
// grf_size_automatic lets the compiler use the large register file when the kernel needs it. grf_size<256> would say the same but is undefined on devices that do not have it.
|
||||
if constexpr (use_large_grf) {
|
||||
cgh.parallel_for(
|
||||
rng,
|
||||
sycl::ext::oneapi::experimental::properties{ sycl::ext::intel::experimental::grf_size_automatic },
|
||||
kernel);
|
||||
} else {
|
||||
cgh.parallel_for(rng, kernel);
|
||||
}
|
||||
cgh.parallel_for(rng, props_kernel<decltype(kernel), warp_size, use_large_grf>{kernel});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -54,6 +54,6 @@ void ggml_sycl_flash_attn_ext_tile(ggml_backend_sycl_context & ctx, ggml_tensor
|
||||
} break;
|
||||
default: {
|
||||
GGML_ABORT("Unsupported head size");
|
||||
} break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,6 +137,47 @@ static void k_get_rows_float(
|
||||
dst_row[i00] = src0_row[i00];
|
||||
}
|
||||
|
||||
// src0 is in reorder layout: all qs of the tensor, then all d
|
||||
template<int qk, int qr, dequantize_kernel_t_reorder dequantize_kernel, typename dst_t>
|
||||
static void k_get_rows_reorder(
|
||||
const void * src0_qs, const void * src0_d, const int32_t * src1, dst_t * dst,
|
||||
int64_t ne00, int64_t ne12,
|
||||
size_t s1, size_t s2, size_t s3,
|
||||
size_t s10, size_t s11, size_t s12,
|
||||
const sycl::nd_item<3> &item_ct1) {
|
||||
|
||||
const int i00 = (item_ct1.get_group(2) * item_ct1.get_local_range(2) +
|
||||
item_ct1.get_local_id(2)) *
|
||||
2;
|
||||
const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) +
|
||||
item_ct1.get_local_id(1);
|
||||
const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) +
|
||||
item_ct1.get_local_id(0)) /
|
||||
ne12;
|
||||
const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) +
|
||||
item_ct1.get_local_id(0)) %
|
||||
ne12;
|
||||
|
||||
if (i00 >= ne00) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
|
||||
|
||||
dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;
|
||||
|
||||
const int64_t ib = (int64_t) i01*(ne00/qk) + i00/qk; // block index in the whole tensor
|
||||
const int iqs = (i00%qk)/qr;
|
||||
const int iybs = i00 - i00%qk;
|
||||
const int y_offset = qr == 1 ? 1 : qk/2;
|
||||
|
||||
dfloat2 v;
|
||||
dequantize_kernel(src0_d, ib, src0_qs, ib*(qk/qr) + iqs, v);
|
||||
|
||||
dst_row[iybs + iqs + 0] = v.x();
|
||||
dst_row[iybs + iqs + y_offset] = v.y();
|
||||
}
|
||||
|
||||
template <int qk, int qr, dequantize_kernel_t dq>
|
||||
static void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1,
|
||||
ggml_tensor *dst, const void *src0_dd,
|
||||
@@ -173,6 +214,93 @@ static void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *sr
|
||||
GGML_UNUSED(ctx);
|
||||
}
|
||||
|
||||
template <int qk, int qr, dequantize_kernel_t_reorder dq>
|
||||
static void get_rows_sycl_reorder(const ggml_tensor *src0, const ggml_tensor *src1,
|
||||
ggml_tensor *dst, const void *src0_dd,
|
||||
const int32_t *src1_dd, float *dst_dd,
|
||||
queue_ptr stream) {
|
||||
|
||||
GGML_TENSOR_BINARY_OP_LOCALS
|
||||
|
||||
// reorder is done only on 2D weights
|
||||
GGML_ASSERT(ne02 == 1 && ne03 == 1);
|
||||
GGML_ASSERT(ne00 % 2 == 0);
|
||||
|
||||
const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE);
|
||||
const int block_num_x = (ne00 + 2*SYCL_GET_ROWS_BLOCK_SIZE - 1) / (2*SYCL_GET_ROWS_BLOCK_SIZE);
|
||||
const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x);
|
||||
|
||||
const size_t s1 = nb1 / ggml_element_size(dst);
|
||||
const size_t s2 = nb2 / ggml_element_size(dst);
|
||||
const size_t s3 = nb3 / ggml_element_size(dst);
|
||||
|
||||
const size_t s10 = nb10 / ggml_element_size(src1);
|
||||
const size_t s11 = nb11 / ggml_element_size(src1);
|
||||
const size_t s12 = nb12 / ggml_element_size(src1);
|
||||
|
||||
const void * src0_d = (const char *) src0_dd + ne00*ne01/qr;
|
||||
|
||||
stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
||||
[=](sycl::nd_item<3> item_ct1) {
|
||||
k_get_rows_reorder<qk, qr, dq>(
|
||||
src0_dd, src0_d, src1_dd, dst_dd, ne00, ne12, s1, s2,
|
||||
s3, s10, s11, s12, item_ct1);
|
||||
});
|
||||
|
||||
GGML_UNUSED(src0);
|
||||
}
|
||||
|
||||
// K-quants in reorder layout: one work-group dequantizes one QK_K block of one row
|
||||
template <int dims, int nthreads, int nlocal, typename F>
|
||||
static void get_rows_sycl_reorder_k(const ggml_tensor *src0, const ggml_tensor *src1,
|
||||
ggml_tensor *dst, const int32_t *src1_dd, float *dst_dd,
|
||||
queue_ptr stream, F dequantize_block) {
|
||||
|
||||
GGML_TENSOR_BINARY_OP_LOCALS
|
||||
|
||||
// reorder is done only on 2D weights
|
||||
GGML_ASSERT(ne02 == 1 && ne03 == 1);
|
||||
GGML_ASSERT(ne00 % QK_K == 0);
|
||||
|
||||
const int64_t nblk = ne00 / QK_K;
|
||||
const int64_t nb = nblk * ne01; // blocks in the whole tensor
|
||||
const int64_t ngroups = ne10 * ne11 * ne12 * nblk;
|
||||
|
||||
const size_t s1 = nb1 / ggml_element_size(dst);
|
||||
const size_t s2 = nb2 / ggml_element_size(dst);
|
||||
const size_t s3 = nb3 / ggml_element_size(dst);
|
||||
|
||||
const size_t s10 = nb10 / ggml_element_size(src1);
|
||||
const size_t s11 = nb11 / ggml_element_size(src1);
|
||||
const size_t s12 = nb12 / ggml_element_size(src1);
|
||||
|
||||
stream->submit([&](sycl::handler & cgh) {
|
||||
sycl::local_accessor<uint8_t, 1> scales_local(sycl::range<1>(nlocal), cgh);
|
||||
|
||||
auto kernel = [=](sycl::nd_item<dims> item_ct1) {
|
||||
const int64_t g = item_ct1.get_group(dims - 1);
|
||||
const int64_t b = g % nblk;
|
||||
const int64_t r = g / nblk;
|
||||
const int64_t i10 = r % ne10;
|
||||
const int64_t i11 = (r / ne10) % ne11;
|
||||
const int64_t i12 = r / (ne10 * ne11);
|
||||
|
||||
const int64_t i01 = src1_dd[i10*s10 + i11*s11 + i12*s12];
|
||||
|
||||
float * y = dst_dd + i10*s1 + i11*s2 + i12*s3 + b*QK_K;
|
||||
dequantize_block(y, i01*nblk + b, get_pointer(scales_local), item_ct1, nb);
|
||||
};
|
||||
|
||||
if constexpr (dims == 1) {
|
||||
cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(ngroups * nthreads), sycl::range<1>(nthreads)), kernel);
|
||||
} else {
|
||||
cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, ngroups * nthreads), sycl::range<3>(1, 1, nthreads)), kernel);
|
||||
}
|
||||
});
|
||||
|
||||
GGML_UNUSED(src0);
|
||||
}
|
||||
|
||||
template <int qk, int qr, dequantize_kernel_f32_t dq>
|
||||
static void get_rows_sycl_f32(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1,
|
||||
ggml_tensor *dst, const void *src0_dd,
|
||||
@@ -332,6 +460,9 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));
|
||||
|
||||
const int32_t * src1_i32 = (const int32_t *) dst->src[1]->data;
|
||||
// a weight shared with MUL_MAT (e.g. tied embeddings) can be in reorder layout
|
||||
const bool src0_reordered = dst->src[0]->extra &&
|
||||
((const ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder;
|
||||
/* TODO: Refactor and remove duplicates */
|
||||
switch (dst->src[0]->type) {
|
||||
case GGML_TYPE_F16:
|
||||
@@ -399,14 +530,37 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
case GGML_TYPE_Q2_K:
|
||||
if (src0_reordered) {
|
||||
const void * vx = dst->src[0]->data;
|
||||
get_rows_sycl_reorder_k<3, 64, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(),
|
||||
[=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) {
|
||||
dequantize_block_q2_K_reorder(vx, y, ib, item_ct1, nb);
|
||||
GGML_UNUSED(scales_local);
|
||||
});
|
||||
break;
|
||||
}
|
||||
get_rows_sycl_f32<QK_K, 1, dequantize_q2_K_f32>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
case GGML_TYPE_Q3_K:
|
||||
if (src0_reordered) {
|
||||
const void * vx = dst->src[0]->data;
|
||||
get_rows_sycl_reorder_k<3, 64, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(),
|
||||
[=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) {
|
||||
dequantize_block_q3_K_reorder(vx, y, ib, item_ct1, nb);
|
||||
GGML_UNUSED(scales_local);
|
||||
});
|
||||
break;
|
||||
}
|
||||
get_rows_sycl<QK_K, 1, dequantize_q3_K>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
case GGML_TYPE_Q4_0:
|
||||
if (src0_reordered) {
|
||||
get_rows_sycl_reorder<QK4_0, QR4_0, dequantize_q4_0_reorder>(dst->src[0], dst->src[1], dst, dst->src[0]->data,
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
}
|
||||
get_rows_sycl<QK4_0, QR4_0, dequantize_q4_0>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
@@ -415,6 +569,14 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
case GGML_TYPE_Q4_K:
|
||||
if (src0_reordered) {
|
||||
const void * vx = dst->src[0]->data;
|
||||
get_rows_sycl_reorder_k<1, 32, K_SCALE_SIZE>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(),
|
||||
[=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<1> & item_ct1, int64_t nb) {
|
||||
dequantize_block_q4_K_reorder(vx, y, ib, scales_local, item_ct1, nb);
|
||||
});
|
||||
break;
|
||||
}
|
||||
get_rows_sycl_f32<QK_K, 1, dequantize_q4_K_f32>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
@@ -427,14 +589,36 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
case GGML_TYPE_Q5_K:
|
||||
if (src0_reordered) {
|
||||
const void * vx = dst->src[0]->data;
|
||||
get_rows_sycl_reorder_k<3, 64, K_SCALE_SIZE>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(),
|
||||
[=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) {
|
||||
dequantize_block_q5_K_reorder(vx, y, ib, scales_local, item_ct1, nb);
|
||||
});
|
||||
break;
|
||||
}
|
||||
get_rows_sycl_f32<QK_K, 1, dequantize_q5_K_f32>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
case GGML_TYPE_Q6_K:
|
||||
if (src0_reordered) {
|
||||
const void * vx = dst->src[0]->data;
|
||||
get_rows_sycl_reorder_k<3, 64, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(),
|
||||
[=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) {
|
||||
dequantize_block_q6_K_reorder(vx, y, ib, item_ct1, nb);
|
||||
GGML_UNUSED(scales_local);
|
||||
});
|
||||
break;
|
||||
}
|
||||
get_rows_sycl<QK_K, 1, dequantize_q6_K>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
case GGML_TYPE_Q8_0:
|
||||
if (src0_reordered) {
|
||||
get_rows_sycl_reorder<QK8_0, QR8_0, dequantize_q8_0_reorder>(dst->src[0], dst->src[1], dst, dst->src[0]->data,
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
}
|
||||
get_rows_sycl<QK8_0, QR8_0, dequantize_q8_0>(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data,
|
||||
src1_i32, (float *)dst->data, ctx.stream());
|
||||
break;
|
||||
@@ -468,6 +652,5 @@ void ggml_sycl_op_get_rows_back(ggml_backend_sycl_context & ctx, ggml_tensor * d
|
||||
break;
|
||||
default:
|
||||
GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0->type));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -388,14 +388,14 @@ static ze_result_t init_zes() {
|
||||
return res;
|
||||
}
|
||||
|
||||
ze_result_t get_zes_init_res() {
|
||||
static ze_result_t get_zes_init_res() {
|
||||
static ze_result_t zes_init_res = init_zes();
|
||||
GGML_SYCL_DEBUG("[SYCL] call %s: zesInit result: %d\n", __func__, (int) zes_init_res);
|
||||
return zes_init_res;
|
||||
}
|
||||
#endif
|
||||
|
||||
void initialize_sycl_begining() {
|
||||
static void initialize_sycl_begining() {
|
||||
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
|
||||
//must be called in initialization stage, before any other Level Zero API calls
|
||||
GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__);
|
||||
@@ -1173,7 +1173,7 @@ static size_t ggml_backend_sycl_buffer_type_get_alignment(ggml_backend_buffer_ty
|
||||
GGML_UNUSED(buft);
|
||||
}
|
||||
|
||||
bool is_bmg_g31_arch(int device) {
|
||||
static bool is_bmg_g31_arch(int device) {
|
||||
return ggml_sycl_info().devices[device].hw_info.arch == gpu_arch::intel_gpu_bmg_g31;
|
||||
}
|
||||
|
||||
@@ -6326,7 +6326,7 @@ catch (sycl::exception const &exc) {
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
bool sycl_get_mem_info(int device, size_t * free, size_t * total) {
|
||||
static bool sycl_get_mem_info(int device, size_t * free, size_t * total) {
|
||||
GGML_SYCL_DEBUG("[SYCL] [%s] g_ggml_sycl_get_mem_api=%d\n",
|
||||
__func__, g_ggml_sycl_get_mem_api);
|
||||
|
||||
@@ -6952,7 +6952,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
case GGML_OP_MUL_MAT:
|
||||
case GGML_OP_MUL_MAT_ID:
|
||||
{
|
||||
@@ -7045,7 +7044,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
(op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32);
|
||||
return res;
|
||||
}
|
||||
break;
|
||||
case GGML_OP_DSV4_HC_PRE:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 &&
|
||||
op->type == GGML_TYPE_F32;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include <sycl/sycl.hpp>
|
||||
|
||||
#include "common.hpp"
|
||||
#include "gla.hpp"
|
||||
|
||||
template <u_int HEAD_SIZE>
|
||||
static void gated_linear_attn_f32_kernel(const dpct::queue_ptr stream, u_int B, u_int T, u_int C, u_int H, float scale,
|
||||
|
||||
@@ -27,7 +27,7 @@ const char * mem_api_int2str(int mem_api) {
|
||||
/*
|
||||
* Depend on to call zesInit(0) before any other Level Zero API calls, otherwise the Level Zero API calls may fail.
|
||||
*/
|
||||
bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & total_bytes) {
|
||||
static bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & total_bytes) {
|
||||
GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using Level Zero API.\n", __func__);
|
||||
|
||||
free_bytes = 0;
|
||||
@@ -94,7 +94,7 @@ bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & tot
|
||||
}
|
||||
#endif
|
||||
|
||||
bool get_memory_size_by_sycl_api(sycl::device dev, size_t & free_bytes, size_t & total_bytes) {
|
||||
static bool get_memory_size_by_sycl_api(sycl::device dev, size_t & free_bytes, size_t & total_bytes) {
|
||||
GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using SYCL API.\n", __func__);
|
||||
total_bytes = dev.get_info<sycl::info::device::global_mem_size>();
|
||||
|
||||
|
||||
@@ -296,7 +296,6 @@ static void ggml_sycl_op_topk_moe(ggml_backend_sycl_context & ctx,
|
||||
break;
|
||||
default:
|
||||
GGML_ASSERT(false && "fatal error");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -183,8 +183,8 @@ static void upscale_f32_bilinear_antialias(const float * src0,
|
||||
}
|
||||
|
||||
namespace bicubic_interpolation {
|
||||
static float weight1(float x, const float &a) { return ((a + 2) * x - (a + 3)) * x * x + 1; };
|
||||
static float weight2(float x, const float &a) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; };
|
||||
static float weight1(float x, const float &a) { return ((a + 2) * x - (a + 3)) * x * x + 1; }
|
||||
static float weight2(float x, const float &a) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; }
|
||||
|
||||
static float bicubic(float p0, float p1, float p2, float p3, float x, float a) {
|
||||
const float w0 = weight2(x + 1, a);
|
||||
@@ -192,7 +192,7 @@ static float bicubic(float p0, float p1, float p2, float p3, float x, float a) {
|
||||
const float w2 = weight1(1 - x, a);
|
||||
const float w3 = weight2(2 - x, a);
|
||||
return p0 * w0 + p1 * w1 + p2 * w2 + p3 * w3;
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -358,7 +358,7 @@ static void upscale_f32_bicubic_sycl(const float * x,
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
static void ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const float * src0_d = (const float *)src0->data;
|
||||
float * dst_d = (float *)dst->data;
|
||||
|
||||
@@ -442,7 +442,7 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0> {
|
||||
}
|
||||
|
||||
return vec_dot_q4_0_q8_1_impl(v, u, d, *q8_1_ds);
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
// Load four contiguous dwords per operand instead of loading each value separately.
|
||||
|
||||
+124
-22
@@ -11,6 +11,9 @@ import platform
|
||||
import shutil
|
||||
import logging
|
||||
import glob
|
||||
import shlex
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
from sdk import validate_windows_sdks
|
||||
|
||||
@@ -19,19 +22,70 @@ logger = logging.getLogger("build")
|
||||
|
||||
def parse_target(target_str):
|
||||
if not target_str:
|
||||
return None, None
|
||||
if target_str.startswith("adb") or target_str.startswith("android"):
|
||||
parts = target_str.split(":", 1)
|
||||
serial = parts[1] if len(parts) > 1 else None
|
||||
return "android", serial
|
||||
elif target_str.startswith("lnx") or target_str.startswith("linux") or target_str.startswith("ubuntu"):
|
||||
parts = target_str.split(":", 1)
|
||||
host = parts[1] if len(parts) > 1 else None
|
||||
return "linux", host
|
||||
return None, None, None
|
||||
|
||||
parts = target_str.split(":")
|
||||
target_name = parts[0]
|
||||
|
||||
if target_name in ("adb", "android"):
|
||||
serial = ":".join(parts[1:]) if len(parts) > 1 else None
|
||||
return "android", serial, "adb"
|
||||
elif target_name in ("lnx", "linux", "ubuntu"):
|
||||
if len(parts) > 1 and parts[1] == "adb":
|
||||
serial = ":".join(parts[2:]) if len(parts) > 2 else None
|
||||
return "linux", serial, "adb"
|
||||
host = ":".join(parts[1:]) if len(parts) > 1 else None
|
||||
return "linux", host, "ssh"
|
||||
elif target_str in ("wos", "windows"):
|
||||
return "windows", None
|
||||
return "windows", None, None
|
||||
else:
|
||||
return None, None
|
||||
return None, None, None
|
||||
|
||||
|
||||
def default_target_dir(target_type, target_transport, home_dir):
|
||||
if target_type == "android":
|
||||
return "/data/local/tmp/llama.cpp"
|
||||
if target_type == "linux":
|
||||
if home_dir:
|
||||
return home_dir.rstrip("/") + "/llama.cpp"
|
||||
if target_transport == "adb":
|
||||
return "/home/llama.cpp"
|
||||
return "~/llama.cpp"
|
||||
return "~/llama.cpp"
|
||||
|
||||
|
||||
def normalize_target_dir(target_dir):
|
||||
target_dir = target_dir.rstrip("/")
|
||||
if not target_dir or target_dir == "/":
|
||||
raise RuntimeError("target directory must not be the filesystem root")
|
||||
return target_dir
|
||||
|
||||
|
||||
def adb_root(adb_cmd):
|
||||
root_cmd = adb_cmd + ["root"]
|
||||
logger.info(f"+ {' '.join(root_cmd)}")
|
||||
res = subprocess.run(root_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB root failed.")
|
||||
sys.exit(res.returncode)
|
||||
|
||||
wait_cmd = adb_cmd + ["wait-for-device"]
|
||||
logger.info(f"+ {' '.join(wait_cmd)}")
|
||||
res = subprocess.run(wait_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB wait-for-device failed.")
|
||||
sys.exit(res.returncode)
|
||||
|
||||
|
||||
def make_package_archive(src_path):
|
||||
fd, archive_path = tempfile.mkstemp(prefix="llama-cpp-", suffix=".tar.gz")
|
||||
os.close(fd)
|
||||
with tarfile.open(archive_path, "w:gz", dereference=False) as tar:
|
||||
for item in os.listdir(src_path):
|
||||
if item.startswith("."):
|
||||
continue
|
||||
tar.add(os.path.join(src_path, item), arcname=item, recursive=True)
|
||||
return archive_path
|
||||
|
||||
|
||||
def get_uid_gid():
|
||||
@@ -45,7 +99,7 @@ def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build llama.cpp for Snapdragon using cross-compilation docker containers or natively."
|
||||
)
|
||||
parser.add_argument("--target", default="android", help="Compilation target and deployment definition (e.g. android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: android)")
|
||||
parser.add_argument("--target", default="android", help="Compilation target and deployment definition (e.g. android[:serial]/adb[:serial], ubuntu:adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: android)")
|
||||
parser.add_argument("--build-dir", help="Build directory name (defaults to build-TARGET[-dbg], e.g. build-android)")
|
||||
parser.add_argument("--install-dir", help="Install directory name (defaults to pkg-TARGET[-dbg], e.g. pkg-android)")
|
||||
parser.add_argument("--jobs", "-j", type=int, help="Number of build jobs (defaults to CPU thread count)")
|
||||
@@ -53,21 +107,22 @@ def main():
|
||||
parser.add_argument("--preset", help="Override the CMake preset to use")
|
||||
parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)")
|
||||
parser.add_argument("--hex-only", "--hexagon-only", action="store_true", help="Only build and push ggml-hexagon and libggml-htp libraries")
|
||||
parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "73", "75", "79", "81"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)")
|
||||
parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "v85", "73", "75", "79", "81", "85"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)")
|
||||
|
||||
# Push options
|
||||
parser.add_argument("--push", action="store_true", help="Push built package to the target device via ADB or SSH/SCP")
|
||||
parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux)")
|
||||
parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux, /home/llama.cpp for Linux via ADB)")
|
||||
parser.add_argument("--home-dir", help="Home directory for Linux targets when deriving the default target directory")
|
||||
|
||||
# Toolchain options
|
||||
parser.add_argument("--toolchain-version", default="v0.7", help="Docker toolchain image version/tag (default: v0.7)")
|
||||
parser.add_argument("--toolchain-version", default="v0.8", help="Docker toolchain image version/tag (default: v0.8)")
|
||||
parser.add_argument("--toolchain-url", default="ghcr.io/snapdragon-toolchain", help="Docker toolchain registry URL/namespace (default: ghcr.io/snapdragon-toolchain)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
target_type, target_val = parse_target(args.target)
|
||||
target_type, target_val, target_transport = parse_target(args.target)
|
||||
if not target_type:
|
||||
logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.")
|
||||
logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], ubuntu:adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.")
|
||||
sys.exit(1)
|
||||
|
||||
if target_type == "windows":
|
||||
@@ -140,7 +195,7 @@ def main():
|
||||
if hex_arch:
|
||||
hex_targets = ["ggml-hexagon", f"htp-{hex_arch}"]
|
||||
else:
|
||||
hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81"]
|
||||
hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81", "htp-v85"]
|
||||
|
||||
if args.no_docker:
|
||||
# Native/local host build
|
||||
@@ -218,8 +273,8 @@ def main():
|
||||
# Resolve target directory on device
|
||||
target_dir = args.target_dir
|
||||
if not target_dir:
|
||||
target_dir = "/data/local/tmp/llama.cpp" if target_type == "android" else "~/llama.cpp"
|
||||
target_dir = target_dir.rstrip("/")
|
||||
target_dir = default_target_dir(target_type, target_transport, args.home_dir)
|
||||
target_dir = normalize_target_dir(target_dir)
|
||||
|
||||
sub_items = [item for item in os.listdir(src_path) if not item.startswith(".")]
|
||||
|
||||
@@ -235,14 +290,23 @@ def main():
|
||||
logger.error(f"Error: no hexagon libraries found in {lib_dir} to push.")
|
||||
sys.exit(1)
|
||||
|
||||
if target_type == "android":
|
||||
if target_transport == "adb":
|
||||
adb_cmd = ["adb"]
|
||||
if target_val: # serial
|
||||
adb_cmd += ["-s", target_val]
|
||||
|
||||
if target_type == "linux":
|
||||
adb_root(adb_cmd)
|
||||
|
||||
if args.hex_only:
|
||||
logger.info("\nPushing Hexagon libraries to Android device via ADB...")
|
||||
logger.info("\nPushing Hexagon libraries to target device via ADB...")
|
||||
target_lib_dir = f"{target_dir}/lib"
|
||||
mkdir_cmd = adb_cmd + ["shell", f"mkdir -p {shlex.quote(target_lib_dir)}"]
|
||||
logger.info(f"+ {' '.join(mkdir_cmd)}")
|
||||
res = subprocess.run(mkdir_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB mkdir failed.")
|
||||
sys.exit(res.returncode)
|
||||
for lib in hex_libs:
|
||||
push_cmd = adb_cmd + ["push", lib, f"{target_lib_dir}/"]
|
||||
logger.info(f"+ {' '.join(push_cmd)}")
|
||||
@@ -251,6 +315,44 @@ def main():
|
||||
logger.error("ADB push failed.")
|
||||
sys.exit(res.returncode)
|
||||
logger.info("Hexagon libraries deployed successfully!")
|
||||
elif target_type == "linux":
|
||||
logger.info("\nDeploying Linux package to target device via ADB tar...")
|
||||
remote_archive = f"/tmp/llama.cpp-package-{os.getpid()}.tar.gz"
|
||||
archive_path = make_package_archive(src_path)
|
||||
try:
|
||||
mkdir_cmd = adb_cmd + ["shell", f"mkdir -p {shlex.quote(target_dir)}"]
|
||||
logger.info(f"+ {' '.join(mkdir_cmd)}")
|
||||
res = subprocess.run(mkdir_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB mkdir failed.")
|
||||
sys.exit(res.returncode)
|
||||
|
||||
if sub_items:
|
||||
clean_paths = " ".join(shlex.quote(f"{target_dir}/{item}") for item in sub_items)
|
||||
clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]
|
||||
logger.info(f"+ {' '.join(clean_cmd)}")
|
||||
subprocess.run(clean_cmd)
|
||||
|
||||
push_cmd = adb_cmd + ["push", archive_path, remote_archive]
|
||||
logger.info(f"+ {' '.join(push_cmd)}")
|
||||
res = subprocess.run(push_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB push failed.")
|
||||
sys.exit(res.returncode)
|
||||
|
||||
extract_cmd = adb_cmd + ["shell", f"tar -xzf {shlex.quote(remote_archive)} -C {shlex.quote(target_dir)} && rm -f {shlex.quote(remote_archive)}"]
|
||||
logger.info(f"+ {' '.join(extract_cmd)}")
|
||||
res = subprocess.run(extract_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB untar failed.")
|
||||
sys.exit(res.returncode)
|
||||
|
||||
chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {shlex.quote(target_dir)}/bin 2>/dev/null || true"]
|
||||
logger.info(f"+ {' '.join(chmod_cmd)}")
|
||||
subprocess.run(chmod_cmd)
|
||||
finally:
|
||||
os.unlink(archive_path)
|
||||
logger.info("ADB tar deploy completed successfully!")
|
||||
else:
|
||||
# Clean stale package files on device
|
||||
if sub_items:
|
||||
|
||||
@@ -255,7 +255,7 @@ class HexagonToolchain:
|
||||
use_docker: bool = False,
|
||||
image_url: str = "ghcr.io/snapdragon-toolchain",
|
||||
image_name: str = "arm64-android",
|
||||
image_ver: str = "v0.7",
|
||||
image_ver: str = "v0.8",
|
||||
):
|
||||
self.repo_root = repo_root
|
||||
self.image = f"{image_url}/{image_name}:{image_ver}"
|
||||
@@ -322,15 +322,14 @@ class HexagonToolchain:
|
||||
if not self.docker_bin:
|
||||
raise RuntimeError("Docker is required but not installed or found on PATH.")
|
||||
|
||||
container_tools_dir = "/opt/hexagon/6.6.0.0/tools/HEXAGON_Tools/19.0.07/Tools/bin"
|
||||
if "objdump" in tool_name:
|
||||
container_tool = f"{container_tools_dir}/hexagon-llvm-objdump"
|
||||
tool_bin = "hexagon-llvm-objdump"
|
||||
elif "addr2line" in tool_name:
|
||||
container_tool = f"{container_tools_dir}/hexagon-addr2line"
|
||||
tool_bin = "hexagon-addr2line"
|
||||
elif "nm" in tool_name:
|
||||
container_tool = f"{container_tools_dir}/llvm-nm"
|
||||
tool_bin = "llvm-nm"
|
||||
else:
|
||||
container_tool = f"{container_tools_dir}/{tool_name}"
|
||||
tool_bin = tool_name
|
||||
|
||||
# Translate file paths from host to /workspace
|
||||
translated_args = [translate_container_arg(arg, self.repo_root) for arg in args]
|
||||
@@ -350,7 +349,9 @@ class HexagonToolchain:
|
||||
if platform.system() != "Windows":
|
||||
docker_cmd += ["-u", f"{os.getuid()}:{os.getgid()}"]
|
||||
|
||||
docker_cmd += [self.image, container_tool] + translated_args
|
||||
default_tools_root = "/opt/hexagon/6.7.0.0/tools/HEXAGON_Tools/21.0.04"
|
||||
sh_cmd = f'exec "${{HEXAGON_TOOLS_ROOT:-{default_tools_root}}}/Tools/bin/{tool_bin}" "$@"'
|
||||
docker_cmd += [self.image, "sh", "-c", sh_cmd, "--"] + translated_args
|
||||
|
||||
res = subprocess.run(docker_cmd, capture_output=True, text=True, input=stdin_data)
|
||||
if res.returncode != 0:
|
||||
@@ -1457,8 +1458,8 @@ def main():
|
||||
)
|
||||
parser.add_argument(
|
||||
"--toolchain-version",
|
||||
default="v0.7",
|
||||
help="Docker toolchain tag (default: v0.7).",
|
||||
default="v0.8",
|
||||
help="Docker toolchain tag (default: v0.8).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--toolchain-url",
|
||||
|
||||
+82
-27
@@ -49,27 +49,46 @@ MANAGED_ENV_NAMES = (
|
||||
"GGML_OPENCL_ADRENO_XMEM_GEMM",
|
||||
"GGML_OPENCL_ADRENO_USE_LARGE_BUFFER",
|
||||
"GGML_SCHED_DEBUG",
|
||||
"ADSP_LIBRARY_PATH",
|
||||
"CDSP_LIBRARY_PATH",
|
||||
"CDSP1_LIBRARY_PATH",
|
||||
"CDSP2_LIBRARY_PATH",
|
||||
"CDSP3_LIBRARY_PATH",
|
||||
"MTMD_BACKEND_DEVICE",
|
||||
"D",
|
||||
"DEVICE",
|
||||
)
|
||||
|
||||
|
||||
DSP_LIBRARY_ENV_NAMES = (
|
||||
"ADSP_LIBRARY_PATH",
|
||||
"CDSP_LIBRARY_PATH",
|
||||
"CDSP1_LIBRARY_PATH",
|
||||
"CDSP2_LIBRARY_PATH",
|
||||
"CDSP3_LIBRARY_PATH",
|
||||
)
|
||||
|
||||
|
||||
def parse_target(target_str):
|
||||
if not target_str:
|
||||
return None, None
|
||||
if target_str.startswith("adb") or target_str.startswith("android"):
|
||||
parts = target_str.split(":", 1)
|
||||
serial = parts[1] if len(parts) > 1 else None
|
||||
return "android", serial
|
||||
elif target_str.startswith("lnx") or target_str.startswith("linux") or target_str.startswith("ubuntu"):
|
||||
parts = target_str.split(":", 1)
|
||||
host = parts[1] if len(parts) > 1 else None
|
||||
return "linux", host
|
||||
return None, None, None
|
||||
|
||||
parts = target_str.split(":")
|
||||
target_name = parts[0]
|
||||
|
||||
if target_name in ("adb", "android"):
|
||||
serial = ":".join(parts[1:]) if len(parts) > 1 else None
|
||||
return "android", serial, "adb"
|
||||
elif target_name in ("lnx", "linux", "ubuntu"):
|
||||
if len(parts) > 1 and parts[1] == "adb":
|
||||
serial = ":".join(parts[2:]) if len(parts) > 2 else None
|
||||
return "linux", serial, "adb"
|
||||
host = ":".join(parts[1:]) if len(parts) > 1 else None
|
||||
return "linux", host, "ssh"
|
||||
elif target_str in ("wos", "windows"):
|
||||
return "windows", None
|
||||
return "windows", None, None
|
||||
else:
|
||||
return None, None
|
||||
return None, None, None
|
||||
|
||||
|
||||
def shlex_join(args_list):
|
||||
@@ -79,6 +98,41 @@ def shlex_join(args_list):
|
||||
return " ".join(pipes.quote(x) for x in args_list)
|
||||
|
||||
|
||||
def default_target_dir(target_type, target_transport, home_dir):
|
||||
if target_type == "android":
|
||||
return "/data/local/tmp/llama.cpp"
|
||||
if target_type == "linux":
|
||||
if home_dir:
|
||||
return home_dir.rstrip("/") + "/llama.cpp"
|
||||
if target_transport == "adb":
|
||||
return "/home/llama.cpp"
|
||||
return "~/llama.cpp"
|
||||
return "~/llama.cpp"
|
||||
|
||||
|
||||
def library_env_parts(lib_dir):
|
||||
env_parts = [f"LD_LIBRARY_PATH={lib_dir}"]
|
||||
for name in DSP_LIBRARY_ENV_NAMES:
|
||||
env_parts.append(f"{name}={lib_dir}")
|
||||
return env_parts
|
||||
|
||||
|
||||
def adb_root(adb_base):
|
||||
root_cmd = adb_base + ["root"]
|
||||
logger.info(f"+ {' '.join(root_cmd)}")
|
||||
res = subprocess.run(root_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB root failed.")
|
||||
sys.exit(res.returncode)
|
||||
|
||||
wait_cmd = adb_base + ["wait-for-device"]
|
||||
logger.info(f"+ {' '.join(wait_cmd)}")
|
||||
res = subprocess.run(wait_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB wait-for-device failed.")
|
||||
sys.exit(res.returncode)
|
||||
|
||||
|
||||
def split_device_list(devices):
|
||||
parts = []
|
||||
curr = []
|
||||
@@ -144,8 +198,9 @@ def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Unified runner for llama.cpp tools on Snapdragon (natively, via ADB, or via SSH)."
|
||||
)
|
||||
parser.add_argument("--target", help="Execution target (e.g. android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: local run)")
|
||||
parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux)")
|
||||
parser.add_argument("--target", help="Execution target (e.g. android[:serial]/adb[:serial], ubuntu:adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: local run)")
|
||||
parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux, /home/llama.cpp for Linux via ADB)")
|
||||
parser.add_argument("--home-dir", help="Home directory for Linux targets when deriving the default target directory")
|
||||
parser.add_argument("--install-dir", help="Install directory name (defaults to pkg-TARGET or pkg-TARGET-dbg prefix based on target)")
|
||||
parser.add_argument("--debug", action="store_true", help="Use debug build (defaults to pkg-TARGET-dbg folder)")
|
||||
parser.add_argument("--devices", "--device", "-d", help="Select execution devices (split into NPU and OpenCL GPUs automatically, default: HTP0)")
|
||||
@@ -201,11 +256,12 @@ def main():
|
||||
|
||||
target_type = None
|
||||
target_val = None
|
||||
target_transport = None
|
||||
target_prefix = None
|
||||
if args.target:
|
||||
target_type, target_val = parse_target(args.target)
|
||||
target_type, target_val, target_transport = parse_target(args.target)
|
||||
if not target_type:
|
||||
logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.")
|
||||
logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], ubuntu:adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.")
|
||||
sys.exit(1)
|
||||
target_prefix = args.target.split(":", 1)[0]
|
||||
|
||||
@@ -435,18 +491,19 @@ def main():
|
||||
# Resolve target directory on device
|
||||
target_dir = args.target_dir
|
||||
if not target_dir:
|
||||
target_dir = "/data/local/tmp/llama.cpp" if target_type == "android" else "~/llama.cpp"
|
||||
target_dir = default_target_dir(target_type, target_transport, args.home_dir)
|
||||
|
||||
if target_type == "android":
|
||||
# Run via ADB
|
||||
if target_transport == "adb":
|
||||
adb_base = ["adb"]
|
||||
if target_val: # serial
|
||||
adb_base += ["-s", target_val]
|
||||
|
||||
env_parts = [
|
||||
"LD_LIBRARY_PATH=./lib",
|
||||
"ADSP_LIBRARY_PATH=./lib"
|
||||
]
|
||||
if target_type == "linux":
|
||||
adb_root(adb_base)
|
||||
|
||||
env_parts = library_env_parts("./lib")
|
||||
if target_type == "linux":
|
||||
env_parts.insert(0, f"HOME={args.home_dir or '/home'}")
|
||||
for k, v in env_vars.items():
|
||||
env_parts.append(f"{k}={v}")
|
||||
env_str = " ".join(env_parts)
|
||||
@@ -466,10 +523,7 @@ def main():
|
||||
sys.exit(1)
|
||||
|
||||
# Linux remote run via SSH
|
||||
env_parts = [
|
||||
"LD_LIBRARY_PATH=./lib",
|
||||
"ADSP_LIBRARY_PATH=./lib"
|
||||
]
|
||||
env_parts = library_env_parts("./lib")
|
||||
for k, v in env_vars.items():
|
||||
env_parts.append(f"{k}={v}")
|
||||
env_str = " ".join(env_parts)
|
||||
@@ -490,7 +544,6 @@ def main():
|
||||
# Run locally
|
||||
local_env = os.environ.copy()
|
||||
lib_dir = os.path.normpath(os.path.join(package_path, "lib"))
|
||||
local_env["ADSP_LIBRARY_PATH"] = lib_dir
|
||||
if platform.system() == "Windows":
|
||||
local_env["PATH"] = lib_dir + os.path.pathsep + local_env.get("PATH", "")
|
||||
else:
|
||||
@@ -498,6 +551,8 @@ def main():
|
||||
|
||||
for k in MANAGED_ENV_NAMES:
|
||||
local_env.pop(k, None)
|
||||
for name in DSP_LIBRARY_ENV_NAMES:
|
||||
local_env[name] = lib_dir
|
||||
for k, v in env_vars.items():
|
||||
local_env[k] = v
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ SDK_CONFIGS = (
|
||||
{
|
||||
"name": "Hexagon SDK",
|
||||
"repo": "snapdragon-toolchain/hexagon-sdk",
|
||||
"default_version": "6.6.0.0",
|
||||
"default_version": "6.7.0.0",
|
||||
"parent_dir": "Hexagon_SDK",
|
||||
"archive_prefix": "hexagon-sdk-v",
|
||||
"markers": ("hexagon_sdk.json",),
|
||||
|
||||
@@ -223,6 +223,7 @@ def patch_config(cfg: dict, families: dict[str, Family], new_vocab: int | None,
|
||||
lfam = next((f for f in families.values() if f.kind == "layer"), None)
|
||||
if lfam is None:
|
||||
continue
|
||||
assert lfam.kept is not None
|
||||
d[key] = sum(1 for i in lfam.kept if i < val)
|
||||
elif "expert" in key and fam is not None and fam.kind == "expert" and not any(x in key for x in EXPERT_KEY_EXCLUDE):
|
||||
d[key] = fam.new_count
|
||||
@@ -236,6 +237,7 @@ def patch_config(cfg: dict, families: dict[str, Family], new_vocab: int | None,
|
||||
# per-layer lists, e.g. layer_types
|
||||
fam = by_card.get(len(val))
|
||||
if fam is not None and fam.kind == "layer" and len(val) in (fam.card, fam.regular_total):
|
||||
assert fam.kept is not None
|
||||
d[key] = [val[i] for i in fam.kept if i < len(val)]
|
||||
log.append(f"{key}: list[{len(val)}] -> list[{len(d[key])}]")
|
||||
return log
|
||||
|
||||
@@ -2453,6 +2453,63 @@ struct test_get_rows : public test_case {
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_GET_ROWS on a weight also used by GGML_OP_MUL_MAT (tied embeddings)
|
||||
// the mul_mat runs first, so a backend that changes the weight layout must still return the right rows
|
||||
struct test_get_rows_after_mul_mat : public test_case {
|
||||
const ggml_type type;
|
||||
const int n; // cols
|
||||
const int m; // rows
|
||||
const int r; // rows to get
|
||||
|
||||
std::string vars() override {
|
||||
return VARS_TO_STR4(type, n, m, r);
|
||||
}
|
||||
|
||||
test_get_rows_after_mul_mat(ggml_type type = GGML_TYPE_Q8_0, int n = 256, int m = 16, int r = 4)
|
||||
: type(type), n(n), m(m), r(r) {}
|
||||
|
||||
double max_nmse_err() override {
|
||||
return 5e-4; // same as test_mul_mat
|
||||
}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * w = ggml_new_tensor_2d(ctx, type, n, m);
|
||||
ggml_set_name(w, "w");
|
||||
|
||||
ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n, 1);
|
||||
ggml_set_name(x, "x");
|
||||
|
||||
ggml_tensor * rows = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, r);
|
||||
ggml_set_name(rows, "rows");
|
||||
|
||||
ggml_tensor * mm = ggml_mul_mat(ctx, w, x);
|
||||
ggml_set_name(mm, "mm");
|
||||
|
||||
ggml_tensor * gr = ggml_get_rows(ctx, w, rows);
|
||||
ggml_set_name(gr, "gr");
|
||||
|
||||
// repeat(mm) is the first source, so mul_mat is computed before get_rows
|
||||
ggml_tensor * out = ggml_add(ctx, ggml_repeat(ctx, mm, gr), gr);
|
||||
ggml_set_name(out, "out");
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
void initialize_tensors(ggml_context * ctx) override {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->type == GGML_TYPE_I32) {
|
||||
std::vector<int> data(ggml_nelements(t));
|
||||
for (size_t i = 0; i < data.size(); i++) {
|
||||
data[i] = rand() % m;
|
||||
}
|
||||
ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(int));
|
||||
} else {
|
||||
init_tensor_uniform(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_GET_ROWS_BACK
|
||||
struct test_get_rows_back : public test_case {
|
||||
const ggml_type type;
|
||||
@@ -9828,6 +9885,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
|
||||
test_cases.emplace_back(new test_get_rows(GGML_TYPE_F32, 1, 8, 2, 1, 1, false));
|
||||
for (ggml_type type : {GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}) {
|
||||
test_cases.emplace_back(new test_get_rows_after_mul_mat(type));
|
||||
}
|
||||
for (ggml_type type : all_types) {
|
||||
for (int b : {1, 7}) {
|
||||
for (bool v : {false, true}) {
|
||||
|
||||
@@ -85,10 +85,18 @@ static void fill_resid_gate(std::vector<float> & dst, const char * name, const t
|
||||
}
|
||||
}
|
||||
|
||||
// a random router bias can tie within the noise between the backends, a ramp keeps the top-k selection the same
|
||||
static void fill_exp_probs_b(std::vector<float> & dst, const char *, const tensor_data_params &) {
|
||||
for (size_t i = 0; i < dst.size(); i++) {
|
||||
dst[i] = 2.0f*i;
|
||||
}
|
||||
}
|
||||
|
||||
// a tensor whose values hold a property of their own, matched on a substring of its name, first match wins
|
||||
static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = {
|
||||
{ "ssm_a", fill_ssm_a },
|
||||
{ "_resid_", fill_resid_gate },
|
||||
{ "ssm_a", fill_ssm_a },
|
||||
{ "_resid_", fill_resid_gate },
|
||||
{ "exp_probs_b", fill_exp_probs_b },
|
||||
};
|
||||
|
||||
static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
|
||||
@@ -934,6 +942,18 @@ static int save_models(const std::string & arch_filter, const size_t seed, const
|
||||
return 0;
|
||||
}
|
||||
|
||||
// the CPU and the GPU of an Apple SoC carry the same description, the name tells them apart (CPU, MTL0)
|
||||
static std::string get_device_label(ggml_backend_dev_t dev) {
|
||||
const char * desc = ggml_backend_dev_description(dev);
|
||||
const char * name = ggml_backend_dev_name(dev);
|
||||
|
||||
if (strcmp(desc, name) == 0) {
|
||||
return desc;
|
||||
}
|
||||
|
||||
return std::string(desc) + " (" + name + ")";
|
||||
}
|
||||
|
||||
static int test_backends(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const char * target_backend) {
|
||||
struct user_data_t {
|
||||
struct {
|
||||
@@ -986,7 +1006,7 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
|
||||
if (target_backend != nullptr && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) {
|
||||
continue;
|
||||
}
|
||||
dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{dev}, ggml_backend_dev_description(dev), LLAMA_SPLIT_MODE_LAYER);
|
||||
dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{dev}, get_device_label(dev), LLAMA_SPLIT_MODE_LAYER);
|
||||
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
|
||||
|
||||
// cpu-based devices cannot be used in tensor split mode
|
||||
|
||||
@@ -66,6 +66,31 @@ MAKE_TEST(test_image_preprocessor_longest_edge) {
|
||||
}
|
||||
}
|
||||
|
||||
MAKE_TEST(test_image_preprocessor_dyn_size) {
|
||||
// { patch_size, image size, expected size }
|
||||
// a side that is an even number plus .5 merged patches rounds down, like Python round() in smart_resize
|
||||
const std::vector<std::tuple<int, clip_image_size, clip_image_size>> cases = {
|
||||
{ 16, { 768, 400 }, { 768, 384 } },
|
||||
{ 16, { 800, 336 }, { 800, 320 } },
|
||||
{ 16, { 400, 400 }, { 384, 384 } },
|
||||
{ 16, { 640, 240 }, { 640, 256 } },
|
||||
{ 14, { 966, 560 }, { 952, 560 } },
|
||||
};
|
||||
|
||||
for (const auto & [patch_size, input, expected] : cases) {
|
||||
clip_hparams hparams;
|
||||
hparams.patch_size = patch_size;
|
||||
hparams.n_merge = 2;
|
||||
hparams.image_round_half_even = true;
|
||||
hparams.set_limit_image_tokens(8, 4096);
|
||||
|
||||
const clip_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input);
|
||||
const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);
|
||||
t.assert_equal(name + " width", expected.width, actual.width);
|
||||
t.assert_equal(name + " height", expected.height, actual.height);
|
||||
}
|
||||
}
|
||||
|
||||
MAKE_TEST(test_image_preprocessor_lfm2) {
|
||||
clip_hparams hparams;
|
||||
hparams.patch_size = 16;
|
||||
|
||||
@@ -65,6 +65,7 @@ struct clip_hparams {
|
||||
int32_t image_max_pixels = -1;
|
||||
resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC;
|
||||
pad_style image_resize_pad = PAD_CEIL; // padding style when resizing
|
||||
bool image_round_half_even = false; // smart_resize rounds .5 to even, like Python round() in the HF processor
|
||||
std::array<uint8_t, 3> image_pad_color = {0, 0, 0};
|
||||
|
||||
// (preprocessor) for llava-uhd style models
|
||||
|
||||
@@ -1634,6 +1634,8 @@ struct clip_model_loader {
|
||||
case PROJECTOR_TYPE_DOTS_OCR:
|
||||
{
|
||||
hparams.rope_theta = 10000.0f;
|
||||
hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
|
||||
hparams.image_round_half_even = true; // smart_resize uses Python round()
|
||||
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
|
||||
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
|
||||
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
|
||||
@@ -1750,6 +1752,8 @@ struct clip_model_loader {
|
||||
{
|
||||
hparams.n_merge = 2; // default value for Qwen 2 and 2.5
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
|
||||
hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
|
||||
hparams.image_round_half_even = true; // smart_resize uses Python round()
|
||||
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
|
||||
get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it
|
||||
// ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json
|
||||
@@ -1825,6 +1829,8 @@ struct clip_model_loader {
|
||||
{
|
||||
hparams.n_merge = 2; // spatial_merge_size
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
|
||||
hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
|
||||
hparams.image_round_half_even = true; // smart_resize uses Python round()
|
||||
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
|
||||
get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
|
||||
// 1D banded sliding-window radius (visual_token_window_size); required
|
||||
@@ -2092,6 +2098,8 @@ struct clip_model_loader {
|
||||
case PROJECTOR_TYPE_EXAONE4_5:
|
||||
{
|
||||
hparams.n_merge = 2;
|
||||
hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
|
||||
hparams.image_round_half_even = true; // smart_resize uses Python round()
|
||||
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
|
||||
get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false);
|
||||
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
|
||||
|
||||
@@ -115,6 +115,7 @@ struct img_tool {
|
||||
int max_pixels = 0; // 0 = disabled
|
||||
// applied before min/max_pixels, so min_pixels can push an edge back above longest_edge
|
||||
int longest_edge = 0; // 0 = disabled
|
||||
bool round_half_even = false; // Python round() instead of std::round
|
||||
};
|
||||
|
||||
// calculate the size of the **resized** image, while preserving the aspect ratio and
|
||||
@@ -127,7 +128,10 @@ struct img_tool {
|
||||
return {0, 0};
|
||||
}
|
||||
|
||||
auto round_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };
|
||||
auto round_by_factor = [f = opts.align_size, even = opts.round_half_even](float x) {
|
||||
const float q = x / static_cast<float>(f);
|
||||
return static_cast<int>(even ? std::nearbyint(q) : std::round(q)) * f;
|
||||
};
|
||||
auto ceil_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };
|
||||
auto floor_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };
|
||||
|
||||
@@ -767,20 +771,24 @@ mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip
|
||||
// mtmd_image_preprocessor_dyn_size
|
||||
//
|
||||
|
||||
mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {
|
||||
clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) {
|
||||
GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0);
|
||||
clip_image_u8 resized_image;
|
||||
const clip_image_size original_size = img.get_size();
|
||||
// the original pixtral model doesn't have n_merge
|
||||
const int cur_merge = hparams.n_merge;
|
||||
const clip_image_size target_size = img_tool::calc_size_preserved_ratio(
|
||||
return img_tool::calc_size_preserved_ratio(
|
||||
original_size,
|
||||
{
|
||||
/* align_size */ hparams.patch_size * cur_merge,
|
||||
/* min_pixels */ hparams.image_min_pixels,
|
||||
/* max_pixels */ hparams.image_max_pixels,
|
||||
/* longest_edge */ 0,
|
||||
/* align_size */ hparams.patch_size * cur_merge,
|
||||
/* min_pixels */ hparams.image_min_pixels,
|
||||
/* max_pixels */ hparams.image_max_pixels,
|
||||
/* longest_edge */ 0,
|
||||
/* round_half_even */ hparams.image_round_half_even,
|
||||
});
|
||||
}
|
||||
|
||||
mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {
|
||||
clip_image_u8 resized_image;
|
||||
const clip_image_size target_size = get_target_size(hparams, img.get_size());
|
||||
img_tool::resize(img, resized_image, target_size,
|
||||
hparams.image_resize_algo,
|
||||
hparams.image_resize_pad,
|
||||
|
||||
@@ -131,6 +131,7 @@ struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {
|
||||
// this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc
|
||||
struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
};
|
||||
|
||||
|
||||
@@ -61,7 +61,9 @@
|
||||
bind:ref
|
||||
bind:value
|
||||
{autofocus}
|
||||
class="max-md:h-10 pl-9 {showClearButton ? 'pr-9' : ''} {size === 'sm' ? 'h-8' : ''}"
|
||||
class="max-md:h-10 pl-9 {showClearButton ? 'pr-9' : ''} {size === 'sm'
|
||||
? 'h-8'
|
||||
: ''} [&::-webkit-search-cancel-button]:hidden"
|
||||
{id}
|
||||
oninput={handleInput}
|
||||
onkeydown={onKeyDown}
|
||||
|
||||
Reference in New Issue
Block a user