Compare commits
72 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 82324fc508 | |||
| 7ceed8737f | |||
| 7d6f5d02bb | |||
| 83078fec0d | |||
| f266648fa9 | |||
| 60199339bc | |||
| b04d4e567c | |||
| 37b53fd454 | |||
| fccf7166fb | |||
| 0bec16e388 | |||
| d4365d9554 | |||
| 0a8b29a607 | |||
| 583926e3ac | |||
| e13469a323 | |||
| 930e2fa599 | |||
| 72b590d65f | |||
| 38a5b42d9a | |||
| 9f31776c37 | |||
| d1d3c3396a | |||
| 6011c34ce6 | |||
| 7609846557 | |||
| 5431581326 | |||
| 9e71716247 | |||
| fc82583e65 | |||
| 77d554b26d | |||
| 6ec1a7e956 | |||
| 1af6c65de0 | |||
| 1e7bcf3da4 | |||
| 0ecb159c9e | |||
| 987498f459 | |||
| 4c9233c034 | |||
| 69eb250670 | |||
| 1bc7a5af0d | |||
| 7cf1c54a96 | |||
| 96ffdc41ce | |||
| bfdc32183d | |||
| 391fac1646 | |||
| 41abbfd599 | |||
| b4fa47d226 | |||
| f3a184b153 | |||
| dfe45163e1 | |||
| b29c606e28 | |||
| d9e03f1074 | |||
| eeea731613 | |||
| bbdd9f246e | |||
| 97e4ca7358 | |||
| 1aca1f9fcd | |||
| be2c6d7d1f | |||
| 3d10bcd197 | |||
| 21f6b0d22c | |||
| 2f539596c6 | |||
| 89fe242405 | |||
| 15d8f2d592 | |||
| 661643e430 | |||
| 093a2f86c3 | |||
| ad6c66839a | |||
| 7a16a6ce32 | |||
| 5f436dddb4 | |||
| e49d2c2760 | |||
| 6978052985 | |||
| 243a3082d4 | |||
| b6b003d2cb | |||
| c95f8e47b8 | |||
| bc52a12b38 | |||
| 4a89937354 | |||
| 37b3a9e0cc | |||
| 002a12ad25 | |||
| f1e44dcc11 | |||
| 56b9eb280a | |||
| 790cf51aab | |||
| 8e330954ad | |||
| acecd56032 |
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"Exclude": ["^\\.gitmodules$", "stb_image\\.h"],
|
||||
"Exclude": ["^\\.gitmodules$", "stb_image\\.h", "examples/test-cmake/build/", "examples/test-cmake/build-subdir/"],
|
||||
"Disable": {
|
||||
"IndentSize": true
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ runs:
|
||||
run: |
|
||||
BUILD_NUMBER="$(git rev-list --count HEAD)"
|
||||
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
|
||||
|
||||
@@ -100,36 +100,36 @@ runs:
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
|
||||
- name: Install Cuda Toolkit 13.3
|
||||
if: ${{ inputs.cuda_version == '13.3' }}
|
||||
- name: Install Cuda Toolkit 13.4 for x64
|
||||
if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3"
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
choco install unzip -y
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.3.33-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.3.29-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.3.33-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.3.33-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.5.1.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.3.33-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.3.29-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.3.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.3.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.3.3.1-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_crt-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_cudart-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvcc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvrtc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libcublas-windows-x86_64-13.5.1.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libnvvm-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvtx-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_profiler_api-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\visual_studio_integration-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cccl-windows-x86_64-13.3.3.3.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.7.0.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvrtc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-x86_64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvtx-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_profiler_api-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\visual_studio_integration-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
|
||||
- name: Install Cuda Toolkit 13.4 for ARM64
|
||||
if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }}
|
||||
|
||||
@@ -221,7 +221,6 @@ jobs:
|
||||
# 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar
|
||||
# $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe)
|
||||
# cd build
|
||||
# $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1
|
||||
# & $sde -future -- ctest -L main -C Release --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
|
||||
@@ -177,7 +177,8 @@ jobs:
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build -S . \
|
||||
-DGGML_MUSA=ON
|
||||
-DGGML_MUSA=ON \
|
||||
-DMUSA_ARCHITECTURES=21
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
- cuda: '12.4'
|
||||
arch: x64
|
||||
defines: '-DGGML_CUDA_CUB_3DOT2=ON'
|
||||
- cuda: '13.3'
|
||||
- cuda: '13.4'
|
||||
arch: x64
|
||||
defines: ''
|
||||
- cuda: '13.4'
|
||||
|
||||
@@ -34,10 +34,15 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
|
||||
ubuntu-24-s390x:
|
||||
name: ubuntu-24-s390x (VXE ${{ matrix.vxe }})
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
vxe: ["ON", "OFF"] # `-DGGML_VXE=ON/OFF`
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
@@ -77,7 +82,8 @@ jobs:
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DGGML_RPC=ON
|
||||
-DGGML_RPC=ON \
|
||||
-DGGML_VXE=${{ matrix.vxe }}
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: Test
|
||||
|
||||
@@ -7,6 +7,7 @@ on:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/build-self-hosted.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
@@ -27,6 +28,7 @@ on:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/build-self-hosted.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
@@ -122,6 +124,7 @@ jobs:
|
||||
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-vulkan-nvidia-cm:
|
||||
# runs-on: "hf-jobs-t4-small:ubuntu26_04"
|
||||
runs-on: [self-hosted, Linux, NVIDIA]
|
||||
|
||||
steps:
|
||||
@@ -129,13 +132,44 @@ jobs:
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
# - name: Install dependencies
|
||||
# run: |
|
||||
# sudo apt update
|
||||
# sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
|
||||
|
||||
# - name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# with:
|
||||
# restore: false
|
||||
# save: false
|
||||
|
||||
# - name: ccache-buckets-restore
|
||||
# uses: ./.github/actions/ccache-buckets
|
||||
# with:
|
||||
# key: self-hosted-vulkan-nvidia-cm
|
||||
# folder: llama.cpp
|
||||
# hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# - name: ccache-buckets-save
|
||||
# if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
# uses: ./.github/actions/ccache-buckets
|
||||
# env:
|
||||
# HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
# with:
|
||||
# key: self-hosted-vulkan-nvidia-cm
|
||||
# folder: llama.cpp
|
||||
# evict-old-files: 1d
|
||||
# hf_bucket: ggml-org/cache
|
||||
# save: true
|
||||
|
||||
gpu-vulkan-nvidia-cm2:
|
||||
# runs-on: "hf-jobs-t4-small:ubuntu26_04"
|
||||
runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
|
||||
|
||||
steps:
|
||||
@@ -143,20 +177,68 @@ jobs:
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
# - name: Install dependencies
|
||||
# run: |
|
||||
# sudo apt update
|
||||
# sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
|
||||
|
||||
# - name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# with:
|
||||
# restore: false
|
||||
# save: false
|
||||
|
||||
# - name: ccache-buckets-restore
|
||||
# uses: ./.github/actions/ccache-buckets
|
||||
# with:
|
||||
# key: self-hosted-vulkan-nvidia-cm2
|
||||
# folder: llama.cpp
|
||||
# hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# - name: ccache-buckets-save
|
||||
# if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
# uses: ./.github/actions/ccache-buckets
|
||||
# env:
|
||||
# HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
# with:
|
||||
# key: self-hosted-vulkan-nvidia-cm2
|
||||
# folder: llama.cpp
|
||||
# evict-old-files: 1d
|
||||
# hf_bucket: ggml-org/cache
|
||||
# save: true
|
||||
|
||||
gpu-webgpu-nvidia:
|
||||
runs-on: [self-hosted, Linux, NVIDIA, X64]
|
||||
runs-on: "hf-jobs-t4-small:ubuntu26_04"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: self-hosted-webgpu-nvidia
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Dawn Dependency
|
||||
id: dawn-depends
|
||||
run: |
|
||||
@@ -178,6 +260,18 @@ jobs:
|
||||
GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
|
||||
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: self-hosted-webgpu-nvidia
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
# TODO: provision AMX-compatible machine
|
||||
#cpu-amx:
|
||||
# runs-on: [self-hosted, Linux, CPU, AMX]
|
||||
@@ -359,7 +453,7 @@ jobs:
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
cpu-arm64-high-perf-graviton4:
|
||||
runs-on: ah-ubuntu_22_04-c8g_8x
|
||||
runs-on: ah-ubuntu_24_04-c8g_8x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -402,7 +496,7 @@ jobs:
|
||||
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
cpu-arm64-graviton4-kleidiai:
|
||||
runs-on: ah-ubuntu_22_04-c8g_8x
|
||||
runs-on: ah-ubuntu_24_04-c8g_8x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -15,7 +15,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
model-naming:
|
||||
runs-on: [self-hosted, fast]
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Check model naming conventions
|
||||
|
||||
@@ -15,7 +15,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
editorconfig:
|
||||
runs-on: [self-hosted, fast]
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: editorconfig-checker/action-editorconfig-checker@840e866d93b8e032123c23bac69dece044d4d84c # v2.2.0
|
||||
|
||||
@@ -9,7 +9,9 @@ on:
|
||||
'.github/workflows/fusion.yml',
|
||||
'ggml/**',
|
||||
'tests/fusion/**',
|
||||
'tests/test-fusion.cpp'
|
||||
'tests/test-fusion.cpp',
|
||||
'tests/test-llama-archs.cpp',
|
||||
'src/models/**'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
@@ -18,7 +20,9 @@ on:
|
||||
'.github/workflows/fusion.yml',
|
||||
'ggml/**',
|
||||
'tests/fusion/**',
|
||||
'tests/test-fusion.cpp'
|
||||
'tests/test-fusion.cpp',
|
||||
'tests/test-llama-archs.cpp',
|
||||
'src/models/**'
|
||||
]
|
||||
|
||||
concurrency:
|
||||
|
||||
@@ -310,6 +310,145 @@ jobs:
|
||||
with:
|
||||
key: release-${{ matrix.os }}-vulkan
|
||||
|
||||
ubuntu-cuda:
|
||||
name: ubuntu-cuda (${{ matrix.label }}, ${{ matrix.build }})
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
# label = short version used in artifact names / release body
|
||||
# cuda = full container image tag
|
||||
- build: 'x64'
|
||||
os: ubuntu-24.04
|
||||
cuda: '12.8.2'
|
||||
label: '12.8'
|
||||
defines: '-DGGML_CUDA_CUB_3DOT2=ON'
|
||||
- build: 'x64'
|
||||
os: ubuntu-24.04
|
||||
cuda: '13.3.1'
|
||||
label: '13.3'
|
||||
defines: ''
|
||||
- build: 'arm64'
|
||||
os: ubuntu-24.04-arm
|
||||
cuda: '13.3.1'
|
||||
label: '13.3'
|
||||
defines: ''
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
|
||||
steps:
|
||||
# the container has no git; install it before checkout so that a real git
|
||||
# repository is created (the get-tag-name action and the build both need it)
|
||||
- name: Install git
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends git
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# checkout runs as the host user; in-container steps run as root, so git
|
||||
# refuses to touch a repo it does not own. Mark the workspace as safe.
|
||||
# use the env var: the github.workspace context holds the HOST path,
|
||||
# GITHUB_WORKSPACE the container path
|
||||
- name: Git safe directory
|
||||
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
|
||||
- name: Download UI build
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
# container jobs default to sh (dash); need bash for the [[ ]] below
|
||||
shell: bash
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends build-essential cmake ninja-build libssl-dev jq python3-venv
|
||||
# the container ships GCC 13, which does not know the 'sme' march
|
||||
# feature used by the armv9.2 CPU variant of GGML_CPU_ALL_VARIANTS
|
||||
if [[ "${{ matrix.build }}" == "arm64" ]]; then
|
||||
apt-get install -y --no-install-recommends gcc-14 g++-14
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
fi
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }}
|
||||
evict-old-files: 1d
|
||||
max-size: "1G"
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
# no CMAKE_CUDA_ARCHITECTURES: use the broad default arch set from
|
||||
# ggml/src/ggml-cuda/CMakeLists.txt so the release binary covers many GPUs
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_INSTALL_RPATH='$ORIGIN' \
|
||||
-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
|
||||
-DGGML_BACKEND_DL=ON \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_CPU_ALL_VARIANTS=ON \
|
||||
-DGGML_CUDA=ON \
|
||||
-DGGML_CUDA_NCCL=OFF \
|
||||
${{ env.CMAKE_ARGS }} ${{ matrix.defines }}
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: Determine tag name
|
||||
id: tag
|
||||
uses: ./.github/actions/get-tag-name
|
||||
|
||||
- name: Pack artifacts
|
||||
id: pack_artifacts
|
||||
run: |
|
||||
cp LICENSE ./build/bin/
|
||||
tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
|
||||
name: llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
|
||||
|
||||
# ship the CUDA runtime libraries the backend links against, mirroring
|
||||
# the windows-cuda cudart zip - extract next to the binaries ($ORIGIN rpath)
|
||||
- name: Pack CUDA runtime
|
||||
id: pack_cuda_runtime
|
||||
run: |
|
||||
major="${{ matrix.label }}"
|
||||
major="${major%%.*}"
|
||||
mkdir -p ./cudart
|
||||
# cp -L dereferences the SONAME symlinks into plain files, so the
|
||||
# tarball holds exactly 3 files with no versioned duplicates
|
||||
cp -L /usr/local/cuda/lib64/libcudart.so.${major} ./cudart/
|
||||
cp -L /usr/local/cuda/lib64/libcublas.so.${major} ./cudart/
|
||||
cp -L /usr/local/cuda/lib64/libcublasLt.so.${major} ./cudart/
|
||||
tar -czvf cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}," -C ./cudart .
|
||||
|
||||
- name: Upload CUDA runtime
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
path: cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
|
||||
name: cudart-llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
with:
|
||||
key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }}
|
||||
|
||||
android-arm64:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
@@ -345,6 +484,7 @@ jobs:
|
||||
uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
|
||||
with:
|
||||
log-accepted-android-sdk-licenses: false
|
||||
packages: 'platform-tools'
|
||||
|
||||
- name: Install NDK
|
||||
run: |
|
||||
@@ -987,7 +1127,7 @@ jobs:
|
||||
- cuda: '12.4'
|
||||
arch: x64
|
||||
defines: '-DGGML_CUDA_CUB_3DOT2=ON'
|
||||
- cuda: '13.3'
|
||||
- cuda: '13.4'
|
||||
arch: x64
|
||||
defines: ''
|
||||
- cuda: '13.4'
|
||||
@@ -1280,7 +1420,7 @@ jobs:
|
||||
matrix:
|
||||
include:
|
||||
- ROCM_VERSION: "10.0.0"
|
||||
gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
|
||||
gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
|
||||
build: 'x64'
|
||||
|
||||
steps:
|
||||
@@ -1572,6 +1712,7 @@ jobs:
|
||||
- ubuntu-24-rocm
|
||||
- ubuntu-cpu
|
||||
- ubuntu-vulkan
|
||||
- ubuntu-cuda
|
||||
- ubuntu-24-openvino
|
||||
- ubuntu-24-sycl
|
||||
- android-arm64
|
||||
@@ -1703,6 +1844,9 @@ jobs:
|
||||
- [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
|
||||
- [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
|
||||
- [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-x64.tar.gz) - [CUDA 13.3 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-arm64.tar.gz) - [CUDA 13.3 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.3-arm64.tar.gz)
|
||||
- [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
|
||||
- [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz)
|
||||
- [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
|
||||
@@ -1716,7 +1860,7 @@ jobs:
|
||||
- [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
|
||||
- [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
|
||||
- [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
|
||||
- [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip)
|
||||
- [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip)
|
||||
- [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
|
||||
- [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
|
||||
- [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip)
|
||||
|
||||
@@ -116,7 +116,6 @@ jobs:
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
cd tools/server/tests
|
||||
export ${{ matrix.extra_args }}
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
|
||||
- name: Slow tests
|
||||
@@ -125,5 +124,4 @@ jobs:
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
cd tools/server/tests
|
||||
export ${{ matrix.extra_args }}
|
||||
PYTEST_WORKERS=1 SLOW_TESTS=1 ./tests.sh
|
||||
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
|
||||
server-kleidiai:
|
||||
runs-on: ah-ubuntu_22_04-c8g_8x
|
||||
runs-on: ah-ubuntu_24_04-c8g_8x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -232,7 +232,7 @@ jobs:
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON
|
||||
cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON -DLLAMA_FATAL_WARNINGS=ON
|
||||
cmake --build build --config Release -j $(nproc) --target llama-server
|
||||
|
||||
- name: Python setup
|
||||
|
||||
+3
-1
@@ -6,6 +6,7 @@ General:
|
||||
- PR and commit titles format: `<module> : <title>`. Lookup recents for examples
|
||||
- Don't try to build or run the code unless you are explicitly asked to do so
|
||||
- Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
|
||||
- When [MODEL] is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user
|
||||
|
||||
Coding:
|
||||
- When in doubt, always refer to the CONTRIBUTING.md file of the project
|
||||
@@ -20,8 +21,9 @@ Pull requests (PRs):
|
||||
- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)
|
||||
- When creating a pull request, look for the repository's PR template and follow it
|
||||
- For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]"
|
||||
- Ask the user to tell you what model was used and write it in place of [MODEL]
|
||||
- If `PI_MODEL_NAME` env var is not set, ask the user to tell you what model was used and write it in place of [MODEL]
|
||||
- Always create the pull requests in draft mode
|
||||
- Never reply to review comments or post comments on issues/PRs without explicit permission from the user
|
||||
|
||||
Commits:
|
||||
- On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ include(CheckIncludeFileCXX)
|
||||
### llama.cpp version
|
||||
set(LLAMA_VERSION_MAJOR 0)
|
||||
set(LLAMA_VERSION_MINOR 4)
|
||||
set(LLAMA_VERSION_PATCH 0)
|
||||
set(LLAMA_VERSION_PATCH 1)
|
||||
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
|
||||
|
||||
# whether this is a development/nightly build
|
||||
|
||||
+2
-2
@@ -20,8 +20,8 @@ If AI is used to generate any portion of the code, contributors must adhere to t
|
||||
|
||||
1. Explicitly disclose the manner in which AI was employed.
|
||||
2. Check for an existing PR addressing the same change; if one exists, comment there to work with its author instead of opening a duplicate.
|
||||
3. Perform a comprehensive manual review prior to submitting the pull request.
|
||||
4. Be prepared to explain every line of code they submitted when asked about it by a maintainer.
|
||||
3. Perform a comprehensive manual review prior to submitting the pull request. A proper code review usually takes something like one hour per 200-400 LOC and you should be spending **at least that much time on code review alone**.
|
||||
4. Be prepared to explain every line of code you submit when asked about it by a maintainer.
|
||||
5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
|
||||
|
||||
For more info, please refer to the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ target_link_libraries(${TARGET} PRIVATE
|
||||
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
||||
|
||||
# Automatically add all files from the 'licenses' directory
|
||||
file(GLOB EXTRA_LICENSES "${CMAKE_SOURCE_DIR}/licenses/LICENSE-*")
|
||||
file(GLOB EXTRA_LICENSES "${PROJECT_SOURCE_DIR}/licenses/LICENSE-*")
|
||||
|
||||
foreach(FILE_PATH ${EXTRA_LICENSES})
|
||||
get_filename_component(FILE_NAME "${FILE_PATH}" NAME)
|
||||
|
||||
@@ -58,8 +58,6 @@ if [ -n "${GG_BUILD_ROCM}" ] && [ -n "${GITHUB_RUN_ID}" ]; then
|
||||
fi
|
||||
|
||||
rm -f $OUT/*.log
|
||||
rm -f $OUT/*.exit
|
||||
rm -f $OUT/*.md
|
||||
|
||||
sd=`dirname $0`
|
||||
cd $sd/../
|
||||
@@ -211,10 +209,6 @@ function gg_wget {
|
||||
cd $cwd
|
||||
}
|
||||
|
||||
function gg_printf {
|
||||
printf -- "$@" >> $OUT/README.md
|
||||
}
|
||||
|
||||
function gg_run {
|
||||
ci=$1
|
||||
|
||||
@@ -223,13 +217,10 @@ function gg_run {
|
||||
|
||||
gg_run_$ci | tee $OUT/$ci.log
|
||||
cur=$?
|
||||
echo "$cur" > $OUT/$ci.exit
|
||||
|
||||
set +x
|
||||
set +o pipefail
|
||||
|
||||
gg_sum_$ci
|
||||
|
||||
ret=$((ret | cur))
|
||||
}
|
||||
|
||||
@@ -255,17 +246,6 @@ function gg_run_ctest_debug {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_ctest_debug {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest in debug mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '\n'
|
||||
}
|
||||
|
||||
# ctest_release
|
||||
|
||||
function gg_run_ctest_release {
|
||||
@@ -290,16 +270,6 @@ function gg_run_ctest_release {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_ctest_release {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest in release mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# test_llama_archs_tensor_split
|
||||
|
||||
function gg_run_test_llama_archs_tensor_split {
|
||||
@@ -324,16 +294,6 @@ function gg_run_test_llama_archs_tensor_split {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_test_llama_archs_tensor_split {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs test-llama-archs with 1 to 4 devices\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# test_llama_archs_models
|
||||
|
||||
function gg_run_test_llama_archs_models {
|
||||
@@ -353,16 +313,6 @@ function gg_run_test_llama_archs_models {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_test_llama_archs_models {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Generates the dummy models used by the model-dependent tests\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# test_scripts
|
||||
|
||||
function gg_run_test_scripts {
|
||||
@@ -376,17 +326,6 @@ function gg_run_test_scripts {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_test_scripts {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs test scripts\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-scripts.log)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '\n'
|
||||
}
|
||||
|
||||
function gg_get_model {
|
||||
#local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-f16.gguf"
|
||||
local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-q4_0.gguf"
|
||||
@@ -430,26 +369,6 @@ function gg_run_ctest_with_model_release {
|
||||
cd ..
|
||||
}
|
||||
|
||||
function gg_sum_ctest_with_model_debug {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest with model files in debug mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
function gg_sum_ctest_with_model_release {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest with model files in release mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# qwen3_0_6b
|
||||
|
||||
function gg_run_qwen3_0_6b {
|
||||
@@ -554,50 +473,24 @@ function gg_run_qwen3_0_6b {
|
||||
return 0
|
||||
}
|
||||
|
||||
check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")"
|
||||
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
||||
check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")"
|
||||
fi
|
||||
check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
#check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log # note: ppl > 20.0 for this quant and model
|
||||
check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
|
||||
cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log
|
||||
check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")"
|
||||
check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")"
|
||||
check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")"
|
||||
check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")"
|
||||
check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")"
|
||||
#check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" # note: ppl > 20.0 for this quant and model
|
||||
check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")"
|
||||
check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")"
|
||||
check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")"
|
||||
check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")"
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_qwen3_0_6b {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Qwen3 0.6B:\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)"
|
||||
gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)"
|
||||
gg_printf '- f16:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
|
||||
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
||||
gg_printf '- bf16:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-bf16.log)"
|
||||
fi
|
||||
gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
|
||||
gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)"
|
||||
gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)"
|
||||
gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)"
|
||||
gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)"
|
||||
gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)"
|
||||
gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)"
|
||||
gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)"
|
||||
gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)"
|
||||
gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)"
|
||||
gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)"
|
||||
}
|
||||
|
||||
# bge-small
|
||||
|
||||
function gg_run_embd_bge_small {
|
||||
@@ -639,15 +532,6 @@ function gg_run_embd_bge_small {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_embd_bge_small {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'BGE Small (BERT):\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
|
||||
gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
|
||||
}
|
||||
|
||||
# rerank_tiny
|
||||
|
||||
function gg_run_rerank_tiny {
|
||||
@@ -704,66 +588,58 @@ function gg_run_rerank_tiny {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_rerank_tiny {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Rerank Tiny (Jina):\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-rk-f16.log)"
|
||||
}
|
||||
|
||||
function gg_check_build_requirements {
|
||||
if ! command -v git &> /dev/null; then
|
||||
gg_printf 'git not found, please install\n'
|
||||
echo 'git not found, please install'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v git-lfs &> /dev/null; then
|
||||
gg_printf 'git-lfs not found, please install\n'
|
||||
echo 'git-lfs not found, please install'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! git config --get filter.lfs.clean &> /dev/null; then
|
||||
gg_printf 'git-lfs not initialized, please run `git lfs install`\n'
|
||||
echo 'git-lfs not initialized, please run `git lfs install`'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v wget &> /dev/null; then
|
||||
gg_printf 'wget not found, please install\n'
|
||||
echo 'wget not found, please install'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v python3 &> /dev/null; then
|
||||
gg_printf 'python3 not found, please install\n'
|
||||
echo 'python3 not found, please install'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v pip3 &> /dev/null; then
|
||||
gg_printf 'pip3 not found, please install\n'
|
||||
echo 'pip3 not found, please install'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! python3 -m ensurepip --help &> /dev/null; then
|
||||
gg_printf 'ensurepip not found, please install python3-venv package\n'
|
||||
echo 'ensurepip not found, please install python3-venv package'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v cmake &> /dev/null; then
|
||||
gg_printf 'cmake not found, please install\n'
|
||||
echo 'cmake not found, please install'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v ccache &> /dev/null; then
|
||||
gg_printf 'ccache not found, please consider installing for faster builds\n'
|
||||
echo 'ccache not found, please consider installing for faster builds'
|
||||
fi
|
||||
|
||||
if ! command -v ctest &> /dev/null; then
|
||||
gg_printf 'ctest not found, please install\n'
|
||||
echo 'ctest not found, please install'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v unzip &> /dev/null; then
|
||||
gg_printf 'unzip not found, please install\n'
|
||||
echo 'unzip not found, please install'
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
@@ -775,7 +651,11 @@ function gg_run_test_backend_ops {
|
||||
|
||||
set -e
|
||||
|
||||
local args_extra="-j $(nproc)"
|
||||
local n_jobs=$(nproc)
|
||||
if [ "${n_jobs}" -gt 2 ]; then
|
||||
n_jobs=2
|
||||
fi
|
||||
local args_extra="-j ${n_jobs}"
|
||||
|
||||
# TODO: fix multi-threaded for ROCm
|
||||
# https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865
|
||||
@@ -789,26 +669,21 @@ function gg_run_test_backend_ops {
|
||||
args_extra=""
|
||||
fi
|
||||
|
||||
# TODO: OpenVINO GPU plugin crashes (CL_OUT_OF_RESOURCES) with 2 concurrent workers on GPU.
|
||||
if [ ! -z "${GG_BUILD_OPENVINO}" ] && [ "${GGML_OPENVINO_DEVICE:-}" = "GPU" ]; then
|
||||
args_extra=""
|
||||
fi
|
||||
|
||||
# TODO: reduce the test-backend-ops timeout to 1800s
|
||||
if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
|
||||
(time ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
|
||||
(time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
|
||||
else
|
||||
(time ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
|
||||
(time timeout 3600 ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
|
||||
fi
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_test_backend_ops {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs test-backend-ops\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops.log)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '\n'
|
||||
}
|
||||
|
||||
## main
|
||||
|
||||
export LLAMA_ARG_LOG_PREFIX=1
|
||||
@@ -856,6 +731,4 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||
test $ret -eq 0 && gg_run ctest_with_model_release
|
||||
fi
|
||||
|
||||
cat $OUT/README.md
|
||||
|
||||
exit $ret
|
||||
|
||||
@@ -84,6 +84,8 @@ add_library(${TARGET}
|
||||
imatrix-loader.cpp
|
||||
imatrix-loader.h
|
||||
json-schema-to-grammar.cpp
|
||||
json-schema.cpp
|
||||
json-schema.h
|
||||
json.cpp
|
||||
json.h
|
||||
llguidance.cpp
|
||||
@@ -134,8 +136,6 @@ set_target_properties(${TARGET} PROPERTIES
|
||||
target_include_directories(${TARGET} PUBLIC .)
|
||||
target_link_libraries (${TARGET} PUBLIC vendor::nlohmann vendor::sheredom)
|
||||
target_compile_features (${TARGET} PUBLIC cxx_std_17)
|
||||
target_precompile_headers (${TARGET} PRIVATE common.h)
|
||||
target_precompile_headers (${TARGET} PRIVATE chat.h)
|
||||
|
||||
if (LLAMA_SUBPROCESS)
|
||||
target_compile_definitions(${TARGET} PUBLIC LLAMA_SUBPROCESS)
|
||||
|
||||
+3
-3
@@ -2277,14 +2277,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"-j", "--json-schema"}, "SCHEMA",
|
||||
"JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
|
||||
"JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, json_schema_to_grammar(json::parse(value))};
|
||||
}
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"-jf", "--json-schema-file"}, "FILE",
|
||||
"File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
|
||||
"File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
|
||||
[](common_params & params, const std::string & value) {
|
||||
std::ifstream file(value);
|
||||
if (!file) {
|
||||
@@ -3875,7 +3875,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
{"--no-log-jsonl"},
|
||||
"Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)",
|
||||
[](common_params &, bool value) {
|
||||
common_log_set_jsonl(common_log_main(), value);
|
||||
common_log_set_jsonl(value);
|
||||
}
|
||||
).set_env("LLAMA_ARG_LOG_JSONL"));
|
||||
add_opt(common_arg(
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "common.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
#include "log.h"
|
||||
#include "parsers/parsers.h"
|
||||
#include "peg-parser.h"
|
||||
|
||||
#include <stdexcept>
|
||||
@@ -12,16 +13,6 @@
|
||||
|
||||
using json = common_json;
|
||||
|
||||
// Helper to iterate over tools/functions
|
||||
static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
|
||||
for (const auto & tool : tools) {
|
||||
if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {
|
||||
continue;
|
||||
}
|
||||
fn(tool);
|
||||
}
|
||||
}
|
||||
|
||||
namespace autoparser {
|
||||
|
||||
parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) :
|
||||
@@ -87,15 +78,6 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
@@ -312,7 +294,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & func = tool.at("function");
|
||||
std::string name = func.at("name");
|
||||
const auto & schema = func.contains("parameters") ? func.at("parameters") : json::object();
|
||||
const auto schema = common_chat_tool_parameters(func);
|
||||
|
||||
// Build call_id parser based on position (if supported)
|
||||
bool have_call_id = false;
|
||||
@@ -383,43 +365,31 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
|
||||
common_peg_parser tool_choice = p.choice();
|
||||
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & func = tool.at("function");
|
||||
std::string name = func.at("name");
|
||||
auto params = func.contains("parameters") ? func.at("parameters") : json::object();
|
||||
const auto & properties = params.contains("properties") ? params.at("properties") : json::object();
|
||||
|
||||
std::set<std::string> required;
|
||||
if (params.contains("required")) {
|
||||
required = params.at("required").get<std::set<std::string>>();
|
||||
}
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
const auto & func = tool.at("function");
|
||||
std::string name = func.at("name");
|
||||
|
||||
// Build parser for each argument, separating required and optional
|
||||
std::vector<common_peg_parser> required_parsers;
|
||||
std::vector<common_peg_parser> optional_parsers;
|
||||
for (const auto & [param_name, param_schema] : properties.items()) {
|
||||
bool is_required = required.find(param_name) != required.end();
|
||||
|
||||
foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
|
||||
auto arg =
|
||||
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) +
|
||||
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
|
||||
arguments.name_suffix) +
|
||||
arguments.value_prefix +
|
||||
(schema_info.resolves_to_string(param_schema) ?
|
||||
(param.schema->may_be_string() ?
|
||||
p.ac(p.tool_arg_string_value(until_suffix) +
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :
|
||||
(p.tool_arg_json_value(p.schema(
|
||||
p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) +
|
||||
p.json(), "tool-" + name + "-arg-" + param.name + "-schema", doc, *param.schema)) +
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix)))));
|
||||
|
||||
auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
|
||||
if (is_required) {
|
||||
auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg);
|
||||
if (param.required) {
|
||||
required_parsers.push_back(named_arg);
|
||||
} else {
|
||||
optional_parsers.push_back(named_arg);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Build required arg sequence in definition order
|
||||
common_peg_parser args_seq = p.eps();
|
||||
|
||||
@@ -488,7 +488,7 @@ common_peg_parser common_chat_peg_builder::standard_constructed_tools(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
|
||||
// Build argument parsers
|
||||
auto args = eps();
|
||||
@@ -565,7 +565,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
|
||||
auto args = eps();
|
||||
if (params.contains("properties") && !params["properties"].empty()) {
|
||||
@@ -640,7 +640,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
|
||||
// Build inner object fields
|
||||
std::vector<common_peg_parser> inner_fields;
|
||||
@@ -726,7 +726,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
|
||||
auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() +
|
||||
atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
|
||||
@@ -795,7 +795,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
|
||||
auto tool_name_ = name_key_parser + space() + literal(":") + space() +
|
||||
atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
|
||||
|
||||
@@ -574,6 +574,16 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
|
||||
return result;
|
||||
}
|
||||
|
||||
json common_chat_tool_parameters(const json & function) {
|
||||
if (function.contains("parameters")) {
|
||||
const auto & params = function.at("parameters");
|
||||
if (!params.is_null() && !(params.is_object() && params.empty())) {
|
||||
return params;
|
||||
}
|
||||
}
|
||||
return json{{"type", "object"}, {"properties", json::object()}};
|
||||
}
|
||||
|
||||
std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & tools) {
|
||||
std::vector<common_chat_tool> result;
|
||||
|
||||
|
||||
@@ -360,6 +360,9 @@ common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg
|
||||
|
||||
common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
|
||||
|
||||
// The parameters schema of a function tool. A tool without parameters, or with an empty {}, takes zero arguments.
|
||||
common_json common_chat_tool_parameters(const common_json & function);
|
||||
|
||||
// get template caps, useful for reporting to server /props endpoint
|
||||
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
|
||||
|
||||
|
||||
+5
-6
@@ -1586,6 +1586,11 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
|
||||
return COMMON_CONTEXT_SEQ_RM_TYPE_NO;
|
||||
}
|
||||
|
||||
if (llama_n_rs_seq(ctx) > 0) {
|
||||
COM_TRC("%s", "the context supports bounded partial sequence removal\n");
|
||||
return COMMON_CONTEXT_SEQ_RM_TYPE_RS;
|
||||
}
|
||||
|
||||
common_context_seq_rm_type res = COMMON_CONTEXT_SEQ_RM_TYPE_PART;
|
||||
|
||||
llama_memory_clear(mem, true);
|
||||
@@ -1602,12 +1607,6 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
|
||||
goto done;
|
||||
}
|
||||
|
||||
if (llama_n_rs_seq(ctx) > 0) {
|
||||
COM_TRC("%s", "the context supports bounded partial sequence removal\n");
|
||||
res = COMMON_CONTEXT_SEQ_RM_TYPE_RS;
|
||||
goto done;
|
||||
}
|
||||
|
||||
// try to remove the last tokens
|
||||
if (!llama_memory_seq_rm(mem, 0, 1, -1)) {
|
||||
COM_TRC("%s", "the context does not support partial sequence removal\n");
|
||||
|
||||
+48
-7
@@ -1,5 +1,6 @@
|
||||
#include "fit.h"
|
||||
|
||||
#include "json.h"
|
||||
#include "log.h"
|
||||
|
||||
#include "../src/llama-ext.h"
|
||||
@@ -191,9 +192,9 @@ static void common_params_fit_impl(
|
||||
uint32_t hp_nct = 0; // hparams.n_ctx_train
|
||||
uint32_t hp_nex = 0; // hparams.n_expert
|
||||
|
||||
// with non-unified kv, we need to take into account n_streams
|
||||
// for example, if memory can hold more than model's trained context size, we must extend the n_ctx to hold enough n_streams
|
||||
const uint32_t n_streams = cparams->kv_unified ? 1 : std::max<uint32_t>(1, cparams->n_seq_max);
|
||||
// size the context for all sequences, but keep minimums and alignment per KV stream
|
||||
const uint32_t n_seq_max = std::max<uint32_t>(1, cparams->n_seq_max);
|
||||
const uint32_t n_streams = cparams->kv_unified ? 1 : n_seq_max;
|
||||
const bool n_ctx_auto = cparams->n_ctx == 0;
|
||||
|
||||
dmds_t dmds_extra; // memory of the extra model, laid out on the devices of the main model
|
||||
@@ -263,15 +264,15 @@ static void common_params_fit_impl(
|
||||
dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
|
||||
// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
|
||||
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
|
||||
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_seq_max, UINT32_MAX);
|
||||
const uint32_t n_ctx_min_total = (uint32_t) std::min<uint64_t>(uint64_t(n_ctx_min) * n_streams, UINT32_MAX);
|
||||
|
||||
// llama_context would use only hp_nct in total for n_ctx == 0, resolve the context before measuring anything else:
|
||||
if (n_ctx_auto) {
|
||||
cparams->n_ctx = n_ctx_max;
|
||||
if (n_streams > 1) {
|
||||
LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
|
||||
__func__, n_ctx_max, n_streams);
|
||||
if (n_seq_max > 1) {
|
||||
LOG_TRC("%s: context size unset -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
|
||||
__func__, n_ctx_max, n_seq_max);
|
||||
dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
|
||||
}
|
||||
}
|
||||
@@ -915,6 +916,9 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
|
||||
std::vector<std::array<std::string, 9>> table_data;
|
||||
table_data.reserve(devices.size());
|
||||
|
||||
// same data as the table below, for --log-jsonl consumers
|
||||
common_json rows = common_json::array();
|
||||
const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n";
|
||||
const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
|
||||
const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n";
|
||||
@@ -989,6 +993,19 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
std::to_string(mb.context / MiB),
|
||||
std::to_string(mb.compute / MiB),
|
||||
std::to_string(unaccounted / static_cast<int64_t>(MiB))});
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "device"},
|
||||
{"name", name},
|
||||
{"description", desc},
|
||||
{"total", total / MiB},
|
||||
{"free", free / MiB},
|
||||
{"self", self / MiB},
|
||||
{"model", mb.model / MiB},
|
||||
{"context", mb.context / MiB},
|
||||
{"compute", mb.compute / MiB},
|
||||
{"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
|
||||
});
|
||||
}
|
||||
|
||||
// print memory breakdown for host:
|
||||
@@ -1004,6 +1021,15 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
std::to_string(mb_host.context / MiB),
|
||||
std::to_string(mb_host.compute / MiB),
|
||||
""}); // unaccounted
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "host"},
|
||||
{"name", "Host"},
|
||||
{"self", self / MiB},
|
||||
{"model", mb_host.model / MiB},
|
||||
{"context", mb_host.context / MiB},
|
||||
{"compute", mb_host.compute / MiB},
|
||||
});
|
||||
}
|
||||
|
||||
// print memory breakdown for all remaining buffer types:
|
||||
@@ -1025,6 +1051,16 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
std::to_string(mb.context / MiB),
|
||||
std::to_string(mb.compute / MiB),
|
||||
""}); // unaccounted
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "buffer_type"},
|
||||
{"name", name},
|
||||
{"self", self / MiB},
|
||||
{"model", mb.model / MiB},
|
||||
{"context", mb.context / MiB},
|
||||
{"compute", mb.compute / MiB},
|
||||
});
|
||||
|
||||
seen_buffer_types.insert(buft);
|
||||
}
|
||||
|
||||
@@ -1042,6 +1078,11 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
__func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(),
|
||||
td[6].c_str(), td[7].c_str(), td[8].c_str());
|
||||
}
|
||||
|
||||
LOG_JSON("fit_memory_breakdown", common_json({
|
||||
{"unit", "MiB"},
|
||||
{"rows", rows},
|
||||
}));
|
||||
}
|
||||
|
||||
void common_fit_print(
|
||||
|
||||
+181
-422
@@ -1,5 +1,7 @@
|
||||
#include "json-schema-to-grammar.h"
|
||||
#include "common.h"
|
||||
#include "trie.h"
|
||||
#include "unicode.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
@@ -336,18 +338,20 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
|
||||
return 2 + n_hex;
|
||||
}
|
||||
|
||||
class common_schema_converter {
|
||||
class common_chat_schema_converter {
|
||||
private:
|
||||
friend class common_schema_info;
|
||||
friend std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options);
|
||||
std::function<json(const std::string &)> _fetch_json;
|
||||
bool _dotall;
|
||||
std::map<std::string, std::string> _rules;
|
||||
std::unordered_map<std::string, json> _refs;
|
||||
std::unordered_set<std::string> _refs_being_resolved;
|
||||
std::vector<std::string> _errors;
|
||||
std::vector<std::string> _warnings;
|
||||
|
||||
template <typename T>
|
||||
static const T & as(const common_chat_schema & node) {
|
||||
return static_cast<const T &>(node);
|
||||
}
|
||||
|
||||
std::string _add_rule(const std::string & name, const std::string & rule) {
|
||||
std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
|
||||
if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
|
||||
@@ -363,11 +367,11 @@ private:
|
||||
return key;
|
||||
}
|
||||
|
||||
std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) {
|
||||
std::string _generate_union_rule(const std::string & name, const std::vector<common_chat_schema_ptr> & alt_schemas) {
|
||||
std::vector<std::string> rules;
|
||||
rules.reserve(alt_schemas.size());
|
||||
for (size_t i = 0; i < alt_schemas.size(); i++) {
|
||||
rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
|
||||
rules.push_back(visit(*alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
|
||||
}
|
||||
return string_join(rules, " | ");
|
||||
}
|
||||
@@ -634,85 +638,68 @@ private:
|
||||
-> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["]
|
||||
*/
|
||||
std::string _not_strings(const std::vector<std::string> & strings) {
|
||||
|
||||
struct TrieNode {
|
||||
std::map<char, TrieNode> children;
|
||||
bool is_end_of_string;
|
||||
|
||||
TrieNode() : is_end_of_string(false) {}
|
||||
|
||||
void insert(const std::string & string) {
|
||||
auto *node = this;
|
||||
for (char c : string) {
|
||||
node = &node->children[c];
|
||||
}
|
||||
node->is_end_of_string = true;
|
||||
}
|
||||
};
|
||||
|
||||
TrieNode trie;
|
||||
for (const auto & s : strings) {
|
||||
trie.insert(s);
|
||||
}
|
||||
common_trie trie(strings);
|
||||
|
||||
std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
|
||||
std::ostringstream out;
|
||||
out << "[\"] ( ";
|
||||
std::function<void(const TrieNode &)> visit = [&](const TrieNode & node) {
|
||||
std::ostringstream rejects;
|
||||
std::function<void(size_t)> visit = [&](size_t idx) {
|
||||
const auto & node = trie.nodes[idx];
|
||||
std::string rejects;
|
||||
auto first = true;
|
||||
for (const auto & kv : node.children) {
|
||||
rejects << kv.first;
|
||||
for (const auto & [cpt, child] : node.children) {
|
||||
std::string c = common_unicode_cpt_to_utf8(cpt);
|
||||
rejects += c;
|
||||
if (first) {
|
||||
first = false;
|
||||
} else {
|
||||
out << " | ";
|
||||
}
|
||||
out << "[" << kv.first << "]";
|
||||
if (!kv.second.children.empty()) {
|
||||
out << "[" << c << "]";
|
||||
if (!trie.nodes[child].children.empty()) {
|
||||
out << " (";
|
||||
visit(kv.second);
|
||||
visit(child);
|
||||
out << ")";
|
||||
} else if (kv.second.is_end_of_string) {
|
||||
} else {
|
||||
out << " " << char_rule << "+";
|
||||
}
|
||||
}
|
||||
if (!node.children.empty()) {
|
||||
if (!first) {
|
||||
out << " | ";
|
||||
}
|
||||
out << "[^\"" << rejects.str() << "] " << char_rule << "*";
|
||||
out << " | [^\"" << rejects << "] " << char_rule << "*";
|
||||
}
|
||||
};
|
||||
visit(trie);
|
||||
visit(0);
|
||||
|
||||
out << " )";
|
||||
if (!trie.is_end_of_string) {
|
||||
if (trie.nodes[0].pattern < 0) {
|
||||
out << "?";
|
||||
}
|
||||
out << " [\"]";
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string _resolve_ref(const std::string & ref) {
|
||||
auto it = ref.find('#');
|
||||
std::string ref_fragment = it != std::string::npos ? ref.substr(it + 1) : ref;
|
||||
std::string _resolve_ref(const common_chat_schema_ref & schema) {
|
||||
auto it = schema.ref.find('#');
|
||||
std::string ref_fragment = it != std::string::npos ? schema.ref.substr(it + 1) : schema.ref;
|
||||
static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)");
|
||||
std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-");
|
||||
if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) {
|
||||
_refs_being_resolved.insert(ref);
|
||||
json resolved = _refs[ref];
|
||||
ref_name = visit(resolved, ref_name);
|
||||
_refs_being_resolved.erase(ref);
|
||||
if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(schema.ref) == _refs_being_resolved.end()) {
|
||||
if (!schema.target) {
|
||||
_errors.push_back("Unresolved $ref " + schema.ref);
|
||||
return "";
|
||||
}
|
||||
_refs_being_resolved.insert(schema.ref);
|
||||
ref_name = visit(*schema.target, ref_name);
|
||||
_refs_being_resolved.erase(schema.ref);
|
||||
}
|
||||
return ref_name;
|
||||
}
|
||||
|
||||
std::string _build_object_rule(
|
||||
const std::vector<std::pair<std::string, json>> & properties,
|
||||
const std::vector<std::pair<std::string, const common_chat_schema *>> & properties,
|
||||
const std::unordered_set<std::string> & required,
|
||||
const std::string & name,
|
||||
const json & additional_properties)
|
||||
const common_chat_schema * additional_properties)
|
||||
{
|
||||
std::vector<std::string> required_props;
|
||||
std::vector<std::string> optional_props;
|
||||
@@ -722,7 +709,7 @@ private:
|
||||
const auto &prop_name = kv.first;
|
||||
const auto &prop_schema = kv.second;
|
||||
|
||||
std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name);
|
||||
std::string prop_rule_name = visit(*prop_schema, name + (name.empty() ? "" : "-") + prop_name);
|
||||
prop_kv_rule_names[prop_name] = _add_rule(
|
||||
name + (name.empty() ? "" : "-") + prop_name + "-kv",
|
||||
format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name
|
||||
@@ -734,10 +721,10 @@ private:
|
||||
}
|
||||
prop_names.push_back(prop_name);
|
||||
}
|
||||
if ((additional_properties.is_boolean() && additional_properties.get<bool>()) || additional_properties.is_object()) {
|
||||
if (additional_properties) {
|
||||
std::string sub_name = name + (name.empty() ? "" : "-") + "additional";
|
||||
std::string value_rule =
|
||||
additional_properties.is_object() ? visit(additional_properties, sub_name + "-value")
|
||||
additional_properties->kind() != common_chat_schema::KIND_ANY ? visit(*additional_properties, sub_name + "-value")
|
||||
: _add_primitive("value", PRIMITIVE_RULES.at("value"));
|
||||
|
||||
auto key_rule =
|
||||
@@ -825,267 +812,163 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
common_schema_converter(
|
||||
const std::function<json(const std::string &)> & fetch_json,
|
||||
bool dotall)
|
||||
: _fetch_json(fetch_json), _dotall(dotall)
|
||||
{
|
||||
explicit common_chat_schema_converter(bool dotall) : _dotall(dotall) {
|
||||
_rules["space"] = SPACE_RULE;
|
||||
}
|
||||
|
||||
void resolve_refs(json & schema, const std::string & url) {
|
||||
/*
|
||||
* Resolves all $ref fields in the given schema, fetching any remote schemas,
|
||||
* replacing each $ref with absolute reference URL and populates _refs with the
|
||||
* respective referenced (sub)schema dictionaries.
|
||||
*/
|
||||
std::function<void(json &)> visit_refs = [&](json & n) {
|
||||
if (n.is_array()) {
|
||||
for (auto & x : n) {
|
||||
visit_refs(x);
|
||||
}
|
||||
} else if (n.is_object()) {
|
||||
if (n.contains("$ref")) {
|
||||
std::string ref = n["$ref"];
|
||||
if (_refs.find(ref) == _refs.end()) {
|
||||
json target;
|
||||
if (ref.find("https://") == 0) {
|
||||
std::string base_url = ref.substr(0, ref.find('#'));
|
||||
auto it = _refs.find(base_url);
|
||||
if (it != _refs.end()) {
|
||||
target = it->second;
|
||||
} else {
|
||||
// Fetch the referenced schema and resolve its refs
|
||||
auto referenced = _fetch_json(ref);
|
||||
resolve_refs(referenced, base_url);
|
||||
_refs[base_url] = referenced;
|
||||
}
|
||||
if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) {
|
||||
return;
|
||||
}
|
||||
} else if (ref.find("#/") == 0) {
|
||||
target = schema;
|
||||
n["$ref"] = url + ref;
|
||||
ref = url + ref;
|
||||
} else {
|
||||
_errors.push_back("Unsupported ref: " + ref);
|
||||
return;
|
||||
}
|
||||
std::string pointer = ref.substr(ref.find('#') + 1);
|
||||
std::vector<std::string> tokens = string_split(pointer, "/");
|
||||
for (size_t i = 1; i < tokens.size(); ++i) {
|
||||
const std::string& sel = tokens[i];
|
||||
if (target.is_object() && target.contains(sel)) {
|
||||
target = target[sel];
|
||||
} else if (target.is_array()) {
|
||||
size_t sel_index;
|
||||
try {
|
||||
sel_index = std::stoull(sel);
|
||||
} catch (const std::invalid_argument & e) {
|
||||
sel_index = target.size();
|
||||
}
|
||||
if (sel_index >= target.size()) {
|
||||
_errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
|
||||
return;
|
||||
}
|
||||
target = target[sel_index];
|
||||
} else {
|
||||
_errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
|
||||
return;
|
||||
}
|
||||
}
|
||||
_refs[ref] = target;
|
||||
}
|
||||
} else {
|
||||
for (const auto & kv : n.items()) {
|
||||
visit_refs(kv.value());
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
visit_refs(schema);
|
||||
std::string add_schema(const std::string & name, const common_chat_schema & schema) {
|
||||
return visit(schema, name);
|
||||
}
|
||||
|
||||
static std::string _generate_constant_rule(const json & value) {
|
||||
return format_literal(value.dump());
|
||||
}
|
||||
|
||||
std::string visit(const json & schema, const std::string & name) {
|
||||
json schema_type = schema.contains("type") ? schema["type"] : json();
|
||||
std::string schema_format = schema.contains("format") ? schema["format"].get<std::string>() : "";
|
||||
std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
|
||||
std::string _visit_primitive(const std::string & rule_name, const std::string & type) {
|
||||
return _add_primitive(rule_name == "root" ? "root" : type, PRIMITIVE_RULES.at(type));
|
||||
}
|
||||
|
||||
if (schema.contains("$ref")) {
|
||||
return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
|
||||
}
|
||||
if (schema.contains("oneOf") || schema.contains("anyOf")) {
|
||||
const json & alts = schema.contains("oneOf") ? schema.at("oneOf") : schema.at("anyOf");
|
||||
std::vector<json> alt_schemas;
|
||||
for (const auto & alt : alts) {
|
||||
alt_schemas.push_back(alt);
|
||||
}
|
||||
return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
|
||||
}
|
||||
if (schema_type.is_array()) {
|
||||
std::vector<json> schema_types;
|
||||
for (const auto & t : schema_type) {
|
||||
json schema_copy(schema);
|
||||
schema_copy["type"] = t;
|
||||
schema_types.push_back(schema_copy);
|
||||
}
|
||||
return _add_rule(rule_name, _generate_union_rule(name, schema_types));
|
||||
}
|
||||
if (schema.contains("const")) {
|
||||
return _add_rule(rule_name, _generate_constant_rule(schema["const"]));
|
||||
}
|
||||
if (schema.contains("enum")) {
|
||||
std::vector<std::string> enum_values;
|
||||
for (const auto & v : schema["enum"]) {
|
||||
enum_values.push_back(_generate_constant_rule(v));
|
||||
}
|
||||
return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")");
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "object")
|
||||
&& (schema.contains("properties") ||
|
||||
(schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {
|
||||
std::unordered_set<std::string> required;
|
||||
if (schema.contains("required") && schema["required"].is_array()) {
|
||||
for (const auto & item : schema["required"]) {
|
||||
if (item.is_string()) {
|
||||
required.insert(item.get<std::string>());
|
||||
std::string _visit_all_of(const common_chat_schema_all_of & schema, const std::string & name, const std::string & rule_name) {
|
||||
std::unordered_set<std::string> required;
|
||||
std::vector<std::pair<std::string, const common_chat_schema *>> properties;
|
||||
std::map<std::string, size_t> enum_values;
|
||||
std::function<void(const common_chat_schema &, bool)> add_component = [&](const common_chat_schema & comp, bool is_required) {
|
||||
if (comp.kind() == common_chat_schema::KIND_REF) {
|
||||
if (const auto * target = as<common_chat_schema_ref>(comp).target) {
|
||||
add_component(*target, is_required);
|
||||
}
|
||||
} else if (comp.kind() == common_chat_schema::KIND_OBJECT) {
|
||||
for (const auto & prop : as<common_chat_schema_object>(comp).properties) {
|
||||
properties.emplace_back(prop.name, prop.schema.get());
|
||||
if (is_required) {
|
||||
required.insert(prop.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
std::vector<std::pair<std::string, json>> properties;
|
||||
if (schema.contains("properties")) {
|
||||
for (const auto & prop : schema["properties"].items()) {
|
||||
properties.emplace_back(prop.key(), prop.value());
|
||||
} else if (comp.kind() == common_chat_schema::KIND_ENUM) {
|
||||
for (const auto & v : as<common_chat_schema_enum>(comp).values) {
|
||||
enum_values[_generate_constant_rule(v)] += 1;
|
||||
}
|
||||
}
|
||||
return _add_rule(rule_name,
|
||||
_build_object_rule(
|
||||
properties, required, name,
|
||||
schema.contains("additionalProperties") ? schema["additionalProperties"] : json()));
|
||||
};
|
||||
for (const auto & child : schema.children) {
|
||||
if (child->kind() == common_chat_schema::KIND_ANY_OF) {
|
||||
for (const auto & alt : as<common_chat_schema_any_of>(*child).children) {
|
||||
add_component(*alt, false);
|
||||
}
|
||||
} else {
|
||||
add_component(*child, true);
|
||||
}
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "object" || schema_type == "string") && schema.contains("allOf")) {
|
||||
std::unordered_set<std::string> required;
|
||||
std::vector<std::pair<std::string, json>> properties;
|
||||
std::map<std::string, size_t> enum_values;
|
||||
const std::string& hybrid_name = name;
|
||||
std::function<void(const json &, bool)> add_component = [&](const json & comp_schema, bool is_required) {
|
||||
if (comp_schema.contains("$ref")) {
|
||||
add_component(_refs[comp_schema["$ref"]], is_required);
|
||||
} else if (comp_schema.contains("properties")) {
|
||||
for (const auto & prop : comp_schema["properties"].items()) {
|
||||
properties.emplace_back(prop.key(), prop.value());
|
||||
if (is_required) {
|
||||
required.insert(prop.key());
|
||||
}
|
||||
}
|
||||
} else if (comp_schema.contains("enum")) {
|
||||
for (const auto & v : comp_schema["enum"]) {
|
||||
const auto rule = _generate_constant_rule(v);
|
||||
if (enum_values.find(rule) == enum_values.end()) {
|
||||
enum_values[rule] = 0;
|
||||
}
|
||||
enum_values[rule] += 1;
|
||||
}
|
||||
} else {
|
||||
// todo warning
|
||||
}
|
||||
};
|
||||
for (const auto & t : schema["allOf"]) {
|
||||
if (t.contains("anyOf")) {
|
||||
for (const auto & tt : t["anyOf"]) {
|
||||
add_component(tt, false);
|
||||
}
|
||||
} else {
|
||||
add_component(t, true);
|
||||
if (!enum_values.empty()) {
|
||||
std::vector<std::string> enum_intersection;
|
||||
for (const auto & p : enum_values) {
|
||||
if (p.second == schema.children.size()) {
|
||||
enum_intersection.push_back(p.first);
|
||||
}
|
||||
}
|
||||
if (!enum_values.empty()) {
|
||||
std::vector<std::string> enum_intersection;
|
||||
for (const auto & p : enum_values) {
|
||||
if (p.second == schema["allOf"].size()) {
|
||||
enum_intersection.push_back(p.first);
|
||||
}
|
||||
}
|
||||
if (!enum_intersection.empty()) {
|
||||
return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")");
|
||||
}
|
||||
if (!enum_intersection.empty()) {
|
||||
return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")");
|
||||
}
|
||||
return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) {
|
||||
json items = schema.contains("items") ? schema["items"] : schema["prefixItems"];
|
||||
if (items.is_array()) {
|
||||
return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr));
|
||||
}
|
||||
|
||||
std::string visit(const common_chat_schema & schema, const std::string & name) {
|
||||
std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
|
||||
std::string sub_name = name + (name.empty() ? "" : "-");
|
||||
|
||||
switch (schema.kind()) {
|
||||
case common_chat_schema::KIND_REF:
|
||||
return _add_rule(rule_name, _resolve_ref(as<common_chat_schema_ref>(schema)));
|
||||
case common_chat_schema::KIND_ANY_OF:
|
||||
return _add_rule(rule_name, _generate_union_rule(name, as<common_chat_schema_any_of>(schema).children));
|
||||
case common_chat_schema::KIND_ALL_OF:
|
||||
return _visit_all_of(as<common_chat_schema_all_of>(schema), name, rule_name);
|
||||
case common_chat_schema::KIND_CONST:
|
||||
return _add_rule(rule_name, _generate_constant_rule(as<common_chat_schema_const>(schema).value));
|
||||
case common_chat_schema::KIND_ENUM: {
|
||||
std::vector<std::string> enum_values;
|
||||
for (const auto & v : as<common_chat_schema_enum>(schema).values) {
|
||||
enum_values.push_back(_generate_constant_rule(v));
|
||||
}
|
||||
return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")");
|
||||
}
|
||||
case common_chat_schema::KIND_OBJECT: {
|
||||
const auto & obj = as<common_chat_schema_object>(schema);
|
||||
if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == common_chat_schema::KIND_ANY) {
|
||||
return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
|
||||
}
|
||||
std::vector<std::pair<std::string, const common_chat_schema *>> properties;
|
||||
std::unordered_set<std::string> required;
|
||||
for (const auto & prop : obj.properties) {
|
||||
properties.emplace_back(prop.name, prop.schema.get());
|
||||
if (prop.required) {
|
||||
required.insert(prop.name);
|
||||
}
|
||||
}
|
||||
return _add_rule(rule_name, _build_object_rule(properties, required, name, obj.additional_properties.get()));
|
||||
}
|
||||
case common_chat_schema::KIND_TUPLE: {
|
||||
const auto & items = as<common_chat_schema_tuple>(schema).items;
|
||||
std::string rule = "\"[\" space ";
|
||||
for (size_t i = 0; i < items.size(); i++) {
|
||||
if (i > 0) {
|
||||
rule += " \",\" space ";
|
||||
}
|
||||
rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i));
|
||||
rule += visit(*items[i], sub_name + "tuple-" + std::to_string(i));
|
||||
}
|
||||
rule += " space \"]\"";
|
||||
return _add_rule(rule_name, rule);
|
||||
}
|
||||
std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item");
|
||||
int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0;
|
||||
json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json();
|
||||
int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : std::numeric_limits<int>::max();
|
||||
|
||||
return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " space \"]\"");
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) {
|
||||
return _visit_pattern(schema["pattern"], rule_name);
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) {
|
||||
return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid"));
|
||||
}
|
||||
if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) {
|
||||
auto prim_name = schema_format + "-string";
|
||||
return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
|
||||
}
|
||||
if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) {
|
||||
std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
|
||||
int min_len = schema.contains("minLength") ? schema["minLength"].get<int>() : 0;
|
||||
int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max();
|
||||
return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\"");
|
||||
}
|
||||
if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) {
|
||||
int64_t min_value = std::numeric_limits<int64_t>::min();
|
||||
int64_t max_value = std::numeric_limits<int64_t>::max();
|
||||
if (schema.contains("minimum")) {
|
||||
min_value = schema["minimum"].get<int64_t>();
|
||||
} else if (schema.contains("exclusiveMinimum")) {
|
||||
min_value = schema["exclusiveMinimum"].get<int64_t>() + 1;
|
||||
case common_chat_schema::KIND_ARRAY: {
|
||||
const auto & arr = as<common_chat_schema_array>(schema);
|
||||
if (arr.items->kind() == common_chat_schema::KIND_ANY && arr.min_items == 0 && arr.max_items < 0) {
|
||||
return _visit_primitive(rule_name, "array");
|
||||
}
|
||||
std::string item_rule_name = visit(*arr.items, sub_name + "item");
|
||||
int max_items = arr.max_items < 0 ? std::numeric_limits<int>::max() : arr.max_items;
|
||||
return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, arr.min_items, max_items, "\",\" space") + " space \"]\"");
|
||||
}
|
||||
if (schema.contains("maximum")) {
|
||||
max_value = schema["maximum"].get<int64_t>();
|
||||
} else if (schema.contains("exclusiveMaximum")) {
|
||||
max_value = schema["exclusiveMaximum"].get<int64_t>() - 1;
|
||||
case common_chat_schema::KIND_STRING: {
|
||||
const auto & str = as<common_chat_schema_string>(schema);
|
||||
if (!str.pattern.empty()) {
|
||||
return _visit_pattern(str.pattern, rule_name);
|
||||
}
|
||||
if (str.format == common_chat_schema::FORMAT_UUID) {
|
||||
return _visit_primitive(rule_name, "uuid");
|
||||
}
|
||||
if (str.format != common_chat_schema::FORMAT_NONE) {
|
||||
std::string prim_name = std::string(str.format == common_chat_schema::FORMAT_DATE ? "date" : str.format == common_chat_schema::FORMAT_TIME ? "time" : "date-time") + "-string";
|
||||
return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
|
||||
}
|
||||
if (str.min_length > 0 || str.max_length >= 0) {
|
||||
std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
|
||||
int max_len = str.max_length < 0 ? std::numeric_limits<int>::max() : str.max_length;
|
||||
return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, str.min_length, max_len) + " \"\\\"\"");
|
||||
}
|
||||
return _visit_primitive(rule_name, "string");
|
||||
}
|
||||
std::stringstream out;
|
||||
out << "(";
|
||||
build_min_max_int(min_value, max_value, out);
|
||||
out << ")";
|
||||
return _add_rule(rule_name, out.str());
|
||||
case common_chat_schema::KIND_INTEGER: {
|
||||
const auto & i = as<common_chat_schema_integer>(schema);
|
||||
if (i.minimum == std::numeric_limits<int64_t>::min() && i.maximum == std::numeric_limits<int64_t>::max()) {
|
||||
return _visit_primitive(rule_name, "integer");
|
||||
}
|
||||
std::stringstream out;
|
||||
out << "(";
|
||||
build_min_max_int(i.minimum, i.maximum, out);
|
||||
out << ")";
|
||||
return _add_rule(rule_name, out.str());
|
||||
}
|
||||
case common_chat_schema::KIND_NUMBER:
|
||||
return _visit_primitive(rule_name, "number");
|
||||
case common_chat_schema::KIND_BOOLEAN:
|
||||
return _visit_primitive(rule_name, "boolean");
|
||||
case common_chat_schema::KIND_NULL:
|
||||
return _visit_primitive(rule_name, "null");
|
||||
case common_chat_schema::KIND_ANY:
|
||||
return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value")));
|
||||
}
|
||||
if (schema.empty() || schema_type == "object") {
|
||||
return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
|
||||
}
|
||||
if (schema_type.is_null() && schema.is_object()) {
|
||||
// No type constraint and no recognized structural keywords (e.g. {"description": "..."}).
|
||||
// Per JSON Schema semantics this is equivalent to {} and accepts any value.
|
||||
return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value")));
|
||||
}
|
||||
if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) {
|
||||
_errors.push_back("Unrecognized schema: " + schema.dump());
|
||||
return "";
|
||||
}
|
||||
// TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
|
||||
return _add_primitive(rule_name == "root" ? "root" : schema_type.get<std::string>(), PRIMITIVE_RULES.at(schema_type.get<std::string>()));
|
||||
return "";
|
||||
}
|
||||
|
||||
void check_errors() {
|
||||
@@ -1106,134 +989,6 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
// common_schema_info implementation (pimpl)
|
||||
|
||||
common_schema_info::common_schema_info()
|
||||
: impl_(std::make_unique<common_schema_converter>(
|
||||
[](const std::string &) { return json(); },
|
||||
false)) {}
|
||||
|
||||
common_schema_info::~common_schema_info() = default;
|
||||
|
||||
common_schema_info::common_schema_info(common_schema_info &&) noexcept = default;
|
||||
common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default;
|
||||
|
||||
void common_schema_info::resolve_refs(common_json & schema) {
|
||||
impl_->resolve_refs(schema, "");
|
||||
}
|
||||
|
||||
// Determines if a JSON schema can resolve to a string type through any path.
|
||||
// Some models emit raw string values rather than JSON-encoded strings for string parameters.
|
||||
// If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns
|
||||
// true, allowing callers to handle the value as a raw string for simplicity.
|
||||
bool common_schema_info::resolves_to_string(const common_json & schema) {
|
||||
std::unordered_set<std::string> visited_refs;
|
||||
|
||||
std::function<bool(const json &)> check = [&](const json & s) -> bool {
|
||||
if (!s.is_object()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Handle $ref
|
||||
if (s.contains("$ref")) {
|
||||
const std::string & ref = s["$ref"];
|
||||
if (visited_refs.find(ref) != visited_refs.end()) {
|
||||
// Circular reference, assume not a string to be safe
|
||||
return false;
|
||||
}
|
||||
visited_refs.insert(ref);
|
||||
auto it = impl_->_refs.find(ref);
|
||||
if (it != impl_->_refs.end()) {
|
||||
return check(it->second);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check type field
|
||||
if (s.contains("type")) {
|
||||
const json & schema_type = s["type"];
|
||||
if (schema_type.is_string()) {
|
||||
if (schema_type == "string") {
|
||||
return true;
|
||||
}
|
||||
} else if (schema_type.is_array()) {
|
||||
// Type can be an array like ["string", "null"]
|
||||
for (const auto & t : schema_type) {
|
||||
if (t == "string") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check oneOf/anyOf - if any alternative can be a string
|
||||
if (s.contains("oneOf")) {
|
||||
for (const auto & alt : s["oneOf"]) {
|
||||
if (check(alt)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (s.contains("anyOf")) {
|
||||
for (const auto & alt : s["anyOf"]) {
|
||||
if (check(alt)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check allOf - all components must be compatible with string type
|
||||
if (s.contains("allOf")) {
|
||||
bool all_string = true;
|
||||
for (const auto & component : s["allOf"]) {
|
||||
if (!check(component)) {
|
||||
all_string = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (all_string) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check const - if the constant value is a string
|
||||
if (s.contains("const")) {
|
||||
if (s["const"].is_string()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check enum - if any enum value is a string
|
||||
if (s.contains("enum")) {
|
||||
for (const auto & val : s["enum"]) {
|
||||
if (val.is_string()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// String-specific keywords imply string type
|
||||
if (s.contains("pattern") || s.contains("minLength") || s.contains("maxLength")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check format - many formats imply string
|
||||
if (s.contains("format")) {
|
||||
const std::string & fmt = s["format"];
|
||||
if (fmt == "date" || fmt == "time" || fmt == "date-time" ||
|
||||
fmt == "uri" || fmt == "email" || fmt == "hostname" ||
|
||||
fmt == "ipv4" || fmt == "ipv6" || fmt == "uuid" ||
|
||||
fmt.find("uuid") == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
return check(schema);
|
||||
}
|
||||
|
||||
std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) {
|
||||
#ifdef LLAMA_USE_LLGUIDANCE
|
||||
if (!force_gbnf) {
|
||||
@@ -1242,25 +997,29 @@ std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf)
|
||||
#else
|
||||
(void)force_gbnf;
|
||||
#endif // LLAMA_USE_LLGUIDANCE
|
||||
return build_grammar([&](const common_grammar_builder & callbacks) {
|
||||
auto copy = schema;
|
||||
callbacks.resolve_refs(copy);
|
||||
callbacks.add_schema("", copy);
|
||||
});
|
||||
try {
|
||||
return json_schema_to_grammar(common_chat_schema_from_json(schema));
|
||||
} catch (const std::runtime_error & e) {
|
||||
throw std::invalid_argument(std::string("JSON schema conversion failed:\n") + e.what());
|
||||
}
|
||||
}
|
||||
|
||||
std::string json_schema_to_grammar(const common_chat_schema_document & schema) {
|
||||
common_chat_schema_converter converter(false);
|
||||
converter.visit(*schema.root, "");
|
||||
converter.check_errors();
|
||||
return converter.format_grammar();
|
||||
}
|
||||
|
||||
std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options) {
|
||||
common_schema_converter converter([&](const std::string &) { return json(); }, options.dotall);
|
||||
common_chat_schema_converter converter(options.dotall);
|
||||
common_grammar_builder builder {
|
||||
/* .add_rule = */ [&](const std::string & name, const std::string & rule) {
|
||||
return converter._add_rule(name, rule);
|
||||
},
|
||||
/* .add_schema = */ [&](const std::string & name, const common_json & schema) {
|
||||
return converter.visit(schema, name == "root" ? "" : name);
|
||||
/* .add_schema = */ [&](const std::string & name, const common_chat_schema & schema) {
|
||||
return converter.add_schema(name == "root" ? "" : name, schema);
|
||||
},
|
||||
/* .resolve_refs = */ [&](common_json & schema) {
|
||||
converter.resolve_refs(schema, "");
|
||||
}
|
||||
};
|
||||
cb(builder);
|
||||
converter.check_errors();
|
||||
|
||||
@@ -1,37 +1,17 @@
|
||||
#pragma once
|
||||
|
||||
#include "json-schema.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
std::string json_schema_to_grammar(const common_json & schema,
|
||||
bool force_gbnf = false);
|
||||
|
||||
class common_schema_converter;
|
||||
|
||||
// Probes a JSON schema to extract information about its structure and type constraints.
|
||||
class common_schema_info {
|
||||
std::unique_ptr<common_schema_converter> impl_;
|
||||
|
||||
public:
|
||||
common_schema_info();
|
||||
~common_schema_info();
|
||||
|
||||
common_schema_info(const common_schema_info &) = delete;
|
||||
common_schema_info & operator=(const common_schema_info &) = delete;
|
||||
common_schema_info(common_schema_info &&) noexcept;
|
||||
common_schema_info & operator=(common_schema_info &&) noexcept;
|
||||
|
||||
void resolve_refs(common_json & schema);
|
||||
bool resolves_to_string(const common_json & schema);
|
||||
};
|
||||
std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf = false);
|
||||
std::string json_schema_to_grammar(const common_chat_schema_document & schema);
|
||||
|
||||
struct common_grammar_builder {
|
||||
std::function<std::string(const std::string &, const std::string &)> add_rule;
|
||||
std::function<std::string(const std::string &, const common_json &)> add_schema;
|
||||
std::function<void(common_json &)> resolve_refs;
|
||||
std::function<std::string(const std::string &, const std::string &)> add_rule;
|
||||
std::function<std::string(const std::string &, const common_chat_schema &)> add_schema;
|
||||
};
|
||||
|
||||
struct common_grammar_options {
|
||||
|
||||
@@ -0,0 +1,514 @@
|
||||
#include "json-schema.h"
|
||||
#include "common.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
class common_chat_schema_builder {
|
||||
const common_json & root_;
|
||||
common_chat_schema_document & doc_;
|
||||
|
||||
// the targets built here, moved into doc_ once the whole schema is built
|
||||
std::map<std::string, common_chat_schema_ptr> refs_;
|
||||
|
||||
// ref nodes get their target once every $ref is built, a cycle would otherwise need it too early
|
||||
std::vector<common_chat_schema_ref *> pending_;
|
||||
|
||||
[[noreturn]] static void fail(const std::string & path, const std::string & msg) {
|
||||
throw std::runtime_error("JSON schema error at " + path + ": " + msg);
|
||||
}
|
||||
|
||||
static int get_count(const common_json & schema, const std::string & key, const std::string & path, int def) {
|
||||
if (!schema.contains(key)) {
|
||||
return def;
|
||||
}
|
||||
const common_json & value = schema.at(key);
|
||||
if (!value.is_number_integer() || value.get<int>() < 0) {
|
||||
fail(path, key + " must be a non-negative integer");
|
||||
}
|
||||
return value.get<int>();
|
||||
}
|
||||
|
||||
// a fractional bound is rounded inwards, towards the integers it still admits
|
||||
static int64_t get_bound(const common_json & schema, const std::string & key, const std::string & path, bool round_up) {
|
||||
const common_json & value = schema.at(key);
|
||||
if (value.is_number_integer()) {
|
||||
return value.get<int64_t>();
|
||||
}
|
||||
if (!value.is_number()) {
|
||||
fail(path, key + " must be a number");
|
||||
}
|
||||
double d = value.get<double>();
|
||||
return (int64_t) (round_up ? std::ceil(d) : std::floor(d));
|
||||
}
|
||||
|
||||
static common_chat_schema::string_format get_format(const common_json & schema, const std::string & path) {
|
||||
if (!schema.contains("format")) {
|
||||
return common_chat_schema::FORMAT_NONE;
|
||||
}
|
||||
const common_json & value = schema.at("format");
|
||||
if (!value.is_string()) {
|
||||
fail(path, "format must be a string");
|
||||
}
|
||||
std::string format = value.get<std::string>();
|
||||
if (format == "date") {
|
||||
return common_chat_schema::FORMAT_DATE;
|
||||
}
|
||||
if (format == "time") {
|
||||
return common_chat_schema::FORMAT_TIME;
|
||||
}
|
||||
if (format == "date-time") {
|
||||
return common_chat_schema::FORMAT_DATE_TIME;
|
||||
}
|
||||
if (format == "uuid" || (format.size() == 5 && format.compare(0, 4, "uuid") == 0 && format[4] >= '1' && format[4] <= '5')) {
|
||||
return common_chat_schema::FORMAT_UUID;
|
||||
}
|
||||
return common_chat_schema::FORMAT_NONE;
|
||||
}
|
||||
|
||||
const common_json & resolve_ref(const std::string & ref, const std::string & path) {
|
||||
const common_json * target = &root_;
|
||||
auto tokens = string_split(ref.substr(1), "/");
|
||||
for (size_t i = 1; i < tokens.size(); i++) {
|
||||
const std::string & sel = tokens[i];
|
||||
if (target->is_object() && target->contains(sel)) {
|
||||
target = &target->at(sel);
|
||||
} else if (target->is_array()) {
|
||||
size_t idx;
|
||||
try {
|
||||
idx = std::stoull(sel);
|
||||
} catch (const std::logic_error &) {
|
||||
idx = target->size();
|
||||
}
|
||||
if (idx >= target->size()) {
|
||||
fail(path, "cannot resolve $ref " + ref + ", " + sel + " is out of range");
|
||||
}
|
||||
target = &target->at(idx);
|
||||
} else {
|
||||
fail(path, "cannot resolve $ref " + ref + ", " + sel + " not found");
|
||||
}
|
||||
}
|
||||
return *target;
|
||||
}
|
||||
|
||||
common_chat_schema_ptr build_ref(const common_json & value, const std::string & path) {
|
||||
if (!value.is_string()) {
|
||||
fail(path, "$ref must be a string");
|
||||
}
|
||||
std::string ref = value.get<std::string>();
|
||||
if (ref.compare(0, 2, "#/") != 0) {
|
||||
fail(path, "unsupported $ref " + ref + ", only references into the same document are supported");
|
||||
}
|
||||
if (refs_.find(ref) == refs_.end()) {
|
||||
// reserve the key first, so that a cycle back to this $ref stops here
|
||||
refs_[ref] = nullptr;
|
||||
refs_[ref] = build_node(resolve_ref(ref, path), ref);
|
||||
}
|
||||
auto node = std::make_unique<common_chat_schema_ref>(ref);
|
||||
pending_.push_back(node.get());
|
||||
return node;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
common_chat_schema_ptr build_alternatives(const common_json & alts, const std::string & path) {
|
||||
if (!alts.is_array()) {
|
||||
fail(path, "must be an array of schemas");
|
||||
}
|
||||
if (alts.empty()) {
|
||||
fail(path, "must not be empty");
|
||||
}
|
||||
auto node = std::make_unique<T>();
|
||||
size_t i = 0;
|
||||
for (const auto & alt : alts) {
|
||||
node->children.push_back(build_node(alt, path + "/" + std::to_string(i++)));
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
common_chat_schema_ptr build_object(const common_json & schema, const std::string & path) {
|
||||
auto node = std::make_unique<common_chat_schema_object>();
|
||||
|
||||
std::unordered_set<std::string> required;
|
||||
if (schema.contains("required") && schema.at("required").is_array()) {
|
||||
for (const auto & name : schema.at("required")) {
|
||||
if (name.is_string()) {
|
||||
required.insert(name.get<std::string>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (schema.contains("properties")) {
|
||||
const common_json & properties = schema.at("properties");
|
||||
if (!properties.is_object()) {
|
||||
fail(path, "properties must be an object");
|
||||
}
|
||||
for (const auto & [name, prop] : properties.items()) {
|
||||
node->properties.push_back({name, build_node(prop, path + "/properties/" + name), required.count(name) > 0});
|
||||
}
|
||||
}
|
||||
|
||||
if (schema.contains("additionalProperties")) {
|
||||
const common_json & additional = schema.at("additionalProperties");
|
||||
if (additional.is_boolean()) {
|
||||
if (additional.get<bool>()) {
|
||||
node->additional_properties = std::make_unique<common_chat_schema_any>();
|
||||
}
|
||||
} else if (additional.is_object()) {
|
||||
node->additional_properties = build_node(additional, path + "/additionalProperties");
|
||||
} else {
|
||||
fail(path, "additionalProperties must be a boolean or a schema");
|
||||
}
|
||||
} else if (!schema.contains("properties")) {
|
||||
// {"type": "object"} on its own accepts any object
|
||||
node->additional_properties = std::make_unique<common_chat_schema_any>();
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
common_chat_schema_ptr build_array(const common_json & schema, const std::string & path) {
|
||||
auto node = std::make_unique<common_chat_schema_array>();
|
||||
if (schema.contains("items") || schema.contains("prefixItems")) {
|
||||
// "items" wins when both are present; as in the converter, a schema instead of an array is the item schema
|
||||
const std::string key = schema.contains("items") ? "items" : "prefixItems";
|
||||
const common_json & items = schema.at(key);
|
||||
if (items.is_array()) {
|
||||
auto tuple = std::make_unique<common_chat_schema_tuple>();
|
||||
size_t i = 0;
|
||||
for (const auto & item : items) {
|
||||
tuple->items.push_back(build_node(item, path + "/" + key + "/" + std::to_string(i++)));
|
||||
}
|
||||
return tuple;
|
||||
}
|
||||
node->items = build_node(items, path + "/" + key);
|
||||
} else {
|
||||
node->items = std::make_unique<common_chat_schema_any>();
|
||||
}
|
||||
node->min_items = get_count(schema, "minItems", path, 0);
|
||||
node->max_items = get_count(schema, "maxItems", path, -1);
|
||||
return node;
|
||||
}
|
||||
|
||||
common_chat_schema_ptr build_string(const common_json & schema, const std::string & path) {
|
||||
auto node = std::make_unique<common_chat_schema_string>();
|
||||
if (schema.contains("pattern")) {
|
||||
const common_json & pattern = schema.at("pattern");
|
||||
if (!pattern.is_string()) {
|
||||
fail(path, "pattern must be a string");
|
||||
}
|
||||
node->pattern = pattern.get<std::string>();
|
||||
}
|
||||
node->format = get_format(schema, path);
|
||||
node->min_length = get_count(schema, "minLength", path, 0);
|
||||
node->max_length = get_count(schema, "maxLength", path, -1);
|
||||
return node;
|
||||
}
|
||||
|
||||
common_chat_schema_ptr build_integer(const common_json & schema, const std::string & path) {
|
||||
auto node = std::make_unique<common_chat_schema_integer>();
|
||||
if (schema.contains("minimum")) {
|
||||
node->minimum = get_bound(schema, "minimum", path, /* round_up */ true);
|
||||
} else if (schema.contains("exclusiveMinimum")) {
|
||||
node->minimum = get_bound(schema, "exclusiveMinimum", path, /* round_up */ false) + 1;
|
||||
}
|
||||
if (schema.contains("maximum")) {
|
||||
node->maximum = get_bound(schema, "maximum", path, /* round_up */ false);
|
||||
} else if (schema.contains("exclusiveMaximum")) {
|
||||
node->maximum = get_bound(schema, "exclusiveMaximum", path, /* round_up */ true) - 1;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
common_chat_schema_ptr build_node(const common_json & schema, const std::string & path) {
|
||||
if (!schema.is_object()) {
|
||||
fail(path, "schema must be an object");
|
||||
}
|
||||
if (schema.contains("$ref")) {
|
||||
return build_ref(schema.at("$ref"), path);
|
||||
}
|
||||
if (schema.contains("oneOf") || schema.contains("anyOf")) {
|
||||
const std::string key = schema.contains("oneOf") ? "oneOf" : "anyOf";
|
||||
return build_alternatives<common_chat_schema_any_of>(schema.at(key), path + "/" + key);
|
||||
}
|
||||
|
||||
common_json type;
|
||||
if (schema.contains("type")) {
|
||||
type = schema.at("type");
|
||||
}
|
||||
if (type.is_array()) {
|
||||
// {"type": ["a", "b"], ...} is {"anyOf": [{"type": "a", ...}, {"type": "b", ...}]}
|
||||
if (type.empty()) {
|
||||
fail(path, "type must not be empty");
|
||||
}
|
||||
auto node = std::make_unique<common_chat_schema_any_of>();
|
||||
size_t i = 0;
|
||||
for (const auto & t : type) {
|
||||
common_json alt = schema;
|
||||
alt["type"] = t;
|
||||
node->children.push_back(build_node(alt, path + "/type/" + std::to_string(i++)));
|
||||
}
|
||||
return node;
|
||||
}
|
||||
if (schema.contains("const")) {
|
||||
return std::make_unique<common_chat_schema_const>(schema.at("const"));
|
||||
}
|
||||
if (schema.contains("enum")) {
|
||||
const common_json & values = schema.at("enum");
|
||||
if (!values.is_array() || values.empty()) {
|
||||
fail(path, "enum must be a non-empty array");
|
||||
}
|
||||
auto node = std::make_unique<common_chat_schema_enum>();
|
||||
for (const auto & value : values) {
|
||||
node->values.push_back(value);
|
||||
}
|
||||
return node;
|
||||
}
|
||||
if (!type.is_null() && !type.is_string()) {
|
||||
fail(path, "type must be a string or an array of strings");
|
||||
}
|
||||
|
||||
const std::string type_name = type.is_string() ? type.get<std::string>() : "";
|
||||
const bool has_properties = schema.contains("properties") ||
|
||||
(schema.contains("additionalProperties") && schema.at("additionalProperties") != true);
|
||||
|
||||
if (type_name.empty()) {
|
||||
// without a type the structural keywords decide, in the same order as the converter
|
||||
if (has_properties) {
|
||||
return build_object(schema, path);
|
||||
}
|
||||
if (schema.contains("allOf")) {
|
||||
return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
|
||||
}
|
||||
if (schema.contains("items") || schema.contains("prefixItems")) {
|
||||
return build_array(schema, path);
|
||||
}
|
||||
if (schema.contains("pattern") || schema.contains("minLength") || schema.contains("maxLength") || get_format(schema, path) != common_chat_schema::FORMAT_NONE) {
|
||||
return build_string(schema, path);
|
||||
}
|
||||
return std::make_unique<common_chat_schema_any>();
|
||||
}
|
||||
if (type_name == "object") {
|
||||
if (!has_properties && schema.contains("allOf")) {
|
||||
return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
|
||||
}
|
||||
return build_object(schema, path);
|
||||
}
|
||||
if (type_name == "string") {
|
||||
if (schema.contains("allOf")) {
|
||||
return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
|
||||
}
|
||||
return build_string(schema, path);
|
||||
}
|
||||
if (type_name == "array") {
|
||||
return build_array(schema, path);
|
||||
}
|
||||
if (type_name == "integer") {
|
||||
return build_integer(schema, path);
|
||||
}
|
||||
if (type_name == "number") {
|
||||
return std::make_unique<common_chat_schema_number>();
|
||||
}
|
||||
if (type_name == "boolean") {
|
||||
return std::make_unique<common_chat_schema_boolean>();
|
||||
}
|
||||
if (type_name == "null") {
|
||||
return std::make_unique<common_chat_schema_null>();
|
||||
}
|
||||
fail(path, "unrecognized type " + type_name);
|
||||
}
|
||||
|
||||
public:
|
||||
common_chat_schema_builder(const common_json & root, common_chat_schema_document & doc) : root_(root), doc_(doc) {}
|
||||
|
||||
common_chat_schema_ptr build() {
|
||||
auto node = build_node(root_, "#");
|
||||
for (auto & entry : refs_) {
|
||||
doc_.refs[entry.first] = std::move(entry.second);
|
||||
}
|
||||
for (auto * ref : pending_) {
|
||||
ref->target = doc_.refs.at(ref->ref).get();
|
||||
}
|
||||
return node;
|
||||
}
|
||||
};
|
||||
|
||||
common_chat_schema_document common_chat_schema_from_json(const common_json & schema) {
|
||||
common_chat_schema_document doc;
|
||||
doc.root = common_chat_schema_builder(schema, doc).build();
|
||||
return doc;
|
||||
}
|
||||
|
||||
static common_chat_schema::value_type json_type(const common_json & value) {
|
||||
if (value.is_null()) {
|
||||
return common_chat_schema::TYPE_NULL;
|
||||
}
|
||||
if (value.is_boolean()) {
|
||||
return common_chat_schema::TYPE_BOOLEAN;
|
||||
}
|
||||
if (value.is_number_integer()) {
|
||||
return common_chat_schema::TYPE_INTEGER;
|
||||
}
|
||||
if (value.is_number()) {
|
||||
return common_chat_schema::TYPE_NUMBER;
|
||||
}
|
||||
if (value.is_string()) {
|
||||
return common_chat_schema::TYPE_STRING;
|
||||
}
|
||||
if (value.is_array()) {
|
||||
return common_chat_schema::TYPE_ARRAY;
|
||||
}
|
||||
return common_chat_schema::TYPE_OBJECT;
|
||||
}
|
||||
|
||||
static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
|
||||
switch (s.kind()) {
|
||||
case common_chat_schema::KIND_ANY:
|
||||
return common_chat_schema::type_set::all();
|
||||
case common_chat_schema::KIND_NULL:
|
||||
return { common_chat_schema::TYPE_NULL };
|
||||
case common_chat_schema::KIND_BOOLEAN:
|
||||
return { common_chat_schema::TYPE_BOOLEAN };
|
||||
case common_chat_schema::KIND_NUMBER:
|
||||
return { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER };
|
||||
case common_chat_schema::KIND_INTEGER:
|
||||
return { common_chat_schema::TYPE_INTEGER };
|
||||
case common_chat_schema::KIND_STRING:
|
||||
return { common_chat_schema::TYPE_STRING };
|
||||
case common_chat_schema::KIND_ARRAY:
|
||||
case common_chat_schema::KIND_TUPLE:
|
||||
return { common_chat_schema::TYPE_ARRAY };
|
||||
case common_chat_schema::KIND_OBJECT:
|
||||
return { common_chat_schema::TYPE_OBJECT };
|
||||
case common_chat_schema::KIND_CONST:
|
||||
return { json_type(static_cast<const common_chat_schema_const &>(s).value) };
|
||||
case common_chat_schema::KIND_ENUM: {
|
||||
common_chat_schema::type_set types;
|
||||
for (const auto & value : static_cast<const common_chat_schema_enum &>(s).values) {
|
||||
types.add(json_type(value));
|
||||
}
|
||||
return types;
|
||||
}
|
||||
case common_chat_schema::KIND_REF: {
|
||||
const auto * target = static_cast<const common_chat_schema_ref &>(s).target;
|
||||
if (!target || !visited.insert(target).second) {
|
||||
// a cycle contributes no type, to be safe
|
||||
return {};
|
||||
}
|
||||
auto types = value_types_impl(*target, visited);
|
||||
visited.erase(target);
|
||||
return types;
|
||||
}
|
||||
case common_chat_schema::KIND_ANY_OF: {
|
||||
common_chat_schema::type_set types;
|
||||
for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
|
||||
types |= value_types_impl(*child, visited);
|
||||
}
|
||||
return types;
|
||||
}
|
||||
case common_chat_schema::KIND_ALL_OF: {
|
||||
auto types = common_chat_schema::type_set::all();
|
||||
for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
|
||||
types &= value_types_impl(*child, visited);
|
||||
}
|
||||
return types;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
common_chat_schema::type_set common_chat_schema::value_types() const {
|
||||
std::unordered_set<const common_chat_schema *> visited;
|
||||
return value_types_impl(*this, visited);
|
||||
}
|
||||
|
||||
static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
|
||||
switch (s.kind()) {
|
||||
case common_chat_schema::KIND_STRING:
|
||||
return true;
|
||||
case common_chat_schema::KIND_CONST:
|
||||
return static_cast<const common_chat_schema_const &>(s).value.is_string();
|
||||
case common_chat_schema::KIND_ENUM:
|
||||
for (const auto & v : static_cast<const common_chat_schema_enum &>(s).values) {
|
||||
if (v.is_string()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
case common_chat_schema::KIND_REF: {
|
||||
// a cycle is taken as not a string, to be safe
|
||||
const auto * target = static_cast<const common_chat_schema_ref &>(s).target;
|
||||
if (!target || !visited.insert(target).second) {
|
||||
return false;
|
||||
}
|
||||
bool result = may_be_string_impl(*target, visited);
|
||||
visited.erase(target);
|
||||
return result;
|
||||
}
|
||||
case common_chat_schema::KIND_ANY_OF:
|
||||
for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
|
||||
if (may_be_string_impl(*child, visited)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
case common_chat_schema::KIND_ALL_OF: {
|
||||
// every child must allow a string, an any child constrains nothing
|
||||
bool any_string = false;
|
||||
for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
|
||||
if (child->kind() == common_chat_schema::KIND_ANY) {
|
||||
continue;
|
||||
}
|
||||
if (!may_be_string_impl(*child, visited)) {
|
||||
return false;
|
||||
}
|
||||
any_string = true;
|
||||
}
|
||||
return any_string;
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool common_chat_schema::may_be_string() const {
|
||||
std::unordered_set<const common_chat_schema *> visited;
|
||||
return may_be_string_impl(*this, visited);
|
||||
}
|
||||
|
||||
const char * common_chat_schema::kind_name(node_kind kind) {
|
||||
switch (kind) {
|
||||
case KIND_ANY: return "any";
|
||||
case KIND_REF: return "ref";
|
||||
case KIND_ANY_OF: return "anyOf";
|
||||
case KIND_ALL_OF: return "allOf";
|
||||
case KIND_CONST: return "const";
|
||||
case KIND_ENUM: return "enum";
|
||||
case KIND_NULL: return "null";
|
||||
case KIND_BOOLEAN: return "boolean";
|
||||
case KIND_NUMBER: return "number";
|
||||
case KIND_INTEGER: return "integer";
|
||||
case KIND_STRING: return "string";
|
||||
case KIND_ARRAY: return "array";
|
||||
case KIND_TUPLE: return "tuple";
|
||||
case KIND_OBJECT: return "object";
|
||||
}
|
||||
return "?";
|
||||
}
|
||||
|
||||
const char * common_chat_schema::type_name(value_type type) {
|
||||
switch (type) {
|
||||
case TYPE_NULL: return "null";
|
||||
case TYPE_BOOLEAN: return "boolean";
|
||||
case TYPE_NUMBER: return "number";
|
||||
case TYPE_INTEGER: return "integer";
|
||||
case TYPE_STRING: return "string";
|
||||
case TYPE_ARRAY: return "array";
|
||||
case TYPE_OBJECT: return "object";
|
||||
}
|
||||
return "?";
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
#pragma once
|
||||
|
||||
#include "json.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <initializer_list>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// JSON schema, covering the subset that json_schema_to_grammar() can convert.
|
||||
|
||||
struct common_chat_schema {
|
||||
enum node_kind {
|
||||
KIND_ANY,
|
||||
KIND_REF,
|
||||
KIND_ANY_OF,
|
||||
KIND_ALL_OF,
|
||||
KIND_CONST,
|
||||
KIND_ENUM,
|
||||
KIND_NULL,
|
||||
KIND_BOOLEAN,
|
||||
KIND_NUMBER,
|
||||
KIND_INTEGER,
|
||||
KIND_STRING,
|
||||
KIND_ARRAY,
|
||||
KIND_TUPLE,
|
||||
KIND_OBJECT,
|
||||
};
|
||||
|
||||
enum value_type {
|
||||
TYPE_NULL,
|
||||
TYPE_BOOLEAN,
|
||||
TYPE_NUMBER,
|
||||
TYPE_INTEGER,
|
||||
TYPE_STRING,
|
||||
TYPE_ARRAY,
|
||||
TYPE_OBJECT,
|
||||
};
|
||||
|
||||
enum string_format {
|
||||
FORMAT_NONE,
|
||||
FORMAT_UUID, // uuid, uuid1 .. uuid5
|
||||
FORMAT_DATE,
|
||||
FORMAT_TIME,
|
||||
FORMAT_DATE_TIME,
|
||||
};
|
||||
|
||||
class type_set {
|
||||
uint32_t mask_ = 0;
|
||||
|
||||
public:
|
||||
type_set() = default;
|
||||
type_set(std::initializer_list<value_type> types) {
|
||||
for (auto type : types) {
|
||||
add(type);
|
||||
}
|
||||
}
|
||||
|
||||
static type_set all() {
|
||||
return { TYPE_NULL, TYPE_BOOLEAN, TYPE_NUMBER, TYPE_INTEGER, TYPE_STRING, TYPE_ARRAY, TYPE_OBJECT };
|
||||
}
|
||||
|
||||
void add(value_type type) { mask_ |= 1u << type; }
|
||||
|
||||
bool has(value_type type) const { return (mask_ & (1u << type)) != 0; }
|
||||
bool is_only(value_type type) const { return mask_ == (1u << type); }
|
||||
bool empty() const { return mask_ == 0; }
|
||||
|
||||
type_set & operator|=(const type_set & other) { mask_ |= other.mask_; return *this; }
|
||||
type_set & operator&=(const type_set & other) { mask_ &= other.mask_; return *this; }
|
||||
|
||||
bool operator==(const type_set & other) const { return mask_ == other.mask_; }
|
||||
bool operator!=(const type_set & other) const { return mask_ != other.mask_; }
|
||||
};
|
||||
|
||||
virtual ~common_chat_schema() = default;
|
||||
virtual node_kind kind() const = 0;
|
||||
|
||||
type_set value_types() const;
|
||||
|
||||
// Whether a value matching the schema may be a string, through any branch of it.
|
||||
bool may_be_string() const;
|
||||
|
||||
static const char * kind_name(node_kind kind);
|
||||
static const char * type_name(value_type type);
|
||||
};
|
||||
|
||||
using common_chat_schema_ptr = std::unique_ptr<common_chat_schema>;
|
||||
|
||||
struct common_chat_schema_any : common_chat_schema {
|
||||
node_kind kind() const override { return KIND_ANY; }
|
||||
};
|
||||
|
||||
// {"$ref": "#/..."}, only references into the same document are supported
|
||||
struct common_chat_schema_ref : common_chat_schema {
|
||||
std::string ref;
|
||||
const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs
|
||||
|
||||
explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {}
|
||||
|
||||
node_kind kind() const override { return KIND_REF; }
|
||||
};
|
||||
|
||||
// oneOf / anyOf, or a "type" array expanded to one alternative per type
|
||||
struct common_chat_schema_any_of : common_chat_schema {
|
||||
std::vector<common_chat_schema_ptr> children;
|
||||
|
||||
node_kind kind() const override { return KIND_ANY_OF; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_all_of : common_chat_schema {
|
||||
std::vector<common_chat_schema_ptr> children;
|
||||
|
||||
node_kind kind() const override { return KIND_ALL_OF; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_const : common_chat_schema {
|
||||
common_json value;
|
||||
|
||||
explicit common_chat_schema_const(common_json value) : value(std::move(value)) {}
|
||||
|
||||
node_kind kind() const override { return KIND_CONST; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_enum : common_chat_schema {
|
||||
std::vector<common_json> values;
|
||||
|
||||
node_kind kind() const override { return KIND_ENUM; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_null : common_chat_schema {
|
||||
node_kind kind() const override { return KIND_NULL; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_boolean : common_chat_schema {
|
||||
node_kind kind() const override { return KIND_BOOLEAN; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_number : common_chat_schema {
|
||||
node_kind kind() const override { return KIND_NUMBER; }
|
||||
};
|
||||
|
||||
// bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in
|
||||
struct common_chat_schema_integer : common_chat_schema {
|
||||
int64_t minimum = INT64_MIN; // INT64_MIN for unbounded
|
||||
int64_t maximum = INT64_MAX; // INT64_MAX for unbounded
|
||||
|
||||
node_kind kind() const override { return KIND_INTEGER; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_string : common_chat_schema {
|
||||
std::string pattern; // empty when absent
|
||||
string_format format = FORMAT_NONE;
|
||||
int min_length = 0;
|
||||
int max_length = -1; // -1 for unbounded
|
||||
|
||||
node_kind kind() const override { return KIND_STRING; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_array : common_chat_schema {
|
||||
common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent
|
||||
int min_items = 0;
|
||||
int max_items = -1; // -1 for unbounded
|
||||
|
||||
node_kind kind() const override { return KIND_ARRAY; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_tuple : common_chat_schema {
|
||||
std::vector<common_chat_schema_ptr> items;
|
||||
|
||||
node_kind kind() const override { return KIND_TUPLE; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_property {
|
||||
std::string name;
|
||||
common_chat_schema_ptr schema;
|
||||
bool required = false;
|
||||
};
|
||||
|
||||
struct common_chat_schema_object : common_chat_schema {
|
||||
std::vector<common_chat_schema_property> properties; // in schema order
|
||||
common_chat_schema_ptr additional_properties; // null when not allowed
|
||||
|
||||
node_kind kind() const override { return KIND_OBJECT; }
|
||||
};
|
||||
|
||||
struct common_chat_schema_document {
|
||||
common_chat_schema_ptr root;
|
||||
std::map<std::string, common_chat_schema_ptr> refs;
|
||||
};
|
||||
|
||||
// A document shared by the PEG parsers built from its nodes, which it keeps alive
|
||||
using common_chat_schema_document_ptr = std::shared_ptr<const common_chat_schema_document>;
|
||||
|
||||
// Throws std::runtime_error when the schema falls outside the supported subset.
|
||||
common_chat_schema_document common_chat_schema_from_json(const common_json & schema);
|
||||
+63
-13
@@ -37,6 +37,16 @@ void common_log_set_verbosity_thold(int verbosity) {
|
||||
common_log_verbosity_thold = verbosity;
|
||||
}
|
||||
|
||||
static bool common_log_jsonl = false;
|
||||
|
||||
bool common_log_get_jsonl(void) {
|
||||
return common_log_jsonl;
|
||||
}
|
||||
|
||||
void common_log_set_jsonl(bool jsonl) {
|
||||
common_log_jsonl = jsonl;
|
||||
}
|
||||
|
||||
static int64_t t_us() {
|
||||
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
@@ -87,6 +97,7 @@ struct common_log_entry {
|
||||
bool is_end { false }; // signals the worker thread to stop
|
||||
bool prefix { false };
|
||||
bool jsonl { false };
|
||||
bool is_json { false }; // msg already holds a serialized JSON object
|
||||
|
||||
common_log_entry(size_t size = 256) : msg(size) { }
|
||||
|
||||
@@ -107,6 +118,12 @@ struct common_log_entry {
|
||||
}
|
||||
|
||||
if (jsonl) {
|
||||
if (is_json) {
|
||||
fprintf(fcur, "%s\n", msg.data());
|
||||
fflush(fcur);
|
||||
return;
|
||||
}
|
||||
|
||||
common_json obj = {
|
||||
{"type", "log"},
|
||||
{"time", timestamp},
|
||||
@@ -156,7 +173,6 @@ struct common_log {
|
||||
file = nullptr;
|
||||
prefix = false;
|
||||
timestamps = false;
|
||||
jsonl = false;
|
||||
running = false;
|
||||
t_start = t_us();
|
||||
|
||||
@@ -184,7 +200,6 @@ private:
|
||||
|
||||
bool prefix;
|
||||
bool timestamps;
|
||||
bool jsonl;
|
||||
bool running;
|
||||
|
||||
int64_t t_start;
|
||||
@@ -273,7 +288,8 @@ public:
|
||||
entry.is_end = false;
|
||||
entry.level = level;
|
||||
entry.prefix = prefix;
|
||||
entry.jsonl = jsonl;
|
||||
entry.jsonl = common_log_jsonl;
|
||||
entry.is_json = false;
|
||||
entry.timestamp = 0;
|
||||
if (timestamps) {
|
||||
entry.timestamp = t_us() - t_start;
|
||||
@@ -283,6 +299,42 @@ public:
|
||||
cv_new.notify_one();
|
||||
}
|
||||
|
||||
void add_json(const char * type, const common_json & obj) {
|
||||
const common_json full = {
|
||||
{"type", type},
|
||||
{"data", obj},
|
||||
};
|
||||
|
||||
const std::string text = full.dump_safe();
|
||||
|
||||
std::unique_lock<std::mutex> lock(mtx);
|
||||
|
||||
// block if the queue is full
|
||||
cv_full.wait(lock, [this]() { return !running || !is_full(); });
|
||||
|
||||
if (!running) {
|
||||
// discard messages while the worker thread is paused
|
||||
return;
|
||||
}
|
||||
|
||||
auto & entry = queue[tail];
|
||||
|
||||
if (entry.msg.size() < text.size() + 1) {
|
||||
entry.msg.resize(text.size() + 1);
|
||||
}
|
||||
memcpy(entry.msg.data(), text.c_str(), text.size() + 1);
|
||||
|
||||
entry.is_end = false;
|
||||
entry.level = GGML_LOG_LEVEL_NONE;
|
||||
entry.prefix = false;
|
||||
entry.jsonl = true;
|
||||
entry.is_json = true;
|
||||
entry.timestamp = 0;
|
||||
|
||||
tail = (tail + 1) % queue.size();
|
||||
cv_new.notify_one();
|
||||
}
|
||||
|
||||
void resume() {
|
||||
std::lock_guard<std::mutex> lock(mtx);
|
||||
|
||||
@@ -388,12 +440,6 @@ public:
|
||||
|
||||
this->timestamps = timestamps;
|
||||
}
|
||||
|
||||
void set_jsonl(bool jsonl) {
|
||||
std::lock_guard<std::mutex> lock(mtx);
|
||||
|
||||
this->jsonl = jsonl;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
@@ -440,6 +486,14 @@ void common_log_add(struct common_log * log, enum ggml_log_level level, const ch
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
void common_log_add_json(struct common_log * log, const char * type, const common_json & obj) {
|
||||
if (!common_log_jsonl) {
|
||||
return;
|
||||
}
|
||||
|
||||
log->add_json(type, obj);
|
||||
}
|
||||
|
||||
void common_log_set_file(struct common_log * log, const char * file) {
|
||||
log->set_file(file);
|
||||
}
|
||||
@@ -467,10 +521,6 @@ void common_log_set_timestamps(struct common_log * log, bool timestamps) {
|
||||
log->set_timestamps(timestamps);
|
||||
}
|
||||
|
||||
void common_log_set_jsonl(struct common_log * log, bool jsonl) {
|
||||
log->set_jsonl(jsonl);
|
||||
}
|
||||
|
||||
void common_log_flush(struct common_log * log) {
|
||||
log->pause();
|
||||
log->resume();
|
||||
|
||||
+18
-1
@@ -43,6 +43,10 @@ int common_log_get_verbosity_thold(void);
|
||||
|
||||
void common_log_set_verbosity_thold(int verbosity); // not thread-safe
|
||||
|
||||
bool common_log_get_jsonl(void);
|
||||
|
||||
void common_log_set_jsonl(bool jsonl); // not thread-safe
|
||||
|
||||
int common_log_get_verbosity(enum ggml_log_level level);
|
||||
|
||||
void common_log_default_callback(enum ggml_log_level level, const char * text, void * user_data);
|
||||
@@ -91,7 +95,6 @@ void common_log_set_file (struct common_log * log, const char * file); // n
|
||||
void common_log_set_colors (struct common_log * log, log_colors colors); // not thread-safe
|
||||
void common_log_set_prefix (struct common_log * log, bool prefix); // whether to output prefix to each log
|
||||
void common_log_set_timestamps(struct common_log * log, bool timestamps); // whether to output timestamps in the prefix
|
||||
void common_log_set_jsonl (struct common_log * log, bool jsonl); // print each log as a JSON object on one line, not thread-safe
|
||||
void common_log_flush (struct common_log * log); // flush all pending log messages
|
||||
|
||||
// helper macros for logging
|
||||
@@ -127,3 +130,17 @@ void common_log_flush (struct common_log * log); // f
|
||||
#define LOG_WRNV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_WARN, verbosity, __VA_ARGS__)
|
||||
#define LOG_ERRV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, verbosity, __VA_ARGS__)
|
||||
#define LOG_CNTV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_CONT, verbosity, __VA_ARGS__)
|
||||
|
||||
class common_json; // defined in common/json.h
|
||||
|
||||
// helper allows different types of json output
|
||||
// no-op if --log-jsonl is not set
|
||||
void common_log_add_json(struct common_log * log, const char * type, const common_json & data);
|
||||
|
||||
// will only print if --log-jsonl is set
|
||||
#define LOG_JSON(type, data) \
|
||||
do { \
|
||||
if (common_log_get_jsonl()) { \
|
||||
common_log_add_json(common_log_main(), type, data); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
@@ -129,15 +129,6 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_template
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -149,39 +149,28 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
const auto & props = params.contains("properties") ? params.at("properties") : json::object();
|
||||
|
||||
std::set<std::string> required;
|
||||
if (params.contains("required")) {
|
||||
required = params.at("required").get<std::set<std::string>>();
|
||||
}
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
std::vector<common_peg_parser> required_parsers;
|
||||
std::vector<common_peg_parser> optional_parsers;
|
||||
for (const auto & [param_name, param_schema] : props.items()) {
|
||||
bool is_required = required.find(param_name) != required.end();
|
||||
bool is_string = schema_info.resolves_to_string(param_schema);
|
||||
foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
|
||||
bool is_string = param.schema->may_be_string();
|
||||
|
||||
auto arg = p.tool_arg(
|
||||
p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) +
|
||||
p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param.name)) +
|
||||
p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
|
||||
(is_string ?
|
||||
p.tool_arg_string_value(p.until(PARAM_END)) :
|
||||
p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema",
|
||||
param_schema, false))) +
|
||||
p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param.name + "-schema",
|
||||
doc, *param.schema))) +
|
||||
p.tool_arg_close(p.literal(PARAM_END)));
|
||||
|
||||
auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
|
||||
if (is_required) {
|
||||
auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg);
|
||||
if (param.required) {
|
||||
required_parsers.push_back(named_arg);
|
||||
} else {
|
||||
optional_parsers.push_back(named_arg);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
common_peg_parser args_seq = p.eps();
|
||||
for (size_t i = 0; i < required_parsers.size(); i++) {
|
||||
@@ -266,15 +255,6 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && !require_tools;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & schema = function.at("parameters");
|
||||
const auto schema = common_chat_tool_parameters(function);
|
||||
|
||||
// Tool format: >>>function_name\n{json_args}
|
||||
auto tool_parser = p.tool(
|
||||
@@ -82,11 +82,6 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te
|
||||
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -291,15 +291,6 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ common_chat_params common_chat_params_init_gigachat_v3(
|
||||
for (const auto & tool : inputs.tools) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & schema = function.at("parameters");
|
||||
const auto schema = common_chat_tool_parameters(function);
|
||||
|
||||
auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\"");
|
||||
auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));
|
||||
@@ -65,11 +65,6 @@ common_chat_params common_chat_params_init_gigachat_v3(
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & params = function.at("parameters");
|
||||
const auto params = common_chat_tool_parameters(function);
|
||||
|
||||
auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name));
|
||||
auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
|
||||
@@ -143,15 +143,6 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template &
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & schema = function.at("parameters");
|
||||
const auto schema = common_chat_tool_parameters(function);
|
||||
|
||||
// Match: functions.<name>:<digits>
|
||||
// Capture the full call id (functions.<name>:<digits>) using tool_id tag
|
||||
@@ -116,11 +116,6 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template &
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -98,7 +98,7 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template &
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const json schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
const json schema = common_chat_tool_parameters(function);
|
||||
|
||||
// arguments come one tag per key, with the JSON type in a type="..."
|
||||
// attribute. the type is taken from the tool schema instead, as it tells
|
||||
@@ -155,13 +155,6 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template &
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
if (function.contains("parameters")) {
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -98,15 +98,6 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template &
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
+19
-33
@@ -71,32 +71,27 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
const std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
std::vector<common_peg_parser> arg_rules;
|
||||
foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
|
||||
auto value_parser = p.eps();
|
||||
if (prop.schema->may_be_string()) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + name + "-arg-" + prop.name + "-schema", doc, *prop.schema)
|
||||
) + p.tool_arg_close(p.literal("</param>"));
|
||||
}
|
||||
|
||||
arg_rules.push_back(p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) +
|
||||
value_parser
|
||||
));
|
||||
});
|
||||
|
||||
auto args = p.eps();
|
||||
if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
auto arg_choice = p.choice();
|
||||
for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {
|
||||
auto value_parser = p.eps();
|
||||
if (schema_info.resolves_to_string(prop_schema)) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)
|
||||
) + p.tool_arg_close(p.literal("</param>"));
|
||||
}
|
||||
|
||||
auto arg_rule = p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) +
|
||||
value_parser
|
||||
);
|
||||
|
||||
arg_choice |= arg_rule;
|
||||
}
|
||||
args = p.zero_or_more(arg_choice + p.space());
|
||||
if (!arg_rules.empty()) {
|
||||
args = p.zero_or_more(p.choice(arg_rules) + p.space());
|
||||
}
|
||||
|
||||
auto tool_parser = p.tool(
|
||||
@@ -123,15 +118,6 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -84,29 +84,18 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
|
||||
auto alternatives_of = [](const json & schema) -> std::optional<json> {
|
||||
for (const auto * keyword : { "oneOf", "anyOf" }) {
|
||||
if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) {
|
||||
return schema.at(keyword);
|
||||
}
|
||||
}
|
||||
return std::nullopt;
|
||||
};
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
auto params = common_chat_tool_parameters(function);
|
||||
auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params));
|
||||
|
||||
// The template expands argument values recursively in XML (see the to_xml() macro)
|
||||
std::function<common_peg_parser(const json &, const std::string &, const std::string &)> value_of;
|
||||
std::function<common_peg_parser(const json &, const std::string &)> members_of;
|
||||
std::function<common_peg_parser(const common_chat_schema &, const std::string &, const std::string &)> value_of;
|
||||
std::function<common_peg_parser(const common_chat_schema_object &, const std::string &)> members_of;
|
||||
|
||||
auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) {
|
||||
auto element_of = [&](const std::string & tag, const common_chat_schema & schema, const std::string & rule_name) {
|
||||
const std::string close = NS + "</" + tag + ">";
|
||||
return p.rule(rule_name,
|
||||
p.tool_arg(
|
||||
@@ -117,69 +106,57 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
|
||||
value_of(schema, rule_name, close)));
|
||||
};
|
||||
|
||||
value_of = [&](const json & schema,
|
||||
value_of = [&](const common_chat_schema & schema,
|
||||
const std::string & rule_name,
|
||||
const std::string & close) -> common_peg_parser {
|
||||
auto close_tag = p.tool_arg_close(p.literal(close));
|
||||
|
||||
// A string accepts anything, so a union with a string alternative is a string
|
||||
if (schema_info.resolves_to_string(schema)) {
|
||||
if (schema.may_be_string()) {
|
||||
return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close);
|
||||
}
|
||||
|
||||
if (auto alternatives = alternatives_of(schema)) {
|
||||
if (schema.kind() == common_chat_schema::KIND_ANY_OF) {
|
||||
std::vector<common_peg_parser> choices;
|
||||
|
||||
size_t index = 0;
|
||||
for (const auto & alternative : *alternatives) {
|
||||
for (const auto & alternative : static_cast<const common_chat_schema_any_of &>(schema).children) {
|
||||
const std::string alt_name = rule_name + "-" + std::to_string(index++);
|
||||
|
||||
// There is a risk that this breaks streaming deltas, but that's a risk we
|
||||
// assume to provide tool arg streaming.
|
||||
choices.push_back(value_of(alternative, alt_name, close));
|
||||
choices.push_back(value_of(*alternative, alt_name, close));
|
||||
}
|
||||
|
||||
return p.choice(choices);
|
||||
}
|
||||
|
||||
const std::string type = schema.contains("type") && schema.at("type").is_string()
|
||||
? schema.at("type").get<std::string>()
|
||||
: "";
|
||||
|
||||
if (type == "object" && schema.contains("properties")) {
|
||||
return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag;
|
||||
if (schema.kind() == common_chat_schema::KIND_OBJECT) {
|
||||
const auto & object = static_cast<const common_chat_schema_object &>(schema);
|
||||
if (!object.properties.empty()) {
|
||||
return p.tag(mm3::TOOL_ARG_OBJECT, members_of(object, rule_name)) + p.space() + close_tag;
|
||||
}
|
||||
}
|
||||
|
||||
if (type == "array" && schema.contains("items")) {
|
||||
if (schema.kind() == common_chat_schema::KIND_ARRAY) {
|
||||
const std::string item_close = NS + "</item>";
|
||||
auto item = p.rule(rule_name + "-item",
|
||||
p.tag(mm3::TOOL_ARG_ITEM,
|
||||
p.literal(NS + "<item>") +
|
||||
value_of(schema.at("items"), rule_name + "-item", item_close)));
|
||||
value_of(*static_cast<const common_chat_schema_array &>(schema).items, rule_name + "-item", item_close)));
|
||||
return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag;
|
||||
}
|
||||
|
||||
return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag;
|
||||
return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, schema)) + close_tag;
|
||||
};
|
||||
|
||||
// Required properties in schema order, then any number of optional ones in any order.
|
||||
members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser {
|
||||
const auto & props = schema.at("properties");
|
||||
|
||||
std::set<std::string> required;
|
||||
if (schema.contains("required")) {
|
||||
required = schema.at("required").get<std::set<std::string>>();
|
||||
}
|
||||
|
||||
members_of = [&](const common_chat_schema_object & object, const std::string & rule_prefix) -> common_peg_parser {
|
||||
std::vector<common_peg_parser> required_elements;
|
||||
std::vector<common_peg_parser> optional_elements;
|
||||
for (const auto & [key, key_schema] : props.items()) {
|
||||
auto element = element_of(key, key_schema, rule_prefix + "-" + key);
|
||||
if (required.find(key) != required.end()) {
|
||||
required_elements.push_back(element);
|
||||
} else {
|
||||
optional_elements.push_back(element);
|
||||
}
|
||||
for (const auto & prop : object.properties) {
|
||||
auto element = element_of(prop.name, *prop.schema, rule_prefix + "-" + prop.name);
|
||||
(prop.required ? required_elements : optional_elements).push_back(element);
|
||||
}
|
||||
|
||||
common_peg_parser members = p.eps();
|
||||
@@ -201,8 +178,10 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
|
||||
return members;
|
||||
};
|
||||
|
||||
common_peg_parser invoke_body =
|
||||
params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps();
|
||||
common_peg_parser invoke_body = p.eps();
|
||||
if (doc->root->kind() == common_chat_schema::KIND_OBJECT) {
|
||||
invoke_body = members_of(static_cast<const common_chat_schema_object &>(*doc->root), "tool-" + name + "-arg");
|
||||
}
|
||||
|
||||
auto func_parser = p.tool(
|
||||
p.tool_open(p.literal(NS + "<invoke name=\"") +
|
||||
@@ -238,15 +217,6 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -89,7 +89,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & schema = function.at("parameters");
|
||||
const auto schema = common_chat_tool_parameters(function);
|
||||
|
||||
tool_choice |=
|
||||
p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") +
|
||||
@@ -114,15 +114,6 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.at("parameters");
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
@@ -74,31 +74,26 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
const std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
std::vector<common_peg_parser> arg_rules;
|
||||
foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
|
||||
auto value_parser = p.eps();
|
||||
if (prop.schema->may_be_string()) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + name + "-arg-" + prop.name + "-schema", doc, *prop.schema))
|
||||
+ p.tool_arg_close(p.literal("</atem:parameter>"));
|
||||
}
|
||||
|
||||
arg_rules.push_back(p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) +
|
||||
value_parser));
|
||||
});
|
||||
|
||||
auto args = p.eps();
|
||||
if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
auto arg_choice = p.choice();
|
||||
for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {
|
||||
auto value_parser = p.eps();
|
||||
if (schema_info.resolves_to_string(prop_schema)) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false))
|
||||
+ p.tool_arg_close(p.literal("</atem:parameter>"));
|
||||
}
|
||||
|
||||
auto arg_rule = p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) +
|
||||
value_parser);
|
||||
|
||||
arg_choice |= arg_rule;
|
||||
}
|
||||
args = p.zero_or_more(arg_choice + p.space());
|
||||
if (!arg_rules.empty()) {
|
||||
args = p.zero_or_more(p.choice(arg_rules) + p.space());
|
||||
}
|
||||
|
||||
auto tool_parser = p.tool(
|
||||
@@ -131,11 +126,6 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
data.grammar_triggers = {
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
|
||||
#include "log.h"
|
||||
|
||||
#include <set>
|
||||
|
||||
void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
|
||||
for (const auto & tool : tools) {
|
||||
if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {
|
||||
@@ -14,21 +12,14 @@ void foreach_function(const json & tools, const std::function<void(const json &)
|
||||
}
|
||||
}
|
||||
|
||||
void foreach_parameter(const json & function, const std::function<void(const std::string &, const json &, bool)> & fn) {
|
||||
if (!function.contains("parameters") || !function.at("parameters").is_object()) {
|
||||
void foreach_parameter(const json & function, const std::function<void(const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn) {
|
||||
auto params = common_chat_tool_parameters(function);
|
||||
auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params));
|
||||
const auto * object = dynamic_cast<const common_chat_schema_object *>(doc->root.get());
|
||||
if (!object) {
|
||||
return;
|
||||
}
|
||||
const auto & params = function.at("parameters");
|
||||
if (!params.contains("properties") || !params.at("properties").is_object()) {
|
||||
return;
|
||||
}
|
||||
const auto & props = params.at("properties");
|
||||
std::set<std::string> required;
|
||||
if (params.contains("required") && params.at("required").is_array()) {
|
||||
required = params.at("required").get<std::set<std::string>>();
|
||||
}
|
||||
for (const auto & [name, prop] : props.items()) {
|
||||
bool is_required = (required.find(name) != required.end());
|
||||
fn(name, prop, is_required);
|
||||
for (const auto & prop : object->properties) {
|
||||
fn(prop, doc);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,8 +20,8 @@ using json = common_json;
|
||||
// iterate over the function tools of an OpenAI-style tools array
|
||||
void foreach_function(const json & tools, const std::function<void(const json &)> & fn);
|
||||
|
||||
// iterate over the parameters of a function tool, flagging the ones listed as required
|
||||
void foreach_parameter(const json & function, const std::function<void(const std::string &, const json &, bool)> & fn);
|
||||
// iterate over the parameters of a function tool, with the document that owns them
|
||||
void foreach_parameter(const json & function, const std::function<void(const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn);
|
||||
|
||||
// render a template; the override arguments let a parser feed in messages, tools or context it has rewritten
|
||||
std::string common_chat_template_direct_apply_impl(
|
||||
|
||||
@@ -23,8 +23,9 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
if (supports_reasoning) {
|
||||
data.thinking_start_tag = "<think>";
|
||||
// Support both </think> and <tool_call> as reasoning end sequences.
|
||||
// The newline variant comes first so it is included in the forced message
|
||||
// <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model
|
||||
data.thinking_end_tags = { "</think>", "<tool_call>" };
|
||||
data.thinking_end_tags = { "\n</think>", "</think>", "<tool_call>" };
|
||||
data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });
|
||||
}
|
||||
|
||||
@@ -93,28 +94,49 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto parameters = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(parameters);
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
std::vector<common_peg_parser> required_args;
|
||||
std::vector<common_peg_parser> optional_args;
|
||||
|
||||
foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) {
|
||||
auto rule_name = "tool-" + name + "-arg-" + param_name;
|
||||
foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
|
||||
auto rule_name = "tool-" + name + "-arg-" + param.name;
|
||||
|
||||
auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param_name)) + ">\n");
|
||||
auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param.name)) + ">\n");
|
||||
|
||||
auto arg_value = schema_info.resolves_to_string(param_schema) ?
|
||||
arg_string :
|
||||
p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close;
|
||||
auto types = param.schema->value_types();
|
||||
|
||||
auto arg_value = p.eps();
|
||||
if (!types.has(common_chat_schema::TYPE_STRING)) {
|
||||
arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
|
||||
} else if (types.is_only(common_chat_schema::TYPE_STRING)) {
|
||||
arg_value = arg_string;
|
||||
} else {
|
||||
// The string alternative accepts any text, so the grammar only keeps the raw string
|
||||
// rule. The parser still tries the JSON alternatives first to type the value.
|
||||
auto json_value = p.choice();
|
||||
if (types.has(common_chat_schema::TYPE_OBJECT)) {
|
||||
json_value |= p.json_object();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_ARRAY)) {
|
||||
json_value |= p.json_array();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) {
|
||||
json_value |= p.json_number();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_BOOLEAN)) {
|
||||
json_value |= p.json_bool();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_NULL)) {
|
||||
json_value |= p.json_null();
|
||||
}
|
||||
arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "xml-arg-string");
|
||||
}
|
||||
|
||||
auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));
|
||||
|
||||
(is_required ? required_args : optional_args).push_back(arg_rule);
|
||||
(param.required ? required_args : optional_args).push_back(arg_rule);
|
||||
});
|
||||
|
||||
// Accept required arguments in any order, as Qwen does not always adhere to the
|
||||
@@ -158,15 +180,6 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
|
||||
+11
-31
@@ -953,7 +953,7 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id
|
||||
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
|
||||
return "Until(" + string_join(p.delimiters, " | ") + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
|
||||
return "Schema(" + dump_impl(p.child, visited) + ", " + (p.schema ? p.schema->dump() : "null") + ")";
|
||||
return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
|
||||
return "Rule(" + p.name + ", " + dump_impl(p.child, visited) + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
|
||||
@@ -1119,8 +1119,13 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes,
|
||||
return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max}));
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw) {
|
||||
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::move(doc), &node, raw}));
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) {
|
||||
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<common_json>(schema), raw}));
|
||||
auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(schema));
|
||||
return this->schema(p, name, doc, *doc->root, raw);
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) {
|
||||
@@ -1573,30 +1578,9 @@ static std::set<std::string> collect_reachable_rules(
|
||||
|
||||
// GBNF generation implementation
|
||||
void common_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const {
|
||||
// A raw string value is parsed by the child rather than constrained by the schema
|
||||
auto schema_delegates = [](const common_peg_schema_parser & s) -> bool {
|
||||
if (!s.schema) {
|
||||
return true;
|
||||
}
|
||||
if (s.raw && s.schema->contains("type")) {
|
||||
const auto & type_val = s.schema->at("type");
|
||||
if (type_val.is_string() && type_val == "string") {
|
||||
return true;
|
||||
}
|
||||
// Handle nullable types like ["string", "null"] - delegate when the
|
||||
// non-null type is string, since the tagged format uses raw text
|
||||
if (type_val.is_array()) {
|
||||
for (const auto & t : type_val) {
|
||||
if (t.is_string() && t.get<std::string>() != "null") {
|
||||
return t.get<std::string>() == "string";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Delegate for enum schemas in raw mode - enum values are literal strings
|
||||
if (s.raw && !s.schema->contains("type") && s.schema->contains("enum")) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
return !s.node || (s.raw && s.node->may_be_string());
|
||||
};
|
||||
|
||||
// Unwrap the parser so we can properly check if it's a sequence or choice
|
||||
@@ -1731,7 +1715,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
|
||||
if (schema_delegates(p)) {
|
||||
return to_gbnf(p.child);
|
||||
}
|
||||
return builder.add_schema(p.name, *p.schema);
|
||||
return builder.add_schema(p.name, *p.node);
|
||||
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
|
||||
return p.name;
|
||||
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
|
||||
@@ -1859,7 +1843,6 @@ static common_json serialize_parser_variant(const common_peg_parser_variant & va
|
||||
{"type", "schema"},
|
||||
{"child", p.child},
|
||||
{"name", p.name},
|
||||
{"schema", p.schema ? *p.schema : json(nullptr)},
|
||||
{"raw", p.raw}
|
||||
};
|
||||
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
|
||||
@@ -1999,15 +1982,12 @@ static common_peg_parser_variant deserialize_parser_variant(const common_json &
|
||||
return common_peg_until_parser{j["delimiters"].get<std::vector<std::string>>()};
|
||||
}
|
||||
if (type == "schema") {
|
||||
if (!j.contains("child") || !j.contains("name") || !j.contains("schema") || !j.contains("raw")) {
|
||||
if (!j.contains("child") || !j.contains("name") || !j.contains("raw")) {
|
||||
throw std::runtime_error("schema parser missing required fields");
|
||||
}
|
||||
common_peg_schema_parser parser;
|
||||
parser.child = j["child"].get<common_peg_parser_id>();
|
||||
parser.name = j["name"];
|
||||
if (!j["schema"].is_null()) {
|
||||
parser.schema = std::make_shared<common_json>(j["schema"]);
|
||||
}
|
||||
parser.raw = j["raw"].get<bool>();
|
||||
return parser;
|
||||
}
|
||||
|
||||
+7
-3
@@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "json-schema.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <memory>
|
||||
@@ -245,7 +246,8 @@ struct common_peg_until_parser {
|
||||
struct common_peg_schema_parser {
|
||||
common_peg_parser_id child;
|
||||
std::string name;
|
||||
std::shared_ptr<common_json> schema;
|
||||
common_chat_schema_document_ptr doc; // owns node
|
||||
const common_chat_schema * node = nullptr;
|
||||
|
||||
// Indicates if the GBNF should accept a raw string that matches the schema.
|
||||
bool raw;
|
||||
@@ -488,8 +490,10 @@ class common_peg_parser_builder {
|
||||
// A marker, i.e. text delimited by a pair of <> or []
|
||||
common_peg_parser marker();
|
||||
|
||||
// Wraps a parser with JSON schema metadata for grammar generation.
|
||||
// Used internally to convert JSON schemas to GBNF grammar rules.
|
||||
// Wraps a parser with the schema its GBNF is generated from, a node of the document that owns it
|
||||
common_peg_parser schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw = false);
|
||||
|
||||
// Parses the JSON schema into a document of its own
|
||||
common_peg_parser schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw = false);
|
||||
|
||||
// Creates a named rule, stores it in the grammar, and returns a ref.
|
||||
|
||||
@@ -123,6 +123,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"HunYuanDenseV1ForCausalLM": "hunyuan",
|
||||
"HunYuanMoEV1ForCausalLM": "hunyuan",
|
||||
"HunYuanVLForConditionalGeneration": "hunyuan",
|
||||
"HrmTextForCausalLM": "hrm_text",
|
||||
"HYV3ForCausalLM": "hunyuan",
|
||||
"HYV4ForCausalLM": "hy_v4",
|
||||
"IQuestCoderForCausalLM": "llama",
|
||||
@@ -168,6 +169,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Mamba2ForCausalLM": "mamba",
|
||||
"MambaForCausalLM": "mamba",
|
||||
"MambaLMHeadModel": "mamba",
|
||||
"MapleForCausalLM": "maple",
|
||||
"MellumForCausalLM": "mellum",
|
||||
"MiMoV2FlashForCausalLM": "mimo",
|
||||
"MiMoV2ForCausalLM": "mimo",
|
||||
|
||||
@@ -1633,6 +1633,9 @@ class TextModel(ModelBase):
|
||||
if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":
|
||||
# ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B
|
||||
res = "lfm2"
|
||||
if chkhsh == "846deafc5b0fa786186fa4ae6c7b49903cf2f1d1895bdb80b9120d60be135252":
|
||||
# ref: https://huggingface.co/danish-foundation-models/DFM-Mimir
|
||||
res = "gemma4"
|
||||
if chkhsh == "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed":
|
||||
# ref: https://huggingface.co/XHToken/Spark-X2.5-1.7B
|
||||
res = "spark2_5"
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from typing import Iterable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("HrmTextForCausalLM")
|
||||
@ModelBase.example("danish-foundation-models/DFM-Mimir")
|
||||
class HrmTextModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.HRM_TEXT
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# training-style configs store the per-stack count in num_hidden_layers,
|
||||
# transformers-style configs keep it in num_layers_per_stack
|
||||
self.layers_per_stack = self.hparams.get("num_layers_per_stack") or self.hparams["num_hidden_layers"]
|
||||
self.h_cycles = self.hparams["H_cycles"]
|
||||
self.l_cycles = self.hparams["L_cycles"]
|
||||
|
||||
# block_count is the expanded cache-slot count; the file only holds
|
||||
# 2 * layers_per_stack physical blocks
|
||||
self.block_count = self.layers_per_stack * self.h_cycles * (self.l_cycles + 1)
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, 2 * self.layers_per_stack)
|
||||
|
||||
def set_vocab(self):
|
||||
self._set_vocab_gpt2()
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
head_dim = self.hparams.get("head_dim") or self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
|
||||
self.gguf_writer.add_rope_dimension_count(head_dim)
|
||||
self.gguf_writer.add_embedding_scale(self.hparams["embedding_scale"])
|
||||
self.gguf_writer.add_hrm_layers_per_stack(self.layers_per_stack)
|
||||
self.gguf_writer.add_hrm_h_cycles(self.h_cycles)
|
||||
self.gguf_writer.add_hrm_l_cycles(self.l_cycles)
|
||||
self.gguf_writer.add_hrm_prefix_lm(bool(self.hparams.get("prefix_lm", False)))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name == "model.embed_tokens.weight":
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), data_torch
|
||||
return
|
||||
if name == "lm_head.weight":
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch
|
||||
return
|
||||
if name == "model.z_L_init":
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.HRM_Z_L_INIT, suffix=""), data_torch
|
||||
return
|
||||
|
||||
match = re.fullmatch(r"model\.([LH])_module\.layers\.(\d+)\.(.+)", name)
|
||||
if match is None:
|
||||
raise ValueError(f"can not map tensor: {name}")
|
||||
|
||||
stack, layer_s, tensor_name = match.groups()
|
||||
# the L stack occupies blocks [0, layers_per_stack), the H stack follows it
|
||||
layer_idx = int(layer_s) + (self.layers_per_stack if stack == "H" else 0)
|
||||
|
||||
if tensor_name == "attn.gqkv_proj.weight":
|
||||
gate, q, k, v = data_torch.chunk(4, dim=0)
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_GATE, layer_idx), gate.contiguous()
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, layer_idx), q.contiguous()
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, layer_idx), k.contiguous()
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, layer_idx), v.contiguous()
|
||||
elif tensor_name == "mlp.gate_up_proj.weight":
|
||||
gate, up = data_torch.chunk(2, dim=0)
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE, layer_idx), gate.contiguous()
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP, layer_idx), up.contiguous()
|
||||
else:
|
||||
if tensor_name.startswith("attn."):
|
||||
tensor_name = "self_attn." + tensor_name[len("attn."):]
|
||||
tensor_name = "model.layers.{bid}." + tensor_name
|
||||
yield from super().modify_tensors(data_torch, tensor_name.format(bid=layer_idx), layer_idx)
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import LazyTorchTensor, ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("MapleForCausalLM")
|
||||
@ModelBase.example("deepgrove/maple-preview")
|
||||
class MapleModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MAPLE
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
hparams = self.hparams
|
||||
|
||||
assert hparams["hidden_act"] == "silu"
|
||||
assert hparams.get("num_shared_experts", 0) == 0
|
||||
assert hparams.get("norm_topk_prob", True)
|
||||
assert hparams.get("nope_on_global_attention", False)
|
||||
|
||||
head_dim = hparams.get("head_dim", hparams["hidden_size"] // hparams["num_attention_heads"])
|
||||
partial_rotary_factor = self.rope_parameters.get("partial_rotary_factor", 1.0)
|
||||
|
||||
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
|
||||
self.gguf_writer.add_rope_dimension_count(int(head_dim * partial_rotary_factor))
|
||||
self.gguf_writer.add_sliding_window(hparams["sliding_window"])
|
||||
self.gguf_writer.add_sliding_window_pattern([layer_type == "sliding_attention" for layer_type in hparams["layer_types"]])
|
||||
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
|
||||
# the reference clamps the MoE SwiGLU gate/up at 7.0 (modeling_maple.py)
|
||||
self.gguf_writer.add_swiglu_clamp_exp([7.0] * self.block_count)
|
||||
|
||||
_experts: list[dict[str, Tensor]] | None = None
|
||||
|
||||
@staticmethod
|
||||
def _stack_experts(tensors: list[Tensor]) -> Tensor:
|
||||
shape = (len(tensors), *tensors[0].shape)
|
||||
dtype = tensors[0].dtype
|
||||
meta = LazyTorchTensor.meta_with_dtype_and_shape(dtype, shape)
|
||||
|
||||
# tensors goes through args, not the closure, so that `func` matches
|
||||
# LazyBase's single-argument shape
|
||||
def stack(ts: list[Tensor]) -> Tensor:
|
||||
result = torch.empty(shape, dtype=dtype)
|
||||
for expert_id, tensor in enumerate(ts):
|
||||
result[expert_id].copy_(LazyTorchTensor.to_eager(tensor))
|
||||
ts.clear()
|
||||
return result
|
||||
|
||||
return cast(torch.Tensor, LazyTorchTensor(meta=meta, args=(tensors,), func=stack))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if "mlp.experts" in name:
|
||||
n_experts = self.hparams["num_experts"]
|
||||
assert bid is not None
|
||||
|
||||
if self._experts is None:
|
||||
self._experts = [{} for _ in range(self.block_count)]
|
||||
|
||||
self._experts[bid][name] = data_torch
|
||||
|
||||
if len(self._experts[bid]) >= n_experts * 3:
|
||||
for weight_name in ("down_proj", "gate_proj", "up_proj"):
|
||||
tensors = []
|
||||
|
||||
for expert_id in range(n_experts):
|
||||
expert_name = f"model.layers.{bid}.mlp.experts.{expert_id}.{weight_name}.weight"
|
||||
tensors.append(self._experts[bid].pop(expert_name))
|
||||
|
||||
merged_name = f"model.layers.{bid}.mlp.experts.{weight_name}.weight"
|
||||
yield from super().modify_tensors(self._stack_experts(tensors), merged_name, bid)
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
def prepare_tensors(self):
|
||||
super().prepare_tensors()
|
||||
|
||||
if self._experts is not None:
|
||||
experts = [name for layer in self._experts for name in layer]
|
||||
if experts:
|
||||
raise ValueError(f"Unprocessed experts: {experts}")
|
||||
@@ -191,6 +191,10 @@ pre_computed_hashes = [
|
||||
{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
|
||||
# lfm2 variants
|
||||
{"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},
|
||||
# hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges
|
||||
# over the whole text (fix_mistral_regex inserts a tekken regex that is a
|
||||
# no-op here); the gemma4 pre (escape ws, split on newlines only) matches it.
|
||||
{"name": "gemma4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/danish-foundation-models/DFM-Mimir", "chkhsh": "846deafc5b0fa786186fa4ae6c7b49903cf2f1d1895bdb80b9120d60be135252"},
|
||||
{"name": "spark2_5", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/XHToken/Spark-X2.5-1.7B", "chkhsh": "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed"},
|
||||
]
|
||||
|
||||
|
||||
@@ -719,10 +719,13 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `
|
||||
| `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Enable stateful KV cache for better performance. Recommended on CPU, GPU. |
|
||||
| `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. |
|
||||
| `GGML_OPENVINO_MEMORY_OPTIMIZE` | Boolean | `0` | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. |
|
||||
| `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. |
|
||||
| `GGML_OPENVINO_RELEASE_WEIGHTS` | Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. |
|
||||
| `GGML_OPENVINO_SPILL_DIR` | String | `not set` | Directory for a disk-backed weight buffer. When set, the repacked weight buffer is mapped from an unlinked file on this path instead of anonymous memory, so its pages are reclaimable under memory pressure instead of staying pinned, cutting the load-time host memory peak. Must point at real storage; a tmpfs mount (e.g. `/tmp` on many systems) backs it with RAM and makes the peak worse. |
|
||||
| `GGML_OPENVINO_REQUANT_KQUANT` | String | `not set` | Requantize Q6_K/Q5_K weights (and matching MoE expert weights) to a 4-bit target instead of the default Q8_0_C, trading accuracy for less memory traffic. One of `q4_sym128` (Q6_K/Q5_K only), `q4_sym128_all` (Q4_K too, drops its per-group zero point), `q4_asym64_all` (Q6_K/Q5_K/Q4_K, keeps a real zero point at group 64), or `native` (no requantization). |
|
||||
| `GGML_OPENVINO_PROFILING` | Boolean | `0` | Enable execution-time profiling. |
|
||||
| `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. |
|
||||
| `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. |
|
||||
|
||||
@@ -790,14 +790,15 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
|
||||
|
||||
| Name | Value | Function |
|
||||
|-------------------|------------------|---------------------------------------------------------------------------------------------------------------------------|
|
||||
| GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG |
|
||||
| GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function: GGML_SYCL_DEBUG() for common debug. |
|
||||
| GGML_SYCL_DEV_DEBUG | 0 (default) or 1 | Enable log function: GGML_SYCL_DEV_DEBUG() for developmental purposes by replacing GGML_SYCL_DEBUG() in special codes. Restore to GGML_SYCL_DEBUG() before committing code.|
|
||||
| GGML_SYCL_DEV2DEV_MEMCPY | 0 (default), 1, 2 | Choose the method of dev2dev memory copy.<br>Value: <br>* 0: SYCL API (default), only support dGPUs.<br>* 1: L0 API -- Better performance, only support dGPUs, found to lead to abnormal crash in some case. <br>* 2: Host Forward -- Most stable method for all cases (including iGPU + dGPU*N), but with lower performance (-2% to -5%).<br>SYCL & L0 API are easy to be impacted by Intel GPU driver issue. When you meet the garbled output or crash issues in multiple GPUs case, try with this debug flag to work around or check the issue.|
|
||||
| GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.|
|
||||
| GGML_SYCL_ENABLE_OPT | 0 or 1 (default)| Enable optimize features for Intel GPUs. (Recommended to 0 for Intel devices older than Gen 10) |
|
||||
| GGML_SYCL_ENABLE_GRAPH | 0 (default) or 1 | Enable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. |
|
||||
| GGML_SYCL_ENABLE_HOST_PINNED_MEM | 0 or 1 (default) | Enable host pinned memory to speed up copy data from host to device. When disable it, host memory will common malloc() on CPU. Disable it when use `--load-model mlock`.|
|
||||
| GGML_SYCL_HOST_PINNED_MEM_2G | 0 (default) or 1 | Limit the max memory allocation to be no more than 2GB when enable host pinned memory. USM allocations above 2 GiB take the relaxed/large-allocation path, which serializes H2D copies with compute and prevents copy/compute overlap. It will impact the startup time. Need more test. Depend on `GGML_SYCL_ENABLE_HOST_PINNED_MEM=1`.|
|
||||
| GGML_SYCL_GET_MEM_API | 0 (default) or 1 | Set to get memory info (free, total) by Level Zero or SYCL API:<br>0 - Level Zero API: support more GPUs, only run on Level Zero running time. When there is an error, fallback to call SYCL API. Depend on GGML_SYCL_SUPPORT_LEVEL_ZERO_API.<br>1 - SYCL API: legacy, support more running time, it can't get the free size of some GPUs (like Arc770). In such case, return total size for free size.|
|
||||
| GGML_SYCL_GET_MEM_API | 0 (default) or 1 | Set to get memory info (free, total) by Level Zero or SYCL API:<br>0 - Level Zero API: support more GPUs, only run on Level Zero running time. When there is an error, fallback to call SYCL API. Depend on GGML_SYCL_SUPPORT_LEVEL_ZERO_API.<br>1 - SYCL API: legacy, support more running time, it can't get the free size of some GPUs (like Arc770). In such case, return the free size as value of total size.|
|
||||
| GGML_SYCL_USE_LEVEL_ZERO_API | 1 (default) or 0 | Use Level Zero API for device memory allocation instead of SYCL. Reduces system RAM usage on Intel dGPUs by avoiding DMA-buf/TTM host memory staging. Requires GGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON at build time. SYCL backend always runs on Level Zero running time even if it's set as OFF (The SYCL api will be usage for memory allocation).|
|
||||
| GGML_SYCL_ENABLE_DNN | 0 or 1 (default)| Enable running computations through oneDNN and always use oneMKL. |
|
||||
| GGML_SYCL_FA_ONEDNN | 1 (default) or 0 | Enable the oneDNN fused SDPA (flash-attention) path on supported GPUs. Set to 0 to always use the native SYCL flash-attention kernel. |
|
||||
|
||||
@@ -28,7 +28,7 @@ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
for (const auto & tool : tools) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto & schema = function.at("parameters");
|
||||
const auto schema = common_chat_tool_parameters(function);
|
||||
|
||||
auto tool_name = p.json_member("name", "\"" + p.literal(name) + "\"");
|
||||
auto tool_args = p.json_member("arguments", p.schema(p.json(), "tool-" + name + "-schema", schema));
|
||||
@@ -108,6 +108,7 @@ For a more complete example, see `test_example_native()` in
|
||||
- **`rule(name, p, trigger)`** - Creates a named rule and returns a reference
|
||||
- **`trigger_rule(name, p)`** - Creates a trigger rule (entry point for lazy grammar generation)
|
||||
- **`schema(p, name, schema, raw)`** - Wraps parser with JSON schema metadata for grammar generation
|
||||
- **`schema(p, name, doc, node, raw)`** - Same, for a node of a `common_chat_schema_document` built earlier, e.g. one tool parameter
|
||||
|
||||
### AST Control
|
||||
|
||||
@@ -121,9 +122,6 @@ some exceptions.
|
||||
|
||||
```cpp
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(params.tools, [&](const json & fn) {
|
||||
builder.resolve_refs(fn.at("parameters"));
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
```
|
||||
@@ -151,7 +149,8 @@ implementation to generate the grammar instead of the underlying parser.
|
||||
|
||||
The `raw` option emits a grammar suitable for a raw string instead of a JSON
|
||||
string. In other words, it won't be wrapped in quotes or require escaping
|
||||
quotes. It should only be used when `type == "string"`.
|
||||
quotes. It only takes effect when the schema may be a string, as reported by
|
||||
`common_chat_schema::may_be_string()`, otherwise the JSON grammar is used.
|
||||
|
||||
The downside is that it can potentially lead to ambiguous grammars. For
|
||||
example, if a user provides the pattern `^.*$`, the following grammar may be
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ Legend:
|
||||
| DSV4_HC_COMB | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_POST | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_PRE | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DUP | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| DUP | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
|
||||
+4
-4
@@ -5000,14 +5000,14 @@
|
||||
"CUDA0","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[1,1,1,2],v=1","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=f32,ne=[10,10,20,1]","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=f16,ne=[10,10,20,1]","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=i32,ne=[10,10,20,1]","support","0","no","CUDA"
|
||||
"CUDA0","DUP","type=i16,ne=[10,10,20,1]","support","0","no","CUDA"
|
||||
"CUDA0","DUP","type=i32,ne=[10,10,20,1]","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=i16,ne=[10,10,20,1]","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=f32,ne=[10,10,5,1],permute=[0,2,1,3]","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=f16,ne=[10,10,5,1],permute=[0,2,1,3]","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=f32,ne=[10,10,5,1],permute=[1,0,2,3]","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=f16,ne=[10,10,5,1],permute=[1,0,2,3]","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[0,2,1,3]","support","0","no","CUDA"
|
||||
"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[1,2,0,3]","support","0","no","CUDA"
|
||||
"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[0,2,1,3]","support","1","yes","CUDA"
|
||||
"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[1,2,0,3]","support","1","yes","CUDA"
|
||||
"CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=1","support","1","yes","CUDA"
|
||||
"CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=2","support","1","yes","CUDA"
|
||||
"CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=3","support","1","yes","CUDA"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -18,7 +18,7 @@ if(LLAMA_BUILD_TESTS)
|
||||
-DDEST=${MODEL_DEST}
|
||||
-DNAME=${MODEL_NAME}
|
||||
-DHASH=${MODEL_HASH}
|
||||
-P ${CMAKE_SOURCE_DIR}/cmake/download-models.cmake
|
||||
-P ${PROJECT_SOURCE_DIR}/cmake/download-models.cmake
|
||||
)
|
||||
set_tests_properties(${TEST_TARGET}-download-model PROPERTIES FIXTURES_SETUP ${TEST_TARGET}-download-model)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND llama-eval-callback -m "${MODEL_DEST}" --prompt hello --seed 42 -ngl 0)
|
||||
|
||||
@@ -1,842 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, List, Optional, Set, Tuple, Union
|
||||
|
||||
def _build_repetition(item_rule, min_items, max_items, separator_rule=None):
|
||||
|
||||
if max_items == 0:
|
||||
return ""
|
||||
|
||||
if min_items == 0 and max_items == 1:
|
||||
return f'{item_rule}?'
|
||||
|
||||
if not separator_rule:
|
||||
if min_items == 1 and max_items is None:
|
||||
return f'{item_rule}+'
|
||||
elif min_items == 0 and max_items is None:
|
||||
return f'{item_rule}*'
|
||||
else:
|
||||
return f'{item_rule}{{{min_items},{max_items if max_items is not None else ""}}}'
|
||||
|
||||
result = item_rule + ' ' + _build_repetition(f'({separator_rule} {item_rule})', min_items - 1 if min_items > 0 else 0, max_items - 1 if max_items is not None else None)
|
||||
return f'({result})?' if min_items == 0 else result
|
||||
|
||||
def _generate_min_max_int(min_value: Optional[int], max_value: Optional[int], out: list, decimals_left: int = 16, top_level: bool = True):
|
||||
def digit_range(from_char: str, to_char: str):
|
||||
out.append("[")
|
||||
if from_char == to_char:
|
||||
out.append(from_char)
|
||||
else:
|
||||
out.append(from_char)
|
||||
out.append("-")
|
||||
out.append(to_char)
|
||||
out.append("]")
|
||||
|
||||
def more_digits(min_digits: int, max_digits: int):
|
||||
out.append("[0-9]")
|
||||
if min_digits == max_digits and min_digits == 1:
|
||||
return
|
||||
out.append("{")
|
||||
out.append(str(min_digits))
|
||||
if max_digits != min_digits:
|
||||
out.append(",")
|
||||
if max_digits != sys.maxsize:
|
||||
out.append(str(max_digits))
|
||||
out.append("}")
|
||||
|
||||
def uniform_range(from_str: str, to_str: str):
|
||||
i = 0
|
||||
while i < len(from_str) and from_str[i] == to_str[i]:
|
||||
i += 1
|
||||
if i > 0:
|
||||
out.append("\"")
|
||||
out.append(from_str[:i])
|
||||
out.append("\"")
|
||||
if i < len(from_str):
|
||||
if i > 0:
|
||||
out.append(" ")
|
||||
sub_len = len(from_str) - i - 1
|
||||
if sub_len > 0:
|
||||
from_sub = from_str[i+1:]
|
||||
to_sub = to_str[i+1:]
|
||||
sub_zeros = "0" * sub_len
|
||||
sub_nines = "9" * sub_len
|
||||
|
||||
to_reached = False
|
||||
out.append("(")
|
||||
if from_sub == sub_zeros:
|
||||
digit_range(from_str[i], chr(ord(to_str[i]) - 1))
|
||||
out.append(" ")
|
||||
more_digits(sub_len, sub_len)
|
||||
else:
|
||||
out.append("[")
|
||||
out.append(from_str[i])
|
||||
out.append("] ")
|
||||
out.append("(")
|
||||
uniform_range(from_sub, sub_nines)
|
||||
out.append(")")
|
||||
if ord(from_str[i]) < ord(to_str[i]) - 1:
|
||||
out.append(" | ")
|
||||
if to_sub == sub_nines:
|
||||
digit_range(chr(ord(from_str[i]) + 1), to_str[i])
|
||||
to_reached = True
|
||||
else:
|
||||
digit_range(chr(ord(from_str[i]) + 1), chr(ord(to_str[i]) - 1))
|
||||
out.append(" ")
|
||||
more_digits(sub_len, sub_len)
|
||||
if not to_reached:
|
||||
out.append(" | ")
|
||||
digit_range(to_str[i], to_str[i])
|
||||
out.append(" ")
|
||||
uniform_range(sub_zeros, to_sub)
|
||||
out.append(")")
|
||||
else:
|
||||
out.append("[")
|
||||
out.append(from_str[i])
|
||||
out.append("-")
|
||||
out.append(to_str[i])
|
||||
out.append("]")
|
||||
|
||||
if min_value is not None and max_value is not None:
|
||||
if min_value < 0 and max_value < 0:
|
||||
out.append("\"-\" (")
|
||||
_generate_min_max_int(-max_value, -min_value, out, decimals_left, top_level=True)
|
||||
out.append(")")
|
||||
return
|
||||
|
||||
if min_value < 0:
|
||||
out.append("\"-\" (")
|
||||
_generate_min_max_int(0, -min_value, out, decimals_left, top_level=True)
|
||||
out.append(") | ")
|
||||
min_value = 0
|
||||
|
||||
min_s = str(min_value)
|
||||
max_s = str(max_value)
|
||||
min_digits = len(min_s)
|
||||
max_digits = len(max_s)
|
||||
|
||||
for digits in range(min_digits, max_digits):
|
||||
uniform_range(min_s, "9" * digits)
|
||||
min_s = "1" + "0" * digits
|
||||
out.append(" | ")
|
||||
uniform_range(min_s, max_s)
|
||||
return
|
||||
|
||||
less_decimals = max(decimals_left - 1, 1)
|
||||
|
||||
if min_value is not None:
|
||||
if min_value < 0:
|
||||
out.append("\"-\" (")
|
||||
_generate_min_max_int(None, -min_value, out, decimals_left, top_level=False)
|
||||
out.append(") | [0] | [1-9] ")
|
||||
more_digits(0, decimals_left - 1)
|
||||
elif min_value == 0:
|
||||
if top_level:
|
||||
out.append("[0] | [1-9] ")
|
||||
more_digits(0, less_decimals)
|
||||
else:
|
||||
more_digits(1, decimals_left)
|
||||
elif min_value <= 9:
|
||||
c = str(min_value)
|
||||
range_start = '1' if top_level else '0'
|
||||
if c > range_start:
|
||||
digit_range(range_start, chr(ord(c) - 1))
|
||||
out.append(" ")
|
||||
more_digits(1, less_decimals)
|
||||
out.append(" | ")
|
||||
digit_range(c, "9")
|
||||
out.append(" ")
|
||||
more_digits(0, less_decimals)
|
||||
else:
|
||||
min_s = str(min_value)
|
||||
length = len(min_s)
|
||||
c = min_s[0]
|
||||
|
||||
if c > "1":
|
||||
digit_range("1" if top_level else "0", chr(ord(c) - 1))
|
||||
out.append(" ")
|
||||
more_digits(length, less_decimals)
|
||||
out.append(" | ")
|
||||
digit_range(c, c)
|
||||
out.append(" (")
|
||||
_generate_min_max_int(int(min_s[1:]), None, out, less_decimals, top_level=False)
|
||||
out.append(")")
|
||||
if c < "9":
|
||||
out.append(" | ")
|
||||
digit_range(chr(ord(c) + 1), "9")
|
||||
out.append(" ")
|
||||
more_digits(length - 1, less_decimals)
|
||||
return
|
||||
|
||||
if max_value is not None:
|
||||
if max_value >= 0:
|
||||
if top_level:
|
||||
out.append("\"-\" [1-9] ")
|
||||
more_digits(0, less_decimals)
|
||||
out.append(" | ")
|
||||
_generate_min_max_int(0, max_value, out, decimals_left, top_level=True)
|
||||
else:
|
||||
out.append("\"-\" (")
|
||||
_generate_min_max_int(-max_value, None, out, decimals_left, top_level=False)
|
||||
out.append(")")
|
||||
return
|
||||
|
||||
raise RuntimeError("At least one of min_value or max_value must be set")
|
||||
|
||||
class BuiltinRule:
|
||||
def __init__(self, content: str, deps: list | None = None):
|
||||
self.content = content
|
||||
self.deps = deps or []
|
||||
|
||||
# Constraining spaces to prevent model "running away".
|
||||
SPACE_RULE = '| " " | "\\n"{1,2} [ \\t]{0,20}'
|
||||
|
||||
PRIMITIVE_RULES = {
|
||||
'boolean' : BuiltinRule('("true" | "false")', []),
|
||||
'decimal-part' : BuiltinRule('[0-9]{1,16}', []),
|
||||
'integral-part': BuiltinRule('[0] | [1-9] [0-9]{0,15}', []),
|
||||
'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)?', ['integral-part', 'decimal-part']),
|
||||
'integer' : BuiltinRule('("-"? integral-part)', ['integral-part']),
|
||||
'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']),
|
||||
'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? space "}"', ['string', 'value']),
|
||||
'array' : BuiltinRule('"[" space ( value ("," space value)* )? space "]"', ['value']),
|
||||
'uuid' : BuiltinRule(r'"\"" [0-9a-fA-F]{8} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{12} "\""', []),
|
||||
'char' : BuiltinRule(r'[^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})', []),
|
||||
'string' : BuiltinRule(r'"\"" char* "\""', ['char']),
|
||||
'null' : BuiltinRule('"null"', []),
|
||||
}
|
||||
|
||||
# TODO: support "uri", "email" string formats
|
||||
STRING_FORMAT_RULES = {
|
||||
'date' : BuiltinRule('[0-9]{4} "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []),
|
||||
'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9]{3} )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []),
|
||||
'date-time' : BuiltinRule('date "T" time', ['date', 'time']),
|
||||
'date-string' : BuiltinRule('"\\"" date "\\""', ['date']),
|
||||
'time-string' : BuiltinRule('"\\"" time "\\""', ['time']),
|
||||
'date-time-string': BuiltinRule('"\\"" date-time "\\""', ['date-time']),
|
||||
}
|
||||
|
||||
DOTALL = '[\\U00000000-\\U0010FFFF]'
|
||||
DOT = '[^\\x0A\\x0D]'
|
||||
|
||||
RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()])
|
||||
|
||||
INVALID_RULE_CHARS_RE = re.compile(r'[^a-zA-Z0-9-]+')
|
||||
GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\\]')
|
||||
GRAMMAR_RANGE_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\]\-\\]')
|
||||
GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]', '\\': '\\\\'}
|
||||
|
||||
NON_LITERAL_SET = set('|.()[]{}*+?')
|
||||
ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('^$.[]()|{}*+?')
|
||||
|
||||
|
||||
class SchemaConverter:
|
||||
def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern):
|
||||
self._prop_order = prop_order
|
||||
self._allow_fetch = allow_fetch
|
||||
self._dotall = dotall
|
||||
self._raw_pattern = raw_pattern
|
||||
self._rules = {
|
||||
'space': SPACE_RULE,
|
||||
}
|
||||
self._refs = {}
|
||||
self._refs_being_resolved = set()
|
||||
|
||||
def _format_literal(self, literal):
|
||||
escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub(
|
||||
lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)) or m.group(0), literal
|
||||
)
|
||||
return f'"{escaped}"'
|
||||
|
||||
def not_literal(self, literal: str, dotall: bool = True, maybe_escaped_underscores = False) -> str:
|
||||
'''
|
||||
not_literal('a') -> '[^a]'
|
||||
not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?'
|
||||
'''
|
||||
assert len(literal) > 0, 'Empty literal not supported'
|
||||
def recurse(i: int):
|
||||
c = literal[i]
|
||||
if maybe_escaped_underscores and c == '_':
|
||||
yield f'[^{c}\\\\]'
|
||||
yield ' | '
|
||||
yield f'"\\\\"? "{c}"'
|
||||
else:
|
||||
yield f'[^{c}]'
|
||||
if i < len(literal) - 1:
|
||||
yield ' | '
|
||||
yield self._format_literal(c)
|
||||
yield ' ('
|
||||
yield from recurse(i + 1)
|
||||
yield ')?'
|
||||
|
||||
return ''.join(('(', *recurse(0), ')'))
|
||||
|
||||
def _not_strings(self, strings):
|
||||
class TrieNode:
|
||||
def __init__(self):
|
||||
self.children = {}
|
||||
self.is_end_of_string = False
|
||||
|
||||
def insert(self, string):
|
||||
node = self
|
||||
for c in string:
|
||||
node = node.children.setdefault(c, TrieNode())
|
||||
node.is_end_of_string = True
|
||||
|
||||
trie = TrieNode()
|
||||
for s in strings:
|
||||
trie.insert(s)
|
||||
|
||||
char_rule = self._add_primitive('char', PRIMITIVE_RULES['char'])
|
||||
out = ['["] ( ']
|
||||
|
||||
def visit(node):
|
||||
rejects = []
|
||||
first = True
|
||||
for c in sorted(node.children.keys()):
|
||||
child = node.children[c]
|
||||
rejects.append(c)
|
||||
if first:
|
||||
first = False
|
||||
else:
|
||||
out.append(' | ')
|
||||
out.append(f'[{c}]')
|
||||
if child.children:
|
||||
out.append(f' (')
|
||||
visit(child)
|
||||
out.append(')')
|
||||
elif child.is_end_of_string:
|
||||
out.append(f' {char_rule}+')
|
||||
if node.children:
|
||||
if not first:
|
||||
out.append(' | ')
|
||||
out.append(f'[^"{"".join(rejects)}] {char_rule}*')
|
||||
visit(trie)
|
||||
|
||||
out.append(f' ){"" if trie.is_end_of_string else "?"} ["]')
|
||||
return ''.join(out)
|
||||
|
||||
def _add_rule(self, name, rule):
|
||||
esc_name = INVALID_RULE_CHARS_RE.sub('-', name)
|
||||
if esc_name not in self._rules or self._rules[esc_name] == rule:
|
||||
key = esc_name
|
||||
else:
|
||||
i = 0
|
||||
while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule:
|
||||
i += 1
|
||||
key = f'{esc_name}{i}'
|
||||
self._rules[key] = rule
|
||||
return key
|
||||
|
||||
def resolve_refs(self, schema: dict, url: str):
|
||||
'''
|
||||
Resolves all $ref fields in the given schema, fetching any remote schemas,
|
||||
replacing $ref with absolute reference URL and populating self._refs with the
|
||||
respective referenced (sub)schema dictionaries.
|
||||
'''
|
||||
def visit(n: dict):
|
||||
if isinstance(n, list):
|
||||
return [visit(x) for x in n]
|
||||
elif isinstance(n, dict):
|
||||
ref = n.get('$ref')
|
||||
if ref is not None and ref not in self._refs:
|
||||
if ref.startswith('https://'):
|
||||
assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)'
|
||||
import requests
|
||||
|
||||
frag_split = ref.split('#')
|
||||
base_url = frag_split[0]
|
||||
|
||||
target = self._refs.get(base_url)
|
||||
if target is None:
|
||||
target = self.resolve_refs(requests.get(ref).json(), base_url)
|
||||
self._refs[base_url] = target
|
||||
|
||||
if len(frag_split) == 1 or frag_split[-1] == '':
|
||||
return target
|
||||
elif ref.startswith('#/'):
|
||||
target = schema
|
||||
ref = f'{url}{ref}'
|
||||
n['$ref'] = ref
|
||||
else:
|
||||
raise ValueError(f'Unsupported ref {ref}')
|
||||
|
||||
for sel in ref.split('#')[-1].split('/')[1:]:
|
||||
assert target is not None, f'Error resolving ref {ref}: {sel} not in {target}'
|
||||
if isinstance(target, list):
|
||||
try:
|
||||
sel_index = int(sel)
|
||||
except ValueError:
|
||||
raise ValueError(f'Error resolving ref {ref}: {sel} not in {target}')
|
||||
assert 0 <= sel_index < len(target), f'Error resolving ref {ref}: {sel} not in {target}'
|
||||
target = target[sel_index]
|
||||
else:
|
||||
assert sel in target, f'Error resolving ref {ref}: {sel} not in {target}'
|
||||
target = target[sel]
|
||||
|
||||
self._refs[ref] = target
|
||||
else:
|
||||
for v in n.values():
|
||||
visit(v)
|
||||
|
||||
return n
|
||||
return visit(schema)
|
||||
|
||||
def _generate_union_rule(self, name, alt_schemas):
|
||||
return ' | '.join((
|
||||
self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}')
|
||||
for i, alt_schema in enumerate(alt_schemas)
|
||||
))
|
||||
|
||||
def _visit_pattern(self, pattern, name):
|
||||
'''
|
||||
Transforms a regular expression pattern into a GBNF rule.
|
||||
|
||||
Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions
|
||||
Output: https://github.com/ggml-org/llama.cpp/blob/master/grammars/README.md
|
||||
|
||||
Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers.
|
||||
|
||||
Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which
|
||||
we define sub-rules to keep the output lean.
|
||||
'''
|
||||
|
||||
assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"'
|
||||
pattern = pattern[1:-1]
|
||||
sub_rule_ids = {}
|
||||
|
||||
i = 0
|
||||
length = len(pattern)
|
||||
|
||||
def to_rule(s: tuple[str, bool]) -> str:
|
||||
(txt, is_literal) = s
|
||||
return "\"" + txt + "\"" if is_literal else txt
|
||||
|
||||
def transform() -> tuple[str, bool]:
|
||||
'''
|
||||
Parse a unit at index i (advancing it), and return its string representation + whether it's a literal.
|
||||
'''
|
||||
nonlocal i
|
||||
nonlocal pattern
|
||||
nonlocal sub_rule_ids
|
||||
|
||||
start = i
|
||||
# For each component of this sequence, store its string representation and whether it's a literal.
|
||||
# We only need a flat structure here to apply repetition operators to the last item, and
|
||||
# to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially
|
||||
# (GBNF's syntax is luckily very close to regular expressions!)
|
||||
seq: list[tuple[str, bool]] = []
|
||||
|
||||
def get_dot():
|
||||
if self._dotall:
|
||||
rule = DOTALL
|
||||
else:
|
||||
# Accept any character... except \n and \r line break chars (\x0A and \xOD)
|
||||
rule = DOT
|
||||
return self._add_rule(f'dot', rule)
|
||||
|
||||
def join_seq():
|
||||
nonlocal seq
|
||||
ret = []
|
||||
for is_literal, g in itertools.groupby(seq, lambda x: x[1]):
|
||||
if is_literal:
|
||||
ret.append((''.join(x[0] for x in g), True))
|
||||
else:
|
||||
ret.extend(g)
|
||||
if len(ret) == 1:
|
||||
return ret[0]
|
||||
return (' '.join(to_rule(x) for x in seq), False)
|
||||
|
||||
while i < length:
|
||||
c = pattern[i]
|
||||
if c == '.':
|
||||
seq.append((get_dot(), False))
|
||||
i += 1
|
||||
elif c == '(':
|
||||
i += 1
|
||||
if i < length:
|
||||
assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/'
|
||||
seq.append((f'({to_rule(transform())})', False))
|
||||
elif c == ')':
|
||||
i += 1
|
||||
assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}'
|
||||
return join_seq()
|
||||
elif c == '[':
|
||||
square_brackets = c
|
||||
i += 1
|
||||
while i < length and pattern[i] != ']':
|
||||
if pattern[i] == '\\':
|
||||
square_brackets += pattern[i:i+2]
|
||||
i += 2
|
||||
else:
|
||||
square_brackets += pattern[i]
|
||||
i += 1
|
||||
assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}'
|
||||
square_brackets += ']'
|
||||
i += 1
|
||||
seq.append((square_brackets, False))
|
||||
elif c == '|':
|
||||
seq.append(('|', False))
|
||||
i += 1
|
||||
elif c in ('*', '+', '?'):
|
||||
seq[-1] = (to_rule(seq[-1]) + c, False)
|
||||
i += 1
|
||||
elif c == '{':
|
||||
curly_brackets = c
|
||||
i += 1
|
||||
while i < length and pattern[i] != '}':
|
||||
curly_brackets += pattern[i]
|
||||
i += 1
|
||||
assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}'
|
||||
curly_brackets += '}'
|
||||
i += 1
|
||||
nums = [s.strip() for s in curly_brackets[1:-1].split(',')]
|
||||
min_times = 0
|
||||
max_times = None
|
||||
try:
|
||||
if len(nums) == 1:
|
||||
min_times = int(nums[0])
|
||||
max_times = min_times
|
||||
else:
|
||||
assert len(nums) == 2
|
||||
min_times = int(nums[0]) if nums[0] else 0
|
||||
max_times = int(nums[1]) if nums[1] else None
|
||||
except ValueError:
|
||||
raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/')
|
||||
|
||||
(sub, sub_is_literal) = seq[-1]
|
||||
|
||||
if not sub_is_literal:
|
||||
id = sub_rule_ids.get(sub)
|
||||
if id is None:
|
||||
id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub)
|
||||
sub_rule_ids[sub] = id
|
||||
sub = id
|
||||
|
||||
seq[-1] = (_build_repetition(f'"{sub}"' if sub_is_literal else sub, min_times, max_times), False)
|
||||
else:
|
||||
literal = ''
|
||||
while i < length:
|
||||
if pattern[i] == '\\' and i < length - 1:
|
||||
next = pattern[i + 1]
|
||||
if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS:
|
||||
i += 1
|
||||
literal += pattern[i]
|
||||
i += 1
|
||||
else:
|
||||
literal += pattern[i:i+2]
|
||||
i += 2
|
||||
elif pattern[i] == '"' and not self._raw_pattern:
|
||||
literal += '\\"'
|
||||
i += 1
|
||||
elif pattern[i] not in NON_LITERAL_SET and \
|
||||
(i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET):
|
||||
literal += pattern[i]
|
||||
i += 1
|
||||
else:
|
||||
break
|
||||
if literal:
|
||||
seq.append((literal, True))
|
||||
|
||||
return join_seq()
|
||||
|
||||
return self._add_rule(
|
||||
name,
|
||||
to_rule(transform()) if self._raw_pattern \
|
||||
else "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\"")
|
||||
|
||||
|
||||
def _resolve_ref(self, ref):
|
||||
ref_fragment = ref.split('#')[-1]
|
||||
ref_name = 'ref' + re.sub(r'[^a-zA-Z0-9-]+', '-', ref_fragment)
|
||||
if ref_name not in self._rules and ref not in self._refs_being_resolved:
|
||||
self._refs_being_resolved.add(ref)
|
||||
resolved = self._refs[ref]
|
||||
ref_name = self.visit(resolved, ref_name)
|
||||
self._refs_being_resolved.remove(ref)
|
||||
return ref_name
|
||||
|
||||
def _generate_constant_rule(self, value):
|
||||
return self._format_literal(json.dumps(value))
|
||||
|
||||
def visit(self, schema, name):
|
||||
schema_type = schema.get('type')
|
||||
schema_format = schema.get('format')
|
||||
rule_name = name + '-' if name in RESERVED_NAMES else name or 'root'
|
||||
|
||||
if (ref := schema.get('$ref')) is not None:
|
||||
return self._add_rule(rule_name, self._resolve_ref(ref))
|
||||
|
||||
elif 'oneOf' in schema or 'anyOf' in schema:
|
||||
return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf']))
|
||||
|
||||
elif isinstance(schema_type, list):
|
||||
return self._add_rule(rule_name, self._generate_union_rule(name, [{**schema, 'type': t} for t in schema_type]))
|
||||
|
||||
elif 'const' in schema:
|
||||
return self._add_rule(rule_name, self._generate_constant_rule(schema['const']))
|
||||
|
||||
elif 'enum' in schema:
|
||||
rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + ')'
|
||||
return self._add_rule(rule_name, rule)
|
||||
|
||||
elif schema_type in (None, 'object') and \
|
||||
('properties' in schema or \
|
||||
('additionalProperties' in schema and schema['additionalProperties'] is not True)):
|
||||
required = set(schema.get('required', []))
|
||||
properties = list(schema.get('properties', {}).items())
|
||||
return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties')))
|
||||
|
||||
elif schema_type in (None, 'object', 'string') and 'allOf' in schema:
|
||||
required = set()
|
||||
properties = []
|
||||
enum_sets = []
|
||||
hybrid_name = name
|
||||
def add_component(comp_schema, is_required):
|
||||
if (ref := comp_schema.get('$ref')) is not None:
|
||||
comp_schema = self._refs[ref]
|
||||
|
||||
if 'properties' in comp_schema:
|
||||
for prop_name, prop_schema in comp_schema['properties'].items():
|
||||
properties.append((prop_name, prop_schema))
|
||||
if is_required:
|
||||
required.add(prop_name)
|
||||
|
||||
if 'enum' in comp_schema:
|
||||
enum_sets.append(set(comp_schema['enum']))
|
||||
|
||||
for t in schema['allOf']:
|
||||
if 'anyOf' in t:
|
||||
for tt in t['anyOf']:
|
||||
add_component(tt, is_required=False)
|
||||
else:
|
||||
add_component(t, is_required=True)
|
||||
|
||||
if enum_sets:
|
||||
enum_intersection = enum_sets[0]
|
||||
for s in enum_sets[1:]:
|
||||
enum_intersection &= s
|
||||
|
||||
if enum_intersection:
|
||||
rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in sorted(enum_intersection))) + ')'
|
||||
return self._add_rule(rule_name, rule)
|
||||
|
||||
return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=None))
|
||||
|
||||
elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema):
|
||||
items = schema.get('items', schema.get('prefixItems'))
|
||||
if isinstance(items, list):
|
||||
return self._add_rule(
|
||||
rule_name,
|
||||
'"[" space ' +
|
||||
' "," space '.join(
|
||||
self.visit(item, f'{name}{"-" if name else ""}tuple-{i}')
|
||||
for i, item in enumerate(items)) +
|
||||
' space "]"')
|
||||
else:
|
||||
item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item')
|
||||
min_items = schema.get("minItems", 0)
|
||||
max_items = schema.get("maxItems")
|
||||
return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' space "]"')
|
||||
|
||||
elif schema_type in (None, 'string') and 'pattern' in schema:
|
||||
return self._visit_pattern(schema['pattern'], rule_name)
|
||||
|
||||
elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''):
|
||||
return self._add_primitive(
|
||||
'root' if rule_name == 'root' else schema_format,
|
||||
PRIMITIVE_RULES['uuid']
|
||||
)
|
||||
|
||||
elif schema_type in (None, 'string') and f'{schema_format}-string' in STRING_FORMAT_RULES:
|
||||
prim_name = f'{schema_format}-string'
|
||||
return self._add_rule(rule_name, self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name]))
|
||||
|
||||
elif schema_type == 'string' and ('minLength' in schema or 'maxLength' in schema):
|
||||
char_rule = self._add_primitive('char', PRIMITIVE_RULES['char'])
|
||||
min_len = schema.get('minLength', 0)
|
||||
max_len = schema.get('maxLength')
|
||||
|
||||
return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\""')
|
||||
|
||||
elif schema_type in (None, 'integer') and \
|
||||
('minimum' in schema or 'exclusiveMinimum' in schema or 'maximum' in schema or 'exclusiveMaximum' in schema):
|
||||
min_value = None
|
||||
max_value = None
|
||||
if 'minimum' in schema:
|
||||
min_value = schema['minimum']
|
||||
elif 'exclusiveMinimum' in schema:
|
||||
min_value = schema['exclusiveMinimum'] + 1
|
||||
if 'maximum' in schema:
|
||||
max_value = schema['maximum']
|
||||
elif 'exclusiveMaximum' in schema:
|
||||
max_value = schema['exclusiveMaximum'] - 1
|
||||
|
||||
out = ["("]
|
||||
_generate_min_max_int(min_value, max_value, out)
|
||||
out.append(")")
|
||||
return self._add_rule(rule_name, ''.join(out))
|
||||
|
||||
elif (schema_type == 'object') or (len(schema) == 0):
|
||||
return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object']))
|
||||
|
||||
elif schema_type is None and isinstance(schema, dict):
|
||||
# No type constraint and no recognized structural keywords (e.g. {"description": "..."}).
|
||||
# Per JSON Schema semantics this is equivalent to {} and accepts any value.
|
||||
return self._add_rule(rule_name, self._add_primitive('value', PRIMITIVE_RULES['value']))
|
||||
|
||||
else:
|
||||
assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}'
|
||||
# TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
|
||||
return self._add_primitive('root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type])
|
||||
|
||||
def _add_primitive(self, name: str, rule: BuiltinRule):
|
||||
n = self._add_rule(name, rule.content)
|
||||
|
||||
for dep in rule.deps:
|
||||
dep_rule = PRIMITIVE_RULES.get(dep) or STRING_FORMAT_RULES.get(dep)
|
||||
assert dep_rule, f'Rule {dep} not known'
|
||||
if dep not in self._rules:
|
||||
self._add_primitive(dep, dep_rule)
|
||||
return n
|
||||
|
||||
def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Optional[Union[bool, Any]]):
|
||||
prop_order = self._prop_order
|
||||
# sort by position in prop_order (if specified) then by original order
|
||||
sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))]
|
||||
|
||||
prop_kv_rule_names = {}
|
||||
for prop_name, prop_schema in properties:
|
||||
prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}')
|
||||
prop_kv_rule_names[prop_name] = self._add_rule(
|
||||
f'{name}{"-" if name else ""}{prop_name}-kv',
|
||||
fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}'
|
||||
)
|
||||
required_props = [k for k in sorted_props if k in required]
|
||||
optional_props = [k for k in sorted_props if k not in required]
|
||||
|
||||
if additional_properties is not None and additional_properties != False:
|
||||
sub_name = f'{name}{"-" if name else ""}additional'
|
||||
value_rule = self.visit(additional_properties, f'{sub_name}-value') if isinstance(additional_properties, dict) else \
|
||||
self._add_primitive('value', PRIMITIVE_RULES['value'])
|
||||
key_rule = self._add_primitive('string', PRIMITIVE_RULES['string']) if not sorted_props \
|
||||
else self._add_rule(f'{sub_name}-k', self._not_strings(sorted_props))
|
||||
|
||||
prop_kv_rule_names["*"] = self._add_rule(
|
||||
f'{sub_name}-kv',
|
||||
f'{key_rule} ":" space {value_rule}'
|
||||
)
|
||||
optional_props.append("*")
|
||||
|
||||
if not required_props and not optional_props:
|
||||
return '"{" space "}"'
|
||||
|
||||
rule = '"{" space '
|
||||
rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props)
|
||||
|
||||
if optional_props:
|
||||
rule += ' ('
|
||||
if required_props:
|
||||
rule += ' "," space ( '
|
||||
|
||||
def get_recursive_refs(ks, first_is_optional):
|
||||
[k, *rest] = ks
|
||||
kv_rule_name = prop_kv_rule_names[k]
|
||||
comma_ref = f'( "," space {kv_rule_name} )'
|
||||
if first_is_optional:
|
||||
res = comma_ref + ('*' if k == '*' else '?')
|
||||
else:
|
||||
res = kv_rule_name + (' ' + comma_ref + "*" if k == '*' else '')
|
||||
if len(rest) > 0:
|
||||
res += ' ' + self._add_rule(
|
||||
f'{name}{"-" if name else ""}{k}-rest',
|
||||
get_recursive_refs(rest, first_is_optional=True)
|
||||
)
|
||||
return res
|
||||
|
||||
rule += ' | '.join(
|
||||
get_recursive_refs(optional_props[i:], first_is_optional=False)
|
||||
for i in range(len(optional_props))
|
||||
)
|
||||
if required_props:
|
||||
rule += ' )'
|
||||
rule += ' )?'
|
||||
|
||||
rule += ' space "}"'
|
||||
|
||||
return rule
|
||||
|
||||
def format_grammar(self):
|
||||
return '\n'.join(
|
||||
f'{name} ::= {rule}'
|
||||
for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0])
|
||||
)
|
||||
|
||||
|
||||
def main(args_in = None):
|
||||
parser = argparse.ArgumentParser(
|
||||
description='''
|
||||
Generates a grammar (suitable for use in ./llama-cli) that produces JSON conforming to a
|
||||
given JSON schema. Only a subset of JSON schema features are supported; more may be
|
||||
added in the future.
|
||||
''',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--prop-order',
|
||||
default=[],
|
||||
type=lambda s: s.split(','),
|
||||
help='''
|
||||
comma-separated property names defining the order of precedence for object properties;
|
||||
properties not specified here are given lower precedence than those that are, and
|
||||
are kept in their original order from the schema. Required properties are always
|
||||
given precedence over optional properties.
|
||||
'''
|
||||
)
|
||||
parser.add_argument(
|
||||
'--allow-fetch',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Whether to allow fetching referenced schemas over HTTPS')
|
||||
parser.add_argument(
|
||||
'--dotall',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Whether to treat dot (".") as matching all chars including line breaks in regular expression patterns')
|
||||
parser.add_argument(
|
||||
'--raw-pattern',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Treats string patterns as raw patterns w/o quotes (or quote escapes)')
|
||||
|
||||
parser.add_argument('schema', help='file containing JSON schema ("-" for stdin)')
|
||||
args = parser.parse_args(args_in)
|
||||
|
||||
if args.schema.startswith('https://'):
|
||||
url = args.schema
|
||||
import requests
|
||||
schema = requests.get(url).json()
|
||||
elif args.schema == '-':
|
||||
url = 'stdin'
|
||||
schema = json.load(sys.stdin)
|
||||
else:
|
||||
url = f'file://{args.schema}'
|
||||
with open(args.schema) as f:
|
||||
schema = json.load(f)
|
||||
converter = SchemaConverter(
|
||||
prop_order={name: idx for idx, name in enumerate(args.prop_order)},
|
||||
allow_fetch=args.allow_fetch,
|
||||
dotall=args.dotall,
|
||||
raw_pattern=args.raw_pattern)
|
||||
schema = converter.resolve_refs(schema, url)
|
||||
converter.visit(schema, '')
|
||||
print(converter.format_grammar())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,20 +0,0 @@
|
||||
import json, subprocess, sys, os
|
||||
|
||||
assert len(sys.argv) >= 2
|
||||
[_, pattern, *rest] = sys.argv
|
||||
|
||||
print(subprocess.check_output(
|
||||
[
|
||||
"python",
|
||||
os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)),
|
||||
"json_schema_to_grammar.py"),
|
||||
*rest,
|
||||
"-",
|
||||
"--raw-pattern",
|
||||
],
|
||||
text=True,
|
||||
input=json.dumps({
|
||||
"type": "string",
|
||||
"pattern": pattern,
|
||||
}, indent=2)))
|
||||
@@ -1,3 +1,4 @@
|
||||
llama-build-install
|
||||
install
|
||||
build
|
||||
build-subdir
|
||||
|
||||
@@ -3,11 +3,20 @@ project(llama-simple)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
|
||||
find_package(llama 0.1.0 REQUIRED)
|
||||
option(LLAMA_TEST_USE_SUBDIR "Use add_subdirectory instead of find_package" OFF)
|
||||
|
||||
if(LLAMA_TEST_USE_SUBDIR)
|
||||
add_subdirectory(../../ llama.cpp)
|
||||
else()
|
||||
find_package(llama 0.1.0 REQUIRED)
|
||||
endif()
|
||||
|
||||
add_executable(test-cmake test-cmake.cpp)
|
||||
target_link_libraries(test-cmake PRIVATE llama)
|
||||
target_compile_definitions(test-cmake PRIVATE
|
||||
LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER}
|
||||
LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}"
|
||||
)
|
||||
|
||||
if(DEFINED LLAMA_BUILD_NUMBER)
|
||||
target_compile_definitions(test-cmake PRIVATE
|
||||
LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER}
|
||||
LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}"
|
||||
)
|
||||
endif()
|
||||
|
||||
@@ -5,17 +5,18 @@ enable troubleshooting issues and exploration. The idea is that this can be used
|
||||
after making changes to llama.cpp installation cmake configuration and then
|
||||
verify it locally.
|
||||
|
||||
### Usage
|
||||
The following will configure, build, and install llama.cpp
|
||||
### find_package
|
||||
The following will configure, build, and install llama.cpp, and the build a
|
||||
project that uses find_package to use the installation.
|
||||
|
||||
Configuring/build/install:
|
||||
```console
|
||||
./build-install.sh
|
||||
```
|
||||
The above command will create a directory named `install` in the current directory
|
||||
which will have the follwing files in its lib directory:
|
||||
which will have the following files in its lib directory:
|
||||
```console
|
||||
(venv) $ ls install/lib/
|
||||
$ ls install/lib/
|
||||
cmake libggml.so libllama-common.so.0 libllama.so.0.1.0 llama.cpp
|
||||
libggml-base.so libggml.so.0 libllama-common.so.0.1.0 libmtmd.so pkgconfig
|
||||
libggml-base.so.0 libggml.so.0.19.0 libllama.so libmtmd.so.0
|
||||
@@ -24,7 +25,7 @@ libggml-base.so.0.19.0 libllama-common.so libllama.so.0 libmtmd.so
|
||||
|
||||
Build/run this project using the installation created above:
|
||||
```console
|
||||
(venv) $ ./build.sh
|
||||
$ ./build.sh
|
||||
-- Configuring done (0.0s)
|
||||
-- Generating done (0.0s)
|
||||
-- Build files have been written to: /path/to/llama.cpp/examples/test-cmake/build
|
||||
@@ -34,3 +35,11 @@ Build/run this project using the installation created above:
|
||||
load_backend: loaded CPU backend from /path/to/llama.cpp/examples/test-cmake/install/lib/llama.cpp/libggml-cpu-alderlake.so
|
||||
[test-cmake] Backend initialized.
|
||||
```
|
||||
|
||||
### add_subdirectory
|
||||
The following will use add_subdirectory to include llama.cpp in a cmake project
|
||||
and is intended to simulate projects that build llama.cpp in this way.
|
||||
|
||||
```console
|
||||
$ USE_SUBDIR=ON ./build.sh
|
||||
```
|
||||
|
||||
@@ -2,6 +2,17 @@
|
||||
|
||||
set -e
|
||||
|
||||
cmake -S . -B build -DCMAKE_PREFIX_PATH="${PWD}/install"
|
||||
cmake --build build
|
||||
LD_LIBRARY_PATH="${PWD}/install/lib/llama.cpp:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ./build/test-cmake
|
||||
if [ "${USE_SUBDIR:-OFF}" = "ON" ]; then
|
||||
BUILD_DIR="build-subdir"
|
||||
CMAKE_ARGS="-DLLAMA_TEST_USE_SUBDIR=ON -DLLAMA_BUILD_COMMON=ON -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON-DLLAMA_BUILD_TESTS=ON"
|
||||
LIB_PATH="${PWD}/${BUILD_DIR}/bin"
|
||||
else
|
||||
BUILD_DIR="build"
|
||||
CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PWD}/install"
|
||||
LIB_PATH="${PWD}/install/lib/llama.cpp"
|
||||
fi
|
||||
|
||||
cmake --fresh -S . -B "${BUILD_DIR}" ${CMAKE_ARGS}
|
||||
cmake --build "${BUILD_DIR}" -j 8
|
||||
|
||||
LD_LIBRARY_PATH="${LIB_PATH}:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "./${BUILD_DIR}/test-cmake"
|
||||
|
||||
@@ -2,8 +2,12 @@
|
||||
#include <cstdio>
|
||||
|
||||
int main(void) {
|
||||
#ifdef LLAMA_BUILD_NUMBER
|
||||
printf("[test-cmake] llama.cpp version: %s, build: %d (%s)\n",
|
||||
llama_version(), LLAMA_BUILD_NUMBER, LLAMA_BUILD_COMMIT);
|
||||
#else
|
||||
printf("[test-cmake] llama.cpp version: %s\n", llama_version());
|
||||
#endif
|
||||
printf("[test-cmake] ggml version: %s, commit: %s\n", ggml_version(), ggml_commit());
|
||||
printf("[test-cmake] Initializing backend...\n");
|
||||
llama_backend_init();
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# ./examples/ts-type-to-grammar.sh "{a:string,b:string,c?:string}"
|
||||
# python examples/json_schema_to_grammar.py https://json.schemastore.org/tsconfig.json
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
readonly type="$1"
|
||||
|
||||
# Create a temporary directory
|
||||
TMPDIR=""
|
||||
trap 'rm -fR "$TMPDIR"' EXIT
|
||||
TMPDIR=$(mktemp -d)
|
||||
|
||||
DTS_FILE="$TMPDIR/type.d.ts"
|
||||
SCHEMA_FILE="$TMPDIR/schema.json"
|
||||
|
||||
echo "export type MyType = $type" > "$DTS_FILE"
|
||||
|
||||
# This is a fork of typescript-json-schema, actively maintained as of March 2024:
|
||||
# https://github.com/vega/ts-json-schema-generator
|
||||
npx ts-json-schema-generator --unstable --no-top-ref --path "$DTS_FILE" --type MyType -e none > "$SCHEMA_FILE"
|
||||
|
||||
# Alternative, not actively maintained as of March 2024:
|
||||
# https://github.com/YousefED/typescript-json-schema
|
||||
# npx typescript-json-schema --defaultProps --required "$DTS_FILE" MyType | tee "$SCHEMA_FILE" >&2
|
||||
|
||||
./examples/json_schema_to_grammar.py "$SCHEMA_FILE"
|
||||
+1
-1
@@ -4,7 +4,7 @@ project("ggml" C CXX ASM)
|
||||
|
||||
### GGML Version
|
||||
set(GGML_VERSION_MAJOR 0)
|
||||
set(GGML_VERSION_MINOR 23)
|
||||
set(GGML_VERSION_MINOR 24)
|
||||
set(GGML_VERSION_PATCH 0)
|
||||
set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define RPC_PROTO_MAJOR_VERSION 6
|
||||
#define RPC_PROTO_MAJOR_VERSION 7
|
||||
#define RPC_PROTO_MINOR_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 0
|
||||
|
||||
|
||||
@@ -2703,11 +2703,21 @@ extern "C" {
|
||||
struct ggml_tensor * x,
|
||||
struct ggml_tensor * weights);
|
||||
|
||||
// hc_pre with a per-element gate (Qwen3.8-Flash-Next): gate [n_embd, hc, n_tokens]
|
||||
// result[i, t] = scale*sum_h x[i, h, t]*sigmoid(gate[i, h, t])
|
||||
//
|
||||
GGML_API struct ggml_tensor * ggml_dsv4_hc_pre_gated(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * x,
|
||||
struct ggml_tensor * gate,
|
||||
float scale);
|
||||
|
||||
// hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens],
|
||||
// post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens]
|
||||
// -> [n_embd, hc, n_tokens]
|
||||
// result[i, dst, t] = x[i, t]*post[dst, t]
|
||||
// + sum_src residual[i, src, t]*comb[dst, src, t]
|
||||
// comb == NULL uses the identity: result[i, dst, t] = x[i, t]*post[dst, t] + residual[i, dst, t]
|
||||
//
|
||||
GGML_API struct ggml_tensor * ggml_dsv4_hc_post(
|
||||
struct ggml_context * ctx,
|
||||
|
||||
@@ -675,12 +675,6 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
|
||||
target_compile_options(${GGML_CPU_NAME} PRIVATE ${ARCH_FLAGS})
|
||||
target_compile_definitions(${GGML_CPU_NAME} PRIVATE ${ARCH_DEFINITIONS})
|
||||
|
||||
if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT GGML_SYSTEM_ARCH STREQUAL "x86")
|
||||
message(STATUS "Skipping PCH for ${GGML_CPU_NAME}: GCC PCH is only enabled for x86 (arch: ${GGML_SYSTEM_ARCH})")
|
||||
else()
|
||||
target_precompile_headers(${GGML_CPU_NAME} PRIVATE ggml-impl.h)
|
||||
endif()
|
||||
|
||||
if (EMSCRIPTEN)
|
||||
set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS "-msimd128")
|
||||
endif()
|
||||
|
||||
@@ -417,6 +417,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo
|
||||
sumf = vec_hsum_f32x4(v_acc);
|
||||
*s = sumf;
|
||||
#else
|
||||
UNUSED(nb);
|
||||
UNUSED(x);
|
||||
UNUSED(y);
|
||||
UNUSED(ib);
|
||||
|
||||
@@ -70,6 +70,7 @@ void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTR
|
||||
#endif
|
||||
}
|
||||
|
||||
#if defined(__VXE__) || defined(__VXE2__)
|
||||
static inline int16x8_t vxe_dot_acc(const int8x16_t v_x, const int8x16_t v_y, const int16x8_t v_acc) {
|
||||
return vec_meadd(v_x, v_y, vec_moadd(v_x, v_y, v_acc));
|
||||
}
|
||||
@@ -84,6 +85,7 @@ static inline int32x4_t vxe_fold(const int16x8_t v_sumi) {
|
||||
const int16x8_t v_ones = vec_splats((int16_t)1);
|
||||
return vec_add(vec_mule(v_sumi, v_ones), vec_mulo(v_sumi, v_ones));
|
||||
}
|
||||
#endif
|
||||
|
||||
void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
|
||||
const int qk = QK8_0;
|
||||
|
||||
+42
-14
@@ -11259,10 +11259,19 @@ static void ggml_compute_forward_dsv4_hc_pre_f32(
|
||||
const int64_t hc = x->ne[1];
|
||||
const int64_t n_tokens = x->ne[2];
|
||||
|
||||
const float scale = ggml_get_op_params_f32(dst, 0);
|
||||
const bool gated = ggml_get_op_params_i32(dst, 1) != 0;
|
||||
|
||||
GGML_ASSERT(dst->ne[0] == n_embd);
|
||||
GGML_ASSERT(dst->ne[1] == n_tokens);
|
||||
GGML_ASSERT(weights->ne[0] == hc);
|
||||
GGML_ASSERT(weights->ne[1] == n_tokens);
|
||||
if (gated) {
|
||||
GGML_ASSERT(weights->ne[0] == n_embd);
|
||||
GGML_ASSERT(weights->ne[1] == hc);
|
||||
GGML_ASSERT(weights->ne[2] == n_tokens);
|
||||
} else {
|
||||
GGML_ASSERT(weights->ne[0] == hc);
|
||||
GGML_ASSERT(weights->ne[1] == n_tokens);
|
||||
}
|
||||
|
||||
GGML_TENSOR_LOCALS(size_t, nbx, x, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbw, weights, nb);
|
||||
@@ -11282,12 +11291,18 @@ static void ggml_compute_forward_dsv4_hc_pre_f32(
|
||||
|
||||
float sum = 0.0f;
|
||||
for (int64_t ih = 0; ih < hc; ++ih) {
|
||||
const float xv = *(const float *) ((const char *) x->data + i0*nbx0 + ih*nbx1 + it*nbx2);
|
||||
const float wv = *(const float *) ((const char *) weights->data + ih*nbw0 + it*nbw1);
|
||||
const float xv = *(const float *) ((const char *) x->data + i0*nbx0 + ih*nbx1 + it*nbx2);
|
||||
float wv;
|
||||
if (gated) {
|
||||
const float gv = *(const float *) ((const char *) weights->data + i0*nbw0 + ih*nbw1 + it*nbw2);
|
||||
wv = 1.0f / (1.0f + expf(-gv));
|
||||
} else {
|
||||
wv = *(const float *) ((const char *) weights->data + ih*nbw0 + it*nbw1);
|
||||
}
|
||||
sum += xv * wv;
|
||||
}
|
||||
|
||||
*(float *) ((char *) dst->data + i0*nbd0 + it*nbd1) = sum;
|
||||
*(float *) ((char *) dst->data + i0*nbd0 + it*nbd1) = scale * sum;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11321,7 +11336,6 @@ static void ggml_compute_forward_dsv4_hc_post_f32(
|
||||
GGML_ASSERT(x->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(residual->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(post->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(comb->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
|
||||
const int64_t n_embd = x->ne[0];
|
||||
@@ -11335,14 +11349,24 @@ static void ggml_compute_forward_dsv4_hc_post_f32(
|
||||
GGML_ASSERT(residual->ne[2] == n_tokens);
|
||||
GGML_ASSERT(post->ne[0] == hc);
|
||||
GGML_ASSERT(post->ne[1] == n_tokens);
|
||||
GGML_ASSERT(comb->ne[0] == hc);
|
||||
GGML_ASSERT(comb->ne[1] == hc);
|
||||
GGML_ASSERT(comb->ne[2] == n_tokens);
|
||||
|
||||
// comb == NULL: identity mixing, each stream keeps its own residual
|
||||
size_t nbc0 = 0;
|
||||
size_t nbc1 = 0;
|
||||
size_t nbc2 = 0;
|
||||
if (comb) {
|
||||
GGML_ASSERT(comb->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(comb->ne[0] == hc);
|
||||
GGML_ASSERT(comb->ne[1] == hc);
|
||||
GGML_ASSERT(comb->ne[2] == n_tokens);
|
||||
nbc0 = comb->nb[0];
|
||||
nbc1 = comb->nb[1];
|
||||
nbc2 = comb->nb[2];
|
||||
}
|
||||
|
||||
GGML_TENSOR_LOCALS(size_t, nbx, x, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbr, residual, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbp, post, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbc, comb, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbd, dst, nb);
|
||||
|
||||
const int ith = params->ith;
|
||||
@@ -11362,10 +11386,14 @@ static void ggml_compute_forward_dsv4_hc_post_f32(
|
||||
const float pv = *(const float *) ((const char *) post->data + idst*nbp0 + it*nbp1);
|
||||
|
||||
float sum = xv * pv;
|
||||
for (int64_t isrc = 0; isrc < hc; ++isrc) {
|
||||
const float rv = *(const float *) ((const char *) residual->data + i0*nbr0 + isrc*nbr1 + it*nbr2);
|
||||
const float cv = *(const float *) ((const char *) comb->data + idst*nbc0 + isrc*nbc1 + it*nbc2);
|
||||
sum += rv * cv;
|
||||
if (comb) {
|
||||
for (int64_t isrc = 0; isrc < hc; ++isrc) {
|
||||
const float rv = *(const float *) ((const char *) residual->data + i0*nbr0 + isrc*nbr1 + it*nbr2);
|
||||
const float cv = *(const float *) ((const char *) comb->data + idst*nbc0 + isrc*nbc1 + it*nbc2);
|
||||
sum += rv * cv;
|
||||
}
|
||||
} else {
|
||||
sum += *(const float *) ((const char *) residual->data + i0*nbr0 + idst*nbr1 + it*nbr2);
|
||||
}
|
||||
|
||||
*(float *) ((char *) dst->data + i0*nbd0 + idst*nbd1 + it*nbd2) = sum;
|
||||
|
||||
+4
-13
@@ -5,10 +5,10 @@
|
||||
//
|
||||
// cache line
|
||||
//
|
||||
|
||||
#if defined(__cpp_lib_hardware_interference_size)
|
||||
#define CACHE_LINE_SIZE std::hardware_destructive_interference_size
|
||||
#else
|
||||
// TODO: rework CACHE_LINE_SIZE so std::hardware_destructive_interference_size
|
||||
// can be used consistently between C and C++ TUs; the previous macro form
|
||||
// diverged based on include order and undersized the work buffer.
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/28882
|
||||
#if defined(__POWER9_VECTOR__)
|
||||
#define CACHE_LINE_SIZE 128
|
||||
#elif defined(__VXE__) || defined(__VXE2__)
|
||||
@@ -16,17 +16,8 @@
|
||||
#else
|
||||
#define CACHE_LINE_SIZE 64
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// -Winterference-size was introduced in GCC 12
|
||||
#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Winterference-size"
|
||||
#endif
|
||||
static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);
|
||||
#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
// Work buffer size for im2col operations in CONV2D
|
||||
#define GGML_IM2COL_WORK_SIZE (16 * 1024 * 1024)
|
||||
|
||||
@@ -639,7 +639,7 @@ static void permute_transpose_impl(const ggml_tensor * src0,
|
||||
}
|
||||
} else if (n_src_stride == sizeof(int16_t)) {
|
||||
for (int64_t bi = ith; bi < batch; bi += nth) {
|
||||
rvv_transposed_s32_mn_to_nm((int8_t *) ((char *) dst->data + bi * batch_stride), n_dst_stride,
|
||||
rvv_transposed_s16_mn_to_nm((int8_t *) ((char *) dst->data + bi * batch_stride), n_dst_stride,
|
||||
(int8_t *) ((char *) src0->data + bi * batch_stride), m_src_stride, m, n);
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#include "allreduce.cuh"
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#if !defined(GGML_USE_MUSA)
|
||||
|
||||
#include "convert.cuh"
|
||||
#include "ggml-impl.h"
|
||||
@@ -11,11 +11,12 @@
|
||||
#include <limits>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CUDA AllReduce for tensor-parallel inference across two GPUs.
|
||||
// AllReduce for tensor-parallel inference across two GPUs (CUDA or
|
||||
// ROCm/HIP).
|
||||
//
|
||||
// Provides an in-place sum reduction over matching tensors on two CUDA
|
||||
// devices in the same process. Used by the tensor-split path alongside
|
||||
// NCCL; targets setups without NVLink, where data is exchanged between the
|
||||
// Provides an in-place sum reduction over matching tensors on two GPUs
|
||||
// in the same process. Used by the tensor-split path alongside NCCL;
|
||||
// targets setups without NVLink/xGMI, where data is exchanged between the
|
||||
// GPUs by staging it through pinned host memory over PCIe.
|
||||
//
|
||||
// Two reduction strategies are selected per call by tensor size:
|
||||
@@ -161,11 +162,14 @@ static __global__ void ggml_cuda_ar_kernel(
|
||||
__threadfence_system(); // make our signal visible system-wide
|
||||
|
||||
while (ggml_cuda_ar_signal_get(other_slot) != token) {
|
||||
#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
|
||||
#ifdef GGML_USE_HIP
|
||||
// Equals ~100ns at 2500 MHz (sleeps for n * [1,64] clock cycles)
|
||||
__builtin_amdgcn_s_sleep(4);
|
||||
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
|
||||
__nanosleep(100);
|
||||
#else
|
||||
NO_DEVICE_CODE;
|
||||
#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
|
||||
#endif // GGML_USE_HIP
|
||||
}
|
||||
}
|
||||
|
||||
@@ -280,7 +284,7 @@ struct ggml_cuda_ar_host_mapping {
|
||||
}
|
||||
rc = cudaHostGetDevicePointer(reinterpret_cast<void **>(&dev), host, 0);
|
||||
if (rc != cudaSuccess) {
|
||||
cudaFreeHost(host);
|
||||
CUDA_CHECK(cudaFreeHost(host));
|
||||
host = nullptr;
|
||||
dev = nullptr;
|
||||
}
|
||||
@@ -289,7 +293,7 @@ struct ggml_cuda_ar_host_mapping {
|
||||
|
||||
void free() {
|
||||
if (host) {
|
||||
cudaFreeHost(host);
|
||||
CUDA_CHECK(cudaFreeHost(host));
|
||||
host = nullptr;
|
||||
dev = nullptr;
|
||||
}
|
||||
@@ -401,7 +405,8 @@ ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(const int * devices, size_t n
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// The chunked kernel uses __nanosleep, which is sm70+ (Volta+).
|
||||
// The chunked kernel uses __nanosleep (NVIDIA, sm70+) or
|
||||
// __builtin_amdgcn_s_sleep (AMD).
|
||||
for (size_t i = 0; i < n_devices; ++i) {
|
||||
const int cc = ggml_cuda_info().devices[devices[i]].cc;
|
||||
if (cc < GGML_CUDA_CC_VOLTA) {
|
||||
@@ -543,7 +548,7 @@ void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline * p) {
|
||||
for (int i = 0; i < p->n_devices; ++i) {
|
||||
if (p->streams[i]) {
|
||||
ggml_cuda_set_device(p->devices[i]);
|
||||
cudaStreamSynchronize(p->streams[i]);
|
||||
CUDA_CHECK(cudaStreamSynchronize(p->streams[i]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -552,28 +557,28 @@ void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline * p) {
|
||||
p->host_large[i].free();
|
||||
if (p->dev_tmp[i]) {
|
||||
ggml_cuda_set_device(p->devices[i]);
|
||||
cudaFree(p->dev_tmp[i]);
|
||||
CUDA_CHECK(cudaFree(p->dev_tmp[i]));
|
||||
}
|
||||
ggml_cuda_set_device(p->devices[i]);
|
||||
for (int s = 0; s < GGML_CUDA_AR_POOL_SIZE; ++s) {
|
||||
if (p->ev_pool[i][s].app) { cudaEventDestroy(p->ev_pool[i][s].app); }
|
||||
if (p->ev_pool[i][s].app) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].app)); }
|
||||
for (int c = 0; c < GGML_CUDA_AR_COPY_MAX_CHUNKS; ++c) {
|
||||
if (p->ev_pool[i][s].cpy[c]) { cudaEventDestroy(p->ev_pool[i][s].cpy[c]); }
|
||||
if (p->ev_pool[i][s].cpy[c]) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].cpy[c])); }
|
||||
}
|
||||
if (p->ev_pool[i][s].h2d) { cudaEventDestroy(p->ev_pool[i][s].h2d); }
|
||||
if (p->ev_pool[i][s].ker) { cudaEventDestroy(p->ev_pool[i][s].ker); }
|
||||
if (p->ev_pool[i][s].h2d) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].h2d)); }
|
||||
if (p->ev_pool[i][s].ker) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].ker)); }
|
||||
}
|
||||
if (p->host_large_read_done[i]) {
|
||||
ggml_cuda_set_device(p->devices[i]);
|
||||
cudaEventDestroy(p->host_large_read_done[i]);
|
||||
CUDA_CHECK(cudaEventDestroy(p->host_large_read_done[i]));
|
||||
}
|
||||
if (p->dev_tmp_kernel_done[i]) {
|
||||
ggml_cuda_set_device(p->devices[i]);
|
||||
cudaEventDestroy(p->dev_tmp_kernel_done[i]);
|
||||
CUDA_CHECK(cudaEventDestroy(p->dev_tmp_kernel_done[i]));
|
||||
}
|
||||
if (p->streams[i]) {
|
||||
ggml_cuda_set_device(p->devices[i]);
|
||||
cudaStreamDestroy(p->streams[i]);
|
||||
CUDA_CHECK(cudaStreamDestroy(p->streams[i]));
|
||||
}
|
||||
}
|
||||
p->arrival.free();
|
||||
@@ -952,13 +957,14 @@ bool ggml_cuda_ar_allreduce(
|
||||
return ok;
|
||||
}
|
||||
|
||||
#else // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
#else // defined(GGML_USE_MUSA)
|
||||
|
||||
// HIP and MUSA lack the host-mapped pinned-memory APIs (cudaHostAllocPortable
|
||||
// / cudaHostAllocMapped / cudaHostGetDevicePointer) and __nanosleep that this
|
||||
// implementation relies on, so the internal AllReduce is a CUDA-only feature.
|
||||
// The dispatcher in ggml-cuda.cu treats a nullptr pipeline as "init failed"
|
||||
// and silently falls back to the meta backend's generic AllReduce.
|
||||
// MUSA lacks the host-mapped pinned-memory APIs (cudaHostAllocPortable
|
||||
// / cudaHostAllocMapped / cudaHostGetDevicePointer) and a device-side
|
||||
// sleep intrinsic that this implementation relies on, so the internal
|
||||
// AllReduce is unavailable there. The dispatcher in ggml-cuda.cu treats
|
||||
// a nullptr pipeline as "init failed" and silently falls back to the meta
|
||||
// backend's generic AllReduce.
|
||||
ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(const int *, size_t) {
|
||||
return nullptr;
|
||||
}
|
||||
@@ -968,4 +974,4 @@ bool ggml_cuda_ar_allreduce(ggml_cuda_ar_pipeline *, ggml_backend_t *, ggml_tens
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#endif // !defined(GGML_USE_MUSA)
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
struct ggml_cuda_ar_pipeline;
|
||||
|
||||
// Allocate a pipeline for n_devices GPUs.
|
||||
// devices[] holds the CUDA device IDs in rank order.
|
||||
// devices[] holds the GPU device IDs in rank order.
|
||||
// Returns nullptr on allocation failure.
|
||||
ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(
|
||||
const int * devices, size_t n_devices);
|
||||
|
||||
@@ -329,6 +329,12 @@ static bool fp16_mma_hardware_available(const int cc) {
|
||||
(GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);
|
||||
}
|
||||
|
||||
// To be used for feature selection of external libraries, e.g. cuBLAS.
|
||||
static bool fast_bf16_hardware_available(const int cc) {
|
||||
return (GGML_CUDA_CC_IS_AMD(cc) && (cc >= GGML_CUDA_CC_RDNA3 || GGML_CUDA_CC_IS_CDNA(cc)))
|
||||
|| (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE);
|
||||
}
|
||||
|
||||
static bool bf16_mma_hardware_available(const int cc) {
|
||||
return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE) ||
|
||||
GGML_CUDA_CC_IS_CDNA(cc) || cc >= GGML_CUDA_CC_RDNA3 ||
|
||||
|
||||
@@ -589,6 +589,14 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
|
||||
ggml_cpy_scalar_cuda<int32_t, int32_t>
|
||||
(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
|
||||
}
|
||||
} else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) {
|
||||
if (can_be_transposed) {
|
||||
ggml_cpy_scalar_cuda<int16_t, int16_t, true>
|
||||
(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
|
||||
} else {
|
||||
ggml_cpy_scalar_cuda<int16_t, int16_t>
|
||||
(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
|
||||
}
|
||||
} else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32) {
|
||||
if (contiguous_srcs) {
|
||||
ggml_cpy_scalar_contiguous_cuda<float, int32_t>
|
||||
|
||||
@@ -100,6 +100,7 @@ static __global__ void dsv4_hc_comb_f32(
|
||||
}
|
||||
}
|
||||
|
||||
template <bool gated>
|
||||
static __global__ void dsv4_hc_pre_f32(
|
||||
const float * x,
|
||||
const float * weights,
|
||||
@@ -112,8 +113,10 @@ static __global__ void dsv4_hc_pre_f32(
|
||||
int64_t sx2,
|
||||
int64_t sw0,
|
||||
int64_t sw1,
|
||||
int64_t sw2,
|
||||
int64_t sd0,
|
||||
int64_t sd1) {
|
||||
int64_t sd1,
|
||||
float scale) {
|
||||
ggml_cuda_pdl_lc();
|
||||
const int64_t ir = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
|
||||
const int64_t nr = n_embd * n_tokens;
|
||||
@@ -127,16 +130,22 @@ static __global__ void dsv4_hc_pre_f32(
|
||||
const int64_t i0 = ir % n_embd;
|
||||
const int64_t it = ir / n_embd;
|
||||
|
||||
float sum = x[i0*sx0 + it*sx2] * weights[it*sw1];
|
||||
for (int64_t ih = 1; ih < hc; ++ih) {
|
||||
float sum = 0.0f;
|
||||
for (int64_t ih = 0; ih < hc; ++ih) {
|
||||
const float xv = x[i0*sx0 + ih*sx1 + it*sx2];
|
||||
const float wv = weights[ih*sw0 + it*sw1];
|
||||
float wv;
|
||||
if constexpr (gated) {
|
||||
wv = 1.0f / (1.0f + expf(-weights[i0*sw0 + ih*sw1 + it*sw2]));
|
||||
} else {
|
||||
wv = weights[ih*sw0 + it*sw1];
|
||||
}
|
||||
sum += xv * wv;
|
||||
}
|
||||
|
||||
dst[i0*sd0 + it*sd1] = sum;
|
||||
dst[i0*sd0 + it*sd1] = scale * sum;
|
||||
}
|
||||
|
||||
template <bool has_comb>
|
||||
static __global__ void dsv4_hc_post_f32(
|
||||
const float * x,
|
||||
const float * residual,
|
||||
@@ -174,8 +183,12 @@ static __global__ void dsv4_hc_post_f32(
|
||||
const int64_t it = ir / (n_embd * hc);
|
||||
|
||||
float sum = x[i0*sx0 + it*sx1] * post[idst*sp0 + it*sp1];
|
||||
for (int64_t isrc = 0; isrc < hc; ++isrc) {
|
||||
sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2];
|
||||
if constexpr (has_comb) {
|
||||
for (int64_t isrc = 0; isrc < hc; ++isrc) {
|
||||
sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2];
|
||||
}
|
||||
} else {
|
||||
sum += residual[i0*sr0 + idst*sr1 + it*sr2];
|
||||
}
|
||||
|
||||
dst[i0*sd0 + idst*sd1 + it*sd2] = sum;
|
||||
@@ -240,18 +253,23 @@ void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst
|
||||
const int64_t hc = x->ne[1];
|
||||
const int64_t n_tokens = x->ne[2];
|
||||
|
||||
const float scale = ggml_get_op_params_f32(dst, 0);
|
||||
const bool gated = ggml_get_op_params_i32(dst, 1) != 0;
|
||||
|
||||
const int block_size = 256;
|
||||
const int64_t nr = n_embd * n_tokens;
|
||||
const dim3 block_dims(block_size, 1, 1);
|
||||
const dim3 grid_dims((nr + block_size - 1) / block_size, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream());
|
||||
|
||||
ggml_cuda_kernel_launch(dsv4_hc_pre_f32, launch_params,
|
||||
auto kernel = gated ? dsv4_hc_pre_f32<true> : dsv4_hc_pre_f32<false>;
|
||||
ggml_cuda_kernel_launch(kernel, launch_params,
|
||||
(const float *) x->data, (const float *) weights->data, (float *) dst->data,
|
||||
n_embd, hc, n_tokens,
|
||||
nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float),
|
||||
nbw0 / sizeof(float), nbw1 / sizeof(float),
|
||||
nbd0 / sizeof(float), nbd1 / sizeof(float));
|
||||
nbw0 / sizeof(float), nbw1 / sizeof(float), nbw2 / sizeof(float),
|
||||
nbd0 / sizeof(float), nbd1 / sizeof(float),
|
||||
scale);
|
||||
}
|
||||
|
||||
void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
@@ -263,15 +281,18 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds
|
||||
GGML_ASSERT(x->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(residual->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(post->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(comb->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(comb == nullptr || comb->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
|
||||
GGML_TENSOR_LOCALS(size_t, nbx, x, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbr, residual, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbp, post, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbc, comb, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbd, dst, nb);
|
||||
|
||||
const size_t nbc0 = comb ? comb->nb[0] : 0;
|
||||
const size_t nbc1 = comb ? comb->nb[1] : 0;
|
||||
const size_t nbc2 = comb ? comb->nb[2] : 0;
|
||||
|
||||
const int64_t n_embd = x->ne[0];
|
||||
const int64_t n_tokens = x->ne[1];
|
||||
const int64_t hc = residual->ne[1];
|
||||
@@ -282,9 +303,10 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds
|
||||
const dim3 grid_dims((nr + block_size - 1) / block_size, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream());
|
||||
|
||||
ggml_cuda_kernel_launch(dsv4_hc_post_f32, launch_params,
|
||||
auto kernel = comb ? dsv4_hc_post_f32<true> : dsv4_hc_post_f32<false>;
|
||||
ggml_cuda_kernel_launch(kernel, launch_params,
|
||||
(const float *) x->data, (const float *) residual->data,
|
||||
(const float *) post->data, (const float *) comb->data, (float *) dst->data,
|
||||
(const float *) post->data, comb ? (const float *) comb->data : nullptr, (float *) dst->data,
|
||||
n_embd, hc, n_tokens,
|
||||
nbx0 / sizeof(float), nbx1 / sizeof(float),
|
||||
nbr0 / sizeof(float), nbr1 / sizeof(float), nbr2 / sizeof(float),
|
||||
|
||||
@@ -181,7 +181,7 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 1, 64, 32, 32, 32, 1, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 32, 32, 32, 1, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 32, 32, 32, 1, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 4, 64, 32, 32, 32, 1, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 3, 64, 32, 32, 32, 1, true);
|
||||
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40, 40, 40, 1, true);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40, 40, 40, 1, true);
|
||||
@@ -1141,7 +1141,7 @@ template<int DV, int ncols> struct mma_tile_sizes {
|
||||
using T_C_KQ = tile<16, 16, float>; // column-major
|
||||
using T_A_VKQ = tile<16, 8, half2>; // row-major
|
||||
using T_B_VKQ = tile<16, 8, half2>; // column-major
|
||||
using T_C_VKQ = tile<16, 8, half2>; // column-major
|
||||
using T_C_VKQ = tile<16, 16, float>; // column-major
|
||||
};
|
||||
#else // Volta
|
||||
template<int DV, int ncols> struct mma_tile_sizes {
|
||||
@@ -1227,7 +1227,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)];
|
||||
#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
|
||||
T_C_VKQ VKQ_C[DV % 32 != 0 ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)];
|
||||
#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
#elif defined(AMD_MFMA_AVAILABLE)
|
||||
T_C_VKQ VKQ_C[ DV/T_C_VKQ::J];
|
||||
#elif defined(AMD_WMMA_AVAILABLE)
|
||||
T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)];
|
||||
#else // Volta
|
||||
T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)];
|
||||
|
||||
@@ -1620,11 +1620,19 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const
|
||||
}
|
||||
|
||||
static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
const int cc = ggml_cuda_info().devices[ctx.device].cc;
|
||||
ggml_type compute_type = src0->type;
|
||||
if (ggml_is_quantized(compute_type)) {
|
||||
compute_type = fast_fp16_hardware_available(ggml_cuda_info().devices[ctx.device].cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
} else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(ggml_cuda_info().devices[ctx.device].cc)) {
|
||||
compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
} else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) {
|
||||
compute_type = GGML_TYPE_F32;
|
||||
} else if (compute_type == GGML_TYPE_BF16 && !fast_bf16_hardware_available(cc)) {
|
||||
if (GGML_CUDA_CC_IS_AMD(cc) && src1->ne[1] > 32) {
|
||||
compute_type = GGML_TYPE_F32;
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && src1->ne[1] > (cc >= GGML_CUDA_CC_VOLTA ? 8 : 128)) {
|
||||
compute_type = GGML_TYPE_F32;
|
||||
}
|
||||
}
|
||||
if (dst->op_params[0] == GGML_PREC_F32) {
|
||||
compute_type = GGML_TYPE_F32;
|
||||
@@ -5290,10 +5298,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return false;
|
||||
} break;
|
||||
case GGML_OP_DUP:
|
||||
{
|
||||
ggml_type src0_type = op->src[0]->type;
|
||||
return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;
|
||||
} break;
|
||||
return true;
|
||||
case GGML_OP_ARGMAX:
|
||||
case GGML_OP_COUNT_EQUAL:
|
||||
{
|
||||
@@ -5461,7 +5466,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return true;
|
||||
#endif
|
||||
case GGML_OP_SUM_ROWS:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]);
|
||||
case GGML_OP_MEAN:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]);
|
||||
case GGML_OP_GROUP_NORM:
|
||||
return ggml_is_contiguous(op->src[0]);
|
||||
case GGML_OP_PAD:
|
||||
@@ -5490,7 +5497,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_DSV4_HC_POST:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 &&
|
||||
op->src[2]->type == GGML_TYPE_F32 && op->src[3]->type == GGML_TYPE_F32 &&
|
||||
op->src[2]->type == GGML_TYPE_F32 && (op->src[3] == nullptr || op->src[3]->type == GGML_TYPE_F32) &&
|
||||
op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
return ggml_cuda_flash_attn_ext_supported(dev_ctx->device, op);
|
||||
|
||||
@@ -7,40 +7,41 @@ template <typename T>
|
||||
static __global__ void im2col_kernel(
|
||||
const float * x, T * dst,
|
||||
int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH,
|
||||
int64_t IC_IH_IW, int64_t IH_IW, int64_t N_OH, int64_t KH_KW, int64_t IC_KH_KW,
|
||||
int64_t N, int64_t IC_IH_IW, int64_t IH_IW, int64_t N_OH, int64_t KH_KW, int64_t IC_KH_KW,
|
||||
int s0, int s1, int p0, int p1, int d0, int d1) {
|
||||
const int64_t i = threadIdx.x + blockIdx.x * blockDim.x;
|
||||
if (i >= IC_KH_KW) {
|
||||
return;
|
||||
}
|
||||
const int tid = threadIdx.x;
|
||||
|
||||
const int64_t iic = i / (KH_KW);
|
||||
const int64_t rem = i - iic * KH_KW;
|
||||
const int64_t ikh = rem / KW;
|
||||
const int64_t ikw = rem - ikh * KW;
|
||||
const int64_t total_channels = IC * KH * KW;
|
||||
const int threads_per_pos = blockDim.x;
|
||||
const int64_t start_ch = tid;
|
||||
const int64_t stride_ch = threads_per_pos;
|
||||
|
||||
for (int64_t iow = blockIdx.y; iow < OW; iow += MAX_GRIDDIM_Y) {
|
||||
for (int64_t iz = blockIdx.z; iz < N_OH; iz += MAX_GRIDDIM_Z) {
|
||||
const int64_t in = iz / OH;
|
||||
const int64_t ioh = iz - in * OH;
|
||||
for (int64_t iow = blockIdx.x; iow < OW; iow += MAX_GRIDDIM_Y) {
|
||||
for (int64_t iz = blockIdx.y; iz < N_OH; iz += MAX_GRIDDIM_Z) {
|
||||
const int64_t in = iz / OH;
|
||||
const int64_t ioh = iz - in * OH;
|
||||
|
||||
const int64_t iiw = iow * s0 + ikw * d0 - p0;
|
||||
const int64_t iih = ioh * s1 + ikh * d1 - p1;
|
||||
for (int64_t iic_khw = start_ch; iic_khw < total_channels; iic_khw += stride_ch) {
|
||||
const int64_t iic = iic_khw / KH_KW;
|
||||
const int64_t rem = iic_khw - iic * KH_KW;
|
||||
const int64_t ikh = rem / KW;
|
||||
const int64_t ikw = rem - ikh * KW;
|
||||
|
||||
const int64_t offset_dst =
|
||||
((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw;
|
||||
const int64_t iiw = iow * s0 + ikw * d0 - p0;
|
||||
const int64_t iih = ioh * s1 + ikh * d1 - p1;
|
||||
|
||||
if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
|
||||
dst[offset_dst] = 0.0f;
|
||||
} else {
|
||||
const int64_t offset_src = iic * IC_IH_IW + in * IH_IW;
|
||||
dst[offset_dst] = x[offset_src + iih * IW + iiw];
|
||||
const int64_t offset_dst =
|
||||
((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw;
|
||||
|
||||
if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
|
||||
dst[offset_dst] = 0.0f;
|
||||
} else {
|
||||
const int64_t offset_src = iic * IC_IH_IW + in * IH_IW;
|
||||
dst[offset_dst] = x[offset_src + iih * IW + iiw];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
GGML_UNUSED(IC);
|
||||
GGML_UNUSED(KH);
|
||||
}
|
||||
|
||||
// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
|
||||
@@ -50,13 +51,15 @@ static void im2col_cuda(const float * x, T* dst,
|
||||
int64_t N, int64_t IC_IH_IW, int64_t IH_IW,
|
||||
int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
|
||||
const int64_t IC_KH_KW = IC * KH * KW;
|
||||
const int64_t num_blocks = (IC_KH_KW + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;
|
||||
const int64_t N_OH = N * OH;
|
||||
const int64_t KH_KW = KW*KH;
|
||||
dim3 block_nums(num_blocks, MIN(OW, MAX_GRIDDIM_Y), MIN(N_OH, MAX_GRIDDIM_Z));
|
||||
im2col_kernel<<<block_nums, MIN(IC_KH_KW, CUDA_IM2COL_BLOCK_SIZE) , 0, stream>>>(x, dst, IC, IW, IH, OH, OW, KW, KH,
|
||||
IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW,
|
||||
s0, s1, p0, p1, d0, d1);
|
||||
const int threads_per_block = MIN((int)IC_KH_KW, CUDA_IM2COL_BLOCK_SIZE);
|
||||
dim3 block_nums(MIN(OW, MAX_GRIDDIM_Y), MIN(N_OH, MAX_GRIDDIM_Z));
|
||||
|
||||
im2col_kernel<<<block_nums, threads_per_block, 0, stream>>>(
|
||||
x, dst, IC, IW, IH, OH, OW, KW, KH,
|
||||
N, IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW,
|
||||
s0, s1, p0, p1, d0, d1);
|
||||
}
|
||||
|
||||
static void im2col_cuda_f16(const float * x, half * dst,
|
||||
|
||||
@@ -18,7 +18,7 @@ void ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(ggml_is_contiguous(src0));
|
||||
GGML_ASSERT(ggml_is_contiguous_rows(src0));
|
||||
|
||||
const int64_t ncols = src0->ne[0];
|
||||
const int64_t nrows = ggml_nrows(src0);
|
||||
@@ -65,13 +65,20 @@ void ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
|
||||
// Heuristic for block size selection to optimize occupancy.
|
||||
// See discussion in: https://github.com/ggml-org/llama.cpp/pull/15132
|
||||
dim3 block_dims;
|
||||
if ((nrows / nsm) < 2) {
|
||||
const dim3 block_dims(512, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
|
||||
ggml_cuda_kernel_launch(reduce_rows_f32</*norm=*/true>, launch_params, src0_d, dst_d, ncols);
|
||||
block_dims = dim3(512, 1, 1);
|
||||
} else {
|
||||
const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
|
||||
ggml_cuda_kernel_launch(reduce_rows_f32</*norm=*/true>, launch_params, src0_d, dst_d, ncols);
|
||||
block_dims = dim3(ncols < 1024 ? 32 : 128, 1, 1);
|
||||
}
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
|
||||
|
||||
if (ggml_is_contiguous(src0)) {
|
||||
ggml_cuda_kernel_launch(reduce_rows_f32</*norm=*/true>, launch_params, src0_d, dst_d, ncols);
|
||||
return;
|
||||
}
|
||||
|
||||
const char * src0_d_bytes = (const char *) src0->data;
|
||||
ggml_cuda_kernel_launch(reduce_rows_f32_strided</*norm=*/true>, launch_params, src0_d_bytes, dst_d, ncols,
|
||||
src0->ne[1], src0->ne[2], src0->nb[1], src0->nb[2], src0->nb[3]);
|
||||
}
|
||||
|
||||
@@ -247,7 +247,7 @@ void ggml_cuda_mul_mat_q(
|
||||
// Each expert only sees ne12*n_expert_used/ne02 tokens on average.
|
||||
// On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.
|
||||
int64_t ncols_opt = ne12;
|
||||
if (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) {
|
||||
if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) {
|
||||
ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
#include "common.cuh"
|
||||
|
||||
// Row reduction kernel template - compute sum (norm=false) or mean (norm=true)
|
||||
template <bool norm>
|
||||
static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, const int ncols) {
|
||||
const float * GGML_CUDA_RESTRICT x = x_ptr;
|
||||
float * GGML_CUDA_RESTRICT dst = dst_ptr;
|
||||
const int row = blockIdx.x;
|
||||
static __device__ __forceinline__ float reduce_row_f32(const float * x, const int ncols) {
|
||||
const int col = threadIdx.x;
|
||||
|
||||
float sum = 0.0f;
|
||||
@@ -17,7 +12,7 @@ static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, con
|
||||
for (int i = col; i < ncols;) {
|
||||
for (int j = 0; j < num_unroll; ++j) {
|
||||
if (i < ncols) {
|
||||
temp[j] = x[row * ncols + i];
|
||||
temp[j] = x[i];
|
||||
} else {
|
||||
temp[j] = 0;
|
||||
}
|
||||
@@ -35,6 +30,40 @@ static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, con
|
||||
__shared__ float shared_vals[32];
|
||||
sum = block_reduce<block_reduce_method::SUM>(sum, shared_vals);
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
// Row reduction kernel template - compute sum (norm=false) or mean (norm=true)
|
||||
template <bool norm>
|
||||
static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, const int ncols) {
|
||||
float * GGML_CUDA_RESTRICT dst = dst_ptr;
|
||||
const int64_t row = blockIdx.x;
|
||||
const int col = threadIdx.x;
|
||||
|
||||
const float * GGML_CUDA_RESTRICT x = x_ptr + row*ncols;
|
||||
const float sum = reduce_row_f32(x, ncols);
|
||||
|
||||
if (col != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
dst[row] = norm ? sum / ncols : sum;
|
||||
}
|
||||
|
||||
template <bool norm>
|
||||
static __global__ void reduce_rows_f32_strided(const char * x_ptr, float * dst_ptr, const int ncols,
|
||||
const int64_t ne1, const int64_t ne2, const int64_t nb1, const int64_t nb2, const int64_t nb3) {
|
||||
float * GGML_CUDA_RESTRICT dst = dst_ptr;
|
||||
const int64_t row = blockIdx.x;
|
||||
const int col = threadIdx.x;
|
||||
|
||||
const int64_t i1 = row % ne1;
|
||||
const int64_t i2 = (row / ne1) % ne2;
|
||||
const int64_t i3 = row / (ne1 * ne2);
|
||||
|
||||
const float * GGML_CUDA_RESTRICT x = (const float *) (x_ptr + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
const float sum = reduce_row_f32(x, ncols);
|
||||
|
||||
if (col != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -24,24 +24,30 @@ void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(ggml_is_contiguous(src0));
|
||||
GGML_ASSERT(ggml_is_contiguous_rows(src0));
|
||||
|
||||
const int64_t ncols = src0->ne[0];
|
||||
const int64_t nrows = ggml_nrows(src0);
|
||||
|
||||
if (ggml_is_contiguous(src0)) {
|
||||
sum_rows_f32_cuda(src0_d, dst_d, ncols, nrows, stream);
|
||||
return;
|
||||
}
|
||||
|
||||
const dim3 block_nums(nrows, 1, 1);
|
||||
|
||||
const int id = ggml_cuda_get_device();
|
||||
const int nsm = ggml_cuda_info().devices[id].nsm;
|
||||
dim3 block_dims;
|
||||
if ((nrows / nsm) < 2) {
|
||||
// Increase num threads to 512 for small nrows to better hide the latency
|
||||
const dim3 block_dims(512, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
|
||||
ggml_cuda_kernel_launch(reduce_rows_f32</*norm=*/false>, launch_params, src0_d, dst_d, ncols);
|
||||
block_dims = dim3(512, 1, 1);
|
||||
} else {
|
||||
// Enough active SMs to hide latency, use smaller blocks to allow better scheduling
|
||||
const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
|
||||
ggml_cuda_kernel_launch(reduce_rows_f32</*norm=*/false>, launch_params, src0_d, dst_d, ncols);
|
||||
block_dims = dim3(ncols < 1024 ? 32 : 128, 1, 1);
|
||||
}
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
|
||||
const char * src0_d_bytes = (const char *) src0->data;
|
||||
ggml_cuda_kernel_launch(reduce_rows_f32_strided</*norm=*/false>, launch_params, src0_d_bytes, dst_d, ncols,
|
||||
src0->ne[1], src0->ne[2], src0->nb[1], src0->nb[2], src0->nb[3]);
|
||||
}
|
||||
|
||||
Vendored
+4
@@ -73,6 +73,10 @@
|
||||
#define cudaGetDeviceProperties hipGetDeviceProperties
|
||||
#define cudaGetErrorString hipGetErrorString
|
||||
#define cudaGetLastError hipGetLastError
|
||||
#define cudaHostAlloc hipHostMalloc
|
||||
#define cudaHostAllocPortable hipHostMallocPortable
|
||||
#define cudaHostAllocMapped hipHostMallocMapped
|
||||
#define cudaHostGetDevicePointer hipHostGetDevicePointer
|
||||
#define cudaHostRegister hipHostRegister
|
||||
#define cudaHostRegisterPortable hipHostRegisterPortable
|
||||
#define cudaHostRegisterReadOnly hipHostRegisterReadOnly
|
||||
|
||||
@@ -4986,7 +4986,9 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess
|
||||
const int mode = op_params[2];
|
||||
const int n_offs = op_params[15];
|
||||
|
||||
if (n_dims <= 0 || n_dims % 2 != 0) {
|
||||
// llama probes weight placement with a dummy rope where every param is 0 (llama-model-loader.cpp).
|
||||
// Rejecting it puts rope_freqs on the CPU, which then splits the graph at every full-attention layer.
|
||||
if (n_dims < 0 || n_dims % 2 != 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -4997,7 +4999,7 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess
|
||||
|
||||
float freq_base;
|
||||
memcpy(&freq_base, op_params + 5, sizeof(float));
|
||||
if (freq_base <= 0.0f) {
|
||||
if (freq_base < 0.0f) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -126,6 +126,13 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
|
||||
const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \
|
||||
if (th_start >= th_end) return; \
|
||||
\
|
||||
if (htp_tensor_is_contiguous(src0, ELEM_SIZE) && htp_tensor_is_contiguous(dst, ELEM_SIZE)) { \
|
||||
hvx_copy_uu((uint8_t *) dst->data + (size_t) th_start * ELEM_SIZE, \
|
||||
(const uint8_t *) src0->data + (size_t) th_start * ELEM_SIZE, \
|
||||
th_end - th_start, ELEM_SIZE); \
|
||||
return; \
|
||||
} \
|
||||
\
|
||||
const uint32_t ne01_ne00 = ne01 * ne00; \
|
||||
const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \
|
||||
const uint32_t ne1_ne0 = ne1 * ne0; \
|
||||
@@ -149,11 +156,21 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
|
||||
char * dst_ptr = (char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \
|
||||
const char * src0_ptr = (const char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \
|
||||
\
|
||||
for (; e < th_end; e++) { \
|
||||
*((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \
|
||||
const bool rows_contig = (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \
|
||||
\
|
||||
dst_ptr += nb0; \
|
||||
if (++i10 == ne0) { \
|
||||
while (e < th_end) { \
|
||||
uint32_t run = 1; \
|
||||
if (rows_contig) { \
|
||||
run = MIN(MIN(ne00 - i00, ne0 - i10), th_end - e); \
|
||||
hvx_copy_uu((uint8_t *) dst_ptr, (const uint8_t *) src0_ptr, run, ELEM_SIZE); \
|
||||
} else { \
|
||||
*((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \
|
||||
} \
|
||||
e += run; \
|
||||
\
|
||||
dst_ptr += run * nb0; \
|
||||
i10 += run; \
|
||||
if (i10 == ne0) { \
|
||||
i10 = 0; \
|
||||
if (++i11 == ne1) { \
|
||||
i11 = 0; \
|
||||
@@ -165,8 +182,9 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
|
||||
dst_ptr = (char *) dst->data + i11*nb1 + i12*nb2 + i13*nb3; \
|
||||
} \
|
||||
\
|
||||
src0_ptr += nb00; \
|
||||
if (++i00 == ne00) { \
|
||||
src0_ptr += run * nb00; \
|
||||
i00 += run; \
|
||||
if (i00 == ne00) { \
|
||||
i00 = 0; \
|
||||
if (++i01 == ne01) { \
|
||||
i01 = 0; \
|
||||
@@ -294,6 +312,18 @@ static inline void cpy_dma_sametype_sameshape(
|
||||
dma_queue_flush(q);
|
||||
}
|
||||
|
||||
static inline void cpy_dma_sametype_reshape_contig(
|
||||
struct htp_ops_context * octx,
|
||||
const struct htp_tensor * dst,
|
||||
const struct htp_tensor * src0,
|
||||
uint32_t total_bytes
|
||||
) {
|
||||
dma_queue * q = octx->ctx->dma[0];
|
||||
dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data),
|
||||
total_bytes, total_bytes, total_bytes, /*nrows=*/ 1);
|
||||
dma_queue_pop(q);
|
||||
}
|
||||
|
||||
static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
|
||||
cpy_preamble;
|
||||
*use_dma = false;
|
||||
@@ -327,6 +357,7 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
|
||||
|
||||
const uint32_t n_threads = octx->n_threads;
|
||||
|
||||
const bool src_is_contiguous = htp_tensor_is_contiguous(src0, ct.src0_type_size);
|
||||
const bool dst_is_contiguous = htp_tensor_is_contiguous(dst, ct.dst_type_size);
|
||||
|
||||
if (sameshape) {
|
||||
@@ -375,6 +406,12 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
|
||||
const uint32_t total_elems = ne0 * ne1 * ne2 * ne3;
|
||||
const uint32_t elems_per_line = (ct.dst_type_size == 4) ? 32 : 64;
|
||||
|
||||
if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) {
|
||||
*use_dma = true;
|
||||
cpy_dma_sametype_reshape_contig(octx, dst, src0, total_elems * ct.dst_type_size);
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
ct.div_ne0 = init_fastdiv_values(ne0);
|
||||
ct.div_ne1_ne0 = init_fastdiv_values(ne1 * ne0);
|
||||
ct.div_ne2_ne1_ne0 = init_fastdiv_values(ne2 * ne1 * ne0);
|
||||
|
||||
@@ -1063,6 +1063,40 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib) {
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
snprintf(base, 256, "kernel_mul_mm_id_amax_part_f32");
|
||||
snprintf(name, 256, "%s", base);
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
|
||||
}
|
||||
|
||||
res.smem = 32*sizeof(float);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib) {
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
snprintf(base, 256, "kernel_mul_mm_id_amax_f32");
|
||||
snprintf(name, 256, "%s", base);
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
|
||||
}
|
||||
|
||||
res.smem = 32*sizeof(float);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(ggml_metal_library_t lib, int ne02, int ne20) {
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
@@ -138,6 +138,8 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ex
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0 (ggml_metal_library_t lib, int ne02, int ne20);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
||||
@@ -1803,6 +1803,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
op->type == GGML_TYPE_F32 &&
|
||||
op->src[0]->ne[1] == 4 &&
|
||||
op->src[1]->ne[0] == 4 &&
|
||||
op->src[1]->ne[2] == 1 &&
|
||||
ggml_is_contiguous_rows(op->src[0]) &&
|
||||
ggml_is_contiguous_rows(op->src[1]);
|
||||
case GGML_OP_DSV4_HC_POST:
|
||||
@@ -1810,6 +1811,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
op->src[0]->type == GGML_TYPE_F32 &&
|
||||
op->src[1]->type == GGML_TYPE_F32 &&
|
||||
op->src[2]->type == GGML_TYPE_F32 &&
|
||||
op->src[3] != NULL &&
|
||||
op->src[3]->type == GGML_TYPE_F32 &&
|
||||
op->type == GGML_TYPE_F32 &&
|
||||
op->src[1]->ne[1] == 4 &&
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
#define N_MM_SIMD_GROUP_X 2
|
||||
#define N_MM_SIMD_GROUP_Y 2
|
||||
|
||||
#define N_MM_NPART_AMAX 256
|
||||
|
||||
// kernel parameters for mat-vec threadgroups
|
||||
//
|
||||
// N_R0: number of src0 rows to process per simdgroup
|
||||
@@ -555,6 +557,14 @@ typedef struct {
|
||||
uint64_t nb21;
|
||||
} ggml_metal_kargs_mul_mm_id_map0;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne00;
|
||||
int32_t ne01;
|
||||
int32_t ne02;
|
||||
uint64_t nb01;
|
||||
uint64_t nb02;
|
||||
} ggml_metal_kargs_mul_mm_id_amax;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne00;
|
||||
int32_t ne02;
|
||||
|
||||
@@ -2631,6 +2631,15 @@ size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) {
|
||||
return ggml_type_size(GGML_TYPE_I32)*ne02*ne21;
|
||||
}
|
||||
|
||||
size_t ggml_metal_op_mul_mat_id_extra_amax(const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_MUL_MAT_ID);
|
||||
|
||||
GGML_UNUSED(op);
|
||||
|
||||
// 2 scaling factors (8 bytes) + N_MM_NPART_AMAX per-threadgroup scales for stage-1
|
||||
return 8 + N_MM_NPART_AMAX*sizeof(float);
|
||||
}
|
||||
|
||||
int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
@@ -2682,6 +2691,36 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_metal_buffer_id bid_ids = bid_tpe;
|
||||
bid_ids.offs += ggml_metal_op_mul_mat_id_extra_tpe(op);
|
||||
|
||||
ggml_metal_buffer_id bid_amax = bid_ids;
|
||||
bid_amax.offs += ggml_metal_op_mul_mat_id_extra_ids(op);
|
||||
|
||||
// src1 rescale factors, computed before the matmul
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/26223
|
||||
{
|
||||
ggml_metal_kargs_mul_mm_id_amax args = {
|
||||
/*.ne00 =*/ ne10,
|
||||
/*.ne01 =*/ ne11,
|
||||
/*.ne02 =*/ ne12,
|
||||
/*.nb01 =*/ nb11,
|
||||
/*.nb02 =*/ nb12,
|
||||
};
|
||||
|
||||
auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax_part(lib);
|
||||
|
||||
const size_t smem = pipeline.smem;
|
||||
|
||||
GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_src1, 1);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_amax, 2);
|
||||
|
||||
ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, N_MM_NPART_AMAX, 1, 1, 256, 1, 1);
|
||||
}
|
||||
|
||||
{
|
||||
ggml_metal_kargs_mul_mm_id_map0 args = {
|
||||
ne02,
|
||||
@@ -2713,7 +2752,18 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, ne02, 1, 1);
|
||||
}
|
||||
|
||||
// this barrier is always needed because the next kernel has to wait for the id maps to be computed
|
||||
ggml_metal_op_concurrency_reset(ctx);
|
||||
|
||||
{
|
||||
auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax(lib);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_amax, 0);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, 32, 1, 1);
|
||||
}
|
||||
|
||||
// the next kernel has to wait for the amax data
|
||||
ggml_metal_op_concurrency_reset(ctx);
|
||||
|
||||
{
|
||||
@@ -2745,6 +2795,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_metal_encoder_set_buffer (enc, bid_tpe, 3);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_ids, 4);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_dst, 5);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_amax, 6);
|
||||
|
||||
const size_t smem = pipeline.smem;
|
||||
|
||||
@@ -2923,6 +2974,7 @@ static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op)
|
||||
const bool dk_dv_ok = (dk == 32 && dv == 32) ||
|
||||
(dk == 64 && dv == 64) ||
|
||||
(dk == 96 && dv == 96) ||
|
||||
(dk == 96 && dv == 64) ||
|
||||
(dk == 128 && dv == 128) ||
|
||||
(dk == 192 && dv == 128) ||
|
||||
(dk == 192 && dv == 192) ||
|
||||
|
||||
@@ -36,6 +36,7 @@ size_t ggml_metal_op_mul_mat_id_extra_tpe(const struct ggml_tensor * op);
|
||||
|
||||
// id map [n_tokens, n_expert]
|
||||
size_t ggml_metal_op_mul_mat_id_extra_ids(const struct ggml_tensor * op);
|
||||
size_t ggml_metal_op_mul_mat_id_extra_amax(const struct ggml_tensor * op);
|
||||
|
||||
// return true if we should use the FA vector kernel for this op
|
||||
bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user