From 4b74cee096f9bed2d5b64a983575c9d9d4c6fe7d Mon Sep 17 00:00:00 2001 From: yeongbba Date: Sun, 19 Jan 2025 01:30:34 +0900 Subject: [PATCH 1/3] making amdgpu work on arm achitecutre with vulkan --- envconfig/config.go | 2 ++ gpu/gpu.go | 2 +- gpu/gpu_linux.go | 4 ++-- llm/generate/gen_common.sh | 4 ++-- llm/generate/gen_linux.sh | 12 +++++++----- llm/generate/gen_windows.ps1 | 2 +- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/envconfig/config.go b/envconfig/config.go index 9c1490a93..239c49fe6 100644 --- a/envconfig/config.go +++ b/envconfig/config.go @@ -175,6 +175,7 @@ var ( CudaVisibleDevices = String("CUDA_VISIBLE_DEVICES") HipVisibleDevices = String("HIP_VISIBLE_DEVICES") RocrVisibleDevices = String("ROCR_VISIBLE_DEVICES") + VkVisibleDevices = String("GGML_VK_VISIBLE_DEVICES") GpuDeviceOrdinal = String("GPU_DEVICE_ORDINAL") HsaOverrideGfxVersion = String("HSA_OVERRIDE_GFX_VERSION") ) @@ -263,6 +264,7 @@ func AsMap() map[string]EnvVar { ret["CUDA_VISIBLE_DEVICES"] = EnvVar{"CUDA_VISIBLE_DEVICES", CudaVisibleDevices(), "Set which NVIDIA devices are visible"} ret["HIP_VISIBLE_DEVICES"] = EnvVar{"HIP_VISIBLE_DEVICES", HipVisibleDevices(), "Set which AMD devices are visible"} ret["ROCR_VISIBLE_DEVICES"] = EnvVar{"ROCR_VISIBLE_DEVICES", RocrVisibleDevices(), "Set which AMD devices are visible"} + ret["GGML_VK_VISIBLE_DEVICES"] = EnvVar{"GGML_VK_VISIBLE_DEVICES", VkVisibleDevices(), "Set which VK AMD devices are visible"} ret["GPU_DEVICE_ORDINAL"] = EnvVar{"GPU_DEVICE_ORDINAL", GpuDeviceOrdinal(), "Set which AMD devices are visible"} ret["HSA_OVERRIDE_GFX_VERSION"] = EnvVar{"HSA_OVERRIDE_GFX_VERSION", HsaOverrideGfxVersion(), "Override the gfx used for all detected AMD GPUs"} ret["OLLAMA_INTEL_GPU"] = EnvVar{"OLLAMA_INTEL_GPU", IntelGPU(), "Enable experimental Intel GPU detection"} diff --git a/gpu/gpu.go b/gpu/gpu.go index 69279867e..1bd337f19 100644 --- a/gpu/gpu.go +++ b/gpu/gpu.go @@ -410,7 +410,7 @@ func GetGPUInfo() GpuInfoList { rocmGPUs = AMDGetGPUInfo() bootstrapped = true - if len(cudaGPUs) == 0 && len(rocmGPUs) == 0 && len(oneapiGPUs) == 0 { + if len(cudaGPUs) == 0 && len(rocmGPUs) == 0 && len(oneapiGPUs) == 0 && len(vulkanGPUs) == 0 { slog.Info("no compatible GPUs were discovered") } } diff --git a/gpu/gpu_linux.go b/gpu/gpu_linux.go index 76df63268..d6f882efb 100644 --- a/gpu/gpu_linux.go +++ b/gpu/gpu_linux.go @@ -53,12 +53,12 @@ var ( ) var VulkanGlobs = []string{ - "/usr/lib/x86_64-linux-gnu/libvulkan.so*", + "/usr/lib/aarch64-linux-gnu/libvulkan.so*", "/usr/lib*/libvulkan.so*", } var capLinuxGlobs = []string{ - "/usr/lib/x86_64-linux-gnu/libcap.so*", + "/usr/lib/aarch64-linux-gnu/libcap.so*", "/usr/lib*/libcap.so*", } diff --git a/llm/generate/gen_common.sh b/llm/generate/gen_common.sh index 3825c155a..2b01e149c 100644 --- a/llm/generate/gen_common.sh +++ b/llm/generate/gen_common.sh @@ -30,7 +30,7 @@ init_vars() { WHOLE_ARCHIVE="-Wl,-force_load" NO_WHOLE_ARCHIVE="" GCC_ARCH="-arch ${ARCH}" - DIST_BASE=../../dist/darwin-${GOARCH}/ + DIST_BASE=../../dist/darwin-${GOARCH} PAYLOAD_BASE=../../build/darwin/${GOARCH} ;; "Linux") @@ -40,7 +40,7 @@ init_vars() { # Cross compiling not supported on linux - Use docker GCC_ARCH="" - DIST_BASE=../../dist/linux-${GOARCH}/ + DIST_BASE=../../dist/linux-${GOARCH} PAYLOAD_BASE=../../build/linux/${GOARCH} ;; *) diff --git a/llm/generate/gen_linux.sh b/llm/generate/gen_linux.sh index 79c449018..17981d543 100755 --- a/llm/generate/gen_linux.sh +++ b/llm/generate/gen_linux.sh @@ -224,9 +224,9 @@ fi if [ -z "${OLLAMA_SKIP_VULKAN_GENERATE}" -a -d "${VULKAN_ROOT}" ] && [ -z "${OLLAMA_SKIP_VULKAN_GENERATE}" -a -d "${CAP_ROOT}" ]; then echo "Vulkan and capabilities libraries detected - building dynamic Vulkan library" init_vars - - CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DLLAMA_VULKAN=1" - BUILD_DIR="../build/linux/${ARCH}/vulkan" + RUNNER=vulkan + CMAKE_DEFS="-DCMAKE_SKIP_RPATH=on -DBUILD_SHARED_LIBS=on -DCMAKE_POSITION_INDEPENDENT_CODE=on -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_OPENMP=off" + BUILD_DIR="../build/linux/${ARCH}/${RUNNER}" EXTRA_LIBS="-L${VULKAN_ROOT} -L${CAP_ROOT} -lvulkan -lcap" build @@ -234,8 +234,10 @@ if [ -z "${OLLAMA_SKIP_VULKAN_GENERATE}" -a -d "${VULKAN_ROOT}" ] && [ -z "${OLL for dep in $(ldd "${BUILD_DIR}/bin/ollama_llama_server" | grep "=>" | cut -f2 -d= | cut -f2 -d' ' | grep -e vulkan -e cap); do cp "${dep}" "${BUILD_DIR}/bin/" done - cp "${VULKAN_ROOT}/libvulkan.so*" "${BUILD_DIR}/bin/" - cp "${CAP_ROOT}/libcap.so*" "${BUILD_DIR}/bin/" + cp ${VULKAN_ROOT}/libvulkan.so* "${BUILD_DIR}/bin/" + cp ${CAP_ROOT}/libcap.so* "${BUILD_DIR}/bin/" + install + dist compress fi diff --git a/llm/generate/gen_windows.ps1 b/llm/generate/gen_windows.ps1 index bb92c5121..e8bba29a5 100644 --- a/llm/generate/gen_windows.ps1 +++ b/llm/generate/gen_windows.ps1 @@ -417,7 +417,7 @@ function build_vulkan() { init_vars $script:buildDir="../build/windows/${script:ARCH}/vulkan" $script:distDir="$script:DIST_BASE\vulkan" - $script:cmakeDefs += @("-A", "x64", "-DLLAMA_VULKAN=1") + $script:cmakeDefs += @("-A", "x64", "-DDGGML_VULKAN=1") write-host "Building Vulkan" build sign From 6d7579b567cdb03154521e43103e9113a8cbd336 Mon Sep 17 00:00:00 2001 From: yeongbba Date: Sun, 19 Jan 2025 12:41:08 +0900 Subject: [PATCH 2/3] add x86_64 lines in VulkanGlobs and capLinuxGlobs --- gpu/gpu_linux.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gpu/gpu_linux.go b/gpu/gpu_linux.go index d6f882efb..1251c6e8e 100644 --- a/gpu/gpu_linux.go +++ b/gpu/gpu_linux.go @@ -53,11 +53,13 @@ var ( ) var VulkanGlobs = []string{ + "/usr/lib/x86_64-linux-gnu/libvulkan.so*", "/usr/lib/aarch64-linux-gnu/libvulkan.so*", "/usr/lib*/libvulkan.so*", } var capLinuxGlobs = []string{ + "/usr/lib/x86_64-linux-gnu/libvulkan.so*", "/usr/lib/aarch64-linux-gnu/libcap.so*", "/usr/lib*/libcap.so*", } From 2bf59a512b938739f3bb8c3cdd034d817692ed45 Mon Sep 17 00:00:00 2001 From: yeongbba Date: Sun, 19 Jan 2025 12:51:10 +0900 Subject: [PATCH 3/3] add aarch64 lines in vulkanGlobs and capLinuxGlobs --- discover/gpu_linux.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/discover/gpu_linux.go b/discover/gpu_linux.go index 840ea435a..a2c1d8715 100644 --- a/discover/gpu_linux.go +++ b/discover/gpu_linux.go @@ -58,11 +58,13 @@ var ( var VulkanGlobs = []string{ "/usr/lib/x86_64-linux-gnu/libvulkan.so*", + "/usr/lib/aarch64-linux-gnu/libvulkan.so*", "/usr/lib*/libvulkan.so*", } var capLinuxGlobs = []string{ "/usr/lib/x86_64-linux-gnu/libcap.so*", + "/usr/lib/aarch64-linux-gnu/libvulkan.so*", "/usr/lib*/libcap.so*", }