...

removing the channel tag from the url so it will always go to the current stable channel.

.dockerignore

@@ -3,7 +3,9 @@ ollama
 app
 macapp
 dist
+build
 .env
 .cache
 test_data
-llama/build
+.git
+

.gitattributes

@@ -15,6 +15,10 @@ ml/backend/**/*.cu linguist-vendored
 ml/backend/**/*.cuh linguist-vendored
 ml/backend/**/*.m linguist-vendored
 ml/backend/**/*.metal linguist-vendored
+ml/backend/**/CMakeLists.txt linguist-vendored
+
+llama/build-info.cpp linguist-generated
+ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-embed.s linguist-generated
 
 * text=auto
 *.go text eol=lf

.github/ISSUE_TEMPLATE/10_bug_report.yml

@@ -9,6 +9,14 @@ body:
       description: What happened? What did you expect to happen?
     validations:
       required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. See [Troubleshooting Guide](https://github.com/ollama/ollama/blob/main/docs/troubleshooting.md#how-to-troubleshoot-issues) for details.
+      render: shell
+    validations:
+      required: false
   - type: dropdown
     id: os
     attributes:

.github/workflows/release.yaml

@@ -1,31 +1,66 @@
 name: release
 
-env:
-  ROCM_WINDOWS_URL: https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe
-  MSYS2_URL: https://github.com/msys2/msys2-installer/releases/download/2024-07-27/msys2-x86_64-20240727.exe
-
 on:
   push:
     tags:
       - 'v*'
 
+env:
+  CGO_CFLAGS: '-O3'
+  CGO_CXXFLAGS: '-O3'
+
 jobs:
-  # Full build of the Mac assets
-  build-darwin:
-    runs-on: macos-13
+  setup-environment:
+    runs-on: ubuntu-latest
     environment: release
+    outputs:
+      GOFLAGS: ${{ steps.goflags.outputs.GOFLAGS }}
     steps:
       - uses: actions/checkout@v4
-      - name: Set Version
-        shell: bash
+      - name: Set environment
+        id: goflags
         run: |
-          echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
-          echo "RELEASE_VERSION=$(echo ${GITHUB_REF_NAME} | cut -f1 -d-)" >> $GITHUB_ENV
-      - name: key
+          echo GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=${GITHUB_REF_NAME#v}\" \"-X=github.com/ollama/ollama/server.mode=release\"'" >>$GITHUB_OUTPUT
+
+  darwin-build:
+    runs-on: macos-13
+    environment: release
+    needs: setup-environment
+    strategy:
+      matrix:
+        os: [darwin]
+        arch: [amd64, arm64]
+    env:
+      GOFLAGS: ${{ needs.setup-environment.outputs.GOFLAGS }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - run: |
+          go build -o dist/ .
         env:
-          MACOS_SIGNING_KEY: ${{ secrets.MACOS_SIGNING_KEY }}
-          MACOS_SIGNING_KEY_PASSWORD: ${{ secrets.MACOS_SIGNING_KEY_PASSWORD }}
+          GOOS: ${{ matrix.os }}
+          GOARCH: ${{ matrix.arch }}
+          CGO_ENABLED: 1
+          CGO_CPPFLAGS: '-mmacosx-version-min=11.3'
+      - if: matrix.arch == 'amd64'
         run: |
+          cmake --preset CPU -DCMAKE_OSX_DEPLOYMENT_TARGET=11.3 -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64
+          cmake --build --parallel --preset CPU
+          cmake --install build --component CPU --strip --parallel 8
+      - uses: actions/upload-artifact@v4
+        with:
+          name: build-${{ matrix.os }}-${{ matrix.arch }}
+          path: dist/*
+
+  darwin-sign:
+    runs-on: macos-13
+    environment: release
+    needs: darwin-build
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
           echo $MACOS_SIGNING_KEY | base64 --decode > certificate.p12
           security create-keychain -p password build.keychain
           security default-keychain -s build.keychain
@@ -33,11 +68,20 @@ jobs:
           security import certificate.p12 -k build.keychain -P $MACOS_SIGNING_KEY_PASSWORD -T /usr/bin/codesign
           security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password build.keychain
           security set-keychain-settings -lut 3600 build.keychain
-      - uses: actions/setup-go@v5
+        env:
+          MACOS_SIGNING_KEY: ${{ secrets.MACOS_SIGNING_KEY }}
+          MACOS_SIGNING_KEY_PASSWORD: ${{ secrets.MACOS_SIGNING_KEY_PASSWORD }}
+      - uses: actions/download-artifact@v4
         with:
-          go-version-file: go.mod
-          cache: true
-      - name: Build Darwin
+          name: build-darwin-amd64
+          path: dist/darwin-amd64
+      - uses: actions/download-artifact@v4
+        with:
+          name: build-darwin-arm64
+          path: dist/darwin-arm64
+      - run: |
+          export VERSION=${GITHUB_REF_NAME#v}
+          ./scripts/build_darwin.sh sign macapp
         env:
           APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY }}
           APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
@@ -45,485 +89,318 @@ jobs:
           APPLE_ID: ${{ vars.APPLE_ID }}
           SDKROOT: /Applications/Xcode_14.1.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
           DEVELOPER_DIR: /Applications/Xcode_14.1.0.app/Contents/Developer
-        run: |
-          ./scripts/build_darwin.sh
-
       - uses: actions/upload-artifact@v4
         with:
           name: dist-darwin
           path: |
             dist/Ollama-darwin.zip
-            dist/ollama-darwin
+            dist/ollama-darwin.tgz
 
-  # Windows builds take a long time to both install the dependencies and build, so parallelize
-  # CPU generation step
-  generate-windows-cpu:
-    environment: release
-    runs-on: windows
-    env:
-      KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set make jobs default
-        run: |
-          echo "MAKEFLAGS=--jobs=$((Get-ComputerInfo -Property CsProcessors).CsProcessors.NumberOfCores)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-      - name: Set Version
-        shell: bash
-        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
-      - name: Add msys paths
-        run: |
-          echo "c:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          echo "C:\msys64\clang64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-      - name: Install msys2 tools
-        run: |
-          Start-Process "c:\msys64\usr\bin\pacman.exe" -ArgumentList @("-S", "--noconfirm", "mingw-w64-clang-x86_64-gcc-compat", "mingw-w64-clang-x86_64-clang") -NoNewWindow -Wait
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-          cache: true
-      - run: |
-          import-module 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
-          Enter-VsDevShell -vsinstallpath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise' -skipautomaticlocation -DevCmdArguments '-arch=x64 -no_logo'
-          if (!(gcc --version | select-string -quiet clang)) { throw "wrong gcc compiler detected - must be clang" }
-          make dist
-        name: make
-      - uses: actions/upload-artifact@v4
-        with:
-          name: generate-windows-cpu
-          path: |
-            dist/windows-amd64/**
-
-  # ROCm generation step
-  generate-windows-rocm:
-    environment: release
-    runs-on: windows
-    env:
-      KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set make jobs default
-        run: |
-          echo "MAKEFLAGS=--jobs=$((Get-ComputerInfo -Property CsProcessors).CsProcessors.NumberOfCores)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-      - name: Set Version
-        shell: bash
-        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
-      - name: Add msys paths
-        run: |
-          echo "c:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          echo "C:\msys64\clang64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-      - name: Install msys2 tools
-        run: |
-          Start-Process "c:\msys64\usr\bin\pacman.exe" -ArgumentList @("-S", "--noconfirm", "mingw-w64-clang-x86_64-gcc-compat", "mingw-w64-clang-x86_64-clang") -NoNewWindow -Wait
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-          cache: true
-      # ROCM installation steps
-      - name: 'Cache ROCm installer'
-        id: cache-rocm
-        uses: actions/cache@v4
-        with:
-          path: rocm-install.exe
-          key: ${{ env.ROCM_WINDOWS_URL }}
-      - name: 'Conditionally Download ROCm'
-        if: steps.cache-rocm.outputs.cache-hit != 'true'
-        run: |
-          $ErrorActionPreference = "Stop"
-          Invoke-WebRequest -Uri "${env:ROCM_WINDOWS_URL}" -OutFile "rocm-install.exe"
-      - name: 'Install ROCm'
-        run: |
-          Start-Process "rocm-install.exe" -ArgumentList '-install' -NoNewWindow -Wait
-      - name: 'Verify ROCm'
-        run: |
-          & 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' --version
-          echo "HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path | select -first 1)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-      - name: make rocm runner
-        run: |
-          import-module 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
-          Enter-VsDevShell -vsinstallpath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise' -skipautomaticlocation -DevCmdArguments '-arch=x64 -no_logo'
-          if (!(gcc --version | select-string -quiet clang)) { throw "wrong gcc compiler detected - must be clang" }
-          make help-runners
-          make dist_rocm
-      - uses: actions/upload-artifact@v4
-        with:
-          name: generate-windows-rocm
-          path: |
-            dist/windows-amd64/**
-
-  # CUDA generation step
-  generate-windows-cuda:
-    environment: release
-    runs-on: windows
+  windows-depends:
     strategy:
       matrix:
-        cuda:
-          - version: "11.3"
-            url: https://developer.download.nvidia.com/compute/cuda/11.3.1/local_installers/cuda_11.3.1_465.89_win10.exe
-          - version: "12.4"
-            url: https://developer.download.nvidia.com/compute/cuda/12.4.0/local_installers/cuda_12.4.0_551.61_windows.exe
-    env:
-      KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set make jobs default
-        run: |
-          echo "MAKEFLAGS=--jobs=$((Get-ComputerInfo -Property CsProcessors).CsProcessors.NumberOfCores)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-      - name: Set Version
-        shell: bash
-        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
-      - name: Install msys2
-        run: |
-          $msys2_url="https://github.com/msys2/msys2-installer/releases/download/2024-07-27/msys2-x86_64-20240727.exe"
-          write-host "Downloading msys2"
-          Invoke-WebRequest -Uri "${msys2_url}" -OutFile "${env:RUNNER_TEMP}\msys2.exe"
-          write-host "Installing msys2"
-          Start-Process "${env:RUNNER_TEMP}\msys2.exe" -ArgumentList @("in", "--confirm-command", "--accept-messages", "--root", "C:/msys64") -NoNewWindow -Wait
-          echo "c:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-      - name: Install msys2 tools
-        run: |
-          Start-Process "c:\msys64\usr\bin\pacman.exe" -ArgumentList @("-S", "--noconfirm", "mingw-w64-clang-x86_64-gcc-compat", "mingw-w64-clang-x86_64-clang", "make") -NoNewWindow -Wait
-          echo "C:\msys64\clang64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-      - name: verify tools
-        run: |
-          get-command gcc
-          gcc --version
-          get-command make
-          make --version
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-          cache: true
-      # CUDA installation steps
-      - name: 'Cache CUDA installer'
-        id: cache-cuda
-        uses: actions/cache@v4
-        with:
-          path: cuda-install.exe
-          key: ${{ matrix.cuda.url }}
-      - name: 'Conditionally Download CUDA'
-        if: steps.cache-cuda.outputs.cache-hit != 'true'
-        run: |
-          $ErrorActionPreference = "Stop"
-          Invoke-WebRequest -Uri "${{ matrix.cuda.url }}" -OutFile "cuda-install.exe"
-      - name: 'Install CUDA'
-        run: |
-          $subpackages = @("cudart", "nvcc", "cublas", "cublas_dev") | foreach-object {"${_}_${{ matrix.cuda.version }}"}
-          Start-Process "cuda-install.exe" -ArgumentList (@("-s") + $subpackages) -NoNewWindow -Wait
-      - name: 'Verify CUDA'
-        run: |
-          & (resolve-path "c:\Program Files\NVIDIA*\CUDA\v*\bin\nvcc.exe")[0] --version
-          $cudaPath=((resolve-path "c:\Program Files\NVIDIA*\CUDA\v*\bin\nvcc.exe")[0].path | split-path | split-path)
-          $cudaVer=($cudaPath | split-path -leaf ) -replace 'v(\d+).(\d+)', '$1_$2' 
-          echo "$cudaPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          echo "CUDA_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-          echo "CUDA_PATH_V${cudaVer}=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-          echo "CUDA_PATH_VX_Y=CUDA_PATH_V${cudaVer}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-
-      - name: make cuda runner
-        run: |
-          import-module 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
-          Enter-VsDevShell -vsinstallpath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise' -skipautomaticlocation -DevCmdArguments '-arch=x64 -no_logo'
-          if (!(gcc --version | select-string -quiet clang)) { throw "wrong gcc compiler detected - must be clang" }
-          make dist_cuda_v$(($env:CUDA_PATH | split-path -leaf) -replace 'v(\d+).*', '$1')
-      - uses: actions/upload-artifact@v4
-        with:
-          name: generate-windows-cuda-${{ matrix.cuda.version }}
-          path: |
-            dist/windows-amd64/**
-
-  # windows arm64 generate, go build, and zip file (no installer)
-  # Output of this build is aggregated into the final x86 build
-  # for a unified windows installer
-  windows-arm64:
-    runs-on: windows-arm64
+        os: [windows]
+        arch: [amd64]
+        preset: ['CPU']
+        include:
+          - os: windows
+            arch: amd64
+            preset: 'CUDA 11'
+            install: https://developer.download.nvidia.com/compute/cuda/11.3.1/local_installers/cuda_11.3.1_465.89_win10.exe
+            cuda-version: '11.3'
+          - os: windows
+            arch: amd64
+            preset: 'CUDA 12'
+            install: https://developer.download.nvidia.com/compute/cuda/12.4.0/local_installers/cuda_12.4.0_551.61_windows.exe
+            cuda-version: '12.4'
+          - os: windows
+            arch: amd64
+            preset: 'ROCm 6'
+            install: https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe
+            rocm-version: '6.1'
+    runs-on: ${{ matrix.arch == 'arm64' && format('{0}-{1}', matrix.os, matrix.arch) || matrix.os }}
     environment: release
     env:
-      KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
+      GOFLAGS: ${{ needs.setup-environment.outputs.GOFLAGS }}
     steps:
-      # The current Windows arm64 beta image has effectively zero dev tools installed...
-      - name: Install git and gzip
+      - name: Install system dependencies
         run: |
+          choco install -y --no-progress ccache ninja
+          ccache -o cache_dir=${{ github.workspace }}\.ccache
+      - if: startsWith(matrix.preset, 'CUDA ') || startsWith(matrix.preset, 'ROCm ')
+        id: cache-install
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA
+            C:\Program Files\AMD\ROCm
+          key: ${{ matrix.install }}
+      - if: startsWith(matrix.preset, 'CUDA ')
+        name: Install CUDA ${{ matrix.cuda-version }}
+        run: |
+          $ErrorActionPreference = "Stop"
+          if ("${{ steps.cache-install.outputs.cache-hit }}" -ne 'true') {
+            Invoke-WebRequest -Uri "${{ matrix.install }}" -OutFile "install.exe"
+            $subpackages = @("cudart", "nvcc", "cublas", "cublas_dev") | Foreach-Object {"${_}_${{ matrix.cuda-version }}"}
+            Start-Process -FilePath .\install.exe -ArgumentList (@("-s") + $subpackages) -NoNewWindow -Wait
+          }
+
+          $cudaPath = (Resolve-Path "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\*").path
+          echo "$cudaPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+      - if: startsWith(matrix.preset, 'ROCm')
+        name: Install ROCm ${{ matrix.rocm-version }}
+        run: |
+          $ErrorActionPreference = "Stop"
+          if ("${{ steps.cache-install.outputs.cache-hit }}" -ne 'true') {
+            Invoke-WebRequest -Uri "${{ matrix.install }}" -OutFile "install.exe"
+            Start-Process -FilePath .\install.exe -ArgumentList '-install' -NoNewWindow -Wait
+          }
+
+          $hipPath = (Resolve-Path "C:\Program Files\AMD\ROCm\*").path
+          echo "$hipPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "CC=$hipPath\bin\clang.exe" | Out-File -FilePath $env:GITHUB_ENV -Append
+          echo "CXX=$hipPath\bin\clang++.exe" | Out-File -FilePath $env:GITHUB_ENV -Append
+      - if: ${{ !cancelled() && steps.cache-install.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA
+            C:\Program Files\AMD\ROCm
+          key: ${{ matrix.install }}
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}\.ccache
+          key: ccache-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.preset }}
+      - name: Build target "${{ matrix.preset }}"
+        run: |
+          Import-Module 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
+          Enter-VsDevShell -VsInstallPath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise' -SkipAutomaticLocation  -DevCmdArguments '-arch=x64 -no_logo'
+          cmake --preset "${{ matrix.preset }}"
+          cmake --build --parallel --preset "${{ matrix.preset }}"
+          cmake --install build --component "${{ startsWith(matrix.preset, 'CUDA ') && 'CUDA' || startsWith(matrix.preset, 'ROCm ') && 'HIP' || 'CPU' }}" --strip --parallel 8
+        env:
+          CMAKE_GENERATOR: Ninja
+      - uses: actions/upload-artifact@v4
+        with:
+          name: depends-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.preset }}
+          path: dist\*
+
+  windows-build:
+    strategy:
+      matrix:
+        os: [windows]
+        arch: [amd64, arm64]
+    runs-on: ${{ matrix.arch == 'arm64' && format('{0}-{1}', matrix.os, matrix.arch) || matrix.os }}
+    environment: release
+    needs: [setup-environment]
+    env:
+      GOFLAGS: ${{ needs.setup-environment.outputs.GOFLAGS }}
+    steps:
+      - name: Install AMD64 system dependencies
+        if: matrix.arch == 'amd64'
+        run: |
+          $ErrorActionPreference = "Stop"
+          Start-Process "C:\msys64\usr\bin\pacman.exe" -ArgumentList @("-S", "--noconfirm", "mingw-w64-clang-x86_64-gcc-compat", "mingw-w64-clang-x86_64-clang") -NoNewWindow -Wait
+          echo "C:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "C:\msys64\clang64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+      - name: Install ARM64 system dependencies
+        if: matrix.arch == 'arm64'
+        run: |
+          $ErrorActionPreference = "Stop"
           Set-ExecutionPolicy Bypass -Scope Process -Force
           [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
           iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+          echo "C:\ProgramData\chocolatey\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
           choco install -y --no-progress git gzip
           echo "C:\Program Files\Git\cmd" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          echo "C:\ProgramData\chocolatey\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-      # pacman is buggy on win arm64, so we avoid using it, but rely on the binary artifacts
-      # we download the sfx (7zip bundle) which isn't fully set up, but the binaries we need to build work
-      - name: Install msys2 x64
-        run: |
-          $url="https://github.com/msys2/msys2-installer/releases/download/2024-07-27/msys2-base-x86_64-20240727.sfx.exe"
-          write-host "Downloading MSYS2"
-          Invoke-WebRequest -Uri "$url" -outfile "${env:RUNNER_TEMP}\msys2.exe"
-          write-host "Installing msys2"
-          Start-Process "${env:RUNNER_TEMP}\msys2.exe" -ArgumentList @(
-              '-y', '-oC:\'
-              ) -NoNewWindow -Wait
-          echo "c:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-      # since pacman isn't reliable, we just download the tar file and extract directly
-      - name: Downloading and extracting msys2 make tar file
-        run: |
-          $url="https://mirror.msys2.org/msys/x86_64/make-4.4.1-2-x86_64.pkg.tar.zst"
-          write-host "Downloading make"
-          Invoke-WebRequest -Uri "$url" -outfile c:\msys64\make.tar.zst
-          cd c:\msys64; tar -xf make.tar.zst
-          rm c:\msys64\make.tar.zst
-      - name: Verify Make works properly
-        run: |
-          echo $env:PATH
-          make --version
-      - name: Install Visual Studio 2022
-        run: |
-          $components = @(
-            "Microsoft.VisualStudio.Component.CoreEditor",
-            "Microsoft.VisualStudio.Workload.CoreEditor",
-            "Microsoft.VisualStudio.Component.Roslyn.Compiler",
-            "Microsoft.Component.MSBuild",
-            "Microsoft.VisualStudio.Component.TextTemplating",
-            "Microsoft.VisualStudio.Component.Debugger.JustInTime",
-            "Microsoft.VisualStudio.Component.VC.CoreIde",
-            "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
-            "Microsoft.VisualStudio.Component.Windows11SDK.22621",
-            "Microsoft.VisualStudio.Component.VC.Tools.ARM64EC",
-            "Microsoft.VisualStudio.Component.VC.Tools.ARM64",
-            "Microsoft.VisualStudio.Component.VC.ATL",
-            "Microsoft.VisualStudio.Component.VC.ATL.ARM64",
-            "Microsoft.VisualStudio.Component.Graphics",
-            "Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
-            "Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
-            "Microsoft.VisualStudio.Component.Windows11Sdk.WindowsPerformanceToolkit",
-            "Microsoft.VisualStudio.Component.CppBuildInsights",
-            "Microsoft.VisualStudio.Component.VC.DiagnosticTools",
-            "Microsoft.VisualStudio.ComponentGroup.WebToolsExtensions.CMake",
-            "Microsoft.VisualStudio.Component.VC.CMake.Project",
-            "Microsoft.VisualStudio.Component.VC.ASAN",
-            "Microsoft.VisualStudio.Component.Vcpkg",
-            "Microsoft.VisualStudio.Workload.NativeDesktop"
-          )
-          $config = @{
-                "version" = "1.0"
-                "components"  = $components
-                "extensions"  = @()
-            }
-          $configPath = "${env:RUNNER_TEMP}\vsconfig"
-          $config | ConvertTo-Json | Out-File -FilePath $configPath
-          $bootstrapperFilePath = "${env:RUNNER_TEMP}\vs_community.exe"
-          write-host "Downloading Visual Studio 2022"
-          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_community.exe" -outfile $bootstrapperFilePath
-          $bootstrapperArgumentList = ('/c', $bootstrapperFilePath, '--config', $configPath, '--quiet', '--wait' )
-          write-host "Installing Visual Studio 2022"
-          $process = Start-Process -FilePath cmd.exe -ArgumentList $bootstrapperArgumentList -Wait -PassThru
-          $exitCode = $process.ExitCode
-          write-host $exitCode
-      # pacman in mingw/msys2 is ~broken on windows arm right now - hangs consistently during attempts to install
-      # so we'll use this alternative GCC binary
-      - name: Install llvm-mingw GCC
-        run: |
-          $gcc_url="https://github.com/mstorsjo/llvm-mingw/releases/download/20240619/llvm-mingw-20240619-ucrt-aarch64.zip"
-          write-host "Downloading llvm-mingw"
-          Invoke-WebRequest -Uri "${gcc_url}" -OutFile "${env:RUNNER_TEMP}\gcc.zip"
-          write-host "Unpacking llvm-mingw"
-          expand-archive -path "${env:RUNNER_TEMP}\gcc.zip" -destinationpath "c:\"
-          mv c:\llvm-mingw-* c:\llvm-mingw
-          echo "c:\llvm-mingw\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-      - name: Verify GCC
-        run: |
-          echo $env:PATH
-          gcc --version
+
+          Invoke-WebRequest -Uri "https://github.com/mstorsjo/llvm-mingw/releases/download/20240619/llvm-mingw-20240619-ucrt-aarch64.zip" -OutFile "${{ runner.temp }}\llvm-mingw-ucrt-aarch64.zip"
+          Expand-Archive -Path ${{ runner.temp }}\llvm-mingw-ucrt-aarch64.zip -DestinationPath "C:\Program Files\"
+          $installPath=(Resolve-Path -Path "C:\Program Files\llvm-mingw-*-ucrt-aarch64").path
+          echo $installPath\bin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
       - uses: actions/checkout@v4
-      - name: Set Version
-        run: |
-          $ver=${env:GITHUB_REF_NAME}.trim("v")
-          echo VERSION=$ver | Out-File -FilePath ${env:GITHUB_ENV} -Encoding utf8 -Append
-      - uses: 'google-github-actions/auth@v2'
-        with:
-          project_id: 'ollama'
-          credentials_json: '${{ secrets.GOOGLE_SIGNING_CREDENTIALS }}'
-      - run: echo "${{ vars.OLLAMA_CERT }}" | Out-File -FilePath ollama_inc.crt -Encoding utf8
-      - name: install Windows SDK 8.1 to get signtool
-        run: |
-          $ErrorActionPreference = "Stop"
-          write-host "downloading SDK"
-          Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?LinkId=323507" -OutFile "${env:RUNNER_TEMP}\sdksetup.exe"
-          Start-Process "${env:RUNNER_TEMP}\sdksetup.exe" -ArgumentList @("/q") -NoNewWindow -Wait
-          write-host "Win SDK 8.1 installed"
-          gci -path 'C:\Program Files (x86)\Windows Kits\' -r -fi 'signtool.exe'
-      - name: install signing plugin
-        run: |
-          $ErrorActionPreference = "Stop"
-          write-host "downloading plugin"
-          Invoke-WebRequest -Uri "https://github.com/GoogleCloudPlatform/kms-integrations/releases/download/cng-v1.0/kmscng-1.0-windows-amd64.zip" -OutFile "${env:RUNNER_TEMP}\plugin.zip"
-          Expand-Archive -Path "${env:RUNNER_TEMP}\plugin.zip" -DestinationPath ${env:RUNNER_TEMP}\plugin\
-          write-host "Installing plugin"
-          & "${env:RUNNER_TEMP}\plugin\*\kmscng.msi" /quiet
-          write-host "plugin installed"
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
-          cache: true
-      - run: go get ./...
       - run: |
-          $gopath=(get-command go).source | split-path -parent
-          $gccpath=(get-command gcc).source | split-path -parent
-          import-module 'C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
-          Enter-VsDevShell -Arch arm64 -vsinstallpath 'C:\Program Files\Microsoft Visual Studio\2022\Community' -skipautomaticlocation
-          $env:PATH="$gopath;$gccpath;$env:PATH"
-          echo $env:PATH
-          $env:ARCH="arm64"
-          .\scripts\build_windows.ps1 buildOllama buildApp gatherDependencies sign distZip
-        name: 'Windows Build'
+          go build -o dist/${{ matrix.os }}-${{ matrix.arch }}/ .
+      - if: matrix.arch == 'arm64'
+        run: |
+          Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.arm64.exe" -OutFile "dist\windows-arm64\vc_redist.arm64.exe"
+      - run: |
+          $env:VERSION='${{ github.ref_name }}' -Replace "v(.*)", '$1'
+          & .\scripts\build_windows.ps1 buildApp
+        env:
+          VCToolsRedistDir: stub
       - uses: actions/upload-artifact@v4
         with:
-          name: windows-arm64
+          name: build-${{ matrix.os }}-${{ matrix.arch }}
           path: |
-            dist/windows-arm64/**
-            dist/windows-arm64-app.exe
-            dist/ollama-windows-arm64.zip
+            dist\${{ matrix.os }}-${{ matrix.arch }}\*.exe
+            dist\${{ matrix.os }}-${{ matrix.arch }}-app.exe
 
-  # Import the prior generation steps plus the full arm64 build, and build the final windows assets
-  build-windows:
+  windows-sign:
+    runs-on: windows-2022
     environment: release
-    runs-on: windows
-    needs:
-      - generate-windows-cuda
-      - generate-windows-rocm
-      - generate-windows-cpu
-      - windows-arm64
-    env:
-      KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
+    needs: [windows-depends, windows-build]
     steps:
       - uses: actions/checkout@v4
+      - uses: google-github-actions/auth@v2
         with:
-          submodules: recursive
-      - name: Set Version
-        shell: bash
-        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
-      - uses: 'google-github-actions/auth@v2'
-        with:
-          project_id: 'ollama'
-          credentials_json: '${{ secrets.GOOGLE_SIGNING_CREDENTIALS }}'
-      - run: echo "${{ vars.OLLAMA_CERT }}" > ollama_inc.crt
-      - name: install Windows SDK 8.1 to get signtool
-        run: |
-          $ErrorActionPreference = "Stop"
-          write-host "downloading SDK"
-          Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?LinkId=323507" -OutFile "${env:RUNNER_TEMP}\sdksetup.exe"
-          Start-Process "${env:RUNNER_TEMP}\sdksetup.exe" -ArgumentList @("/q") -NoNewWindow -Wait
-          write-host "Win SDK 8.1 installed"
-          gci -path 'C:\Program Files (x86)\Windows Kits\' -r -fi 'signtool.exe'
-      - name: install signing plugin
-        run: |
-          $ErrorActionPreference = "Stop"
-          write-host "downloading plugin"
-          Invoke-WebRequest -Uri "https://github.com/GoogleCloudPlatform/kms-integrations/releases/download/cng-v1.0/kmscng-1.0-windows-amd64.zip" -OutFile "${env:RUNNER_TEMP}\plugin.zip"
-          Expand-Archive -Path "${env:RUNNER_TEMP}\plugin.zip" -DestinationPath ${env:RUNNER_TEMP}\plugin\
-          write-host "Installing plugin"
-          & "${env:RUNNER_TEMP}\plugin\*\kmscng.msi" /quiet
-          write-host "plugin installed"
-      - name: Install msys2
-        run: |
-          $msys2_url="https://github.com/msys2/msys2-installer/releases/download/2024-07-27/msys2-x86_64-20240727.exe"
-          write-host "Downloading msys2"
-          Invoke-WebRequest -Uri "${msys2_url}" -OutFile "${env:RUNNER_TEMP}\msys2.exe"
-          write-host "Installing msys2"
-          Start-Process "${env:RUNNER_TEMP}\msys2.exe" -ArgumentList @("in", "--confirm-command", "--accept-messages", "--root", "C:/msys64") -NoNewWindow -Wait
-          echo "c:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-      - name: Install msys2 tools
-        run: |
-          Start-Process "c:\msys64\usr\bin\pacman.exe" -ArgumentList @("-S", "--noconfirm", "mingw-w64-clang-x86_64-gcc-compat", "mingw-w64-clang-x86_64-clang", "make") -NoNewWindow -Wait
-          echo "C:\msys64\clang64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-      - name: verify tools
-        run: |
-          get-command gcc
-          gcc --version
-          get-command make
-          make --version
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-          cache: true
-      - run: go get
-      - uses: actions/download-artifact@v4
-        with:
-          name: generate-windows-cpu
-          path: dist/windows-amd64/
-      - uses: actions/download-artifact@v4
-        with:
-          name: generate-windows-cuda-11.3
-          path: dist/windows-amd64/
-      - uses: actions/download-artifact@v4
-        with:
-          name: generate-windows-cuda-12.4
-          path: dist/windows-amd64/
-      - uses: actions/download-artifact@v4
-        with:
-          name: generate-windows-rocm
-          path: dist/windows-amd64/
-      - uses: actions/download-artifact@v4
-        with:
-          name: windows-arm64
-          path: dist
+          project_id: ollama
+          credentials_json: ${{ secrets.GOOGLE_SIGNING_CREDENTIALS }}
       - run: |
-          import-module 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
-          Enter-VsDevShell -vsinstallpath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise' -skipautomaticlocation -DevCmdArguments '-arch=x64 -no_logo'
-          $env:OLLAMA_SKIP_GENERATE="1"
-          $env:ARCH="amd64"
-          if (!(gcc --version | select-string -quiet clang)) { throw "wrong gcc compiler detected - must be clang" }
-          & .\scripts\build_windows.ps1
+          $ErrorActionPreference = "Stop"
+          Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?LinkId=323507" -OutFile "${{ runner.temp }}\sdksetup.exe"
+          Start-Process "${{ runner.temp }}\sdksetup.exe" -ArgumentList @("/q") -NoNewWindow -Wait
+
+          Invoke-WebRequest -Uri "https://github.com/GoogleCloudPlatform/kms-integrations/releases/download/cng-v1.0/kmscng-1.0-windows-amd64.zip" -OutFile "${{ runner.temp }}\plugin.zip"
+          Expand-Archive -Path "${{ runner.temp }}\plugin.zip" -DestinationPath "${{ runner.temp }}\plugin\"
+          & "${{ runner.temp }}\plugin\*\kmscng.msi" /quiet
+
+          echo "${{ vars.OLLAMA_CERT }}" >ollama_inc.crt
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: build-windows-*
+          path: dist\
+          merge-multiple: true
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: depends-windows-amd64-*
+          path: dist\windows-amd64\
+          merge-multiple: true
+      - run: |
+          & .\scripts\build_windows.ps1 gatherDependencies sign buildInstaller distZip
+        env:
+          KEY_CONTAINER: ${{ vars.KEY_CONTAINER }}
       - uses: actions/upload-artifact@v4
         with:
           name: dist-windows
           path: |
-            dist/OllamaSetup.exe
-            dist/ollama-windows-*.zip
+            dist\OllamaSetup.exe
+            dist\ollama-windows-*.zip
 
-  build-linux:
-    environment: release
-    runs-on: linux
+  linux-build:
     strategy:
       matrix:
         include:
           - os: linux
             arch: amd64
-            targets: [archive, rocm]
+            target: archive
+          - os: linux
+            arch: amd64
+            target: rocm
           - os: linux
             arch: arm64
-            targets: [archive]
+            target: archive
+    runs-on: ${{ matrix.arch == 'arm64' && format('{0}-{1}', matrix.os, matrix.arch) || matrix.os }}
+    environment: release
+    needs: setup-environment
+    env:
+      GOFLAGS: ${{ needs.setup-environment.outputs.GOFLAGS }}
     steps:
       - uses: actions/checkout@v4
-      - uses: docker/setup-qemu-action@v3
       - uses: docker/setup-buildx-action@v3
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: ${{ matrix.os }}/${{ matrix.arch }}
+          target: ${{ matrix.target }}
+          build-args: |
+            GOFLAGS=${{ env.GOFLAGS }}
+            CGO_CFLAGS=${{ env.CGO_CFLAGS }}
+            CGO_CXXFLAGS=${{ env.CGO_CXXFLAGS }}
+          outputs: type=local,dest=dist/${{ matrix.os }}-${{ matrix.arch }}
+          cache-from: type=registry,ref=ollama/ollama:latest
+          cache-to: type=inline
       - run: |
-          apt-get update && apt-get install pigz
-          for TARGET in ${{ matrix.targets }}; do docker buildx build --platform $PLATFORM --target $TARGET --output type=local,dest=dist/$PLATFORM .; done
-          tar c -C dist/$PLATFORM . | pigz -9cv >dist/ollama-${PLATFORM//\//-}.tar.gz
-        env:
-          PLATFORM: ${{ matrix.os }}/${{ matrix.arch }}
+          for COMPONENT in bin/* lib/ollama/*; do
+            case "$COMPONENT" in
+              bin/ollama)               echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;
+              lib/ollama/*.so)          echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;
+              lib/ollama/cuda_v11)      echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;
+              lib/ollama/cuda_v12)      echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;
+              lib/ollama/cuda_jetpack5) echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}-jetpack5.tar.in ;;
+              lib/ollama/cuda_jetpack6) echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}-jetpack6.tar.in ;;
+              lib/ollama/rocm)          echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}-rocm.tar.in ;;
+            esac
+          done
+        working-directory: dist/${{ matrix.os }}-${{ matrix.arch }}
+      - run: |
+          for ARCHIVE in dist/${{ matrix.os }}-${{ matrix.arch }}/*.tar.in; do tar c -C dist/${{ matrix.os }}-${{ matrix.arch }} -T $ARCHIVE | pigz -9vc >$(basename ${ARCHIVE//.*/}.tgz); done
       - uses: actions/upload-artifact@v4
         with:
-          name: dist-${{ matrix.os }}-${{ matrix.arch }}
+          name: dist-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}
           path: |
-            dist/ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.gz
+            *.tgz
 
-  build-docker:
-    environment: release
-    runs-on: linux
+  # Build each Docker variant (OS, arch, and flavor) separately. Using QEMU is unreliable and slower.
+  docker-build-push:
     strategy:
       matrix:
         include:
-          - flavor: |
-              latest=auto
-            platforms: linux/amd64,linux/arm64
-            build-args: [GOFLAGS]
-          - flavor: |
-              suffix=-rocm,onlatest=false
-            platforms: linux/amd64
-            build-args: [GOFLAGS, FLAVOR=rocm]
+          - os: linux
+            arch: arm64
+            build-args: |
+              CGO_CFLAGS
+              CGO_CXXFLAGS
+              GOFLAGS
+          - os: linux
+            arch: amd64
+            build-args: |
+              CGO_CFLAGS
+              CGO_CXXFLAGS
+              GOFLAGS
+          - os: linux
+            arch: amd64
+            suffix: '-rocm'
+            build-args: |
+              CGO_CFLAGS
+              CGO_CXXFLAGS
+              GOFLAGS
+              FLAVOR=rocm
+    runs-on: ${{ matrix.arch == 'arm64' && format('{0}-{1}', matrix.os, matrix.arch) || matrix.os }}
+    environment: release
+    needs: setup-environment
+    env:
+      GOFLAGS: ${{ needs.setup-environment.outputs.GOFLAGS }}
     steps:
       - uses: actions/checkout@v4
-      - uses: docker/setup-qemu-action@v2
-      - uses: docker/setup-buildx-action@v2
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+      - id: build-push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: ${{ matrix.os }}/${{ matrix.arch }}
+          build-args: ${{ matrix.build-args }}
+          outputs: type=image,name=ollama/ollama,push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=registry,ref=ollama/ollama:latest
+          cache-to: type=inline
+      - run: |
+          mkdir -p ${{ matrix.os }}-${{ matrix.arch }}
+          echo "${{ steps.build-push.outputs.digest }}" >${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.suffix }}.txt
+        working-directory: ${{ runner.temp }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: digest-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.suffix }}
+          path: |
+            ${{ runner.temp }}/${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.suffix }}.txt
+
+  # Merge Docker images for the same flavor into a single multi-arch manifest
+  docker-merge-push:
+    strategy:
+      matrix:
+        suffix: ['', '-rocm']
+    runs-on: linux
+    environment: release
+    needs: [docker-build-push]
+    steps:
       - uses: docker/login-action@v3
         with:
           username: ${{ vars.DOCKER_USER }}
@@ -531,32 +408,27 @@ jobs:
       - id: metadata
         uses: docker/metadata-action@v4
         with:
-          flavor: ${{ matrix.flavor }}
+          flavor: |
+            latest=false
+            suffix=${{ matrix.suffix }}
           images: |
             ollama/ollama
           tags: |
+            type=ref,enable=true,priority=600,prefix=pr-,event=pr
             type=semver,pattern={{version}}
-      - uses: docker/build-push-action@v6
+      - uses: actions/download-artifact@v4
         with:
-          context: .
-          push: true
-          platforms: ${{ matrix.platforms }}
-          build-args: ${{ matrix.build-args }}
-          tags: ${{ steps.metadata.outputs.tags }}
-          labels: ${{ steps.metadata.outputs.labels }}
-          cache-from: type=registry,ref=ollama/ollama:latest
-          cache-to: type=inline
-          provenance: false
-        env:
-          GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=${{ steps.metadata.outputs.version }}\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
+          pattern: digest-*
+          path: ${{ runner.temp }}
+          merge-multiple: true
+      - run: |
+          docker buildx imagetools create $(echo '${{ steps.metadata.outputs.json }}' | jq -cr '.tags | map("-t", .) | join(" ")') $(cat *-${{ matrix.suffix }}.txt | xargs printf 'ollama/ollama@%s ')
+          docker buildx imagetools inspect ollama/ollama:${{ steps.metadata.outputs.version }}
+        working-directory: ${{ runner.temp }}
 
   # Aggregate all the assets and ship a release
   release:
-    needs:
-      - build-darwin
-      - build-windows
-      - build-linux-amd64
-      - build-linux-arm64
+    needs: [darwin-sign, windows-sign, linux-build]
     runs-on: linux
     environment: release
     permissions:
@@ -565,33 +437,34 @@ jobs:
       GH_TOKEN: ${{ github.token }}
     steps:
       - uses: actions/checkout@v4
-      - name: Set Version
-        shell: bash
-        run: |
-          echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
-          echo "RELEASE_VERSION=$(echo ${GITHUB_REF_NAME} | cut -f1 -d-)" >> $GITHUB_ENV
-      - name: Retrieve built artifact
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
+          name: dist-darwin
+          path: dist
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist-windows
+          path: dist
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: dist-linux-*
           path: dist
-          pattern: dist-*
           merge-multiple: true
-      - run: |
-          ls -lh dist/
-          (cd dist; find . -type f | xargs sha256sum > ../sha256sum.txt)
-          mv sha256sum.txt dist/
-          cat dist/sha256sum.txt
+      - run: find . -type f -not -name 'sha256sum.txt' | xargs sha256sum | tee sha256sum.txt
+        working-directory: dist
       - name: Create or update Release
         run: |
-          echo "Looking for existing release for ${{ env.RELEASE_VERSION }}"
-          OLD_TAG=$(gh release ls --json name,tagName | jq -r ".[] | select(.name == \"${{ env.RELEASE_VERSION }}\") | .tagName")
+          RELEASE_VERSION="$(echo ${GITHUB_REF_NAME} | cut -f1 -d-)"
+
+          echo "Looking for existing release for ${RELEASE_VERSION}"
+          OLD_TAG=$(gh release ls --json name,tagName | jq -r ".[] | select(.name == \"${RELEASE_VERSION}\") | .tagName")
           if [ -n "$OLD_TAG" ]; then
-            echo "Updating release ${{ env.RELEASE_VERSION }} to point to new tag ${GITHUB_REF_NAME}"
+            echo "Updating release ${RELEASE_VERSION} to point to new tag ${GITHUB_REF_NAME}"
             gh release edit ${OLD_TAG} --tag ${GITHUB_REF_NAME}
           else
-            echo "Creating new release ${{ env.RELEASE_VERSION }} pointing to tag ${GITHUB_REF_NAME}"
+            echo "Creating new release ${RELEASE_VERSION} pointing to tag ${GITHUB_REF_NAME}"
             gh release create ${GITHUB_REF_NAME} \
-              --title ${{ env.RELEASE_VERSION }} \
+              --title ${RELEASE_VERSION} \
               --draft \
               --generate-notes \
               --prerelease

.github/workflows/test.yaml

@@ -40,32 +40,105 @@ jobs:
 
   linux:
     needs: [changes]
-    if: ${{ needs.changes.outputs.changed == 'True' }}
+    if: needs.changes.outputs.changed == 'True'
     strategy:
       matrix:
         include:
-          - container: nvidia/cuda:11.8.0-devel-ubuntu22.04
-            preset: CUDA
-          - container: rocm/dev-ubuntu-22.04:6.1.2
-            preset: ROCm
+          - preset: CPU
+          - preset: CUDA
+            container: nvidia/cuda:11.8.0-devel-ubuntu22.04
+            flags: '-DCMAKE_CUDA_ARCHITECTURES=87'
+          - preset: ROCm
+            container: rocm/dev-ubuntu-22.04:6.1.2
             extra-packages: rocm-libs
-    runs-on: ubuntu-latest
+            flags: '-DAMDGPU_TARGETS=gfx1010 -DCMAKE_PREFIX_PATH=/opt/rocm'
+    runs-on: linux
     container: ${{ matrix.container }}
     steps:
       - uses: actions/checkout@v4
       - run: |
-          apt-get update
-          apt-get install -y cmake pkg-config ccache ${{ matrix.extra-packages }}
-          ccache -o cache_dir=${{ github.workspace }}\.ccache
+          [ -n "${{ matrix.container }}" ] || sudo=sudo
+          $sudo apt-get update
+          $sudo apt-get install -y cmake ccache ${{ matrix.extra-packages }}
         env:
           DEBIAN_FRONTEND: noninteractive
+      - uses: actions/cache@v4
+        with:
+          path: /github/home/.cache/ccache
+          key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.preset }}
+      - run: |
+          cmake --preset ${{ matrix.preset }} ${{ matrix.flags }}
+          cmake --build --preset ${{ matrix.preset }} --parallel
+
+  windows:
+    needs: [changes]
+    if: needs.changes.outputs.changed == 'True'
+    strategy:
+      matrix:
+        include:
+          - preset: CPU
+          - preset: CUDA
+            install: https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_522.06_windows.exe
+            flags: '-DCMAKE_CUDA_ARCHITECTURES=87'
+          - preset: ROCm
+            install: https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe
+            flags: '-DAMDGPU_TARGETS=gfx1010'
+    runs-on: windows
+    steps:
+      - run: |
+          choco install -y --no-progress ccache ninja
+          ccache -o cache_dir=${{ github.workspace }}\.ccache
+      - if: matrix.preset == 'CUDA' || matrix.preset == 'ROCm'
+        id: cache-install
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA
+            C:\Program Files\AMD\ROCm
+          key: ${{ matrix.install }}
+      - if: matrix.preset == 'CUDA'
+        name: Install CUDA ${{ matrix.cuda-version }}
+        run: |
+          $ErrorActionPreference = "Stop"
+          if ("${{ steps.cache-install.outputs.cache-hit }}" -ne 'true') {
+            Invoke-WebRequest -Uri "${{ matrix.install }}" -OutFile "install.exe"
+            Start-Process -FilePath .\install.exe -ArgumentList (@("-s", "cudart_11.8", "nvcc_11.8", "cublas_11.8", "cublas_dev_11.8")) -NoNewWindow -Wait
+          }
+
+          $cudaPath = (Resolve-Path "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\*").path
+          echo "$cudaPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+      - if: matrix.preset == 'ROCm'
+        name: Install ROCm ${{ matrix.rocm-version }}
+        run: |
+          $ErrorActionPreference = "Stop"
+          if ("${{ steps.cache-install.outputs.cache-hit }}" -ne 'true') {
+            Invoke-WebRequest -Uri "${{ matrix.install }}" -OutFile "install.exe"
+            Start-Process -FilePath .\install.exe -ArgumentList '-install' -NoNewWindow -Wait
+          }
+
+          $hipPath = (Resolve-Path "C:\Program Files\AMD\ROCm\*").path
+          echo "$hipPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "CC=$hipPath\bin\clang.exe" | Out-File -FilePath $env:GITHUB_ENV -Append
+          echo "CXX=$hipPath\bin\clang++.exe" | Out-File -FilePath $env:GITHUB_ENV -Append
+      - if: ${{ !cancelled() && steps.cache-install.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA
+            C:\Program Files\AMD\ROCm
+          key: ${{ matrix.install }}
+      - uses: actions/checkout@v4
       - uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}\.ccache
           key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.preset }}
       - run: |
-          cmake --preset ${{ matrix.preset }}
-          cmake --build --preset ${{ matrix.preset }} --parallel
+          Import-Module 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
+          Enter-VsDevShell -VsInstallPath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise' -SkipAutomaticLocation  -DevCmdArguments '-arch=x64 -no_logo'
+          cmake --preset "${{ matrix.preset }}" ${{ matrix.flags }}
+          cmake --build --parallel --preset "${{ matrix.preset }}"
+        env:
+          CMAKE_GENERATOR: Ninja
 
   test:
     strategy:
@@ -90,5 +163,5 @@ jobs:
       - uses: actions/checkout@v4
       - name: Verify patches apply cleanly and do not change files
         run: |
-          make -f Makefile2 clean checkout sync
+          make -f Makefile.sync clean sync
           git diff --compact-summary --exit-code

.gitignore

@@ -4,12 +4,13 @@
 .venv
 .swp
 dist
+build
 ollama
 .cache
 *.exe
 .idea
 test_data
 *.crt
-llama/build
 __debug_bin*
-llama/vendor
+llama/build
+llama/vendor

CMakeLists.txt

@@ -21,12 +21,28 @@ set(GGML_BACKEND_SHARED ON)
 set(GGML_SCHED_MAX_COPIES 4)
 
 set(GGML_LLAMAFILE ON)
-set(GGML_CPU_ALL_VARIANTS ON)
 set(GGML_CUDA_PEER_MAX_BATCH_SIZE 128)
 set(GGML_CUDA_GRAPHS ON)
 
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+if((NOT CMAKE_OSX_ARCHITECTURES MATCHES "arm64")
+    OR (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm|aarch64|ARM64|ARMv[0-9]+"))
+    set(GGML_CPU_ALL_VARIANTS ON)
+endif()
+
+if (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64")
+    set(CMAKE_BUILD_RPATH "@loader_path")
+    set(CMAKE_INSTALL_RPATH "@loader_path")
+endif()
+
+set(OLLAMA_BUILD_DIR ${CMAKE_BINARY_DIR}/lib/ollama)
+set(OLLAMA_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/ollama)
+
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY         ${OLLAMA_BUILD_DIR})
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG   ${OLLAMA_BUILD_DIR})
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${OLLAMA_BUILD_DIR})
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY         ${OLLAMA_BUILD_DIR})
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG   ${OLLAMA_BUILD_DIR})
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE ${OLLAMA_BUILD_DIR})
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ml/backend/ggml/ggml/src)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ml/backend/ggml/ggml/src/include)
@@ -37,18 +53,73 @@ set(GGML_CPU ON)
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ml/backend/ggml/ggml/src)
 set_property(TARGET ggml PROPERTY EXCLUDE_FROM_ALL TRUE)
 
+get_target_property(CPU_VARIANTS ggml-cpu MANUALLY_ADDED_DEPENDENCIES)
+if(NOT CPU_VARIANTS)
+    set(CPU_VARIANTS "ggml-cpu")
+endif()
+
+install(TARGETS ggml-base ${CPU_VARIANTS}
+    RUNTIME_DEPENDENCIES
+        PRE_EXCLUDE_REGEXES ".*"
+    RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT CPU
+    LIBRARY DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT CPU
+    FRAMEWORK DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT CPU
+)
+
 check_language(CUDA)
 if(CMAKE_CUDA_COMPILER)
     if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24" AND NOT CMAKE_CUDA_ARCHITECTURES)
         set(CMAKE_CUDA_ARCHITECTURES "native")
     endif()
 
+    find_package(CUDAToolkit)
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ml/backend/ggml/ggml/src/ggml-cuda)
+    set(OLLAMA_CUDA_INSTALL_DIR ${OLLAMA_INSTALL_DIR}/cuda_v${CUDAToolkit_VERSION_MAJOR})
+    install(TARGETS ggml-cuda
+        RUNTIME_DEPENDENCIES
+            DIRECTORIES ${CUDAToolkit_BIN_DIR} ${CUDAToolkit_LIBRARY_DIR}
+            PRE_INCLUDE_REGEXES cublas cublasLt cudart
+            PRE_EXCLUDE_REGEXES ".*"
+        RUNTIME DESTINATION ${OLLAMA_CUDA_INSTALL_DIR} COMPONENT CUDA
+        LIBRARY DESTINATION ${OLLAMA_CUDA_INSTALL_DIR} COMPONENT CUDA
+    )
 endif()
 
+set(WINDOWS_AMDGPU_TARGETS_EXCLUDE_REGEX "^gfx(906|908|90a):xnack[+-]$"
+    CACHE STRING
+    "Regular expression describing AMDGPU_TARGETS not supported on Windows. Override to force building these targets. Default \"^gfx(906|908|90a):xnack[+-]$\"."
+)
+
 check_language(HIP)
 if(CMAKE_HIP_COMPILER)
     set(HIP_PLATFORM "amd")
 
-    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ml/backend/ggml/ggml/src/ggml-hip)
+    find_package(hip REQUIRED)
+    if(NOT AMDGPU_TARGETS)
+        list(FILTER AMDGPU_TARGETS INCLUDE REGEX "^gfx(900|94[012]|101[02]|1030|110[012])$")
+    elseif(WIN32 AND WINDOWS_AMDGPU_TARGETS_EXCLUDE_REGEX)
+        list(FILTER AMDGPU_TARGETS EXCLUDE REGEX ${WINDOWS_AMDGPU_TARGETS_EXCLUDE_REGEX})
+    endif()
+
+    if(AMDGPU_TARGETS)
+        add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ml/backend/ggml/ggml/src/ggml-hip)
+
+        set(OLLAMA_HIP_INSTALL_DIR ${OLLAMA_INSTALL_DIR}/rocm)
+        install(TARGETS ggml-hip
+            RUNTIME_DEPENDENCIES
+                DIRECTORIES ${HIP_BIN_INSTALL_DIR} ${HIP_LIB_INSTALL_DIR}
+                PRE_INCLUDE_REGEXES hipblas rocblas amdhip64 rocsolver amd_comgr hsa-runtime64 rocsparse tinfo rocprofiler-register drm drm_amdgpu numa elf
+                PRE_EXCLUDE_REGEXES ".*"
+                POST_EXCLUDE_REGEXES "system32"
+            RUNTIME DESTINATION ${OLLAMA_HIP_INSTALL_DIR} COMPONENT HIP
+            LIBRARY DESTINATION ${OLLAMA_HIP_INSTALL_DIR} COMPONENT HIP
+        )
+
+        foreach(HIP_LIB_BIN_INSTALL_DIR IN ITEMS ${HIP_BIN_INSTALL_DIR} ${HIP_LIB_INSTALL_DIR})
+            if(EXISTS ${HIP_LIB_BIN_INSTALL_DIR}/rocblas)
+                install(DIRECTORY ${HIP_LIB_BIN_INSTALL_DIR}/rocblas DESTINATION ${OLLAMA_HIP_INSTALL_DIR} COMPONENT HIP)
+                break()
+            endif()
+        endforeach()
+    endif()
 endif()

CMakePresets.json

@@ -4,6 +4,7 @@
     {
       "name": "Default",
       "binaryDir": "${sourceDir}/build",
+      "installDir": "${sourceDir}/dist",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release"
       }
@@ -55,7 +56,7 @@
       "name": "ROCm 6",
       "inherits": [ "ROCm" ],
       "cacheVariables": {
-        "AMDGPU_TARGETS": "gfx900;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102"
+        "AMDGPU_TARGETS": "gfx900;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-"
       }
     }
   ],

Dockerfile

@@ -33,25 +33,33 @@ FROM base AS cpu
 RUN if [ "$(uname -m)" = "x86_64" ]; then yum install -y devtoolset-11-gcc devtoolset-11-gcc-c++; fi
 ENV PATH=/opt/rh/devtoolset-11/root/usr/bin:$PATH
 RUN --mount=type=cache,target=/root/.ccache \
-    cmake --preset 'CPU' && cmake --build --parallel --preset 'CPU'
+    cmake --preset 'CPU' \
+        && cmake --build --parallel --preset 'CPU' \
+        && cmake --install build --component CPU --strip --parallel 8
 
 FROM base AS cuda-11
 ARG CUDA11VERSION=11.3
 RUN yum install -y cuda-toolkit-${CUDA11VERSION//./-}
 ENV PATH=/usr/local/cuda-11/bin:$PATH
 RUN --mount=type=cache,target=/root/.ccache \
-    cmake --preset 'CUDA 11' && cmake --build --parallel --preset 'CUDA 11'
+    cmake --preset 'CUDA 11' \
+        && cmake --build --parallel --preset 'CUDA 11' \
+        && cmake --install build --component CUDA --strip --parallel 8
 
 FROM base AS cuda-12
 ARG CUDA12VERSION=12.4
 RUN yum install -y cuda-toolkit-${CUDA12VERSION//./-}
 ENV PATH=/usr/local/cuda-12/bin:$PATH
 RUN --mount=type=cache,target=/root/.ccache \
-    cmake --preset 'CUDA 12' && cmake --build --parallel --preset 'CUDA 12'
+    cmake --preset 'CUDA 12' \
+        && cmake --build --parallel --preset 'CUDA 12' \
+        && cmake --install build --component CUDA --strip --parallel 8
 
 FROM base AS rocm-6
 RUN --mount=type=cache,target=/root/.ccache \
-    cmake --preset 'ROCm 6' && cmake --build --parallel --preset 'ROCm 6'
+    cmake --preset 'ROCm 6' \
+        && cmake --build --parallel --preset 'ROCm 6' \
+        && cmake --install build --component HIP --strip --parallel 8
 
 FROM --platform=linux/arm64 nvcr.io/nvidia/l4t-jetpack:${JETPACK5VERSION} AS jetpack-5
 ARG CMAKEVERSION
@@ -60,7 +68,9 @@ RUN apt-get update && apt-get install -y curl ccache \
 COPY CMakeLists.txt CMakePresets.json .
 COPY ml/backend/ggml/ggml ml/backend/ggml/ggml
 RUN --mount=type=cache,target=/root/.ccache \
-    cmake --preset 'JetPack 5' && cmake --build --parallel --preset 'JetPack 5'
+    cmake --preset 'JetPack 5' \
+        && cmake --build --parallel --preset 'JetPack 5' \
+        && cmake --install build --component CUDA --strip --parallel 8
 
 FROM --platform=linux/arm64 nvcr.io/nvidia/l4t-jetpack:${JETPACK6VERSION} AS jetpack-6
 ARG CMAKEVERSION
@@ -69,7 +79,9 @@ RUN apt-get update && apt-get install -y curl ccache \
 COPY CMakeLists.txt CMakePresets.json .
 COPY ml/backend/ggml/ggml ml/backend/ggml/ggml
 RUN --mount=type=cache,target=/root/.ccache \
-    cmake --preset 'JetPack 6' && cmake --build --parallel --preset 'JetPack 6'
+    cmake --preset 'JetPack 6' \
+        && cmake --build --parallel --preset 'JetPack 6' \
+        && cmake --install build --component CUDA --strip --parallel 8
 
 FROM base AS build
 ARG GOVERSION=1.23.4
@@ -83,65 +95,20 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     go build -trimpath -buildmode=pie -o /bin/ollama .
 
 FROM --platform=linux/amd64 scratch AS amd64
-COPY --from=cuda-11 --chmod=644 \
-    build/lib/libggml-cuda.so \
-    /usr/local/cuda/lib64/libcublas.so.11 \
-    /usr/local/cuda/lib64/libcublasLt.so.11 \
-    /usr/local/cuda/lib64/libcudart.so.11.0 \
-    /lib/ollama/cuda_v11/
-COPY --from=cuda-12 --chmod=644 \
-    build/lib/libggml-cuda.so \
-    /usr/local/cuda/lib64/libcublas.so.12 \
-    /usr/local/cuda/lib64/libcublasLt.so.12 \
-    /usr/local/cuda/lib64/libcudart.so.12 \
-    /lib/ollama/cuda_v12/
+COPY --from=cuda-11 dist/lib/ollama/cuda_v11 /lib/ollama/cuda_v11
+COPY --from=cuda-12 dist/lib/ollama/cuda_v12 /lib/ollama/cuda_v12
 
 FROM --platform=linux/arm64 scratch AS arm64
-COPY --from=cuda-11 --chmod=644 \
-    build/lib/libggml-cuda.so \
-    /usr/local/cuda/lib64/libcublas.so.11 \
-    /usr/local/cuda/lib64/libcublasLt.so.11 \
-    /usr/local/cuda/lib64/libcudart.so.11.0 \
-    /lib/ollama/cuda_v11/
-COPY --from=cuda-12 --chmod=644 \
-    build/lib/libggml-cuda.so \
-    /usr/local/cuda/lib64/libcublas.so.12 \
-    /usr/local/cuda/lib64/libcublasLt.so.12 \
-    /usr/local/cuda/lib64/libcudart.so.12 \
-    /lib/ollama/cuda_v12/
-COPY --from=jetpack-5 --chmod=644 \
-    build/lib/libggml-cuda.so \
-    /usr/local/cuda/lib64/libcublas.so.11 \
-    /usr/local/cuda/lib64/libcublasLt.so.11 \
-    /usr/local/cuda/lib64/libcudart.so.11.0 \
-    /lib/ollama/cuda_jetpack5/
-COPY --from=jetpack-6 --chmod=644 \
-    build/lib/libggml-cuda.so \
-    /usr/local/cuda/lib64/libcublas.so.12 \
-    /usr/local/cuda/lib64/libcublasLt.so.12 \
-    /usr/local/cuda/lib64/libcudart.so.12 \
-    /lib/ollama/cuda_jetpack6/
+COPY --from=cuda-11 dist/lib/ollama/cuda_v11 /lib/ollama/cuda_v11
+COPY --from=cuda-12 dist/lib/ollama/cuda_v12 /lib/ollama/cuda_v12
+COPY --from=jetpack-5 dist/lib/ollama/cuda_v11 lib/ollama/cuda_jetpack5
+COPY --from=jetpack-6 dist/lib/ollama/cuda_v12 lib/ollama/cuda_jetpack6
 
 FROM --platform=linux/arm64 scratch AS rocm
-COPY --from=rocm-6 --chmod=644 \
-    build/lib/libggml-hip.so \
-    /opt/rocm/lib/libamdhip64.so.6 \
-    /opt/rocm/lib/libhipblas.so.2 \
-    /opt/rocm/lib/librocblas.so.4 \
-    /opt/rocm/lib/libamd_comgr.so.2 \
-    /opt/rocm/lib/libhsa-runtime64.so.1 \
-    /opt/rocm/lib/librocprofiler-register.so.0 \
-    /opt/amdgpu/lib64/libdrm_amdgpu.so.1 \
-    /opt/amdgpu/lib64/libdrm.so.2 \
-    /usr/lib64/libnuma.so.1 \
-    /lib/ollama/rocm/
-COPY --from=rocm-6 /opt/rocm/lib/rocblas/ /lib/ollama/rocm/rocblas/
+COPY --from=rocm-6 dist/lib/ollama/rocm /lib/ollama/rocm
 
 FROM ${FLAVOR} AS archive
-COPY --from=cpu --chmod=644 \
-    build/lib/libggml-base.so \
-    build/lib/libggml-cpu-*.so \
-    /lib/ollama/
+COPY --from=cpu dist/lib/ollama /lib/ollama
 COPY --from=build /bin/ollama /bin/ollama
 
 FROM ubuntu:20.04
@@ -149,10 +116,10 @@ RUN apt-get update \
     && apt-get install -y ca-certificates \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=archive /bin/ /usr/bin/
+COPY --from=archive /bin /usr/bin
 ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-COPY --from=archive /lib/ollama/ /usr/lib/ollama/
-ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/ollama
+COPY --from=archive /lib/ollama /usr/lib/ollama
+ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV OLLAMA_HOST=0.0.0.0:11434

Makefile.sync

@@ -0,0 +1,60 @@
+UPSTREAM=https://github.com/ggerganov/llama.cpp.git
+WORKDIR=llama/vendor
+FETCH_HEAD=46e3556e01b824e52395fb050b29804b6cff2a7c
+
+.PHONY: help
+help:
+	@echo "Available targets:"
+	@echo "    sync                 Sync with upstream repositories"
+	@echo "    checkout             Checkout upstream repository"
+	@echo "    apply-patches        Apply patches to local repository"
+	@echo "    format-patches       Format patches from local repository"
+	@echo "    clean                Clean local repository"
+	@echo
+	@echo "Example:"
+	@echo "    make -f $(lastword $(MAKEFILE_LIST)) clean sync"
+
+.PHONY: sync
+sync: llama/build-info.cpp llama/llama.cpp ml/backend/ggml/ggml apply-patches
+
+.PHONY: llama/build-info.cpp
+llama/build-info.cpp: llama/build-info.cpp.in
+	sed -e 's|@FETCH_HEAD@|$(FETCH_HEAD)|' $< > $@
+
+.PHONY: llama/llama.cpp
+llama/llama.cpp: llama/vendor/ apply-patches
+	rsync -arvzc -f "merge $@/.rsync-filter" $< $@
+
+.PHONY: ml/backend/ggml/ggml apply-patches
+ml/backend/ggml/ggml: llama/vendor/ggml/ apply-patches
+	rsync -arvzc -f "merge $@/.rsync-filter" $< $@
+
+PATCHES=$(wildcard llama/patches/*.patch)
+
+.PHONY: apply-patches
+.NOTPARALLEL:
+apply-patches: $(addsuffix ed, $(PATCHES))
+
+%.patched: %.patch
+	@if git -c user.name=nobody -c 'user.email=<>' -C $(WORKDIR) am -3 $(realpath $<); then touch $@; else git -C $(WORKDIR) am --abort; exit 1; fi
+
+.PHONY: checkout
+checkout: $(WORKDIR)
+	git -C $(WORKDIR) fetch
+	git -C $(WORKDIR) checkout -f $(FETCH_HEAD)
+
+$(WORKDIR):
+	git clone $(UPSTREAM) $(WORKDIR)
+
+.PHONE: format-patches
+format-patches: llama/patches
+	git -C $(WORKDIR) format-patch \
+		--no-signature \
+		--no-numbered \
+		--zero-commit \
+		-o $(realpath $<) \
+		$(FETCH_HEAD)
+
+.PHONE: clean
+clean: checkout
+	$(RM) $(addsuffix ed, $(PATCHES))

Makefile2

@@ -1,46 +0,0 @@
-UPSTREAM=https://github.com/ggerganov/llama.cpp.git
-WORKDIR=llama/vendor
-FETCH_HEAD=46e3556e01b824e52395fb050b29804b6cff2a7c
-
-all: sync
-
-.PHONY: sync
-sync: llama/llama.cpp ml/backend/ggml/ggml
-
-.PHONY: llama/llama.cpp
-llama/llama.cpp: llama/vendor/ apply_patches
-	rsync -arvzc -f "merge $@/.rsync-filter" $< $@
-
-.PHONY: ml/backend/ggml/ggml apply_patches
-ml/backend/ggml/ggml: llama/vendor/ggml/ apply_patches
-	rsync -arvzc -f "merge $@/.rsync-filter" $< $@
-
-PATCHES=$(wildcard llama/patches/*.patch)
-
-.PHONY: apply_patches
-.NOTPARALLEL:
-apply_patches: $(addsuffix ed, $(PATCHES))
-
-%.patched: %.patch
-	@if git -c user.name=nobody -c 'user.email=<>' -C $(WORKDIR) am -3 $(realpath $<); then touch $@; else git -C $(WORKDIR) am --abort; exit 1; fi
-
-.PHONY: checkout
-checkout: $(WORKDIR)
-	git -C $(WORKDIR) fetch
-	git -C $(WORKDIR) checkout -f $(FETCH_HEAD)
-
-$(WORKDIR):
-	git clone $(UPSTREAM) $(WORKDIR)
-
-.PHONE: format_patches
-format_patches: llama/patches
-	git -C $(WORKDIR) format-patch \
-		--no-signature \
-		--no-numbered \
-		--zero-commit \
-		-o $(realpath $<) \
-		$(FETCH_HEAD)
-
-.PHONE: clean
-clean: checkout
-	$(RM) $(addsuffix ed, $(PATCHES))

README.md

@@ -18,7 +18,7 @@ Get up and running with large language models.
 
 ### Linux
 
-```
+```shell
 curl -fsSL https://ollama.com/install.sh | sh
 ```
 
@@ -42,7 +42,7 @@ The official [Ollama Docker image](https://hub.docker.com/r/ollama/ollama) `olla
 
 To run and chat with [Llama 3.2](https://ollama.com/library/llama3.2):
 
-```
+```shell
 ollama run llama3.2
 ```
 
@@ -54,6 +54,8 @@ Here are some example models that can be downloaded:
 
 | Model              | Parameters | Size  | Download                         |
 | ------------------ | ---------- | ----- | -------------------------------- |
+| DeepSeek-R1        | 7B         | 4.7GB | `ollama run deepseek-r1`         |
+| DeepSeek-R1        | 671B       | 404GB | `ollama run deepseek-r1:671b`    |
 | Llama 3.3          | 70B        | 43GB  | `ollama run llama3.3`            |
 | Llama 3.2          | 3B         | 2.0GB | `ollama run llama3.2`            |
 | Llama 3.2          | 1B         | 1.3GB | `ollama run llama3.2:1b`         |
@@ -92,13 +94,13 @@ Ollama supports importing GGUF models in the Modelfile:
 
 2. Create the model in Ollama
 
-   ```
+   ```shell
    ollama create example -f Modelfile
    ```
 
 3. Run the model
 
-   ```
+   ```shell
    ollama run example
    ```
 
@@ -110,7 +112,7 @@ See the [guide](docs/import.md) on importing models for more information.
 
 Models from the Ollama library can be customized with a prompt. For example, to customize the `llama3.2` model:
 
-```
+```shell
 ollama pull llama3.2
 ```
 
@@ -145,13 +147,13 @@ For more information on working with a Modelfile, see the [Modelfile](docs/model
 
 `ollama create` is used to create a model from a Modelfile.
 
-```
+```shell
 ollama create mymodel -f ./Modelfile
 ```
 
 ### Pull a model
 
-```
+```shell
 ollama pull llama3.2
 ```
 
@@ -159,13 +161,13 @@ ollama pull llama3.2
 
 ### Remove a model
 
-```
+```shell
 ollama rm llama3.2
 ```
 
 ### Copy a model
 
-```
+```shell
 ollama cp llama3.2 my-model
 ```
 
@@ -184,37 +186,39 @@ I'm a basic program that prints the famous "Hello, world!" message to the consol
 
 ```
 ollama run llava "What's in this image? /Users/jmorgan/Desktop/smile.png"
-The image features a yellow smiley face, which is likely the central focus of the picture.
 ```
 
+> **Output**: The image features a yellow smiley face, which is likely the central focus of the picture.
+
 ### Pass the prompt as an argument
 
+```shell
+ollama run llama3.2 "Summarize this file: $(cat README.md)"
 ```
-$ ollama run llama3.2 "Summarize this file: $(cat README.md)"
- Ollama is a lightweight, extensible framework for building and running language models on the local machine. It provides a simple API for creating, running, and managing models, as well as a library of pre-built models that can be easily used in a variety of applications.
-```
+
+> **Output**: Ollama is a lightweight, extensible framework for building and running language models on the local machine. It provides a simple API for creating, running, and managing models, as well as a library of pre-built models that can be easily used in a variety of applications.
 
 ### Show model information
 
-```
+```shell
 ollama show llama3.2
 ```
 
 ### List models on your computer
 
-```
+```shell
 ollama list
 ```
 
 ### List which models are currently loaded
 
-```
+```shell
 ollama ps
 ```
 
 ### Stop a model which is currently running
 
-```
+```shell
 ollama stop llama3.2
 ```
 
@@ -230,13 +234,13 @@ See the [developer guide](https://github.com/ollama/ollama/blob/main/docs/develo
 
 Next, start the server:
 
-```
+```shell
 ./ollama serve
 ```
 
 Finally, in a separate shell, run a model:
 
-```
+```shell
 ./ollama run llama3.2
 ```
 
@@ -246,7 +250,7 @@ Ollama has a REST API for running and managing models.
 
 ### Generate a response
 
-```
+```shell
 curl http://localhost:11434/api/generate -d '{
   "model": "llama3.2",
   "prompt":"Why is the sky blue?"
@@ -255,7 +259,7 @@ curl http://localhost:11434/api/generate -d '{
 
 ### Chat with a model
 
-```
+```shell
 curl http://localhost:11434/api/chat -d '{
   "model": "llama3.2",
   "messages": [
@@ -353,6 +357,7 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [Web management](https://github.com/lemonit-eric-mao/ollama-web-management) (Web management page)
 - [Promptery](https://github.com/promptery/promptery) (desktop client for Ollama.)
 - [Ollama App](https://github.com/JHubi1/ollama-app) (Modern and easy-to-use multi-platform client for Ollama)
+- [chat-ollama](https://github.com/annilq/chat-ollama) (a React Native client for Ollama)
 - [SpaceLlama](https://github.com/tcsenpai/spacellama) (Firefox and Chrome extension to quickly summarize web pages with ollama in a sidebar)
 - [YouLama](https://github.com/tcsenpai/youlama) (Webapp to quickly summarize any YouTube video, supporting Invidious as well)
 - [DualMind](https://github.com/tcsenpai/dualmind) (Experimental app allowing two models to talk to each other in the terminal or in a web interface)
@@ -369,6 +374,12 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [Minima](https://github.com/dmayboroda/minima) (RAG with on-premises or fully local workflow)
 - [aidful-ollama-model-delete](https://github.com/AidfulAI/aidful-ollama-model-delete) (User interface for simplified model cleanup)
 - [Perplexica](https://github.com/ItzCrazyKns/Perplexica) (An AI-powered search engine & an open-source alternative to Perplexity AI)
+- [Ollama Chat WebUI for Docker ](https://github.com/oslook/ollama-webui) (Support for local docker deployment, lightweight ollama webui)
+- [AI Toolkit for Visual Studio Code](https://aka.ms/ai-tooklit/ollama-docs) (Microsoft-official VSCode extension to chat, test, evaluate models with Ollama support, and use them in your AI applications.)
+- [MinimalNextOllamaChat](https://github.com/anilkay/MinimalNextOllamaChat) (Minimal Web UI for Chat and Model Control)
+- [Chipper](https://github.com/TilmanGriesel/chipper) AI interface for tinkerers (Ollama, Haystack RAG, Python)
+- [ChibiChat](https://github.com/CosmicEventHorizon/ChibiChat) (Kotlin-based Android app to chat with Ollama and Koboldcpp API endpoints)
+- [LocalLLM](https://github.com/qusaismael/localllm) (Minimal Web-App to run ollama models on it with a GUI)
 
 ### Cloud
 
@@ -426,9 +437,10 @@ See the [API documentation](./docs/api.md) for all endpoints.
 
 - [Pacman](https://archlinux.org/packages/extra/x86_64/ollama/)
 - [Gentoo](https://github.com/gentoo/guru/tree/master/app-misc/ollama)
+- [Homebrew](https://formulae.brew.sh/formula/ollama)
 - [Helm Chart](https://artifacthub.io/packages/helm/ollama-helm/ollama)
 - [Guix channel](https://codeberg.org/tusharhero/ollama-guix)
-- [Nix package](https://search.nixos.org/packages?channel=24.05&show=ollama&from=0&size=50&sort=relevance&type=packages&query=ollama)
+- [Nix package](https://search.nixos.org/packages?show=ollama&from=0&size=50&sort=relevance&type=packages&query=ollama)
 - [Flox](https://flox.dev/blog/ollama-part-one)
 
 ### Libraries
@@ -481,6 +493,9 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [GoLamify](https://github.com/prasad89/golamify)
 - [Ollama for Haskell](https://github.com/tusharad/ollama-haskell)
 - [multi-llm-ts](https://github.com/nbonamy/multi-llm-ts) (A Typescript/JavaScript library allowing access to different LLM in unified API)
+- [LlmTornado](https://github.com/lofcz/llmtornado) (C# library providing a unified interface for major FOSS & Commercial inference APIs)
+- [Ollama for Zig](https://github.com/dravenk/ollama-zig)
+- [Abso](https://github.com/lunary-ai/abso) (OpenAI-compatible TypeScript SDK for any LLM provider)
 
 ### Mobile
 
@@ -531,13 +546,15 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [TextCraft](https://github.com/suncloudsmoon/TextCraft) (Copilot in Word alternative using Ollama)
 - [Alfred Ollama](https://github.com/zeitlings/alfred-ollama) (Alfred Workflow)
 - [TextLLaMA](https://github.com/adarshM84/TextLLaMA) A Chrome Extension that helps you write emails, correct grammar, and translate into any language
+- [Simple-Discord-AI](https://github.com/zyphixor/simple-discord-ai)
 
 ### Supported backends
 
 - [llama.cpp](https://github.com/ggerganov/llama.cpp) project founded by Georgi Gerganov.
 
 ### Observability
-
+- [Lunary](https://lunary.ai/docs/integrations/ollama) is the leading open-source LLM observability platform. It provides a variety of enterprise-grade features such as real-time analytics, prompt templates management, PII masking, and comprehensive agent tracing.
 - [OpenLIT](https://github.com/openlit/openlit) is an OpenTelemetry-native tool for monitoring Ollama Applications & GPUs using traces and metrics.
 - [HoneyHive](https://docs.honeyhive.ai/integrations/ollama) is an AI observability and evaluation platform for AI agents. Use HoneyHive to evaluate agent performance, interrogate failures, and monitor quality in production.
 - [Langfuse](https://langfuse.com/docs/integrations/ollama) is an open source LLM observability platform that enables teams to collaboratively monitor, evaluate and debug AI applications.
+- [MLflow Tracing](https://mlflow.org/docs/latest/llms/tracing/index.html#automatic-tracing) is an open source LLM observability tool with a convenient API to log and visualize traces, making it easy to debug and evaluate GenAI applications.

api/examples/README.md

@@ -2,9 +2,10 @@
 
 Run the examples in this directory with:
 
-```
+```shell
 go run example_name/main.go
 ```
+
 ## Chat - Chat with a model
 - [chat/main.go](chat/main.go)
 

api/types.go

@@ -77,6 +77,8 @@ type GenerateRequest struct {
 	// request, for multimodal models.
 	Images []ImageData `json:"images,omitempty"`
 
+	LogProbs int `json:"logprobs,omitempty"`
+
 	// Options lists model-specific options. For example, temperature can be
 	// set through this field, if the model supports it.
 	Options map[string]interface{} `json:"options"`
@@ -103,6 +105,8 @@ type ChatRequest struct {
 	// Tools is an optional list of tools the model has access to.
 	Tools `json:"tools,omitempty"`
 
+	LogProbs int `json:"logprobs,omitempty"`
+
 	// Options lists model-specific options.
 	Options map[string]interface{} `json:"options"`
 }
@@ -182,13 +186,20 @@ func (t *ToolFunction) String() string {
 	return string(bts)
 }
 
+type TokenProbs struct {
+	TokenID int     `json:"id"`
+	LogProb float32 `json:"logprob"`
+	Token   string  `json:"token"`
+}
+
 // ChatResponse is the response returned by [Client.Chat]. Its fields are
 // similar to [GenerateResponse].
 type ChatResponse struct {
-	Model      string    `json:"model"`
-	CreatedAt  time.Time `json:"created_at"`
-	Message    Message   `json:"message"`
-	DoneReason string    `json:"done_reason,omitempty"`
+	Model      string       `json:"model"`
+	CreatedAt  time.Time    `json:"created_at"`
+	Message    Message      `json:"message"`
+	DoneReason string       `json:"done_reason,omitempty"`
+	LogProbs   []TokenProbs `json:"logprobs,omitempty"`
 
 	Done bool `json:"done"`
 
@@ -452,6 +463,8 @@ type GenerateResponse struct {
 	// can be sent in the next request to keep a conversational memory.
 	Context []int `json:"context,omitempty"`
 
+	LogProbs []TokenProbs `json:"logprobs,omitempty"`
+
 	Metrics
 }
 

app/README.md

@@ -17,6 +17,6 @@ If you want to build the installer, youll need to install
 In the top directory of this repo, run the following powershell script
 to build the ollama CLI, ollama app, and ollama installer.
 
-```
+```powershell
 powershell -ExecutionPolicy Bypass -File .\scripts\build_windows.ps1
 ```

benchmark/README.md

@@ -1,25 +0,0 @@
-# Benchmark
-
-Performance benchmarking for Ollama.
-
-## Prerequisites
-- Ollama server running locally (`127.0.0.1:11434`)
-- Desired models pre-downloaded (e.g., `llama3.2:1b`)
-
-## Run Benchmark
-```bash
-# Run all tests
-go test -bench=. -timeout 30m ./...
-```
-
-## New Runner Benchmark
-```bash
-go test -bench=Runner
-```
-
-or to test multiple models:
-```bash
-# run this from within the benchmark directory
-# requires: llama3.2:1b, llama3.1:8b, llama3.3:70b
-sh new_runner.sh
-```

benchmark/new_runner.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-
-kill_process_tree() {
-    local pid=$1
-    # Get all child processes using pgrep
-    local children=$(pgrep -P $pid)
-    
-    # Kill children first
-    for child in $children; do
-        kill_process_tree $child
-    done
-    
-    # Kill the parent process
-    kill -9 $pid 2>/dev/null || true
-}
-
-# Function to run the runner and benchmark for a given model
-run_benchmark() {
-    local model=$1
-    
-    echo "Starting runner with model: $model"
-    # Start the runner in background and save its PID
-    go run ../cmd/runner/main.go --new-runner -model "$model" &
-    runner_pid=$!
-    
-    # Wait for the runner to initialize (adjust sleep time as needed)
-    sleep 5
-    
-    echo "Running benchmark..."
-    # Run test and wait for it to complete
-    go test -bench=Runner
-    test_exit_code=$?
-    
-    echo "Stopping runner process..."
-    # Kill the runner process and all its children
-    kill_process_tree $runner_pid
-    
-    # Wait for the process to fully terminate
-    wait $runner_pid 2>/dev/null || true
-    
-    # Make sure no processes are still listening on port 8080
-    lsof -t -i:8080 | xargs kill -9 2>/dev/null || true
-    
-    # Additional sleep to ensure port is freed
-    sleep 2
-    
-    # Check if test failed
-    if [ $test_exit_code -ne 0 ]; then
-        echo "Warning: Benchmark test failed with exit code $test_exit_code"
-    fi
-    
-    echo "Benchmark complete for model: $model"
-    echo "----------------------------------------"
-}
-
-
-HOME_DIR="$HOME"
-# llama3.2:1b:  ~/.ollama/models/blobs/sha256-74701a8c35f6c8d9a4b91f3f3497643001d63e0c7a84e085bed452548fa88d45
-# llama3.1:8b:  ~/.ollama/models/blobs/sha256-667b0c1932bc6ffc593ed1d03f895bf2dc8dc6df21db3042284a6f4416b06a29
-# llama3.3:70b: ~/.ollama/models/blobs/sha256-4824460d29f2058aaf6e1118a63a7a197a09bed509f0e7d4e2efb1ee273b447d
-models=(
-    "${HOME_DIR}/.ollama/models/blobs/sha256-74701a8c35f6c8d9a4b91f3f3497643001d63e0c7a84e085bed452548fa88d45"
-    "${HOME_DIR}/.ollama/models/blobs/sha256-667b0c1932bc6ffc593ed1d03f895bf2dc8dc6df21db3042284a6f4416b06a29"
-    # "${HOME_DIR}/.ollama/models/blobs/sha256-4824460d29f2058aaf6e1118a63a7a197a09bed509f0e7d4e2efb1ee273b447d"
-)
-
-# Run benchmarks for each model
-for model in "${models[@]}"; do
-    run_benchmark "$model"
-done
-
-echo "All benchmarks completed!"

benchmark/new_runner_benchmark_test.go

@@ -1,175 +0,0 @@
-package benchmark
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"testing"
-	"time"
-)
-
-const (
-	runnerURL     = "http://localhost:8080"
-	warmupPrompts = 2  // Number of warm-up requests per test case
-	warmupTokens  = 50 // Smaller token count for warm-up requests
-)
-
-var runnerMetrics []BenchmarkMetrics
-
-// CompletionRequest represents the request body for the completion endpoint
-type CompletionRequest struct {
-	Prompt      string  `json:"prompt"`
-	NumPredict  int     `json:"n_predict"`
-	Temperature float32 `json:"temperature"`
-}
-
-// CompletionResponse represents a single response chunk from the streaming API
-type CompletionResponse struct {
-	Content string `json:"content"`
-	Stop    bool   `json:"stop"`
-	Timings struct {
-		PredictedN  int `json:"predicted_n"`
-		PredictedMs int `json:"predicted_ms"`
-		PromptN     int `json:"prompt_n"`
-		PromptMs    int `json:"prompt_ms"`
-	} `json:"timings"`
-}
-
-// warmUp performs warm-up requests before the actual benchmark
-func warmUp(b *testing.B, tt TestCase) {
-	b.Logf("Warming up for test case %s", tt.name)
-	warmupTest := TestCase{
-		name:      tt.name + "_warmup",
-		prompt:    tt.prompt,
-		maxTokens: warmupTokens,
-	}
-
-	for i := 0; i < warmupPrompts; i++ {
-		runCompletion(context.Background(), warmupTest, b)
-		time.Sleep(100 * time.Millisecond) // Brief pause between warm-up requests
-	}
-	b.Logf("Warm-up complete")
-}
-
-func BenchmarkRunnerInference(b *testing.B) {
-	b.Logf("Starting benchmark suite")
-
-	// Verify server availability
-	if _, err := http.Get(runnerURL + "/health"); err != nil {
-		b.Fatalf("Runner unavailable: %v", err)
-	}
-	b.Log("Runner available")
-
-	tests := []TestCase{
-		{
-			name:      "short_prompt",
-			prompt:    formatPrompt("Write a long story"),
-			maxTokens: 100,
-		},
-		{
-			name:      "medium_prompt",
-			prompt:    formatPrompt("Write a detailed economic analysis"),
-			maxTokens: 500,
-		},
-		{
-			name:      "long_prompt",
-			prompt:    formatPrompt("Write a comprehensive AI research paper"),
-			maxTokens: 1000,
-		},
-	}
-
-	// Register cleanup handler for results reporting
-	b.Cleanup(func() { reportMetrics(metrics) })
-
-	// Main benchmark loop
-	for _, tt := range tests {
-		b.Run(tt.name, func(b *testing.B) {
-			// Perform warm-up requests
-			warmUp(b, tt)
-
-			// Wait a bit after warm-up before starting the actual benchmark
-			time.Sleep(500 * time.Millisecond)
-
-			m := make([]BenchmarkMetrics, b.N)
-
-			for i := 0; i < b.N; i++ {
-				b.ResetTimer()
-				m[i] = runCompletion(context.Background(), tt, b)
-			}
-			metrics = append(metrics, m...)
-		})
-	}
-}
-
-func formatPrompt(text string) string {
-	return fmt.Sprintf("<|start_header_id|>system<|end_header_id|>\n\nCutting Knowledge Date: December 2023\n\n<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n%s<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", text)
-}
-
-func runCompletion(ctx context.Context, tt TestCase, b *testing.B) BenchmarkMetrics {
-	start := time.Now()
-	var ttft time.Duration
-	var tokens int
-	lastToken := start
-
-	// Create request body
-	reqBody := CompletionRequest{
-		Prompt:      tt.prompt,
-		NumPredict:  tt.maxTokens,
-		Temperature: 0.1,
-	}
-	jsonData, err := json.Marshal(reqBody)
-	if err != nil {
-		b.Fatalf("Failed to marshal request: %v", err)
-	}
-
-	// Create HTTP request
-	req, err := http.NewRequestWithContext(ctx, "POST", runnerURL+"/completion", bytes.NewBuffer(jsonData))
-	if err != nil {
-		b.Fatalf("Failed to create request: %v", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-
-	// Execute request
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		b.Fatalf("Request failed: %v", err)
-	}
-	defer resp.Body.Close()
-
-	// Process streaming response
-	decoder := json.NewDecoder(resp.Body)
-	for {
-		var chunk CompletionResponse
-		if err := decoder.Decode(&chunk); err != nil {
-			if err == io.EOF {
-				break
-			}
-			b.Fatalf("Failed to decode response: %v", err)
-		}
-
-		if ttft == 0 && chunk.Content != "" {
-			ttft = time.Since(start)
-		}
-
-		if chunk.Content != "" {
-			tokens++
-			lastToken = time.Now()
-		}
-
-		if chunk.Stop {
-			break
-		}
-	}
-
-	totalTime := lastToken.Sub(start)
-	return BenchmarkMetrics{
-		testName:        tt.name,
-		ttft:            ttft,
-		totalTime:       totalTime,
-		totalTokens:     tokens,
-		tokensPerSecond: float64(tokens) / totalTime.Seconds(),
-	}
-}

benchmark/server_benchmark_test.go

@@ -1,293 +0,0 @@
-// Package benchmark provides tools for performance testing of Ollama inference server and supported models.
-package benchmark
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-	"os"
-	"testing"
-	"text/tabwriter"
-	"time"
-
-	"github.com/ollama/ollama/api"
-)
-
-// ServerURL is the default Ollama server URL for benchmarking
-const serverURL = "http://127.0.0.1:11434"
-
-// metrics collects all benchmark results for final reporting
-var metrics []BenchmarkMetrics
-
-// models contains the list of model names to benchmark
-var models = []string{
-	"llama3.2:1b",
-	// "qwen2.5:7b",
-	// "llama3.3:70b",
-}
-
-// TestCase defines a benchmark test scenario with prompt characteristics
-type TestCase struct {
-	name      string // Human-readable test name
-	prompt    string // Input prompt text
-	maxTokens int    // Maximum tokens to generate
-}
-
-// BenchmarkMetrics contains performance measurements for a single test run
-type BenchmarkMetrics struct {
-	model           string        // Model being tested
-	scenario        string        // cold_start or warm_start
-	testName        string        // Name of the test case
-	ttft            time.Duration // Time To First Token (TTFT)
-	totalTime       time.Duration // Total time for complete response
-	totalTokens     int           // Total generated tokens
-	tokensPerSecond float64       // Calculated throughput
-}
-
-// ScenarioType defines the initialization state for benchmarking
-type ScenarioType int
-
-const (
-	ColdStart ScenarioType = iota // Model is loaded from cold state
-	WarmStart                     // Model is already loaded in memory
-)
-
-// String implements fmt.Stringer for ScenarioType
-func (s ScenarioType) String() string {
-	return [...]string{"cold_start", "warm_start"}[s]
-}
-
-// BenchmarkServerInference is the main entry point for benchmarking Ollama inference performance.
-// It tests all configured models with different prompt lengths and start scenarios.
-func BenchmarkServerInference(b *testing.B) {
-	b.Logf("Starting benchmark suite with %d models", len(models))
-
-	// Verify server availability
-	if _, err := http.Get(serverURL + "/api/version"); err != nil {
-		b.Fatalf("Server unavailable: %v", err)
-	}
-	b.Log("Server available")
-
-	tests := []TestCase{
-		{"short_prompt", "Write a long story", 100},
-		{"medium_prompt", "Write a detailed economic analysis", 500},
-		{"long_prompt", "Write a comprehensive AI research paper", 1000},
-	}
-
-	// Register cleanup handler for results reporting
-	b.Cleanup(func() { reportMetrics(metrics) })
-
-	// Main benchmark loop
-	for _, model := range models {
-		client := api.NewClient(mustParse(serverURL), http.DefaultClient)
-		// Verify model availability
-		if _, err := client.Show(context.Background(), &api.ShowRequest{Model: model}); err != nil {
-			b.Fatalf("Model unavailable: %v", err)
-		}
-
-		for _, tt := range tests {
-			testName := fmt.Sprintf("%s/%s/%s", model, ColdStart, tt.name)
-			b.Run(testName, func(b *testing.B) {
-				m := runBenchmark(b, tt, model, ColdStart, client)
-				metrics = append(metrics, m...)
-			})
-		}
-
-		for _, tt := range tests {
-			testName := fmt.Sprintf("%s/%s/%s", model, WarmStart, tt.name)
-			b.Run(testName, func(b *testing.B) {
-				m := runBenchmark(b, tt, model, WarmStart, client)
-				metrics = append(metrics, m...)
-			})
-		}
-	}
-}
-
-// runBenchmark executes multiple iterations of a specific test case and scenario.
-// Returns collected metrics for all iterations.
-func runBenchmark(b *testing.B, tt TestCase, model string, scenario ScenarioType, client *api.Client) []BenchmarkMetrics {
-	results := make([]BenchmarkMetrics, b.N)
-
-	// Run benchmark iterations
-	for i := 0; i < b.N; i++ {
-		switch scenario {
-		case WarmStart:
-			// Pre-warm the model by generating some tokens
-			for i := 0; i < 2; i++ {
-				client.Generate(
-					context.Background(),
-					&api.GenerateRequest{
-						Model:   model,
-						Prompt:  tt.prompt,
-						Options: map[string]interface{}{"num_predict": tt.maxTokens, "temperature": 0.1},
-					},
-					func(api.GenerateResponse) error { return nil },
-				)
-			}
-		case ColdStart:
-			unloadModel(client, model, b)
-		}
-		b.ResetTimer()
-
-		results[i] = runSingleIteration(context.Background(), client, tt, model, b)
-		results[i].scenario = scenario.String()
-	}
-	return results
-}
-
-// unloadModel forces model unloading using KeepAlive: -1 parameter.
-// Includes short delay to ensure unloading completes before next test.
-func unloadModel(client *api.Client, model string, b *testing.B) {
-	req := &api.GenerateRequest{
-		Model:     model,
-		KeepAlive: &api.Duration{Duration: 0},
-	}
-	if err := client.Generate(context.Background(), req, func(api.GenerateResponse) error { return nil }); err != nil {
-		b.Logf("Unload error: %v", err)
-	}
-	time.Sleep(100 * time.Millisecond)
-}
-
-// runSingleIteration measures performance metrics for a single inference request.
-// Captures TTFT, total generation time, and calculates tokens/second.
-func runSingleIteration(ctx context.Context, client *api.Client, tt TestCase, model string, b *testing.B) BenchmarkMetrics {
-	start := time.Now()
-	var ttft time.Duration
-	var tokens int
-	lastToken := start
-
-	req := &api.GenerateRequest{
-		Model:   model,
-		Prompt:  tt.prompt,
-		Options: map[string]interface{}{"num_predict": tt.maxTokens, "temperature": 0.1},
-	}
-
-	if b != nil {
-		b.Logf("Prompt length: %d chars", len(tt.prompt))
-	}
-
-	// Execute generation request with metrics collection
-	client.Generate(ctx, req, func(resp api.GenerateResponse) error {
-		if ttft == 0 {
-			ttft = time.Since(start)
-		}
-		if resp.Response != "" {
-			tokens++
-			lastToken = time.Now()
-		}
-		return nil
-	})
-
-	totalTime := lastToken.Sub(start)
-	return BenchmarkMetrics{
-		model:           model,
-		testName:        tt.name,
-		ttft:            ttft,
-		totalTime:       totalTime,
-		totalTokens:     tokens,
-		tokensPerSecond: float64(tokens) / totalTime.Seconds(),
-	}
-}
-
-// reportMetrics processes collected metrics and prints formatted results.
-// Generates both human-readable tables and CSV output with averaged statistics.
-func reportMetrics(results []BenchmarkMetrics) {
-	if len(results) == 0 {
-		return
-	}
-
-	// Aggregate results by test case
-	type statsKey struct {
-		model    string
-		scenario string
-		testName string
-	}
-	stats := make(map[statsKey]*struct {
-		ttftSum      time.Duration
-		totalTimeSum time.Duration
-		tokensSum    int
-		iterations   int
-	})
-
-	for _, m := range results {
-		key := statsKey{m.model, m.scenario, m.testName}
-		if _, exists := stats[key]; !exists {
-			stats[key] = &struct {
-				ttftSum      time.Duration
-				totalTimeSum time.Duration
-				tokensSum    int
-				iterations   int
-			}{}
-		}
-
-		stats[key].ttftSum += m.ttft
-		stats[key].totalTimeSum += m.totalTime
-		stats[key].tokensSum += m.totalTokens
-		stats[key].iterations++
-	}
-
-	// Calculate averages
-	var averaged []BenchmarkMetrics
-	for key, data := range stats {
-		count := data.iterations
-		averaged = append(averaged, BenchmarkMetrics{
-			model:           key.model,
-			scenario:        key.scenario,
-			testName:        key.testName,
-			ttft:            data.ttftSum / time.Duration(count),
-			totalTime:       data.totalTimeSum / time.Duration(count),
-			totalTokens:     data.tokensSum / count,
-			tokensPerSecond: float64(data.tokensSum) / data.totalTimeSum.Seconds(),
-		})
-	}
-
-	// Print formatted results
-	printTableResults(averaged)
-	printCSVResults(averaged)
-}
-
-// printTableResults displays averaged metrics in a formatted table
-func printTableResults(averaged []BenchmarkMetrics) {
-	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
-	fmt.Fprintln(w, "\nAVERAGED BENCHMARK RESULTS")
-	fmt.Fprintln(w, "Model\tScenario\tTest Name\tTTFT (ms)\tTotal Time (ms)\tTokens\tTokens/sec")
-	for _, m := range averaged {
-		fmt.Fprintf(w, "%s\t%s\t%s\t%.2f\t%.2f\t%d\t%.2f\n",
-			m.model,
-			m.scenario,
-			m.testName,
-			float64(m.ttft.Milliseconds()),
-			float64(m.totalTime.Milliseconds()),
-			m.totalTokens,
-			m.tokensPerSecond,
-		)
-	}
-	w.Flush()
-}
-
-// printCSVResults outputs averaged metrics in CSV format
-func printCSVResults(averaged []BenchmarkMetrics) {
-	fmt.Println("\nCSV OUTPUT")
-	fmt.Println("model,scenario,test_name,ttft_ms,total_ms,tokens,tokens_per_sec")
-	for _, m := range averaged {
-		fmt.Printf("%s,%s,%s,%.2f,%.2f,%d,%.2f\n",
-			m.model,
-			m.scenario,
-			m.testName,
-			float64(m.ttft.Milliseconds()),
-			float64(m.totalTime.Milliseconds()),
-			m.totalTokens,
-			m.tokensPerSecond,
-		)
-	}
-}
-
-// mustParse is a helper function to parse URLs with panic on error
-func mustParse(rawURL string) *url.URL {
-	u, err := url.Parse(rawURL)
-	if err != nil {
-		panic(err)
-	}
-	return u
-}

cache/cache.go

@@ -1,420 +0,0 @@
-package cache
-
-import (
-	"errors"
-	"fmt"
-	"log/slog"
-	"math"
-	"slices"
-
-	"github.com/ollama/ollama/ml"
-)
-
-var ErrNotSupported = errors.New("model does not support operation")
-
-type Cache interface {
-	// ** used by model implementations **
-
-	// Returns an instance of the cache for layer 'i'
-	Sub(i int) Cache
-
-	// Returns the history of key and value tensors plus a mask
-	//
-	// The tensors are of shape embed dim, kv heads, batch size
-	// The mask is of shape history size, batch size
-	Get(ctx ml.Context) (ml.Tensor, ml.Tensor, ml.Tensor)
-
-	// Stores a batch of key and value in the cache
-	//
-	// The tensors must be of shape embed dim, kv heads, batch size
-	Put(ctx ml.Context, key, value ml.Tensor)
-
-	// ** cache management **
-
-	// Closes the cache and frees resources associated with it
-	Close()
-
-	// Called before the start of the model's forward pass. For each
-	// token in the coming batch, there must be a corresponding entry
-	// in positions and seqs.
-	StartForward(ctx ml.Context, positions []int32, seqs []int) error
-
-	// Copies tokens in the range [0, len) from srcSeq to dstSeq
-	CopyPrefix(srcSeq, dstSeq int, len int32)
-
-	// Removes tokens in the range [beginIndex, endIndex) from seq. Set
-	// endIndex to math.MaxInt32 to remove everything starting at beginIndex
-	Remove(seq int, beginIndex, endIndex int32) error
-}
-
-type Causal struct {
-	DType    ml.DType
-	Capacity int32
-
-	// current forward pass
-	curLayer     int
-	curLoc       int
-	curBatchSize int
-	curMask      ml.Tensor
-	curCellRange cellRange
-
-	// metadata
-	cells      []cacheCell
-	cellRanges map[int]cellRange
-
-	// cache data storage
-	backend      ml.Backend
-	cacheCtx     ml.Context
-	keys, values []ml.Tensor
-}
-
-type seqCell struct {
-	seq int
-	pos int32
-}
-
-type cacheCell struct {
-	sequences []seqCell
-}
-
-type cellRange struct {
-	min int
-	max int
-}
-
-func (cell cacheCell) findSeq(seq int) *seqCell {
-	for i := range cell.sequences {
-		if cell.sequences[i].seq == seq {
-			return &cell.sequences[i]
-		}
-	}
-	return nil
-}
-
-func NewCausalCache(backend ml.Backend, dtype ml.DType, capacity int32) Cache {
-	return &Causal{
-		Capacity:   capacity,
-		DType:      dtype,
-		cells:      make([]cacheCell, capacity),
-		cellRanges: make(map[int]cellRange),
-		backend:    backend,
-		cacheCtx:   backend.NewContext(),
-	}
-}
-
-func (c *Causal) Close() {
-	c.cacheCtx.Close()
-}
-
-var ErrKvCacheFull = errors.New("could not find a kv cache slot")
-
-func (c *Causal) StartForward(ctx ml.Context, positions []int32, seqs []int) error {
-	if len(positions) != len(seqs) {
-		return fmt.Errorf("length of positions (%v) must match length of seqs (%v)", len(positions), len(seqs))
-	}
-
-	c.curBatchSize = len(positions)
-
-	if c.curBatchSize < 1 {
-		return errors.New("batch size cannot be less than 1")
-	}
-
-	var err error
-	c.curLoc, err = c.findStartLoc()
-	if errors.Is(err, ErrKvCacheFull) {
-		c.defrag()
-		c.curLoc, err = c.findStartLoc()
-	}
-	if err != nil {
-		return err
-	}
-
-	c.curCellRange = newRange()
-	for i, pos := range positions {
-		seq := seqs[i]
-
-		c.cells[c.curLoc+i] = cacheCell{sequences: []seqCell{{seq: seq, pos: pos}}}
-
-		ranges, ok := c.cellRanges[seq]
-		if !ok {
-			ranges = newRange()
-		}
-
-		if c.curLoc+i > ranges.max {
-			ranges.max = c.curLoc + i
-		}
-		if ranges.max > c.curCellRange.max {
-			c.curCellRange.max = ranges.max
-		}
-
-		if c.curLoc+i < ranges.min {
-			ranges.min = c.curLoc + i
-		}
-		if ranges.min < c.curCellRange.min {
-			c.curCellRange.min = ranges.min
-		}
-		c.cellRanges[seq] = ranges
-	}
-
-	c.curMask, err = c.buildMask(ctx, positions, seqs)
-
-	return err
-}
-
-func newRange() cellRange {
-	return cellRange{
-		min: math.MaxInt,
-		max: 0,
-	}
-}
-
-func (c *Causal) findStartLoc() (int, error) {
-	var start, count int
-	for i := range c.cells {
-		if len(c.cells[i].sequences) == 0 {
-			count++
-			if count >= c.curBatchSize {
-				return start, nil
-			}
-		} else {
-			start = i + 1
-			count = 0
-		}
-	}
-
-	return 0, fmt.Errorf("%w (length: %v)", ErrKvCacheFull, c.Capacity)
-}
-
-func (c *Causal) buildMask(ctx ml.Context, positions []int32, seqs []int) (ml.Tensor, error) {
-	// TODO(jessegross): This makes a number of simplifications such as no padding,
-	// which could be an issue for CUDA graphs and/or flash attention
-	len := c.curCellRange.max - c.curCellRange.min + 1
-	mask := make([]float32, c.curBatchSize*len)
-
-	for i := range c.curBatchSize {
-		for j := c.curCellRange.min; j <= c.curCellRange.max; j++ {
-			cellSeq := c.cells[j].findSeq(seqs[i])
-			if cellSeq == nil || cellSeq.pos > positions[i] {
-				mask[i*len+(j-c.curCellRange.min)] = float32(math.Inf(-1))
-			}
-		}
-	}
-
-	return ctx.FromFloatSlice(mask, len, c.curBatchSize)
-}
-
-func moveCell(ctx ml.Context, objs []ml.Tensor, src, dst, len int) {
-	for _, obj := range objs {
-		srcView := obj.View(ctx, int(obj.Stride(2))*src, int(obj.Dim(0)*obj.Dim(1))*len)
-		dstView := obj.View(ctx, int(obj.Stride(2))*dst, int(obj.Dim(0)*obj.Dim(1))*len)
-
-		ctx.Forward(srcView.Copy(ctx, dstView))
-	}
-}
-
-func (c *Causal) defrag() {
-	slog.Debug("defragmenting kv cache")
-
-	// Defrag strategy:
-	// - Search for empty holes at the beginning of the cache,
-	//   filling them with active data starting at the end
-	// - If there are contiguous elements that need to be moved,
-	//   combine them into a single operation by holding new moves
-	//   until we see the next one is non-contiguous
-	// - Fill up the context with the maximum number of operations it
-	//   can hold then compute that and continue with a new context
-	//
-	// We could try to optimize placement by grouping blocks from
-	// the same sequences together but most likely the next forward
-	// pass will disrupt this anyways, so the real world benefit
-	// seems limited as this time.
-
-	ctx := c.backend.NewContext()
-
-	// For every move, 6 tensors are required per layer (2 views and a
-	// copy for each of k and v). For efficiency, we try to group
-	// multiple contiguous blocks into a single move. However, if we
-	// exceed the maximum number of tensors then we need to compute
-	// what we have and start a new batch.
-	maxMoves := ctx.MaxTensors() / (6 * len(c.keys))
-	moves := 0
-
-	var pendingSrc, pendingDst, pendingLen int
-
-	for dst := range c.cells {
-		if len(c.cells[dst].sequences) == 0 {
-			for src := len(c.cells) - 1; src > dst; src-- {
-				if len(c.cells[src].sequences) != 0 {
-					c.cells[dst] = c.cells[src]
-					c.cells[src] = cacheCell{}
-
-					if pendingLen > 0 {
-						if src == pendingSrc-pendingLen && dst == pendingDst+pendingLen {
-							pendingSrc = src
-							pendingLen++
-							break
-						} else {
-							moveCell(ctx, c.keys, pendingSrc, pendingDst, pendingLen)
-							moveCell(ctx, c.values, pendingSrc, pendingDst, pendingLen)
-							moves++
-						}
-					}
-
-					pendingSrc = src
-					pendingDst = dst
-					pendingLen = 1
-
-					break
-				}
-			}
-		}
-
-		if moves >= maxMoves {
-			ctx.Compute(nil)
-			ctx.Close()
-			ctx = c.backend.NewContext()
-
-			moves = 0
-		}
-	}
-
-	if pendingLen > 0 {
-		moveCell(ctx, c.keys, pendingSrc, pendingDst, pendingLen)
-		moveCell(ctx, c.values, pendingSrc, pendingDst, pendingLen)
-		moves++
-	}
-
-	if moves > 0 {
-		ctx.Compute(nil)
-	}
-	ctx.Close()
-
-	for seq := range c.cellRanges {
-		seqRange := newRange()
-
-		for i, cell := range c.cells {
-			if cell.findSeq(seq) != nil {
-				if i < seqRange.min {
-					seqRange.min = i
-				}
-				if i > seqRange.max {
-					seqRange.max = i
-				}
-			}
-		}
-
-		c.cellRanges[seq] = seqRange
-	}
-}
-
-func (c *Causal) Sub(i int) Cache {
-	if i >= len(c.keys) {
-		c.keys = append(c.keys, make([]ml.Tensor, i-len(c.keys)+1)...)
-		c.values = append(c.values, make([]ml.Tensor, i-len(c.values)+1)...)
-	}
-
-	c.curLayer = i
-
-	return c
-}
-
-func (c *Causal) Get(ctx ml.Context) (ml.Tensor, ml.Tensor, ml.Tensor) {
-	key := c.keys[c.curLayer]
-	value := c.values[c.curLayer]
-
-	key = key.View(ctx, int(key.Stride(2))*c.curCellRange.min,
-		int(key.Dim(0)), int(key.Stride(1)),
-		int(key.Dim(1)), int(key.Stride(2)),
-		int(c.curMask.Dim(0)),
-	)
-
-	value = value.View(ctx, int(key.Stride(2))*c.curCellRange.min,
-		int(value.Dim(0)), int(value.Stride(1)),
-		int(value.Dim(1)), int(value.Stride(2)),
-		int(c.curMask.Dim(0)),
-	)
-
-	return key, value, c.curMask
-}
-
-func (c *Causal) Put(ctx ml.Context, key, value ml.Tensor) {
-	if c.curBatchSize != int(key.Dim(2)) {
-		panic(fmt.Errorf("inconsistent batch sizes (layer: %v, batch size: %v layer batch size: %v)", c.curLayer, c.curBatchSize, int(key.Dim(2))))
-	}
-
-	if c.keys[c.curLayer] == nil || c.values[c.curLayer] == nil {
-		c.keys[c.curLayer] = c.cacheCtx.Zeros(c.DType, key.Dim(0), key.Dim(1), int64(c.Capacity))
-		c.values[c.curLayer] = c.cacheCtx.Zeros(c.DType, value.Dim(0), value.Dim(1), int64(c.Capacity))
-	}
-
-	ctx.Forward(key.Copy(ctx, c.keys[c.curLayer].View(ctx, int(key.Stride(2))*c.curLoc, int(key.Dim(0)*key.Dim(1)*key.Dim(2)))))
-	ctx.Forward(value.Copy(ctx, c.values[c.curLayer].View(ctx, int(value.Stride(2))*c.curLoc, int(value.Dim(0)*value.Dim(1)*value.Dim(2)))))
-}
-
-func (c *Causal) CopyPrefix(srcSeq, dstSeq int, len int32) {
-	seqRange := newRange()
-
-	for i := range c.cells {
-		srcCellSeq := c.cells[i].findSeq(srcSeq)
-		dstCellSeq := c.cells[i].findSeq(dstSeq)
-
-		if dstCellSeq != nil {
-			c.cells[i].sequences = slices.DeleteFunc(c.cells[i].sequences, func(s seqCell) bool { return s.seq == dstSeq })
-		}
-
-		if srcCellSeq != nil && srcCellSeq.pos < len {
-			c.cells[i].sequences = append(c.cells[i].sequences, seqCell{seq: dstSeq, pos: srcCellSeq.pos})
-			if i < seqRange.min {
-				seqRange.min = i
-			}
-			if i > seqRange.max {
-				seqRange.max = i
-			}
-		}
-	}
-
-	c.cellRanges[dstSeq] = seqRange
-}
-
-func (c *Causal) shift(seq int, beginIndex, offset int32) error {
-	panic("Shift not yet implemented")
-}
-
-func (c *Causal) Remove(seq int, beginIndex, endIndex int32) error {
-	var offset int32
-	if endIndex != math.MaxInt32 {
-		offset = beginIndex - endIndex
-	}
-
-	seqRange := newRange()
-
-	for i := range c.cells {
-		cellSeq := c.cells[i].findSeq(seq)
-		if cellSeq != nil {
-			if cellSeq.pos >= beginIndex && cellSeq.pos < endIndex {
-				c.cells[i].sequences = slices.DeleteFunc(c.cells[i].sequences, func(s seqCell) bool { return s.seq == seq })
-			} else {
-				if cellSeq.pos >= endIndex {
-					cellSeq.pos += offset
-				}
-				if i < seqRange.min {
-					seqRange.min = i
-				}
-				if i > seqRange.max {
-					seqRange.max = i
-				}
-			}
-		}
-	}
-
-	if endIndex != math.MaxInt32 {
-		err := c.shift(seq, endIndex, offset)
-		if err != nil {
-			return err
-		}
-	}
-
-	c.cellRanges[seq] = seqRange
-
-	return nil
-}

cache/tensor.go

@@ -1,47 +0,0 @@
-package cache
-
-import (
-	"github.com/ollama/ollama/ml"
-)
-
-type TensorCache struct {
-	curLayer int
-
-	cacheCtx     ml.Context
-	keys, values []ml.Tensor
-}
-
-func NewTensorCache(backend ml.Backend) *TensorCache {
-	return &TensorCache{
-		cacheCtx: backend.NewContext(),
-	}
-}
-
-func (c *TensorCache) Close() {
-	c.cacheCtx.Close()
-}
-
-func (c *TensorCache) Sub(i int) *TensorCache {
-	if i >= len(c.keys) {
-		c.keys = append(c.keys, make([]ml.Tensor, i-len(c.keys)+1)...)
-		c.values = append(c.values, make([]ml.Tensor, i-len(c.values)+1)...)
-	}
-
-	c.curLayer = i
-
-	return c
-}
-
-func (c *TensorCache) Get(ctx ml.Context) (ml.Tensor, ml.Tensor, ml.Tensor) {
-	return c.keys[c.curLayer], c.values[c.curLayer], nil
-}
-
-func (c *TensorCache) Put(ctx ml.Context, key, value ml.Tensor) {
-	if c.keys[c.curLayer] == nil || c.values[c.curLayer] == nil {
-		c.keys[c.curLayer] = c.cacheCtx.Zeros(key.DType(), key.Shape()...)
-		c.values[c.curLayer] = c.cacheCtx.Zeros(value.DType(), value.Shape()...)
-	}
-
-	ctx.Forward(key.Copy(ctx, c.keys[c.curLayer]))
-	ctx.Forward(value.Copy(ctx, c.values[c.curLayer]))
-}

cmd/cmd.go

@@ -35,9 +35,9 @@ import (
 	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
 	"github.com/ollama/ollama/llama"
+	"github.com/ollama/ollama/llama/runner"
 	"github.com/ollama/ollama/parser"
 	"github.com/ollama/ollama/progress"
-	"github.com/ollama/ollama/runner"
 	"github.com/ollama/ollama/server"
 	"github.com/ollama/ollama/types/model"
 	"github.com/ollama/ollama/version"
@@ -338,10 +338,7 @@ func RunHandler(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// TODO(jessegross): We should either find another way to know if this is
-	// a vision model or remove the logic. Also consider that other modalities will
-	// need different behavior anyways.
-	opts.MultiModal = true
+	opts.MultiModal = len(info.ProjectorInfo) != 0
 	opts.ParentModel = info.Details.ParentModel
 
 	if interactive {

cmd/runner/main.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/ollama/ollama/runner"
+	"github.com/ollama/ollama/llama/runner"
 )
 
 func main() {

convert/convert.go

@@ -9,7 +9,7 @@ import (
 	"log/slog"
 	"strings"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type ModelParameters struct {
@@ -27,8 +27,8 @@ type AdapterParameters struct {
 	} `json:"lora_parameters"`
 }
 
-func (ModelParameters) KV(t *Tokenizer) ggml.KV {
-	kv := ggml.KV{
+func (ModelParameters) KV(t *Tokenizer) llm.KV {
+	kv := llm.KV{
 		"general.file_type":            uint32(1),
 		"general.quantization_version": uint32(2),
 		"tokenizer.ggml.pre":           t.Pre,
@@ -54,7 +54,7 @@ func (ModelParameters) KV(t *Tokenizer) ggml.KV {
 	return kv
 }
 
-func (p AdapterParameters) KV() ggml.KV {
+func (p AdapterParameters) KV() llm.KV {
 	var alpha float32
 	if p.LoraParameters.Alpha == 0 {
 		alpha = float32(p.Alpha)
@@ -62,7 +62,7 @@ func (p AdapterParameters) KV() ggml.KV {
 		alpha = p.LoraParameters.Alpha
 	}
 
-	kv := ggml.KV{
+	kv := llm.KV{
 		"adapter.lora.alpha": alpha,
 		"adapter.type":       "lora",
 		"general.file_type":  uint32(1),
@@ -79,19 +79,19 @@ func (ModelParameters) specialTokenTypes() []string {
 	}
 }
 
-func (ModelParameters) writeFile(ws io.WriteSeeker, kv ggml.KV, ts []ggml.Tensor) error {
-	return ggml.WriteGGUF(ws, kv, ts)
+func (ModelParameters) writeFile(ws io.WriteSeeker, kv llm.KV, ts []llm.Tensor) error {
+	return llm.WriteGGUF(ws, kv, ts)
 }
 
-func (AdapterParameters) writeFile(ws io.WriteSeeker, kv ggml.KV, ts []ggml.Tensor) error {
-	return ggml.WriteGGUF(ws, kv, ts)
+func (AdapterParameters) writeFile(ws io.WriteSeeker, kv llm.KV, ts []llm.Tensor) error {
+	return llm.WriteGGUF(ws, kv, ts)
 }
 
 type ModelConverter interface {
 	// KV maps parameters to LLM key-values
-	KV(*Tokenizer) ggml.KV
+	KV(*Tokenizer) llm.KV
 	// Tensors maps input tensors to LLM tensors. Model specific modifications can be done here.
-	Tensors([]Tensor) []ggml.Tensor
+	Tensors([]Tensor) []llm.Tensor
 	// Replacements returns a list of string pairs to replace in tensor names.
 	// See [strings.Replacer](https://pkg.go.dev/strings#Replacer) for details
 	Replacements() []string
@@ -99,7 +99,7 @@ type ModelConverter interface {
 	// specialTokenTypes returns any special token types the model uses
 	specialTokenTypes() []string
 	// writeFile writes the model to the provided io.WriteSeeker
-	writeFile(io.WriteSeeker, ggml.KV, []ggml.Tensor) error
+	writeFile(io.WriteSeeker, llm.KV, []llm.Tensor) error
 }
 
 type moreParser interface {
@@ -108,17 +108,17 @@ type moreParser interface {
 
 type AdapterConverter interface {
 	// KV maps parameters to LLM key-values
-	KV(ggml.KV) ggml.KV
+	KV(llm.KV) llm.KV
 	// Tensors maps input tensors to LLM tensors. Adapter specific modifications can be done here.
-	Tensors([]Tensor) []ggml.Tensor
+	Tensors([]Tensor) []llm.Tensor
 	// Replacements returns a list of string pairs to replace in tensor names.
 	// See [strings.Replacer](https://pkg.go.dev/strings#Replacer) for details
 	Replacements() []string
 
-	writeFile(io.WriteSeeker, ggml.KV, []ggml.Tensor) error
+	writeFile(io.WriteSeeker, llm.KV, []llm.Tensor) error
 }
 
-func ConvertAdapter(fsys fs.FS, ws io.WriteSeeker, baseKV ggml.KV) error {
+func ConvertAdapter(fsys fs.FS, ws io.WriteSeeker, baseKV llm.KV) error {
 	bts, err := fs.ReadFile(fsys, "adapter_config.json")
 	if err != nil {
 		return err

convert/convert_bert.go

@@ -8,7 +8,7 @@ import (
 	"slices"
 	"strings"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type bertModel struct {
@@ -85,7 +85,7 @@ func (p *bertModel) parseMore(fsys fs.FS) error {
 	return nil
 }
 
-func (p *bertModel) KV(t *Tokenizer) ggml.KV {
+func (p *bertModel) KV(t *Tokenizer) llm.KV {
 	kv := p.ModelParameters.KV(t)
 	kv["general.architecture"] = "bert"
 	kv["bert.attention.causal"] = false
@@ -132,8 +132,8 @@ func (p *bertModel) KV(t *Tokenizer) ggml.KV {
 	return kv
 }
 
-func (p *bertModel) Tensors(ts []Tensor) []ggml.Tensor {
-	var out []ggml.Tensor
+func (p *bertModel) Tensors(ts []Tensor) []llm.Tensor {
+	var out []llm.Tensor
 	for _, t := range ts {
 		if slices.Contains([]string{
 			"embeddings.position_ids",
@@ -143,7 +143,7 @@ func (p *bertModel) Tensors(ts []Tensor) []ggml.Tensor {
 			continue
 		}
 
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     t.Name(),
 			Kind:     t.Kind(),
 			Shape:    t.Shape(),

convert/convert_commandr.go

@@ -3,7 +3,7 @@ package convert
 import (
 	"cmp"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type commandrModel struct {
@@ -24,7 +24,7 @@ type commandrModel struct {
 
 var _ ModelConverter = (*commandrModel)(nil)
 
-func (p *commandrModel) KV(t *Tokenizer) ggml.KV {
+func (p *commandrModel) KV(t *Tokenizer) llm.KV {
 	kv := p.ModelParameters.KV(t)
 	kv["general.architecture"] = "command-r"
 	kv["general.name"] = "command-r"
@@ -43,10 +43,10 @@ func (p *commandrModel) KV(t *Tokenizer) ggml.KV {
 	return kv
 }
 
-func (p *commandrModel) Tensors(ts []Tensor) []ggml.Tensor {
-	var out []ggml.Tensor
+func (p *commandrModel) Tensors(ts []Tensor) []llm.Tensor {
+	var out []llm.Tensor
 	for _, t := range ts {
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     t.Name(),
 			Kind:     t.Kind(),
 			Shape:    t.Shape(),

convert/convert_gemma.go

@@ -6,7 +6,7 @@ import (
 	"github.com/pdevine/tensor"
 	"github.com/pdevine/tensor/native"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type gemmaModel struct {
@@ -23,7 +23,7 @@ type gemmaModel struct {
 
 var _ ModelConverter = (*gemmaModel)(nil)
 
-func (p *gemmaModel) KV(t *Tokenizer) ggml.KV {
+func (p *gemmaModel) KV(t *Tokenizer) llm.KV {
 	kv := p.ModelParameters.KV(t)
 	kv["general.architecture"] = "gemma"
 	kv["gemma.context_length"] = p.MaxPositionEmbeddings
@@ -42,14 +42,14 @@ func (p *gemmaModel) KV(t *Tokenizer) ggml.KV {
 	return kv
 }
 
-func (p *gemmaModel) Tensors(ts []Tensor) []ggml.Tensor {
-	var out []ggml.Tensor
+func (p *gemmaModel) Tensors(ts []Tensor) []llm.Tensor {
+	var out []llm.Tensor
 	for _, t := range ts {
 		if strings.HasSuffix(t.Name(), "_norm.weight") {
 			t.SetRepacker(p.addOne)
 		}
 
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     t.Name(),
 			Kind:     t.Kind(),
 			Shape:    t.Shape(),

convert/convert_gemma2.go

@@ -1,6 +1,8 @@
 package convert
 
-import "github.com/ollama/ollama/fs/ggml"
+import (
+	"github.com/ollama/ollama/llm"
+)
 
 type gemma2Model struct {
 	gemmaModel
@@ -9,7 +11,7 @@ type gemma2Model struct {
 	FinalLogitSoftcap     float32 `json:"final_logit_softcapping"`
 }
 
-func (p *gemma2Model) KV(t *Tokenizer) ggml.KV {
+func (p *gemma2Model) KV(t *Tokenizer) llm.KV {
 	kv := p.ModelParameters.KV(t)
 	kv["general.architecture"] = "gemma2"
 	kv["gemma2.context_length"] = p.MaxPositionEmbeddings

convert/convert_gemma2_adapter.go

@@ -6,7 +6,7 @@ import (
 	"github.com/pdevine/tensor"
 	"github.com/pdevine/tensor/native"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type gemma2Adapter struct {
@@ -15,14 +15,14 @@ type gemma2Adapter struct {
 
 var _ AdapterConverter = (*gemma2Adapter)(nil)
 
-func (p *gemma2Adapter) KV(baseKV ggml.KV) ggml.KV {
+func (p *gemma2Adapter) KV(baseKV llm.KV) llm.KV {
 	kv := p.AdapterParameters.KV()
 	kv["general.architecture"] = "gemma2"
 	return kv
 }
 
-func (p *gemma2Adapter) Tensors(ts []Tensor) []ggml.Tensor {
-	var out []ggml.Tensor
+func (p *gemma2Adapter) Tensors(ts []Tensor) []llm.Tensor {
+	var out []llm.Tensor
 	for _, t := range ts {
 		shape := t.Shape()
 		if (strings.HasSuffix(t.Name(), "weight.lora_a") && shape[0] > shape[1]) ||
@@ -31,7 +31,7 @@ func (p *gemma2Adapter) Tensors(ts []Tensor) []ggml.Tensor {
 			t.SetRepacker(p.repack)
 		}
 
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     t.Name(),
 			Kind:     t.Kind(),
 			Shape:    t.Shape(),

convert/convert_llama.go

@@ -9,7 +9,7 @@ import (
 	"github.com/pdevine/tensor"
 	"github.com/pdevine/tensor/native"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type llamaModel struct {
@@ -46,7 +46,7 @@ type llamaModel struct {
 
 var _ ModelConverter = (*llamaModel)(nil)
 
-func (p *llamaModel) KV(t *Tokenizer) ggml.KV {
+func (p *llamaModel) KV(t *Tokenizer) llm.KV {
 	kv := p.ModelParameters.KV(t)
 	kv["general.architecture"] = "llama"
 	kv["llama.vocab_size"] = p.VocabSize
@@ -120,11 +120,11 @@ func (p *llamaModel) KV(t *Tokenizer) ggml.KV {
 	return kv
 }
 
-func (p *llamaModel) Tensors(ts []Tensor) []ggml.Tensor {
-	var out []ggml.Tensor
+func (p *llamaModel) Tensors(ts []Tensor) []llm.Tensor {
+	var out []llm.Tensor
 
 	if p.RopeScaling.factors != nil {
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     "rope_freqs.weight",
 			Kind:     0,
 			Shape:    []uint64{uint64(len(p.RopeScaling.factors))},
@@ -138,7 +138,7 @@ func (p *llamaModel) Tensors(ts []Tensor) []ggml.Tensor {
 			t.SetRepacker(p.repack)
 		}
 
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     t.Name(),
 			Kind:     t.Kind(),
 			Shape:    t.Shape(),

convert/convert_llama_adapter.go

@@ -7,7 +7,7 @@ import (
 	"github.com/pdevine/tensor"
 	"github.com/pdevine/tensor/native"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type llamaAdapter struct {
@@ -18,7 +18,7 @@ type llamaAdapter struct {
 
 var _ AdapterConverter = (*llamaAdapter)(nil)
 
-func (p *llamaAdapter) KV(baseKV ggml.KV) ggml.KV {
+func (p *llamaAdapter) KV(baseKV llm.KV) llm.KV {
 	kv := p.AdapterParameters.KV()
 	kv["general.architecture"] = "llama"
 	kv["llama.attention.head_count"] = baseKV["llama.attention.head_count"]
@@ -29,8 +29,8 @@ func (p *llamaAdapter) KV(baseKV ggml.KV) ggml.KV {
 	return kv
 }
 
-func (p *llamaAdapter) Tensors(ts []Tensor) []ggml.Tensor {
-	var out []ggml.Tensor
+func (p *llamaAdapter) Tensors(ts []Tensor) []llm.Tensor {
+	var out []llm.Tensor
 	for _, t := range ts {
 		shape := t.Shape()
 		if (strings.HasSuffix(t.Name(), "weight.lora_a") && shape[0] > shape[1]) ||
@@ -41,7 +41,7 @@ func (p *llamaAdapter) Tensors(ts []Tensor) []ggml.Tensor {
 			t.SetRepacker(p.repack)
 		}
 
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     t.Name(),
 			Kind:     t.Kind(),
 			Shape:    shape,

convert/convert_mixtral.go

@@ -6,7 +6,7 @@ import (
 	"slices"
 	"strings"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type mixtralModel struct {
@@ -15,7 +15,7 @@ type mixtralModel struct {
 	NumExpertsPerToken uint32 `json:"num_experts_per_tok"`
 }
 
-func (p *mixtralModel) KV(t *Tokenizer) ggml.KV {
+func (p *mixtralModel) KV(t *Tokenizer) llm.KV {
 	kv := p.llamaModel.KV(t)
 
 	if p.NumLocalExperts > 0 {
@@ -29,7 +29,7 @@ func (p *mixtralModel) KV(t *Tokenizer) ggml.KV {
 	return kv
 }
 
-func (p *mixtralModel) Tensors(ts []Tensor) []ggml.Tensor {
+func (p *mixtralModel) Tensors(ts []Tensor) []llm.Tensor {
 	oldnew := []string{
 		"model.layers", "blk",
 		"w1", "ffn_gate_exps",
@@ -56,10 +56,10 @@ func (p *mixtralModel) Tensors(ts []Tensor) []ggml.Tensor {
 		return true
 	})
 
-	var out []ggml.Tensor
+	var out []llm.Tensor
 	for n, e := range experts {
 		// TODO(mxyng): sanity check experts
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     n,
 			Kind:     e[0].Kind(),
 			Shape:    append([]uint64{uint64(len(e))}, e[0].Shape()...),

convert/convert_phi3.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type phi3Model struct {
@@ -37,7 +37,7 @@ type phi3Model struct {
 
 var _ ModelConverter = (*phi3Model)(nil)
 
-func (p *phi3Model) KV(t *Tokenizer) ggml.KV {
+func (p *phi3Model) KV(t *Tokenizer) llm.KV {
 	kv := p.ModelParameters.KV(t)
 	kv["general.architecture"] = "phi3"
 	kv["phi3.context_length"] = p.MaxPositionEmbeddings
@@ -68,19 +68,19 @@ func (p *phi3Model) KV(t *Tokenizer) ggml.KV {
 	return kv
 }
 
-func (p *phi3Model) Tensors(ts []Tensor) []ggml.Tensor {
+func (p *phi3Model) Tensors(ts []Tensor) []llm.Tensor {
 	var addRopeFactors sync.Once
 
-	out := make([]ggml.Tensor, 0, len(ts)+2)
+	out := make([]llm.Tensor, 0, len(ts)+2)
 	for _, t := range ts {
 		if strings.HasPrefix(t.Name(), "blk.0.") {
 			addRopeFactors.Do(func() {
-				out = append(out, ggml.Tensor{
+				out = append(out, llm.Tensor{
 					Name:     "rope_factors_long.weight",
 					Kind:     0,
 					Shape:    []uint64{uint64(len(p.RopeScaling.LongFactor))},
 					WriterTo: p.RopeScaling.LongFactor,
-				}, ggml.Tensor{
+				}, llm.Tensor{
 					Name:     "rope_factors_short.weight",
 					Kind:     0,
 					Shape:    []uint64{uint64(len(p.RopeScaling.ShortFactor))},
@@ -89,7 +89,7 @@ func (p *phi3Model) Tensors(ts []Tensor) []ggml.Tensor {
 			})
 		}
 
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     t.Name(),
 			Kind:     t.Kind(),
 			Shape:    t.Shape(),

convert/convert_qwen2.go

@@ -1,7 +1,6 @@
 package convert
 
-import "github.com/ollama/ollama/fs/ggml"
-
+import "github.com/ollama/ollama/llm"
 
 type qwen2Model struct {
 	ModelParameters
@@ -22,7 +21,7 @@ type qwen2Model struct {
 
 var _ ModelConverter = (*qwen2Model)(nil)
 
-func (q *qwen2Model) KV(t *Tokenizer) ggml.KV {
+func (q *qwen2Model) KV(t *Tokenizer) llm.KV {
 	kv := q.ModelParameters.KV(t)
 	kv["general.architecture"] = "qwen2"
 	kv["qwen2.block_count"] = q.HiddenLayers
@@ -46,10 +45,10 @@ func (q *qwen2Model) KV(t *Tokenizer) ggml.KV {
 	return kv
 }
 
-func (q *qwen2Model) Tensors(ts []Tensor) []ggml.Tensor {
-	var out []ggml.Tensor
+func (q *qwen2Model) Tensors(ts []Tensor) []llm.Tensor {
+	var out []llm.Tensor
 	for _, t := range ts {
-		out = append(out, ggml.Tensor{
+		out = append(out, llm.Tensor{
 			Name:     t.Name(),
 			Kind:     t.Kind(),
 			Shape:    t.Shape(),

convert/convert_test.go

@@ -20,7 +20,7 @@ import (
 
 	"golang.org/x/exp/maps"
 
-	"github.com/ollama/ollama/fs/ggml"
+	"github.com/ollama/ollama/llm"
 )
 
 type tensorData struct {
@@ -29,7 +29,7 @@ type tensorData struct {
 	Shape   []int  `json:"shape"`
 }
 
-func convertFull(t *testing.T, fsys fs.FS) (*os.File, ggml.KV, ggml.Tensors) {
+func convertFull(t *testing.T, fsys fs.FS) (*os.File, llm.KV, *llm.Tensors) {
 	t.Helper()
 
 	f, err := os.CreateTemp(t.TempDir(), "f16")
@@ -48,7 +48,7 @@ func convertFull(t *testing.T, fsys fs.FS) (*os.File, ggml.KV, ggml.Tensors) {
 	}
 	t.Cleanup(func() { r.Close() })
 
-	m, _, err := ggml.Decode(r, math.MaxInt)
+	m, _, err := llm.DecodeGGML(r, math.MaxInt)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -60,7 +60,7 @@ func convertFull(t *testing.T, fsys fs.FS) (*os.File, ggml.KV, ggml.Tensors) {
 	return r, m.KV(), m.Tensors()
 }
 
-func generateResultsJSON(t *testing.T, f *os.File, kv ggml.KV, tensors ggml.Tensors) map[string]string {
+func generateResultsJSON(t *testing.T, f *os.File, kv llm.KV, tensors *llm.Tensors) map[string]string {
 	actual := make(map[string]string)
 	for k, v := range kv {
 		if s, ok := v.(json.Marshaler); !ok {
@@ -75,7 +75,7 @@ func generateResultsJSON(t *testing.T, f *os.File, kv ggml.KV, tensors ggml.Tens
 		}
 	}
 
-	for _, tensor := range tensors.Items() {
+	for _, tensor := range tensors.Items {
 		sha256sum := sha256.New()
 		sr := io.NewSectionReader(f, int64(tensors.Offset+tensor.Offset), int64(tensor.Size()))
 		if _, err := io.Copy(sha256sum, sr); err != nil {
@@ -332,7 +332,7 @@ func TestConvertAdapter(t *testing.T) {
 			}
 			defer r.Close()
 
-			m, _, err := ggml.Decode(r, math.MaxInt)
+			m, _, err := llm.DecodeGGML(r, math.MaxInt)
 			if err != nil {
 				t.Fatal(err)
 			}

discover/amd_common.go

@@ -9,8 +9,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-
-	"github.com/ollama/ollama/envconfig"
 )
 
 // Determine if the given ROCm lib directory is usable by checking for existence of some glob patterns
@@ -41,13 +39,10 @@ func commonAMDValidateLibDir() (string, error) {
 	// Favor our bundled version
 
 	// Installer payload location if we're running the installed binary
-	exe, err := os.Executable()
-	if err == nil {
-		rocmTargetDir := filepath.Join(filepath.Dir(exe), envconfig.LibRelativeToExe(), "lib", "ollama")
-		if rocmLibUsable(rocmTargetDir) {
-			slog.Debug("detected ROCM next to ollama executable " + rocmTargetDir)
-			return rocmTargetDir, nil
-		}
+	rocmTargetDir := filepath.Join(LibOllamaPath, "rocm")
+	if rocmLibUsable(rocmTargetDir) {
+		slog.Debug("detected ROCM next to ollama executable " + rocmTargetDir)
+		return rocmTargetDir, nil
 	}
 
 	// Prefer explicit HIP env var

discover/amd_linux.go

@@ -77,8 +77,7 @@ func AMDGetGPUInfo() ([]RocmGPUInfo, error) {
 
 	gfxOverride := envconfig.HsaOverrideGfxVersion()
 	var supported []string
-	depPaths := LibraryDirs()
-	libDir := ""
+	var libDir string
 
 	// The amdgpu driver always exposes the host CPU(s) first, but we have to skip them and subtract
 	// from the other IDs to get alignment with the HIP libraries expectations (zero is the first GPU, not the CPU)
@@ -353,9 +352,8 @@ func AMDGetGPUInfo() ([]RocmGPUInfo, error) {
 				})
 				return nil, err
 			}
-			depPaths = append(depPaths, libDir)
 		}
-		gpuInfo.DependencyPath = depPaths
+		gpuInfo.DependencyPath = []string{libDir}
 
 		if gfxOverride == "" {
 			// Only load supported list once

discover/amd_windows.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
-	"os"
 	"path/filepath"
 	"slices"
 	"strconv"
@@ -50,14 +49,13 @@ func AMDGetGPUInfo() ([]RocmGPUInfo, error) {
 		slog.Info(err.Error())
 		return nil, err
 	}
-	depPaths := LibraryDirs()
+
 	libDir, err := AMDValidateLibDir()
 	if err != nil {
 		err = fmt.Errorf("unable to verify rocm library: %w", err)
 		slog.Warn(err.Error())
 		return nil, err
 	}
-	depPaths = append(depPaths, libDir)
 
 	var supported []string
 	gfxOverride := envconfig.HsaOverrideGfxVersion()
@@ -113,7 +111,7 @@ func AMDGetGPUInfo() ([]RocmGPUInfo, error) {
 				UnreliableFreeMemory: true,
 
 				ID:             strconv.Itoa(i), // TODO this is probably wrong if we specify visible devices
-				DependencyPath: depPaths,
+				DependencyPath: []string{libDir},
 				MinimumMemory:  rocmMinimumMemory,
 				Name:           name,
 				Compute:        gfx,
@@ -164,9 +162,7 @@ func AMDValidateLibDir() (string, error) {
 	}
 
 	// Installer payload (if we're running from some other location)
-	localAppData := os.Getenv("LOCALAPPDATA")
-	appDir := filepath.Join(localAppData, "Programs", "Ollama")
-	rocmTargetDir := filepath.Join(appDir, envconfig.LibRelativeToExe(), "lib", "ollama")
+	rocmTargetDir := filepath.Join(LibOllamaPath, "rocm")
 	if rocmLibUsable(rocmTargetDir) {
 		slog.Debug("detected ollama installed ROCm at " + rocmTargetDir)
 		return rocmTargetDir, nil

discover/gpu.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
-	"github.com/ollama/ollama/runners"
 )
 
 type cudaHandles struct {
@@ -101,15 +100,7 @@ func initCudaHandles() *cudaHandles {
 
 	// Aligned with driver, we can't carry as payloads
 	nvcudaMgmtPatterns := NvcudaGlobs
-
-	if runtime.GOOS == "windows" {
-		localAppData := os.Getenv("LOCALAPPDATA")
-		cudartMgmtPatterns = []string{filepath.Join(localAppData, "Programs", "Ollama", CudartMgmtName)}
-	}
-	libDirs := LibraryDirs()
-	for _, d := range libDirs {
-		cudartMgmtPatterns = append(cudartMgmtPatterns, filepath.Join(d, CudartMgmtName))
-	}
+	cudartMgmtPatterns = append(cudartMgmtPatterns, filepath.Join(LibOllamaPath, "cuda_v*", CudartMgmtName))
 	cudartMgmtPatterns = append(cudartMgmtPatterns, CudartGlobs...)
 
 	if len(NvmlGlobs) > 0 {
@@ -240,7 +231,7 @@ func GetGPUInfo() GpuInfoList {
 		if err != nil {
 			slog.Warn("error looking up system memory", "error", err)
 		}
-		depPaths := LibraryDirs()
+
 		details, err := GetCPUDetails()
 		if err != nil {
 			slog.Warn("failed to lookup CPU details", "error", err)
@@ -248,11 +239,9 @@ func GetGPUInfo() GpuInfoList {
 		cpus = []CPUInfo{
 			{
 				GpuInfo: GpuInfo{
-					memInfo:        mem,
-					Library:        "cpu",
-					Variant:        runners.GetCPUCapability().String(),
-					ID:             "0",
-					DependencyPath: depPaths,
+					memInfo: mem,
+					Library: "cpu",
+					ID:      "0",
 				},
 				CPUs: details,
 			},
@@ -294,17 +283,13 @@ func GetGPUInfo() GpuInfoList {
 				gpuInfo.DriverMajor = driverMajor
 				gpuInfo.DriverMinor = driverMinor
 				variant := cudaVariant(gpuInfo)
-				if depPaths != nil {
-					gpuInfo.DependencyPath = depPaths
-					// Check for variant specific directory
-					if variant != "" {
-						for _, d := range depPaths {
-							if _, err := os.Stat(filepath.Join(d, "cuda_"+variant)); err == nil {
-								// Put the variant directory first in the search path to avoid runtime linking to the wrong library
-								gpuInfo.DependencyPath = append([]string{filepath.Join(d, "cuda_"+variant)}, gpuInfo.DependencyPath...)
-								break
-							}
-						}
+
+				// Start with our bundled libraries
+				if variant != "" {
+					variantPath := filepath.Join(LibOllamaPath, "cuda_"+variant)
+					if _, err := os.Stat(variantPath); err == nil {
+						// Put the variant directory first in the search path to avoid runtime linking to the wrong library
+						gpuInfo.DependencyPath = append([]string{variantPath}, gpuInfo.DependencyPath...)
 					}
 				}
 				gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
@@ -376,7 +361,7 @@ func GetGPUInfo() GpuInfoList {
 						gpuInfo.FreeMemory = uint64(memInfo.free)
 						gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
 						gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
-						gpuInfo.DependencyPath = depPaths
+						gpuInfo.DependencyPath = []string{LibOllamaPath}
 						oneapiGPUs = append(oneapiGPUs, gpuInfo)
 					}
 				}
@@ -512,33 +497,30 @@ func GetGPUInfo() GpuInfoList {
 
 func FindGPULibs(baseLibName string, defaultPatterns []string) []string {
 	// Multiple GPU libraries may exist, and some may not work, so keep trying until we exhaust them
-	var ldPaths []string
 	gpuLibPaths := []string{}
 	slog.Debug("Searching for GPU library", "name", baseLibName)
 
-	// Start with our bundled libraries
-	patterns := []string{}
-	for _, d := range LibraryDirs() {
-		patterns = append(patterns, filepath.Join(d, baseLibName))
-	}
+	// search our bundled libraries first
+	patterns := []string{filepath.Join(LibOllamaPath, baseLibName)}
 
+	var ldPaths []string
 	switch runtime.GOOS {
 	case "windows":
-		ldPaths = strings.Split(os.Getenv("PATH"), ";")
+		ldPaths = strings.Split(os.Getenv("PATH"), string(os.PathListSeparator))
 	case "linux":
-		ldPaths = strings.Split(os.Getenv("LD_LIBRARY_PATH"), ":")
-	default:
-		return gpuLibPaths
+		ldPaths = strings.Split(os.Getenv("LD_LIBRARY_PATH"), string(os.PathListSeparator))
 	}
 
-	// Then with whatever we find in the PATH/LD_LIBRARY_PATH
-	for _, ldPath := range ldPaths {
-		d, err := filepath.Abs(ldPath)
+	// then search the system's LD_LIBRARY_PATH
+	for _, p := range ldPaths {
+		p, err := filepath.Abs(p)
 		if err != nil {
 			continue
 		}
-		patterns = append(patterns, filepath.Join(d, baseLibName))
+		patterns = append(patterns, filepath.Join(p, baseLibName))
 	}
+
+	// finally, search the default patterns provided by the caller
 	patterns = append(patterns, defaultPatterns...)
 	slog.Debug("gpu library search", "globs", patterns)
 	for _, pattern := range patterns {
@@ -715,23 +697,6 @@ func (l GpuInfoList) GetVisibleDevicesEnv() (string, string) {
 	}
 }
 
-func LibraryDirs() []string {
-	// dependencies can exist wherever we found the runners (e.g. build tree for developers) and relative to the executable
-	// This can be simplified once we no longer carry runners as payloads
-	exe, err := os.Executable()
-	if err != nil {
-		slog.Warn("failed to lookup executable path", "error", err)
-		return nil
-	}
-
-	lib := filepath.Join(filepath.Dir(exe), envconfig.LibRelativeToExe(), "lib", "ollama")
-	if _, err := os.Stat(lib); err != nil {
-		return nil
-	}
-
-	return []string{lib}
-}
-
 func GetSystemInfo() SystemInfo {
 	gpus := GetGPUInfo()
 	gpuMutex.Lock()

discover/gpu_darwin.go

@@ -15,7 +15,6 @@ import (
 	"syscall"
 
 	"github.com/ollama/ollama/format"
-	"github.com/ollama/ollama/runners"
 )
 
 const (
@@ -28,7 +27,6 @@ func GetGPUInfo() GpuInfoList {
 		return []GpuInfo{
 			{
 				Library: "cpu",
-				Variant: runners.GetCPUCapability().String(),
 				memInfo: mem,
 			},
 		}
@@ -51,7 +49,6 @@ func GetCPUInfo() GpuInfoList {
 	return []GpuInfo{
 		{
 			Library: "cpu",
-			Variant: runners.GetCPUCapability().String(),
 			memInfo: mem,
 		},
 	}

discover/path.go

@@ -0,0 +1,57 @@
+package discover
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+// LibPath is a path to lookup dynamic libraries
+// in development it's usually 'build/lib/ollama'
+// in distribution builds it's 'lib/ollama' on Windows
+// '../lib/ollama' on Linux and the executable's directory on macOS
+// note: distribution builds, additional GPU-specific libraries are
+// found in subdirectories of the returned path, such as
+// 'cuda_v11', 'cuda_v12', 'rocm', etc.
+var LibOllamaPath string = func() string {
+	exe, err := os.Executable()
+	if err != nil {
+		return ""
+	}
+
+	exe, err = filepath.EvalSymlinks(exe)
+	if err != nil {
+		return ""
+	}
+
+	var libPath string
+	switch runtime.GOOS {
+	case "windows":
+		libPath = filepath.Join(filepath.Dir(exe), "lib", "ollama")
+	case "linux":
+		libPath = filepath.Join(filepath.Dir(exe), "..", "lib", "ollama")
+	case "darwin":
+		libPath = filepath.Dir(exe)
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return ""
+	}
+
+	paths := []string{
+		libPath,
+
+		// build paths for development
+		filepath.Join(filepath.Dir(exe), "build", "lib", "ollama"),
+		filepath.Join(cwd, "build", "lib", "ollama"),
+	}
+
+	for _, p := range paths {
+		if _, err := os.Stat(p); err == nil {
+			return p
+		}
+	}
+
+	return filepath.Dir(exe)
+}()

discover/types.go

@@ -5,7 +5,6 @@ import (
 	"log/slog"
 
 	"github.com/ollama/ollama/format"
-	"github.com/ollama/ollama/runners"
 )
 
 type memInfo struct {
@@ -107,7 +106,7 @@ func (l GpuInfoList) ByLibrary() []GpuInfoList {
 	for _, info := range l {
 		found := false
 		requested := info.Library
-		if info.Variant != runners.CPUCapabilityNone.String() {
+		if info.Variant != "" {
 			requested += "_" + info.Variant
 		}
 		for i, lib := range libs {

docs/api.md

@@ -31,7 +31,7 @@ Certain endpoints stream responses as JSON objects. Streaming can be disabled by
 
 ## Generate a completion
 
-```shell
+```
 POST /api/generate
 ```
 
@@ -306,7 +306,7 @@ curl http://localhost:11434/api/generate -d '{
 
 #### Response
 
-```
+```json
 {
   "model": "llava",
   "created_at": "2023-11-03T15:36:02.583064Z",
@@ -485,7 +485,7 @@ A single JSON object is returned:
 
 ## Generate a chat completion
 
-```shell
+```
 POST /api/chat
 ```
 
@@ -495,14 +495,14 @@ Generate the next message in a chat with a provided model. This is a streaming e
 
 - `model`: (required) the [model name](#model-names)
 - `messages`: the messages of the chat, this can be used to keep a chat memory
-- `tools`: tools for the model to use if supported. Requires `stream` to be set to `false`
+- `tools`: list of tools in JSON for the model to use if supported
 
 The `message` object has the following fields:
 
 - `role`: the role of the message, either `system`, `user`, `assistant`, or `tool`
 - `content`: the content of the message
 - `images` (optional): a list of images to include in the message (for multimodal models such as `llava`)
-- `tool_calls` (optional): a list of tools the model wants to use
+- `tool_calls` (optional): a list of tools in JSON that the model wants to use
 
 Advanced parameters (optional):
 
@@ -795,7 +795,7 @@ curl http://localhost:11434/api/chat -d '{
 
 ##### Request
 
-```
+```shell
 curl http://localhost:11434/api/chat -d '{
   "model": "llama3.2",
   "messages": [
@@ -870,7 +870,7 @@ If the messages array is empty, the model will be loaded into memory.
 
 ##### Request
 
-```
+```shell
 curl http://localhost:11434/api/chat -d '{
   "model": "llama3.2",
   "messages": []
@@ -878,6 +878,7 @@ curl http://localhost:11434/api/chat -d '{
 ```
 
 ##### Response
+
 ```json
 {
   "model": "llama3.2",
@@ -897,7 +898,7 @@ If the messages array is empty and the `keep_alive` parameter is set to `0`, a m
 
 ##### Request
 
-```
+```shell
 curl http://localhost:11434/api/chat -d '{
   "model": "llama3.2",
   "messages": [],
@@ -924,7 +925,7 @@ A single JSON object is returned:
 
 ## Create a Model
 
-```shell
+```
 POST /api/create
 ```
 
@@ -1020,7 +1021,7 @@ curl http://localhost:11434/api/create -d '{
 
 A stream of JSON objects is returned:
 
-```
+```json
 {"status":"quantizing F16 model to Q4_K_M"}
 {"status":"creating new layer sha256:667b0c1932bc6ffc593ed1d03f895bf2dc8dc6df21db3042284a6f4416b06a29"}
 {"status":"using existing layer sha256:11ce4ee3e170f6adebac9a991c22e22ab3f8530e154ee669954c4bc73061c258"}
@@ -1051,7 +1052,7 @@ curl http://localhost:11434/api/create -d '{
 
 A stream of JSON objects is returned:
 
-```
+```json
 {"status":"parsing GGUF"}
 {"status":"using existing layer sha256:432f310a77f4650a88d0fd59ecdd7cebed8d684bafea53cbff0473542964f0c3"}
 {"status":"writing manifest"}
@@ -1118,7 +1119,7 @@ Return 200 OK if the blob exists, 404 Not Found if it does not.
 
 ## Push a Blob
 
-```shell
+```
 POST /api/blobs/:digest
 ```
 
@@ -1142,7 +1143,7 @@ Return 201 Created if the blob was successfully created, 400 Bad Request if the
 
 ## List Local Models
 
-```shell
+```
 GET /api/tags
 ```
 
@@ -1195,7 +1196,7 @@ A single JSON object will be returned.
 
 ## Show Model Information
 
-```shell
+```
 POST /api/show
 ```
 
@@ -1261,7 +1262,7 @@ curl http://localhost:11434/api/show -d '{
 
 ## Copy a Model
 
-```shell
+```
 POST /api/copy
 ```
 
@@ -1284,7 +1285,7 @@ Returns a 200 OK if successful, or a 404 Not Found if the source model doesn't e
 
 ## Delete a Model
 
-```shell
+```
 DELETE /api/delete
 ```
 
@@ -1310,7 +1311,7 @@ Returns a 200 OK if successful, 404 Not Found if the model to be deleted doesn't
 
 ## Pull a Model
 
-```shell
+```
 POST /api/pull
 ```
 
@@ -1382,7 +1383,7 @@ if `stream` is set to false, then the response is a single JSON object:
 
 ## Push a Model
 
-```shell
+```
 POST /api/push
 ```
 
@@ -1447,7 +1448,7 @@ If `stream` is set to `false`, then the response is a single JSON object:
 
 ## Generate Embeddings
 
-```shell
+```
 POST /api/embed
 ```
 
@@ -1515,7 +1516,7 @@ curl http://localhost:11434/api/embed -d '{
 ```
 
 ## List Running Models
-```shell
+```
 GET /api/ps
 ```
 
@@ -1562,7 +1563,7 @@ A single JSON object will be returned.
 
 > Note: this endpoint has been superseded by `/api/embed`
 
-```shell
+```
 POST /api/embeddings
 ```
 
@@ -1602,7 +1603,7 @@ curl http://localhost:11434/api/embeddings -d '{
 
 ## Version
 
-```shell
+```
 GET /api/version
 ```
 

docs/development.md

@@ -1,165 +1,131 @@
 # Development
 
-Install required tools:
+Install prerequisites:
 
-- go version 1.22 or higher
-- OS specific C/C++ compiler (see below)
-- GNU Make
+- [Go](https://go.dev/doc/install)
+- C/C++ Compiler e.g. Clang on macOS, [TDM-GCC](https://github.com/jmeubank/tdm-gcc/releases/latest) (Windows amd64) or [llvm-mingw](https://github.com/mstorsjo/llvm-mingw) (Windows arm64), GCC/Clang on Linux.
 
+Then build and run Ollama from the root directory of the repository:
 
-## Overview
-
-Ollama uses a mix of Go and C/C++ code to interface with GPUs.  The C/C++ code is compiled with both CGO and GPU library specific compilers.  A set of GNU Makefiles are used to compile the project.  GPU Libraries are auto-detected based on the typical environment variables used by the respective libraries, but can be overridden if necessary.  The default make target will build the runners and primary Go Ollama application that will run within the repo directory.  Throughout the examples below `-j 5` is suggested for 5 parallel jobs to speed up the build.  You can adjust the job count based on your CPU Core count to reduce build times.  If you want to relocate the built binaries, use the `dist` target and recursively copy the files in `./dist/$OS-$ARCH/` to your desired location. To learn more about the other make targets use `make help`
-
-Once you have built the GPU/CPU runners, you can compile the main application with `go build .` 
-
-### MacOS
-
-[Download Go](https://go.dev/dl/)
-
-```bash
-make -j 5
+```shell
+go run . serve
 ```
 
-Now you can run `ollama`:
+## macOS (Apple Silicon)
 
-```bash
-./ollama
+macOS Apple Silicon supports Metal which is built-in to the Ollama binary. No additional steps are required.
+
+## macOS (Intel)
+
+Install prerequisites:
+
+- [CMake](https://cmake.org/download/) or `brew install cmake`
+
+Then, configure and build the project:
+
+```shell
+cmake -B build
+cmake --build build
 ```
 
-#### Xcode 15 warnings
+Lastly, run Ollama:
 
-If you are using Xcode newer than version 14, you may see a warning during `go build` about `ld: warning: ignoring duplicate libraries: '-lobjc'` due to Golang issue https://github.com/golang/go/issues/67799 which can be safely ignored.  You can suppress the warning with `export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries"`
-
-### Linux
-
-#### Linux CUDA (NVIDIA)
-
-_Your operating system distribution may already have packages for NVIDIA CUDA. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!_
-
-Install `make`, `gcc` and `golang` as well as [NVIDIA CUDA](https://developer.nvidia.com/cuda-downloads)
-development and runtime packages.
-
-Typically the makefile will auto-detect CUDA, however, if your Linux distro
-or installation approach uses alternative paths, you can specify the location by
-overriding `CUDA_PATH` to the location of the CUDA toolkit. You can customize
-a set of target CUDA architectures by setting `CUDA_ARCHITECTURES` (e.g. `CUDA_ARCHITECTURES=50;60;70`)
-
-```
-make -j 5
+```shell
+go run . serve
 ```
 
-If both v11 and v12 tookkits are detected, runners for both major versions will be built by default.  You can build just v12 with `make cuda_v12`
+## Windows
 
-#### Older Linux CUDA (NVIDIA)
+Install prerequisites:
 
-To support older GPUs with Compute Capability 3.5 or 3.7, you will need to use an older version of the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11).  When you build Ollama, you will need to set two make variable to adjust the minimum compute capability Ollama supports via `make -j 5 CUDA_ARCHITECTURES="35;37;50;52" EXTRA_GOLDFLAGS="\"-X=github.com/ollama/ollama/discover.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/discover.CudaComputeMinorMin=5\""`.  To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus).
+- [CMake](https://cmake.org/download/)
+- [Visual Studio 2022](https://visualstudio.microsoft.com/downloads/) including the Native Desktop Workload
+- (Optional) AMD GPU support
+    - [ROCm](https://rocm.github.io/install.html)
+    - [Ninja](https://github.com/ninja-build/ninja/releases)
+- (Optional) NVIDIA GPU support
+    - [CUDA SDK](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=11&target_type=exe_network)
 
-#### Linux ROCm (AMD)
+> [!IMPORTANT]
+> Ensure prerequisites are in `PATH` before running CMake.
 
-_Your operating system distribution may already have packages for AMD ROCm. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!_
+> [!IMPORTANT]
+> ROCm is not compatible with Visual Studio CMake generators. Use `-GNinja` when configuring the project.
 
-Install [ROCm](https://rocm.docs.amd.com/en/latest/) development packages first, as well as `make`, `gcc`, and `golang`.
+> [!IMPORTANT]
+> CUDA is only compatible with Visual Studio CMake generators.
 
-Typically the build scripts will auto-detect ROCm, however, if your Linux distro
-or installation approach uses unusual paths, you can specify the location by
-specifying an environment variable `HIP_PATH` to the location of the ROCm
-install (typically `/opt/rocm`). You can also customize
-the AMD GPU targets by setting HIP_ARCHS (e.g. `HIP_ARCHS=gfx1101;gfx1102`)
+Then, configure and build the project:
 
-```
-make -j 5
+```shell
+cmake -B build
+cmake --build build --config Release
 ```
 
-ROCm requires elevated privileges to access the GPU at runtime. On most distros you can add your user account to the `render` group, or run as root.
+Lastly, run Ollama:
 
-#### Containerized Linux Build
-
-If you have Docker and buildx available, you can build linux binaries with `./scripts/build_linux.sh` which has the CUDA and ROCm dependencies included. The resulting artifacts are placed in `./dist`  and by default the script builds both arm64 and amd64 binaries.  If you want to build only amd64, you can build with `PLATFORM=linux/amd64 ./scripts/build_linux.sh`
-
-### Windows
-
-The following tools are required as a minimal development environment to build CPU inference support.
-
-- Go version 1.22 or higher
-  - https://go.dev/dl/
-- Git
-  - https://git-scm.com/download/win
-- clang with gcc compat and Make.  There are multiple options on how to go about installing these tools on Windows.  We have verified the following, but others may work as well:  
-  - [MSYS2](https://www.msys2.org/)
-    - After installing, from an MSYS2 terminal, run `pacman -S mingw-w64-clang-x86_64-gcc-compat mingw-w64-clang-x86_64-clang make` to install the required tools
-  - Assuming you used the default install prefix for msys2 above, add `C:\msys64\clang64\bin` and `c:\msys64\usr\bin` to your environment variable `PATH` where you will perform the build steps below (e.g. system-wide, account-level, powershell, cmd, etc.)
-
-> [!NOTE]  
-> Due to bugs in the GCC C++ library for unicode support, Ollama should be built with clang on windows.
-
-```
-make -j 5
+```shell
+go run . serve
 ```
 
-#### GPU Support
+## Windows (ARM)
 
-The GPU tools require the Microsoft native build tools.  To build either CUDA or ROCm, you must first install MSVC via Visual Studio:
+Windows ARM does not support additional acceleration libraries at this time.
 
-- Make sure to select `Desktop development with C++` as a Workload during the Visual Studio install
-- You must complete the Visual Studio install and run it once **BEFORE** installing CUDA or ROCm for the tools to properly register
-- Add the location of the **64 bit (x64)** compiler (`cl.exe`) to your `PATH`
-- Note: the default Developer Shell may configure the 32 bit (x86) compiler which will lead to build failures.  Ollama requires a 64 bit toolchain.
+## Linux
 
-#### Windows CUDA (NVIDIA)
+Install prerequisites:
 
-In addition to the common Windows development tools and MSVC described above:
+- [CMake](https://cmake.org/download/) or `sudo apt install cmake` or `sudo dnf install cmake`
+- (Optional) AMD GPU support
+    - [ROCm](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/quick-start.html)
+- (Optional) NVIDIA GPU support
+    - [CUDA SDK](https://developer.nvidia.com/cuda-downloads)
 
-- [NVIDIA CUDA](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html)
+> [!IMPORTANT]
+> Ensure prerequisites are in `PATH` before running CMake.
 
-#### Windows ROCm (AMD Radeon)
 
-In addition to the common Windows development tools and MSVC described above:
+Then, configure and build the project:
 
-- [AMD HIP](https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html)
-
-#### Windows arm64
-
-The default `Developer PowerShell for VS 2022` may default to x86 which is not what you want.  To ensure you get an arm64 development environment, start a plain PowerShell terminal and run:
-
-```powershell
-import-module 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\Tools\\Microsoft.VisualStudio.DevShell.dll'
-Enter-VsDevShell -Arch arm64 -vsinstallpath 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Community' -skipautomaticlocation
+```shell
+cmake -B build
+cmake --build build
 ```
 
-You can confirm with `write-host $env:VSCMD_ARG_TGT_ARCH`
+Lastly, run Ollama:
 
-Follow the instructions at https://www.msys2.org/wiki/arm64/ to set up an arm64 msys2 environment.  Ollama requires gcc and mingw32-make to compile, which is not currently available on Windows arm64, but a gcc compatibility adapter is available via `mingw-w64-clang-aarch64-gcc-compat`. At a minimum you will need to install the following:
-
-```
-pacman -S mingw-w64-clang-aarch64-clang mingw-w64-clang-aarch64-gcc-compat mingw-w64-clang-aarch64-make make
+```shell
+go run . serve
 ```
 
-You will need to ensure your PATH includes go, cmake, gcc and clang mingw32-make to build ollama from source. (typically `C:\msys64\clangarm64\bin\`)
+## Docker
 
-
-## Advanced CPU Vector Settings
-
-On x86, running `make` will compile several CPU runners which can run on different CPU families. At runtime, Ollama will auto-detect the best variation to load.  If GPU libraries are present at build time, Ollama also compiles GPU runners with the `AVX` CPU vector feature enabled.  This provides a good performance balance when loading large models that split across GPU and CPU with broad compatibility.  Some users may prefer no vector extensions (e.g. older Xeon/Celeron processors, or hypervisors that mask the vector features) while other users may prefer turning on many more vector extensions to further improve performance for split model loads.
-
-To customize the set of CPU vector features enabled for a CPU runner and all GPU runners, use CUSTOM_CPU_FLAGS during the build.
-
-To build without any vector flags:
-
-```
-make CUSTOM_CPU_FLAGS=""
+```shell
+docker build .
 ```
 
-To build with both AVX and AVX2:
-```
-make CUSTOM_CPU_FLAGS=avx,avx2
+### ROCm
+
+```shell
+docker build --build-arg FLAVOR=rocm .
 ```
 
-To build with AVX512 features turned on:
+## Running tests
 
-```
-make CUSTOM_CPU_FLAGS=avx,avx2,avx512,avx512vbmi,avx512vnni,avx512bf16
+To run tests, use `go test`:
+
+```shell
+go test ./...
 ```
 
-> [!NOTE]  
-> If you are experimenting with different flags, make sure to do a `make clean` between each change to ensure everything is rebuilt with the new compiler flags
+## Library detection
+
+Ollama looks for acceleration libraries in the following paths relative to the `ollama` executable:
+
+* `./lib/ollama` (Windows)
+* `../lib/ollama` (Linux)
+* `.` (macOS)
+* `build/lib/ollama` (for development)
+
+If the libraries are not found, Ollama will not run with any acceleration libraries.

docs/docker.md

@@ -2,7 +2,7 @@
 
 ### CPU only
 
-```bash
+```shell
 docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
 ```
 
@@ -11,42 +11,46 @@ Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-
 
 #### Install with Apt
 1.  Configure the repository
-```bash
-curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
-    | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
-curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
-    | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
-    | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
-sudo apt-get update
-```
+
+    ```shell
+    curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
+        | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+    curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
+        | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
+        | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
+    sudo apt-get update
+    ```
+
 2.  Install the NVIDIA Container Toolkit packages
-```bash
-sudo apt-get install -y nvidia-container-toolkit
-```
+
+    ```shell
+    sudo apt-get install -y nvidia-container-toolkit
+    ```
 
 #### Install with Yum or Dnf
 1.  Configure the repository
 
-```bash
-curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo \
-    | sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo
-```
+    ```shell
+    curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo \
+        | sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo
+    ```
 
 2. Install the NVIDIA Container Toolkit packages
 
-```bash
-sudo yum install -y nvidia-container-toolkit
-```
+    ```shell
+    sudo yum install -y nvidia-container-toolkit
+    ```
 
 #### Configure Docker to use Nvidia driver
-```
+
+```shell
 sudo nvidia-ctk runtime configure --runtime=docker
 sudo systemctl restart docker
 ```
 
 #### Start the container
 
-```bash
+```shell
 docker run -d --gpus=all -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
 ```
 
@@ -57,7 +61,7 @@ docker run -d --gpus=all -v ollama:/root/.ollama -p 11434:11434 --name ollama ol
 
 To run Ollama using Docker with AMD GPUs, use the `rocm` tag and the following command:
 
-```
+```shell
 docker run -d --device /dev/kfd --device /dev/dri -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama:rocm
 ```
 
@@ -65,7 +69,7 @@ docker run -d --device /dev/kfd --device /dev/dri -v ollama:/root/.ollama -p 114
 
 Now you can run a model:
 
-```
+```shell
 docker exec -it ollama ollama run llama3.2
 ```
 

docs/faq.md

@@ -24,7 +24,7 @@ By default, Ollama uses a context window size of 2048 tokens.
 
 To change this when using `ollama run`, use `/set parameter`:
 
-```
+```shell
 /set parameter num_ctx 4096
 ```
 
@@ -46,10 +46,15 @@ Use the `ollama ps` command to see what models are currently loaded into memory.
 
 ```shell
 ollama ps
-NAME      	ID          	SIZE 	PROCESSOR	UNTIL
-llama3:70b	bcfb190ca3a7	42 GB	100% GPU 	4 minutes from now
 ```
 
+> **Output**:
+>
+> ```
+> NAME      	ID          	SIZE 	PROCESSOR	UNTIL
+> llama3:70b	bcfb190ca3a7	42 GB	100% GPU 	4 minutes from now
+> ```
+
 The `Processor` column will show which memory the model was loaded in to:
 * `100% GPU` means the model was loaded entirely into the GPU
 * `100% CPU` means the model was loaded entirely in system memory
@@ -66,7 +71,7 @@ If Ollama is run as a macOS application, environment variables should be set usi
 1. For each environment variable, call `launchctl setenv`.
 
     ```bash
-    launchctl setenv OLLAMA_HOST "0.0.0.0"
+    launchctl setenv OLLAMA_HOST "0.0.0.0:11434"
     ```
 
 2. Restart Ollama application.
@@ -81,14 +86,14 @@ If Ollama is run as a systemd service, environment variables should be set using
 
     ```ini
     [Service]
-    Environment="OLLAMA_HOST=0.0.0.0"
+    Environment="OLLAMA_HOST=0.0.0.0:11434"
     ```
 
 3. Save and exit.
 
 4. Reload `systemd` and restart Ollama:
 
-   ```bash
+   ```shell
    systemctl daemon-reload
    systemctl restart ollama
    ```
@@ -221,16 +226,19 @@ properties.
 If you are using the API you can preload a model by sending the Ollama server an empty request. This works with both the `/api/generate` and `/api/chat` API endpoints.
 
 To preload the mistral model using the generate endpoint, use:
+
 ```shell
 curl http://localhost:11434/api/generate -d '{"model": "mistral"}'
 ```
 
 To use the chat completions endpoint, use:
+
 ```shell
 curl http://localhost:11434/api/chat -d '{"model": "mistral"}'
 ```
 
 To preload a model using the CLI, use the command:
+
 ```shell
 ollama run llama3.2 ""
 ```
@@ -250,11 +258,13 @@ If you're using the API, use the `keep_alive` parameter with the `/api/generate`
 * '0' which will unload the model immediately after generating a response
 
 For example, to preload a model and leave it in memory use:
+
 ```shell
 curl http://localhost:11434/api/generate -d '{"model": "llama3.2", "keep_alive": -1}'
 ```
 
 To unload the model and free up memory use:
+
 ```shell
 curl http://localhost:11434/api/generate -d '{"model": "llama3.2", "keep_alive": 0}'
 ```

docs/import.md

@@ -20,13 +20,13 @@ Make sure that you use the same base model in the `FROM` command as you used to
 
 Now run `ollama create` from the directory where the `Modelfile` was created:
 
-```bash
+```shell
 ollama create my-model
 ```
 
 Lastly, test the model:
 
-```bash
+```shell
 ollama run my-model
 ```
 

docs/linux.md

@@ -119,7 +119,7 @@ sudo systemctl status ollama
 
 To customize the installation of Ollama, you can edit the systemd service file or the environment variables by running:
 
-```
+```shell
 sudo systemctl edit ollama
 ```
 
@@ -152,7 +152,7 @@ Use `OLLAMA_VERSION` environment variable with the install script to install a s
 For example:
 
 ```shell
-curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.3.9 sh
+curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.5.7 sh
 ```
 
 ## Viewing logs
@@ -186,3 +186,9 @@ sudo rm -r /usr/share/ollama
 sudo userdel ollama
 sudo groupdel ollama
 ```
+
+Remove installed libraries:
+
+```shell
+sudo rm -rf /usr/local/lib/ollama
+```

docs/modelfile.md

@@ -28,7 +28,7 @@ A model file is the blueprint to create and share models with Ollama.
 
 The format of the `Modelfile`:
 
-```modelfile
+```
 # comment
 INSTRUCTION arguments
 ```
@@ -49,7 +49,7 @@ INSTRUCTION arguments
 
 An example of a `Modelfile` creating a mario blueprint:
 
-```modelfile
+```
 FROM llama3.2
 # sets the temperature to 1 [higher is more creative, lower is more coherent]
 PARAMETER temperature 1
@@ -67,28 +67,32 @@ To use this:
 3. `ollama run choose-a-model-name`
 4. Start using the model!
 
-More examples are available in the [examples directory](../examples).
-
 To view the Modelfile of a given model, use the `ollama show --modelfile` command.
 
-  ```bash
-  > ollama show --modelfile llama3.2
-  # Modelfile generated by "ollama show"
-  # To build a new Modelfile based on this one, replace the FROM line with:
-  # FROM llama3.2:latest
-  FROM /Users/pdevine/.ollama/models/blobs/sha256-00e1317cbf74d901080d7100f57580ba8dd8de57203072dc6f668324ba545f29
-  TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>
+```shell
+ollama show --modelfile llama3.2
+```
 
-  {{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>
+> **Output**:
+>
+> ```
+> # Modelfile generated by "ollama show"
+> # To build a new Modelfile based on this one, replace the FROM line with:
+> # FROM llama3.2:latest
+> FROM /Users/pdevine/.ollama/models/blobs/sha256-00e1317cbf74d901080d7100f57580ba8dd8de57203072dc6f668324ba545f29
+> TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>
+>
+> {{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>
+>
+> {{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
+>
+> {{ .Response }}<|eot_id|>"""
+> PARAMETER stop "<|start_header_id|>"
+> PARAMETER stop "<|end_header_id|>"
+> PARAMETER stop "<|eot_id|>"
+> PARAMETER stop "<|reserved_special_token"
+> ```
 
-  {{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
-
-  {{ .Response }}<|eot_id|>"""
-  PARAMETER stop "<|start_header_id|>"
-  PARAMETER stop "<|end_header_id|>"
-  PARAMETER stop "<|eot_id|>"
-  PARAMETER stop "<|reserved_special_token"
-  ```
 
 ## Instructions
 
@@ -96,13 +100,13 @@ To view the Modelfile of a given model, use the `ollama show --modelfile` comman
 
 The `FROM` instruction defines the base model to use when creating a model.
 
-```modelfile
+```
 FROM <model name>:<tag>
 ```
 
 #### Build from existing model
 
-```modelfile
+```
 FROM llama3.2
 ```
 
@@ -113,7 +117,7 @@ Additional models can be found at:
 
 #### Build from a Safetensors model
 
-```modelfile
+```
 FROM <model directory>
 ```
 
@@ -127,7 +131,7 @@ Currently supported model architectures:
 
 #### Build from a GGUF file
 
-```modelfile
+```
 FROM ./ollama-model.gguf
 ```
 
@@ -138,7 +142,7 @@ The GGUF file location should be specified as an absolute path or relative to th
 
 The `PARAMETER` instruction defines a parameter that can be set when the model is run.
 
-```modelfile
+```
 PARAMETER <parameter> <parametervalue>
 ```
 
@@ -185,7 +189,7 @@ TEMPLATE """{{ if .System }}<|im_start|>system
 
 The `SYSTEM` instruction specifies the system message to be used in the template, if applicable.
 
-```modelfile
+```
 SYSTEM """<system message>"""
 ```
 
@@ -195,7 +199,7 @@ The `ADAPTER` instruction specifies a fine tuned LoRA adapter that should apply
 
 #### Safetensor adapter
 
-```modelfile
+```
 ADAPTER <path to safetensor adapter>
 ```
 
@@ -206,7 +210,7 @@ Currently supported Safetensor adapters:
 
 #### GGUF adapter
 
-```modelfile
+```
 ADAPTER ./ollama-lora.gguf
 ```
 
@@ -214,7 +218,7 @@ ADAPTER ./ollama-lora.gguf
 
 The `LICENSE` instruction allows you to specify the legal license under which the model used with this Modelfile is shared or distributed.
 
-```modelfile
+```
 LICENSE """
 <license text>
 """
@@ -224,7 +228,7 @@ LICENSE """
 
 The `MESSAGE` instruction allows you to specify a message history for the model to use when responding. Use multiple iterations of the MESSAGE command to build up a conversation which will guide the model to answer in a similar way.
 
-```modelfile
+```
 MESSAGE <role> <message>
 ```
 
@@ -239,7 +243,7 @@ MESSAGE <role> <message>
 
 #### Example conversation
 
-```modelfile
+```
 MESSAGE user Is Toronto in Canada?
 MESSAGE assistant yes
 MESSAGE user Is Sacramento in Canada?

docs/openai.md

@@ -1,6 +1,7 @@
 # OpenAI compatibility
 
-> **Note:** OpenAI compatibility is experimental and is subject to major adjustments including breaking changes. For fully-featured access to the Ollama API, see the Ollama [Python library](https://github.com/ollama/ollama-python), [JavaScript library](https://github.com/ollama/ollama-js) and [REST API](https://github.com/ollama/ollama/blob/main/docs/api.md).
+> [!NOTE]
+> OpenAI compatibility is experimental and is subject to major adjustments including breaking changes. For fully-featured access to the Ollama API, see the Ollama [Python library](https://github.com/ollama/ollama-python), [JavaScript library](https://github.com/ollama/ollama-js) and [REST API](https://github.com/ollama/ollama/blob/main/docs/api.md).
 
 Ollama provides experimental compatibility with parts of the [OpenAI API](https://platform.openai.com/docs/api-reference) to help connect existing applications to Ollama.
 
@@ -59,8 +60,10 @@ embeddings = client.embeddings.create(
     input=["why is the sky blue?", "why is the grass green?"],
 )
 ```
+
 #### Structured outputs
-```py
+
+```python
 from pydantic import BaseModel
 from openai import OpenAI
 
@@ -144,7 +147,7 @@ const embedding = await openai.embeddings.create({
 
 ### `curl`
 
-``` shell
+```shell
 curl http://localhost:11434/v1/chat/completions \
     -H "Content-Type: application/json" \
     -d '{
@@ -319,7 +322,7 @@ ollama pull llama3.2
 
 For tooling that relies on default OpenAI model names such as `gpt-3.5-turbo`, use `ollama cp` to copy an existing model name to a temporary name:
 
-```
+```shell
 ollama cp llama3.2 gpt-3.5-turbo
 ```
 
@@ -343,7 +346,7 @@ curl http://localhost:11434/v1/chat/completions \
 
 The OpenAI API does not have a way of setting the context size for a model. If you need to change the context size, create a `Modelfile` which looks like:
 
-```modelfile
+```
 FROM <some model>
 PARAMETER num_ctx <context size>
 ```

docs/troubleshooting.md

@@ -17,6 +17,7 @@ When you run Ollama in a **container**, the logs go to stdout/stderr in the cont
 ```shell
 docker logs <container-name>
 ```
+
 (Use `docker ps` to find the container name)
 
 If manually running `ollama serve` in a terminal, the logs will be on that terminal.
@@ -28,6 +29,7 @@ When you run Ollama on **Windows**, there are a few different locations. You can
 - `explorer %TEMP%` where temporary executable files are stored in one or more `ollama*` directories
 
 To enable additional debug logging to help troubleshoot problems, first **Quit the running app from the tray menu** then in a powershell terminal
+
 ```powershell
 $env:OLLAMA_DEBUG="1"
 & "ollama app.exe"
@@ -49,12 +51,13 @@ Dynamic LLM libraries [rocm_v6 cpu cpu_avx cpu_avx2 cuda_v11 rocm_v5]
 
 You can set OLLAMA_LLM_LIBRARY to any of the available LLM libraries to bypass autodetection, so for example, if you have a CUDA card, but want to force the CPU LLM library with AVX2 vector support, use:
 
-```
+```shell
 OLLAMA_LLM_LIBRARY="cpu_avx2" ollama serve
 ```
 
 You can see what features your CPU has with the following.
-```
+
+```shell
 cat /proc/cpuinfo| grep flags | head -1
 ```
 
@@ -62,8 +65,8 @@ cat /proc/cpuinfo| grep flags | head -1
 
 If you run into problems on Linux and want to install an older version, or you'd like to try out a pre-release before it's officially released, you can tell the install script which version to install.
 
-```sh
-curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION="0.1.29" sh
+```shell
+curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.5.7 sh
 ```
 
 ## Linux tmp noexec 

docs/windows.md

@@ -47,6 +47,7 @@ If Ollama is already running, Quit the tray application and relaunch it from the
 ## API Access
 
 Here's a quick example showing API access from `powershell`
+
 ```powershell
 (Invoke-WebRequest -method POST -Body '{"model":"llama3.2", "prompt":"Why is the sky blue?", "stream": false}' -uri http://localhost:11434/api/generate ).Content | ConvertFrom-json
 ```

envconfig/config.go

@@ -165,8 +165,6 @@ var (
 	IntelGPU = Bool("OLLAMA_INTEL_GPU")
 	// MultiUserCache optimizes prompt caching for multi-user scenarios
 	MultiUserCache = Bool("OLLAMA_MULTIUSER_CACHE")
-	// Enable the new Ollama engine
-	NewRunners = Bool("OLLAMA_NEW_RUNNERS")
 )
 
 func String(s string) func() string {
@@ -252,7 +250,6 @@ func AsMap() map[string]EnvVar {
 		"OLLAMA_ORIGINS":           {"OLLAMA_ORIGINS", Origins(), "A comma separated list of allowed origins"},
 		"OLLAMA_SCHED_SPREAD":      {"OLLAMA_SCHED_SPREAD", SchedSpread(), "Always schedule model across all GPUs"},
 		"OLLAMA_MULTIUSER_CACHE":   {"OLLAMA_MULTIUSER_CACHE", MultiUserCache(), "Optimize prompt caching for multi-user scenarios"},
-		"OLLAMA_NEW_RUNNERS":       {"OLLAMA_NEW_RUNNERS", NewRunners(), "Enable the new Ollama engine"},
 
 		// Informational
 		"HTTP_PROXY":  {"HTTP_PROXY", String("HTTP_PROXY")(), "HTTP proxy"},
@@ -291,12 +288,3 @@ func Values() map[string]string {
 func Var(key string) string {
 	return strings.Trim(strings.TrimSpace(os.Getenv(key)), "\"'")
 }
-
-// On windows, we keep the binary at the top directory, but
-// other platforms use a "bin" directory, so this returns ".."
-func LibRelativeToExe() string {
-	if runtime.GOOS == "windows" {
-		return "."
-	}
-	return ".."
-}

format/bytes.go

@@ -40,8 +40,6 @@ func HumanBytes(b int64) string {
 	}
 
 	switch {
-	case value >= 100:
-		return fmt.Sprintf("%d %s", int(value), unit)
 	case value >= 10:
 		return fmt.Sprintf("%d %s", int(value), unit)
 	case value != math.Trunc(value):

format/bytes_test.go

@@ -0,0 +1,91 @@
+package format
+
+import (
+	"testing"
+)
+
+func TestHumanBytes(t *testing.T) {
+	type testCase struct {
+		input    int64
+		expected string
+	}
+
+	tests := []testCase{
+		// Test bytes (B)
+		{0, "0 B"},
+		{1, "1 B"},
+		{999, "999 B"},
+
+		// Test kilobytes (KB)
+		{1000, "1 KB"},
+		{1500, "1.5 KB"},
+		{999999, "999 KB"},
+
+		// Test megabytes (MB)
+		{1000000, "1 MB"},
+		{1500000, "1.5 MB"},
+		{999999999, "999 MB"},
+
+		// Test gigabytes (GB)
+		{1000000000, "1 GB"},
+		{1500000000, "1.5 GB"},
+		{999999999999, "999 GB"},
+
+		// Test terabytes (TB)
+		{1000000000000, "1 TB"},
+		{1500000000000, "1.5 TB"},
+		{1999999999999, "2.0 TB"},
+
+		// Test fractional values
+		{1234, "1.2 KB"},
+		{1234567, "1.2 MB"},
+		{1234567890, "1.2 GB"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.expected, func(t *testing.T) {
+			result := HumanBytes(tc.input)
+			if result != tc.expected {
+				t.Errorf("Expected %s, got %s", tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestHumanBytes2(t *testing.T) {
+	type testCase struct {
+		input    uint64
+		expected string
+	}
+
+	tests := []testCase{
+		// Test bytes (B)
+		{0, "0 B"},
+		{1, "1 B"},
+		{1023, "1023 B"},
+
+		// Test kibibytes (KiB)
+		{1024, "1.0 KiB"},
+		{1536, "1.5 KiB"},
+		{1048575, "1024.0 KiB"},
+
+		// Test mebibytes (MiB)
+		{1048576, "1.0 MiB"},
+		{1572864, "1.5 MiB"},
+		{1073741823, "1024.0 MiB"},
+
+		// Test gibibytes (GiB)
+		{1073741824, "1.0 GiB"},
+		{1610612736, "1.5 GiB"},
+		{2147483648, "2.0 GiB"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.expected, func(t *testing.T) {
+			result := HumanBytes2(tc.input)
+			if result != tc.expected {
+				t.Errorf("Expected %s, got %s", tc.expected, result)
+			}
+		})
+	}
+}

llama/README.md

@@ -1,156 +1,52 @@
 # `llama`
 
-This package integrates the [llama.cpp](https://github.com/ggerganov/llama.cpp) library as a Go package and makes it easy to build it with tags for different CPU and GPU processors.
-
-Supported:
-
-- [x] CPU
-- [x] avx, avx2
-- [x] macOS Metal
-- [x] Windows CUDA
-- [x] Windows ROCm
-- [x] Linux CUDA
-- [x] Linux ROCm
-- [x] Llava
-
-Extra build steps are required for CUDA and ROCm on Windows since `nvcc` and `hipcc` both require using msvc as the host compiler. For these shared libraries are created:
-
-- `ggml_cuda.dll` on Windows or `ggml_cuda.so` on Linux
-- `ggml_hipblas.dll` on Windows or `ggml_hipblas.so` on Linux
-
-> Note: it's important that memory is allocated and freed by the same compiler (e.g. entirely by code compiled with msvc or mingw). Issues from this should be rare, but there are some places where pointers are returned by the CUDA or HIP runtimes and freed elsewhere, causing a a crash. In a future change the same runtime should be used in both cases to avoid crashes.
-
-## Building
-
-```
-go build .
-```
-
-### AVX
-
-```shell
-go build -tags avx .
-```
-
-### AVX2
-
-```shell
-# go doesn't recognize `-mfma` as a valid compiler flag
-# see https://github.com/golang/go/issues/17895
-go env -w "CGO_CPPFLAGS_ALLOW=-mfma|-mf16c"
-go build -tags=avx,avx2 .
-```
-
-## Linux
-
-### CUDA
-
-Install the [CUDA toolkit v11.3.1](https://developer.nvidia.com/cuda-11-3-1-download-archive):
-
-```shell
-make ggml_cuda.so
-go build -tags avx,cuda .
-```
-
-### ROCm
-
-Install [ROCm](https://rocm.docs.amd.com/en/latest/).
-
-```shell
-make ggml_hipblas.so
-go build -tags avx,rocm .
-```
-
-## Windows
-
-Download [w64devkit](https://github.com/skeeto/w64devkit/releases/latest) for a simple MinGW development environment.
-
-### CUDA
-
-Install the [CUDA toolkit v11.3.1](https://developer.nvidia.com/cuda-11-3-1-download-archive) then build the cuda code:
-
-```shell
-make ggml_cuda.dll
-go build -tags avx,cuda .
-```
-
-### ROCm
-
-Install [ROCm](https://rocm.docs.amd.com/en/latest/).
-
-```shell
-make ggml_hipblas.dll
-go build -tags avx,rocm .
-```
-
-## Building runners
-
-```shell
-# build all runners for this platform
-make -j
-```
+This package provides Go bindings to [llama.cpp](https://github.com/ggerganov/llama.cpp).
 
 ## Vendoring
 
-Ollama currently vendors [llama.cpp](https://github.com/ggerganov/llama.cpp/) and [ggml](https://github.com/ggerganov/ggml) through a vendoring model. While we generally strive to contribute changes back upstream to avoid drift, we cary a small set of patches which are applied to the tracking commit. A set of make targets are available to aid developers in updating to a newer tracking commit, or to work on changes.
+Ollama vendors [llama.cpp](https://github.com/ggerganov/llama.cpp/) and [ggml](https://github.com/ggerganov/llama.cpp/tree/master/ggml/src). While we generally strive to contribute changes back upstream to avoid drift, we carry a small set of patches which are applied to the tracking commit.
 
 If you update the vendoring code, start by running the following command to establish the tracking llama.cpp repo in the `./vendor/` directory.
 
-```
-make apply-patches
+```shell
+make -f Makefile.sync apply-patches
 ```
 
 ### Updating Base Commit
 
 **Pin to new base commit**
 
-To update to a newer base commit, select the upstream git tag or commit and update `llama/vendoring`
-
-#### Applying patches
+To change the base commit, update `FETCH_HEAD` in Makefile.sync.
 
 When updating to a newer base commit, the existing patches may not apply cleanly and require manual merge resolution.
 
 Start by applying the patches. If any of the patches have conflicts, the `git am` will stop at the first failure.
 
-```
-make apply-patches
+```shell
+make -f Makefile.sync apply-patches
 ```
 
-If you see an error message about a conflict, go into the `./vendor/` directory, and perform merge resolution using your preferred tool to the patch commit which failed. Save the file(s) and continue the patch series with `git am --continue` . If any additional patches fail, follow the same pattern until the full patch series is applied. Once finished, run a final `create-patches` and `sync` target to ensure everything is updated.
+If there are conflicts, you will see an error message. Resolve the conflicts in `./vendor/`, and continue the patch series with `git am --continue` and rerun `make -f Makefile.sync apply-patches`. Repeat until all patches are successfully applied.
 
-```
-make create-patches sync
-```
+Once all patches are applied, commit the changes to the tracking repository.
 
-Build and test Ollama, and make any necessary changes to the Go code based on the new base commit. Submit your PR to the Ollama repo.
+```shell
+make -f Makefile.sync format-patches sync
+```
 
 ### Generating Patches
 
 When working on new fixes or features that impact vendored code, use the following model. First get a clean tracking repo with all current patches applied:
 
+```shell
+make -f Makefile.sync clean apply-patches
 ```
-make apply-patches
-```
-
-Now edit the upstream native code in the `./vendor/` directory. You do not need to commit every change in order to build, a dirty working tree in the tracking repo is OK while developing. Simply save in your editor, and run the following to refresh the vendored code with your changes, build the backend(s) and build ollama:
-
-```
-make sync
-make -j 8
-go build .
-```
-
-> [!IMPORTANT]
-> Do **NOT** run `apply-patches` while you're iterating as that will reset the tracking repo. It will detect a dirty tree and abort, but if your tree is clean and you accidentally ran this target, use `git reflog` to recover your commit(s).
 
 Iterate until you're ready to submit PRs. Once your code is ready, commit a change in the `./vendor/` directory, then generate the patches for ollama with
 
+```shell
+make -f Makefile.sync format-patches
 ```
-make create-patches
-```
-
-> [!IMPORTANT]
-> Once you have completed this step, it is safe to run `apply-patches` since your change is preserved in the patches.
 
 In your `./vendor/` directory, create a branch, and cherry-pick the new commit to that branch, then submit a PR upstream to llama.cpp.
 

llama/build-info.cpp

@@ -1,4 +1,4 @@
 int LLAMA_BUILD_NUMBER = 0;
-char const *LLAMA_COMMIT = "ba1cb19cdd0d92e012e0f6e009e0620f854b6afd";
+char const *LLAMA_COMMIT = "46e3556e01b824e52395fb050b29804b6cff2a7c";
 char const *LLAMA_COMPILER = "";
 char const *LLAMA_BUILD_TARGET = "";

llama/build-info.cpp.in

@@ -0,0 +1,4 @@
+int LLAMA_BUILD_NUMBER = 0;
+char const *LLAMA_COMMIT = "@FETCH_HEAD@";
+char const *LLAMA_COMPILER = "";
+char const *LLAMA_BUILD_TARGET = "";

llama/llama.cpp/examples/llava/clip.cpp

@@ -1235,35 +1235,15 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
         }
     }
 
-#ifdef GGML_USE_CUDA
-   new_clip->backend = ggml_backend_cuda_init(0);
-   LOG_INF("%s: CLIP using CUDA backend\n", __func__);
-#endif
-
-#ifdef GGML_USE_METAL
-   new_clip->backend = ggml_backend_metal_init();
-   LOG_INF("%s: CLIP using Metal backend\n", __func__);
-#endif
-
-#ifdef GGML_USE_CANN
-   new_clip->backend = ggml_backend_cann_init(0);
-   LOG_INF("%s: CLIP using CANN backend\n", __func__);
-#endif
-
-#ifdef GGML_USE_VULKAN
-   new_clip->backend = ggml_backend_vk_init(0);
-   LOG_INF("%s: CLIP using Vulkan backend\n", __func__);
-#endif
-
-#ifdef GGML_USE_SYCL
-   new_clip->backend = ggml_backend_sycl_init(0);
-   LOG_INF("%s: CLIP using SYCL backend\n", __func__);
-#endif
-
-    if (!new_clip->backend) {
-        new_clip->backend = ggml_backend_cpu_init();
-        LOG_INF("%s: CLIP using CPU backend\n", __func__);
+    ggml_backend_t backend = ggml_backend_init_best();
+    if (backend == nullptr) {
+        LOG_ERR("%s: failed to initialize backend\n", __func__);
+        clip_free(new_clip);
+        gguf_free(ctx);
+        return nullptr;
     }
+    LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend));
+    new_clip->backend = backend;
 
     // model size and capabilities
     {

llama/llama.go

@@ -50,7 +50,7 @@ import (
 	_ "github.com/ollama/ollama/llama/llama.cpp/common"
 	_ "github.com/ollama/ollama/llama/llama.cpp/examples/llava"
 	_ "github.com/ollama/ollama/llama/llama.cpp/src"
-	"github.com/ollama/ollama/ml/backend/ggml/ggml/src"
+	ggml "github.com/ollama/ollama/ml/backend/ggml/ggml/src"
 )
 
 func BackendInit() {
@@ -199,21 +199,38 @@ func (c *Context) KvCacheDefrag() {
 
 // Get the embeddings for a sequence id
 func (c *Context) GetEmbeddingsSeq(seqId int) []float32 {
-	embeddings := unsafe.Pointer(C.llama_get_embeddings_seq(c.c, C.int(seqId)))
-	if embeddings == nil {
+	e := unsafe.Pointer(C.llama_get_embeddings_seq(c.c, C.int(seqId)))
+	if e == nil {
 		return nil
 	}
 
-	return unsafe.Slice((*float32)(embeddings), c.Model().NEmbd())
+	embeddings := make([]float32, c.Model().NEmbd())
+	_ = copy(embeddings, unsafe.Slice((*float32)(e), c.Model().NEmbd()))
+	return embeddings
 }
 
 func (c *Context) GetEmbeddingsIth(i int) []float32 {
-	embeddings := unsafe.Pointer(C.llama_get_embeddings_ith(c.c, C.int32_t(i)))
-	if embeddings == nil {
+	e := unsafe.Pointer(C.llama_get_embeddings_ith(c.c, C.int32_t(i)))
+	if e == nil {
 		return nil
 	}
 
-	return unsafe.Slice((*float32)(embeddings), c.Model().NEmbd())
+	embeddings := make([]float32, c.Model().NEmbd())
+	_ = copy(embeddings, unsafe.Slice((*float32)(e), c.Model().NEmbd()))
+	return embeddings
+}
+
+// GetLogits returns the logits from the last decode operation.
+// The returned slice has length equal to the vocabulary size.
+func (c *Context) GetLogits() []float32 {
+	logits := unsafe.Pointer(C.llama_get_logits(c.c))
+	if logits == nil {
+		return nil
+	}
+
+	// Get the number of vocabulary tokens to determine array size
+	vocabSize := c.Model().NumVocab()
+	return unsafe.Slice((*float32)(logits), vocabSize)
 }
 
 type ModelParams struct {

llama/mllama.cpp

@@ -558,30 +558,15 @@ struct mllama_ctx *mllama_model_load(const char *fname, const int verbosity = 1)
 
     mllama_ctx *new_mllama = new mllama_ctx{};
 
-#ifdef GGML_USE_CUDA
-    new_mllama->backend = ggml_backend_cuda_init(0);
-    LOG("vision using CUDA backend");
-#endif
-
-#ifdef GGML_USE_METAL
-    new_mllama->backend = ggml_backend_metal_init();
-    LOG("vision using Metal backend");
-#endif
-
-#ifdef GGML_USE_CANN
-    new_mllama->backend = ggml_backend_cann_init(0);
-    LOG("vision using CANN backend");
-#endif
-
-#ifdef GGML_USE_VULKAN
-    new_mllama->backend = ggml_backend_vk_init(0);
-    LOG("vision using Vulkan backend");
-#endif
-
-    if (!new_mllama->backend) {
-        new_mllama->backend = ggml_backend_cpu_init();
-        LOG("vision using CPU backend");
+    ggml_backend_t backend = ggml_backend_init_best();
+    if (backend == nullptr) {
+        LOG("%s: failed to initialize backend\n", __func__);
+        mllama_free(new_mllama);
+        gguf_free(ctx);
+        return nullptr;
     }
+    LOG("%s: using %s backend\n", __func__, ggml_backend_name(backend));
+    new_mllama->backend = backend;
 
     // load tensors
     {

llama/patches/0013-use-dynamic-backend-loading-for-clip.patch

@@ -1,14 +1,14 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: jmorganca <jmorganca@gmail.com>
 Date: Sat, 4 Jan 2025 22:52:48 -0800
-Subject: [PATCH] re-enable gpu for clip
+Subject: [PATCH] use dynamic backend loading for clip
 
 ---
- examples/llava/clip.cpp | 86 ++++++++++++++++++++---------------------
- 1 file changed, 43 insertions(+), 43 deletions(-)
+ examples/llava/clip.cpp | 74 +++++++++++++++--------------------------
+ 1 file changed, 27 insertions(+), 47 deletions(-)
 
 diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
-index b3c1829f..718052e1 100644
+index b3c1829f..86b91d5c 100644
 --- a/examples/llava/clip.cpp
 +++ b/examples/llava/clip.cpp
 @@ -8,25 +8,25 @@
@@ -56,7 +56,7 @@ index b3c1829f..718052e1 100644
  
  #define STB_IMAGE_IMPLEMENTATION
  #include "stb_image.h"
-@@ -1235,30 +1235,30 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
+@@ -1235,35 +1235,15 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
          }
      }
  
@@ -84,30 +84,19 @@ index b3c1829f..718052e1 100644
 -//    new_clip->backend = ggml_backend_sycl_init(0);
 -//    LOG_INF("%s: CLIP using SYCL backend\n", __func__);
 -//#endif
-+#ifdef GGML_USE_CUDA
-+   new_clip->backend = ggml_backend_cuda_init(0);
-+   LOG_INF("%s: CLIP using CUDA backend\n", __func__);
-+#endif
-+
-+#ifdef GGML_USE_METAL
-+   new_clip->backend = ggml_backend_metal_init();
-+   LOG_INF("%s: CLIP using Metal backend\n", __func__);
-+#endif
-+
-+#ifdef GGML_USE_CANN
-+   new_clip->backend = ggml_backend_cann_init(0);
-+   LOG_INF("%s: CLIP using CANN backend\n", __func__);
-+#endif
-+
-+#ifdef GGML_USE_VULKAN
-+   new_clip->backend = ggml_backend_vk_init(0);
-+   LOG_INF("%s: CLIP using Vulkan backend\n", __func__);
-+#endif
-+
-+#ifdef GGML_USE_SYCL
-+   new_clip->backend = ggml_backend_sycl_init(0);
-+   LOG_INF("%s: CLIP using SYCL backend\n", __func__);
-+#endif
+-
+-    if (!new_clip->backend) {
+-        new_clip->backend = ggml_backend_cpu_init();
+-        LOG_INF("%s: CLIP using CPU backend\n", __func__);
++    ggml_backend_t backend = ggml_backend_init_best();
++    if (backend == nullptr) {
++        LOG_ERR("%s: failed to initialize backend\n", __func__);
++        clip_free(new_clip);
++        gguf_free(ctx);
++        return nullptr;
+     }
++    LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend));
++    new_clip->backend = backend;
  
-     if (!new_clip->backend) {
-         new_clip->backend = ggml_backend_cpu_init();
+     // model size and capabilities
+     {

llama/patches/0016-remove-sgemm-global-variables.patch

@@ -0,0 +1,55 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: jmorganca <jmorganca@gmail.com>
+Date: Sun, 9 Feb 2025 17:22:15 -0800
+Subject: [PATCH] remove sgemm global variables
+
+removes the 'iq4nlt' global variable in sgemm.cpp that causes
+a runtime crash when calling dlopen on ggml-cpu libraries as
+its initialization depends on AVX instructions the host machine
+may not have
+---
+ ggml/src/ggml-cpu/llamafile/sgemm.cpp | 17 +++++++++--------
+ 1 file changed, 9 insertions(+), 8 deletions(-)
+
+diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
+index 8fce576c..3f260ce5 100644
+--- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp
++++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
+@@ -279,14 +279,6 @@ template <> inline __m256bh load(const float *p) {
+ }
+ #endif
+ 
+-////////////////////////////////////////////////////////////////////////////////////////////////////
+-// CONSTANTS
+-
+-#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+-static const int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
+-static const __m128i iq4nlt = _mm_loadu_si128((const __m128i *) kvalues_iq4nl);
+-#endif
+-
+ ////////////////////////////////////////////////////////////////////////////////////////////////////
+ // FLOATING POINT MATRIX MULTIPLICATION
+ 
+@@ -613,6 +605,14 @@ class tinyBLAS_Q0_AVX {
+                     TC *C, int64_t ldc,
+                     int ith, int nth)
+         : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
++        const int8_t kvalues_iq4nl[16] = {
++            -127, -104, -83, -65,
++            -49,  -35,  -22, -10,
++              1,   13,   25,  38,
++             53,   69,   89, 113
++        };
++
++        iq4nlt = _mm_loadu_si128((const __m128i *)kvalues_iq4nl);
+     }
+ 
+     void matmul(int64_t m, int64_t n) {
+@@ -1037,6 +1037,7 @@ class tinyBLAS_Q0_AVX {
+     const int64_t ldc;
+     const int ith;
+     const int nth;
++    __m128i iq4nlt;
+ };
+ #endif // __AVX__
+ 

llama/patches/0017-try-catch-backend-load.patch

@@ -0,0 +1,69 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Tue, 11 Feb 2025 14:06:36 -0800
+Subject: [PATCH] try/catch backend load
+
+---
+ ggml/src/ggml-backend-reg.cpp | 45 ++++++++++++++++++-----------------
+ 1 file changed, 23 insertions(+), 22 deletions(-)
+
+diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp
+index ac5cda07..374c3b21 100644
+--- a/ggml/src/ggml-backend-reg.cpp
++++ b/ggml/src/ggml-backend-reg.cpp
+@@ -512,32 +512,33 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
+         }
+         fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);
+         for (const auto & entry : dir_it) {
+-            if (entry.is_regular_file()) {
+-                std::wstring filename = entry.path().filename().wstring();
+-                std::wstring ext = entry.path().extension().wstring();
+-                if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) {
+-                    dl_handle_ptr handle { dl_load_library(entry.path().wstring()) };
+-                    if (!handle && !silent) {
+-                        GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
+-                    }
+-                    if (handle) {
++            try {
++                if (entry.is_regular_file()) {
++                    std::wstring filename = entry.path().filename().wstring();
++                    std::wstring ext = entry.path().extension().wstring();
++                    if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) {
++                        dl_handle_ptr handle { dl_load_library(entry.path().wstring()) };
++                        if (!handle) {
++                            GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
++                            continue;
++                        }
++
+                         auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");
+-                        if (score_fn) {
+-                            int s = score_fn();
+-#ifndef NDEBUG
+-                            GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s);
+-#endif
+-                            if (s > best_score) {
+-                                best_score = s;
+-                                best_path = entry.path().wstring();
+-                            }
+-                        } else {
+-                            if (!silent) {
+-                                GGML_LOG_INFO("%s: failed to find ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
+-                            }
++                        if (!score_fn) {
++                            GGML_LOG_DEBUG("%s: failed to find ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
++                            continue;
++                        }
++
++                        int s = score_fn();
++                        GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s);
++                        if (s > best_score) {
++                            best_score = s;
++                            best_path = entry.path().wstring();
+                         }
+                     }
+                 }
++            } catch (const std::exception & e) {
++                GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), e.what());
+             }
+         }
+     }

llama/runner/README.md

@@ -4,18 +4,18 @@
 
 A minimial runner for loading a model and running inference via a http web server.
 
-```
+```shell
 ./runner -model <model binary>
 ```
 
 ### Completion
 
-```
+```shell
 curl -X POST -H "Content-Type: application/json" -d '{"prompt": "hi"}' http://localhost:8080/completion
 ```
 
 ### Embeddings
 
-```
+```shell
 curl -X POST -H "Content-Type: application/json" -d '{"prompt": "turn me into an embedding"}' http://localhost:8080/embedding
 ```

llama/runner/cache.go

@@ -1,4 +1,4 @@
-package oldrunner
+package runner
 
 import (
 	"errors"

llama/runner/cache_test.go

@@ -1,4 +1,4 @@
-package oldrunner
+package runner
 
 import (
 	"testing"

llama/runner/image.go

@@ -1,4 +1,4 @@
-package oldrunner
+package runner
 
 import (
 	"errors"

llama/runner/image_test.go

@@ -1,4 +1,4 @@
-package oldrunner
+package runner
 
 import (
 	"reflect"

llama/runner/runner.go

@@ -1,4 +1,4 @@
-package oldrunner
+package runner
 
 import (
 	"context"
@@ -8,12 +8,14 @@ import (
 	"fmt"
 	"log"
 	"log/slog"
+	"math"
 	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -24,7 +26,6 @@ import (
 
 	"github.com/ollama/ollama/api"
 	"github.com/ollama/ollama/llama"
-	"github.com/ollama/ollama/runner/common"
 )
 
 // input is an element of the prompt to process, either
@@ -49,8 +50,9 @@ type Sequence struct {
 	// inputs that have been added to a batch but not yet submitted to Decode
 	pendingInputs []input
 
+	// TODO: update this comment
 	// tokens that have been generated but not returned yet (e.g. for stop sequences)
-	pendingResponses []string
+	pendingResponses []CompletionResponse
 
 	// input cache being used by this sequence
 	cache *InputCacheSlot
@@ -60,7 +62,7 @@ type Sequence struct {
 	crossAttention bool
 
 	// channel to send responses over
-	responses chan string
+	responses chan CompletionResponse
 
 	// channel to stop decoding (such as if the remote connection is closed)
 	quit chan bool
@@ -84,6 +86,11 @@ type Sequence struct {
 
 	doneReason string
 
+	logits []float32
+
+	// number of logprobs to return with the completion response
+	logprobs int
+
 	// Metrics
 	startProcessingTime time.Time
 	startGenerationTime time.Time
@@ -97,6 +104,7 @@ type NewSequenceParams struct {
 	numKeep        int
 	samplingParams *llama.SamplingParams
 	embedding      bool
+	logprobs       int
 }
 
 func (s *Server) NewSequence(prompt string, images []ImageData, params NewSequenceParams) (*Sequence, error) {
@@ -149,14 +157,15 @@ func (s *Server) NewSequence(prompt string, images []ImageData, params NewSequen
 		numPromptInputs:     len(inputs),
 		startProcessingTime: startTime,
 		numPredict:          params.numPredict,
-		pendingResponses:    make([]string, 0),
-		responses:           make(chan string, 100),
+		pendingResponses:    make([]CompletionResponse, 0),
+		responses:           make(chan CompletionResponse, 100),
 		quit:                make(chan bool, 1),
 		embedding:           make(chan []float32, 1),
 		samplingCtx:         sc,
 		embeddingOnly:       params.embedding,
 		stop:                params.stop,
 		numKeep:             params.numKeep,
+		logprobs:            params.logprobs,
 	}, nil
 }
 
@@ -275,29 +284,37 @@ func (s *Server) allNil() bool {
 }
 
 func flushPending(seq *Sequence) bool {
-	joined := strings.Join(seq.pendingResponses, "")
-	seq.pendingResponses = []string{}
-
-	// Check if there are any partial UTF-8 characters remaining.
-	// We already check and queue as we are generating but some may
-	// still make it here:
-	// - Sequence is ending, e.g. generation limit has been hit
-	// - Invalid characters in the middle of a string
-	// This is a stricter check to ensure we never output invalid Unicode.
-	for !utf8.ValidString(joined) {
-		joined = joined[:len(joined)-1]
-	}
-
-	if len(joined) == 0 {
+	if len(seq.pendingResponses) == 0 {
 		return true
 	}
-
-	select {
-	case seq.responses <- joined:
-		return true
-	case <-seq.quit:
-		return false
+	resps := []CompletionResponse{}
+	for _, resp := range seq.pendingResponses {
+		resps = append(resps, resp)
 	}
+	seq.pendingResponses = []CompletionResponse{}
+
+	// TODO: figure out this result logic
+	result := false
+	for _, resp := range resps {
+		// Check if there are any partial UTF-8 characters remaining.
+		// We already check and queue as we are generating but some may
+		// still make it here:
+		// - Sequence is ending, e.g. generation limit has been hit
+		// - Invalid characters in the middle of a string
+		// This is a stricter check to ensure we never output invalid Unicode.
+		for !utf8.ValidString(resp.Content) {
+			resp.Content = resp.Content[:len(resp.Content)-1]
+		}
+
+		select {
+		case seq.responses <- resp:
+			result = true
+		case <-seq.quit:
+			result = false
+		}
+	}
+
+	return result
 }
 
 func (s *Server) removeSequence(seqIndex int, reason string) {
@@ -351,6 +368,63 @@ func (s *Server) run(ctx context.Context) {
 	}
 }
 
+// TokenProbs represents probability information for a token
+type TokenProbs struct {
+	TokenID int     `json:"id"`
+	Logit   float32 `json:"logit"`
+	Prob    float32 `json:"prob"`
+	LogProb float32 `json:"logprob"`
+	Token   string  `json:"token"`
+}
+
+// probs returns sorted token probabilities for a specific token index
+func probs(logits []float32, vocabSize int) []TokenProbs {
+	probs := make([]TokenProbs, vocabSize)
+
+	// Initialize token data with logits
+	for i := 0; i < vocabSize; i++ {
+		probs[i] = TokenProbs{
+			TokenID: i,
+			Logit:   logits[i],
+		}
+	}
+
+	// Sort tokens by logits in descending order
+	sort.Slice(probs, func(i, j int) bool {
+		return probs[i].Logit > probs[j].Logit
+	})
+
+	// Apply softmax
+	maxLogit := probs[0].Logit
+	var sum float32 = 0.0
+
+	for i := range probs {
+		p := float32(math.Exp(float64(probs[i].Logit - maxLogit)))
+		probs[i].Prob = p
+		sum += p
+	}
+
+	// Normalize probabilities and calculate log probs
+	for i := range probs {
+		prob := probs[i].Prob / sum
+		probs[i].Prob = prob
+		probs[i].LogProb = float32(math.Log(float64(prob)))
+	}
+
+	return probs
+}
+
+// probs returns sorted token probabilities for a specific token index
+func (s *Server) probs(seq *Sequence) []TokenProbs {
+	// Get logits for the specific token index
+	logits := s.lc.GetLogits()
+	seq.logits = make([]float32, len(logits))
+	copy(seq.logits, logits)
+
+	vocabSize := s.model.NumVocab()
+	return probs(logits, vocabSize)
+}
+
 // TODO (jmorganca): processBatch should be simplified, removing:
 // * sampling
 // * stop token checking
@@ -484,6 +558,19 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
 
 		seq.numPredicted++
 
+		resp := CompletionResponse{Content: piece}
+
+		if seq.logprobs > 0 {
+			// TODO: return selected token in logprobs always
+			resp.LogProbs = s.probs(seq)
+			// TODO: fix this logprobs limit
+			resp.LogProbs = resp.LogProbs[:min(len(resp.LogProbs), seq.logprobs)]
+			for i := range resp.LogProbs {
+				// decode the token id to a piece
+				resp.LogProbs[i].Token = s.model.TokenToPiece(resp.LogProbs[i].TokenID)
+			}
+		}
+
 		// if it's an end of sequence token, break
 		if s.model.TokenIsEog(token) {
 			// TODO (jmorganca): we should send this back
@@ -496,16 +583,21 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
 
 		seq.inputs = []input{{token: token}}
 
-		seq.pendingResponses = append(seq.pendingResponses, piece)
-		sequence := strings.Join(seq.pendingResponses, "")
+		// TODO: add probs here
+		seq.pendingResponses = append(seq.pendingResponses, resp)
+		var sequence string
+		for _, r := range seq.pendingResponses {
+			sequence += r.Content
+		}
 
-		if ok, stop := common.FindStop(sequence, seq.stop); ok {
+		if ok, stop := findStop(sequence, seq.stop); ok {
 			slog.Debug("hit stop token", "pending", seq.pendingResponses, "stop", stop)
 
+			// TODO: fix this stop sequence caching
 			var tokenTruncated bool
-			origLen := len(seq.pendingResponses)
-			seq.pendingResponses, tokenTruncated = common.TruncateStop(seq.pendingResponses, stop)
-			newLen := len(seq.pendingResponses)
+			origLen := len(sequence)
+			sequence, tokenTruncated = truncateStop(sequence, stop)
+			newLen := len(sequence)
 
 			// Update the cache based on the tokens that will be returned:
 			// - We have 1 token more than is currently in the cache because
@@ -525,11 +617,11 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
 			continue
 		}
 
-		if common.ContainsStopSuffix(sequence, seq.stop) {
+		if containsStopSuffix(sequence, seq.stop) {
 			continue
 		}
 
-		if common.IncompleteUnicode(sequence) {
+		if incompleteUnicode(sequence) {
 			continue
 		}
 
@@ -576,6 +668,7 @@ type CompletionRequest struct {
 	Images      []ImageData `json:"image_data"`
 	Grammar     string      `json:"grammar"`
 	CachePrompt bool        `json:"cache_prompt"`
+	Logprobs    int         `json:"logprobs,omitempty"`
 
 	Options
 }
@@ -591,8 +684,10 @@ type CompletionResponse struct {
 	Content string `json:"content"`
 	Stop    bool   `json:"stop"`
 
-	Model        string  `json:"model,omitempty"`
-	Prompt       string  `json:"prompt,omitempty"`
+	Model    string       `json:"model,omitempty"`
+	Prompt   string       `json:"prompt,omitempty"`
+	LogProbs []TokenProbs `json:"logprobs,omitempty"`
+
 	StoppedLimit bool    `json:"stopped_limit,omitempty"`
 	PredictedN   int     `json:"predicted_n,omitempty"`
 	PredictedMS  float64 `json:"predicted_ms,omitempty"`
@@ -610,10 +705,6 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Set the headers to indicate streaming
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Transfer-Encoding", "chunked")
-
 	flusher, ok := w.(http.Flusher)
 	if !ok {
 		http.Error(w, "Streaming not supported", http.StatusInternalServerError)
@@ -642,6 +733,7 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) {
 		numKeep:        req.NumKeep,
 		samplingParams: &samplingParams,
 		embedding:      false,
+		logprobs:       req.Logprobs,
 	})
 	if err != nil {
 		http.Error(w, fmt.Sprintf("Failed to create new sequence: %v", err), http.StatusInternalServerError)
@@ -689,11 +781,10 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) {
 		case <-r.Context().Done():
 			close(seq.quit)
 			return
-		case content, ok := <-seq.responses:
+		case resp, ok := <-seq.responses:
 			if ok {
-				if err := json.NewEncoder(w).Encode(&CompletionResponse{
-					Content: content,
-				}); err != nil {
+				fmt.Println("response", resp)
+				if err := json.NewEncoder(w).Encode(&resp); err != nil {
 					http.Error(w, fmt.Sprintf("failed to encode response: %v", err), http.StatusInternalServerError)
 					close(seq.quit)
 					return
@@ -886,6 +977,9 @@ func (s *Server) loadModel(
 }
 
 func Execute(args []string) error {
+	if args[0] == "runner" {
+		args = args[1:]
+	}
 	fs := flag.NewFlagSet("runner", flag.ExitOnError)
 	mpath := fs.String("model", "", "Path to model binary file")
 	ppath := fs.String("mmproj", "", "Path to projector binary file")

llama/runner/runner_test.go

@@ -0,0 +1,58 @@
+package runner
+
+import (
+	"math"
+	"testing"
+)
+
+func TestProbs(t *testing.T) {
+	// Input test data
+	logits := []float32{1.0, 2.0, 0.5, -1.0}
+	vocabSize := 4
+	want := []TokenProbs{
+		{TokenID: 1, Logit: 2.0},  // Highest logit
+		{TokenID: 0, Logit: 1.0},  // Second highest
+		{TokenID: 2, Logit: 0.5},  // Third
+		{TokenID: 3, Logit: -1.0}, // Lowest
+	}
+
+	got := probs(logits, vocabSize)
+
+	// Test 1: Check sorting order
+	for i := 0; i < len(got)-1; i++ {
+		if got[i].Logit < got[i+1].Logit {
+			t.Errorf("probs not properly sorted: logit at pos %d (%f) < logit at pos %d (%f)",
+				i, got[i].Logit, i+1, got[i+1].Logit)
+		}
+	}
+
+	// Test 2: Check probability normalization
+	var sum float32
+	for _, p := range got {
+		sum += p.Prob
+	}
+	if math.Abs(float64(sum-1.0)) > 1e-6 {
+		t.Errorf("probabilities do not sum to 1: got %v", sum)
+	}
+
+	// Test 3: Check token IDs match expected order
+	for i, want := range want {
+		if got[i].TokenID != want.TokenID {
+			t.Errorf("wrong token ID at position %d: got %d, want %d",
+				i, got[i].TokenID, want.TokenID)
+		}
+		if got[i].Logit != want.Logit {
+			t.Errorf("wrong logit at position %d: got %f, want %f",
+				i, got[i].Logit, want.Logit)
+		}
+	}
+
+	// Test 4: Check log probs are correctly calculated
+	for i, p := range got {
+		expectedLogProb := float32(math.Log(float64(p.Prob)))
+		if math.Abs(float64(p.LogProb-expectedLogProb)) > 1e-6 {
+			t.Errorf("wrong log prob at position %d: got %f, want %f",
+				i, p.LogProb, expectedLogProb)
+		}
+	}
+}

llama/runner/stop.go

@@ -0,0 +1,68 @@
+package runner
+
+import (
+	"strings"
+)
+
+func findStop(sequence string, stops []string) (bool, string) {
+	for _, stop := range stops {
+		if strings.Contains(sequence, stop) {
+			return true, stop
+		}
+	}
+
+	return false, ""
+}
+
+func containsStopSuffix(sequence string, stops []string) bool {
+	for _, stop := range stops {
+		for i := 1; i <= len(stop); i++ {
+			if strings.HasSuffix(sequence, stop[:i]) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// truncateStop removes the provided stop string from sequence,
+// returning both the truncated sequence and a bool indicating if truncation occurred
+func truncateStop(sequence string, stop string) (string, bool) {
+	index := strings.Index(sequence, stop)
+	if index == -1 {
+		return sequence, false
+	}
+
+	return sequence[:index], true
+}
+
+func incompleteUnicode(token string) bool {
+	incomplete := false
+
+	// check if there is incomplete UTF-8 character at the end
+	for i := 1; i < 5 && i <= len(token); i++ {
+		c := token[len(token)-i]
+
+		if (c & 0xc0) == 0x80 {
+			// continuation byte: 10xxxxxx
+			continue
+		}
+
+		if (c & 0xe0) == 0xc0 {
+			// 2-byte character: 110xxxxx ...
+			incomplete = i < 2
+		} else if (c & 0xf0) == 0xe0 {
+			// 3-byte character: 1110xxxx ...
+			incomplete = i < 3
+		} else if (c & 0xf8) == 0xf0 {
+			// 4-byte character: 11110xxx ...
+			incomplete = i < 4
+		}
+
+		// else 1-byte character or invalid byte
+		break
+	}
+
+	return incomplete
+}

llama/runner/stop_test.go

@@ -1,60 +1,60 @@
-package common
+package runner
 
 import (
-	"reflect"
 	"testing"
 )
 
 func TestTruncateStop(t *testing.T) {
 	tests := []struct {
 		name          string
-		pieces        []string
+		sequence      string
 		stop          string
-		expected      []string
+		expected      string
 		expectedTrunc bool
 	}{
 		{
 			name:          "Single word",
-			pieces:        []string{"hello", "world"},
+			sequence:      "helloworld",
 			stop:          "world",
-			expected:      []string{"hello"},
-			expectedTrunc: false,
+			expected:      "hello",
+			expectedTrunc: true,
 		},
 		{
 			name:          "Partial",
-			pieces:        []string{"hello", "wor"},
+			sequence:      "hellowor",
 			stop:          "or",
-			expected:      []string{"hello", "w"},
+			expected:      "hellow",
 			expectedTrunc: true,
 		},
 		{
 			name:          "Suffix",
-			pieces:        []string{"Hello", " there", "!"},
+			sequence:      "Hello there!",
 			stop:          "!",
-			expected:      []string{"Hello", " there"},
-			expectedTrunc: false,
-		},
-		{
-			name:          "Suffix partial",
-			pieces:        []string{"Hello", " the", "re!"},
-			stop:          "there!",
-			expected:      []string{"Hello", " "},
+			expected:      "Hello there",
 			expectedTrunc: true,
 		},
 		{
 			name:          "Middle",
-			pieces:        []string{"hello", " wor"},
+			sequence:      "hello wor",
 			stop:          "llo w",
-			expected:      []string{"he"},
+			expected:      "he",
 			expectedTrunc: true,
 		},
+		{
+			name:          "No stop found",
+			sequence:      "hello world",
+			stop:          "xyz",
+			expected:      "hello world",
+			expectedTrunc: false,
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result, resultTrunc := TruncateStop(tt.pieces, tt.stop)
-			if !reflect.DeepEqual(result, tt.expected) || resultTrunc != tt.expectedTrunc {
-				t.Errorf("truncateStop(%v, %s): have %v (%v); want %v (%v)", tt.pieces, tt.stop, result, resultTrunc, tt.expected, tt.expectedTrunc)
+			result, truncated := truncateStop(tt.sequence, tt.stop)
+			if result != tt.expected || truncated != tt.expectedTrunc {
+				t.Errorf("truncateStop(%q, %q): have %q (%v); want %q (%v)",
+					tt.sequence, tt.stop, result, truncated, tt.expected, tt.expectedTrunc)
 			}
 		})
 	}
@@ -120,7 +120,7 @@ func TestIncompleteUnicode(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result := IncompleteUnicode(tt.input)
+			result := incompleteUnicode(tt.input)
 			if result != tt.expected {
 				t.Errorf("incompleteUnicode(%s): have %v; want %v", tt.input, result, tt.expected)
 			}

llm/filetype.go

@@ -1,4 +1,4 @@
-package ggml
+package llm
 
 import "fmt"
 
@@ -32,9 +32,10 @@ const (
 	fileTypeIQ1_S
 	fileTypeIQ4_NL
 	fileTypeIQ3_S
+	fileTypeIQ3_M
 	fileTypeIQ2_S
-	fileTypeIQ4_XS
 	fileTypeIQ2_M
+	fileTypeIQ4_XS
 	fileTypeIQ1_M
 	fileTypeBF16
 
@@ -93,6 +94,8 @@ func ParseFileType(s string) (fileType, error) {
 		return fileTypeIQ4_NL, nil
 	case "IQ3_S":
 		return fileTypeIQ3_S, nil
+	case "IQ3_M":
+		return fileTypeIQ3_M, nil
 	case "IQ2_S":
 		return fileTypeIQ2_S, nil
 	case "IQ4_XS":
@@ -160,6 +163,8 @@ func (t fileType) String() string {
 		return "IQ4_NL"
 	case fileTypeIQ3_S:
 		return "IQ3_S"
+	case fileTypeIQ3_M:
+		return "IQ3_M"
 	case fileTypeIQ2_S:
 		return "IQ2_S"
 	case fileTypeIQ4_XS:

llm/ggla.go

@@ -0,0 +1,149 @@
+package llm
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+	"slices"
+)
+
+type containerGGLA struct {
+	version uint32
+}
+
+func (c *containerGGLA) Name() string {
+	return "ggla"
+}
+
+func (c *containerGGLA) Decode(rs io.ReadSeeker) (model, error) {
+	if err := binary.Read(rs, binary.LittleEndian, &c.version); err != nil {
+		return nil, err
+	}
+
+	switch c.version {
+	case 1:
+	default:
+		return nil, errors.New("invalid version")
+	}
+
+	model := newGGLA(c)
+	err := model.decode(rs)
+	return model, err
+}
+
+type ggla struct {
+	*containerGGLA
+
+	kv      KV
+	tensors []*Tensor
+
+	tensorOffset uint64
+}
+
+func newGGLA(container *containerGGLA) *ggla {
+	return &ggla{
+		containerGGLA: container,
+		kv:            make(KV),
+	}
+}
+
+func (llm *ggla) KV() KV {
+	return llm.kv
+}
+
+func (llm *ggla) Tensors() *Tensors {
+	return &Tensors{
+		Items:  llm.tensors,
+		Offset: llm.tensorOffset,
+	}
+}
+
+func (llm *ggla) decode(rs io.ReadSeeker) (retErr error) {
+	var r uint32
+	if err := binary.Read(rs, binary.LittleEndian, &r); err != nil {
+		return err
+	}
+	llm.kv["r"] = r
+
+	var alpha uint32
+	if err := binary.Read(rs, binary.LittleEndian, &alpha); err != nil {
+		return err
+	}
+	llm.kv["alpha"] = alpha
+
+	offset, err := rs.Seek(0, io.SeekCurrent)
+	if err != nil {
+		return err
+	}
+
+	llm.tensorOffset = uint64(offset)
+
+	for {
+		var dims uint32
+		if err := binary.Read(rs, binary.LittleEndian, &dims); err != nil {
+			if errors.Is(err, io.EOF) {
+				return nil
+			}
+			return err
+		}
+
+		defer func() {
+			if errors.Is(retErr, io.EOF) {
+				retErr = io.ErrUnexpectedEOF
+			}
+		}()
+
+		var namesize uint32
+		if err := binary.Read(rs, binary.LittleEndian, &namesize); err != nil {
+			return err
+		}
+
+		var t Tensor
+		if err := binary.Read(rs, binary.LittleEndian, &t.Kind); err != nil {
+			return err
+		}
+
+		t.Shape = make([]uint64, dims)
+		for i := 0; uint32(i) < dims; i++ {
+			var shape32 uint32
+			if err := binary.Read(rs, binary.LittleEndian, &shape32); err != nil {
+				return err
+			}
+
+			t.Shape[i] = uint64(shape32)
+		}
+
+		// ggla tensor shape is reversed
+		// ref: https://github.com/ggerganov/llama.cpp/blob/29ae62d2ae163e2b68aa0ad3bf2ab4636de0c957/convert-lora-to-ggml.py#L44
+		slices.Reverse(t.Shape)
+
+		name := make([]byte, namesize)
+		if err := binary.Read(rs, binary.LittleEndian, &name); err != nil {
+			return err
+		}
+
+		t.Name = string(name)
+
+		offset, err := rs.Seek(0, io.SeekCurrent)
+		if err != nil {
+			return err
+		}
+
+		if _, err := rs.Seek((offset+31)&-32-offset, io.SeekCurrent); err != nil {
+			return err
+		}
+
+		offset, err = rs.Seek(0, io.SeekCurrent)
+		if err != nil {
+			return err
+		}
+
+		t.Offset = uint64(offset)
+
+		if _, err := rs.Seek(int64(t.Size()), io.SeekCurrent); err != nil {
+			return err
+		}
+
+		llm.tensors = append(llm.tensors, &t)
+	}
+}

llm/ggml.go

@@ -1,15 +1,15 @@
-package ggml
+package llm
 
 import (
 	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
-	"log/slog"
 	"slices"
 	"strings"
+	"sync"
 
-	"github.com/ollama/ollama/fs/util/bufioutil"
+	"github.com/ollama/ollama/util/bufioutil"
 )
 
 type GGML struct {
@@ -19,168 +19,145 @@ type GGML struct {
 
 type model interface {
 	KV() KV
-	Tensors() Tensors
+	Tensors() *Tensors
 }
 
 type KV map[string]any
 
+func (kv KV) u64(key string) uint64 {
+	switch v := kv[key].(type) {
+	case uint64:
+		return v
+	case uint32:
+		return uint64(v)
+	case float64:
+		return uint64(v)
+	default:
+		return 0
+	}
+}
+
 func (kv KV) Architecture() string {
-	return kv.String("general.architecture", "unknown")
+	if s, ok := kv["general.architecture"].(string); ok {
+		return s
+	}
+
+	return "unknown"
 }
 
 func (kv KV) Kind() string {
-	return kv.String("general.type", "unknown")
+	if s, ok := kv["general.type"].(string); ok {
+		return s
+	}
+
+	return "unknown"
 }
 
 func (kv KV) ParameterCount() uint64 {
-	return keyValue[uint64](kv, "general.parameter_count")
+	return kv.u64("general.parameter_count")
 }
 
 func (kv KV) FileType() fileType {
-	if t := kv.Uint("general.file_type"); t > 0 {
-		return fileType(t)
+	if u64 := kv.u64("general.file_type"); u64 > 0 {
+		return fileType(uint32(u64))
 	}
 
 	return fileTypeUnknown
 }
 
 func (kv KV) BlockCount() uint64 {
-	return uint64(kv.Uint("block_count"))
-}
-
-func (kv KV) EmbeddingLength() uint64 {
-	return uint64(kv.Uint("embedding_length"))
+	return kv.u64(fmt.Sprintf("%s.block_count", kv.Architecture()))
 }
 
 func (kv KV) HeadCount() uint64 {
-	return uint64(kv.Uint("attention.head_count"))
+	return kv.u64(fmt.Sprintf("%s.attention.head_count", kv.Architecture()))
 }
 
 func (kv KV) HeadCountKV() uint64 {
-	return uint64(kv.Uint("attention.head_count_kv", 1))
+	if headCountKV := kv.u64(fmt.Sprintf("%s.attention.head_count_kv", kv.Architecture())); headCountKV > 0 {
+		return headCountKV
+	}
+
+	return 1
 }
 
 func (kv KV) EmbeddingHeadCount() uint64 {
 	if heads := kv.HeadCount(); heads > 0 {
-		return kv.EmbeddingLength() / heads
+		return kv.EmbeddingLength() / kv.HeadCount()
 	}
 
 	return 0
 }
 
 func (kv KV) EmbeddingHeadCountK() uint64 {
-	return uint64(kv.Uint("attention.key_length", uint32(kv.EmbeddingHeadCount())))
+	if k := kv.u64(fmt.Sprintf("%s.attention.key_length", kv.Architecture())); k > 0 {
+		return k
+	}
+
+	return kv.EmbeddingHeadCount()
 }
 
 func (kv KV) EmbeddingHeadCountV() uint64 {
-	return uint64(kv.Uint("attention.value_length", uint32(kv.EmbeddingHeadCount())))
+	if v := kv.u64(fmt.Sprintf("%s.attention.value_length", kv.Architecture())); v > 0 {
+		return v
+	}
+
+	return kv.EmbeddingHeadCount()
 }
 
 func (kv KV) GQA() uint64 {
 	return kv.HeadCount() / kv.HeadCountKV()
 }
 
+func (kv KV) EmbeddingLength() uint64 {
+	return kv.u64(fmt.Sprintf("%s.embedding_length", kv.Architecture()))
+}
+
 func (kv KV) ContextLength() uint64 {
-	return uint64(kv.Uint("context_length"))
+	return kv.u64(fmt.Sprintf("%s.context_length", kv.Architecture()))
 }
 
 func (kv KV) ChatTemplate() string {
-	return kv.String("tokenizer.chat_template")
-}
-
-func (kv KV) String(key string, defaultValue ...string) string {
-	return keyValue(kv, key, append(defaultValue, "")...)
-}
-
-func (kv KV) Uint(key string, defaultValue ...uint32) uint32 {
-	return keyValue(kv, key, append(defaultValue, 0)...)
-}
-
-func (kv KV) Float(key string, defaultValue ...float32) float32 {
-	return keyValue(kv, key, append(defaultValue, 0)...)
-}
-
-func (kv KV) Strings(key string, defaultValue ...[]string) []string {
-	r := keyValue(kv, key, &array{})
-	s := make([]string, r.size)
-	for i := range r.size {
-		s[i] = r.values[i].(string)
-	}
-
+	s, _ := kv["tokenizer.chat_template"].(string)
 	return s
 }
 
-func (kv KV) Uints(key string, defaultValue ...[]uint32) []uint32 {
-	r := keyValue(kv, key, &array{})
-	s := make([]uint32, r.size)
-	for i := range r.size {
-		s[i] = uint32(r.values[i].(int32))
-	}
-
-	return s
-}
-
-func keyValue[T string | uint32 | uint64 | float32 | *array](kv KV, key string, defaultValue ...T) T {
-	if !strings.HasPrefix(key, "tokenizer.") && !strings.HasPrefix(key, "general.") {
-		key = kv.Architecture() + "." + key
-	}
-
-	if val, ok := kv[key]; ok {
-		return val.(T)
-	}
-
-	slog.Warn("key not found", "key", key, "default", defaultValue[0])
-	return defaultValue[0]
-}
-
 type Tensors struct {
-	items  []*Tensor
+	Items  []*Tensor
 	Offset uint64
+
+	layers     map[string]Layer
+	layersOnce sync.Once
 }
 
-func (s Tensors) Items(prefix ...string) []*Tensor {
-	if len(prefix) == 0 {
-		return s.items
-	}
+func (ts *Tensors) Layers() map[string]Layer {
+	ts.layersOnce.Do(func() {
+		ts.layers = make(map[string]Layer)
+		for _, t := range ts.Items {
+			parts := strings.Split(t.Name, ".")
+			if index := slices.IndexFunc(parts, func(s string) bool { return s == "blk" || s == "mm" }); index != -1 {
+				if len(parts) > index+2 {
+					// blk and mm should have a number after them, join it
+					parts = append(
+						[]string{strings.Join(parts[:index+2], ".")},
+						parts[index+2:]...)
+				}
+			}
 
-	var items []*Tensor
-	for _, t := range s.items {
-		if strings.HasPrefix(t.Name, prefix[0]) {
-			items = append(items, t)
+			if _, ok := ts.layers[parts[0]]; !ok {
+				ts.layers[parts[0]] = make(Layer)
+			}
+
+			ts.layers[parts[0]][strings.Join(parts[1:], ".")] = t
 		}
-	}
+	})
 
-	return items
-}
-
-func (ts Tensors) Layers() map[string]Layer {
-	layers := make(map[string]Layer)
-	for _, t := range ts.items {
-		parts := strings.Split(t.Name, ".")
-		if i := slices.Index(parts, "blk"); i > 0 {
-			parts = append([]string{
-				strings.Join(parts[:i], "."),
-				strings.Join(parts[i:i+2], "."),
-			}, parts[i+2:]...)
-		} else if i == 0 {
-			parts = append([]string{
-				strings.Join(parts[i:i+2], "."),
-			}, parts[i+2:]...)
-		}
-
-		if _, ok := layers[parts[0]]; !ok {
-			layers[parts[0]] = make(Layer)
-		}
-
-		layers[parts[0]][strings.Join(parts[1:], ".")] = t
-	}
-
-	return layers
+	return ts.layers
 }
 
 type Layer map[string]*Tensor
 
-func (l Layer) Size() (size uint64) {
+func (l Layer) size() (size uint64) {
 	for _, t := range l {
 		size += t.Size()
 	}
@@ -278,6 +255,8 @@ func (t Tensor) typeSize() uint64 {
 		return 8
 	case 29: // IQ1_M
 		return blockSize/8 + blockSize/16 + blockSize/32
+	case 30: // BF16
+		return 2
 	default:
 		return 0
 	}
@@ -316,7 +295,7 @@ const (
 
 var ErrUnsupportedFormat = errors.New("unsupported model format")
 
-func DetectContentType(b []byte) string {
+func DetectGGMLType(b []byte) string {
 	switch binary.LittleEndian.Uint32(b[:4]) {
 	case FILE_MAGIC_GGML:
 		return "ggml"
@@ -333,12 +312,12 @@ func DetectContentType(b []byte) string {
 	}
 }
 
-// Decode decodes a GGML model from the given reader.
+// DecodeGGML decodes a GGML model from the given reader.
 //
 // It collects array values for arrays with a size less than or equal to
 // maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If
 // the maxArraySize is negative, all arrays are collected.
-func Decode(rs io.ReadSeeker, maxArraySize int) (*GGML, int64, error) {
+func DecodeGGML(rs io.ReadSeeker, maxArraySize int) (*GGML, int64, error) {
 	if maxArraySize == 0 {
 		maxArraySize = 1024
 	}
@@ -352,6 +331,10 @@ func Decode(rs io.ReadSeeker, maxArraySize int) (*GGML, int64, error) {
 
 	var c container
 	switch magic {
+	case FILE_MAGIC_GGML, FILE_MAGIC_GGMF, FILE_MAGIC_GGJT:
+		return nil, 0, ErrUnsupportedFormat
+	case FILE_MAGIC_GGLA:
+		c = &containerGGLA{}
 	case FILE_MAGIC_GGUF_LE:
 		c = &containerGGUF{ByteOrder: binary.LittleEndian, maxArraySize: maxArraySize}
 	case FILE_MAGIC_GGUF_BE:
@@ -547,20 +530,21 @@ func (llm GGML) GraphSize(context, batch uint64, kvCacheType string) (kv, partia
 }
 
 // SupportsKVCacheType checks if the requested cache type is supported
-func (llm GGML) SupportsKVCacheType(cacheType string) bool {
-	return slices.Contains([]string{"f16", "q8_0", "q4_0"}, cacheType)
+func (ggml GGML) SupportsKVCacheType(cacheType string) bool {
+	validKVCacheTypes := []string{"f16", "q8_0", "q4_0"}
+	return slices.Contains(validKVCacheTypes, cacheType)
 }
 
 // SupportsFlashAttention checks if the model supports flash attention
-func (llm GGML) SupportsFlashAttention() bool {
-	_, isEmbedding := llm.KV()[fmt.Sprintf("%s.pooling_type", llm.KV().Architecture())]
+func (ggml GGML) SupportsFlashAttention() bool {
+	_, isEmbedding := ggml.KV()[fmt.Sprintf("%s.pooling_type", ggml.KV().Architecture())]
 	if isEmbedding {
 		return false
 	}
 
 	// Check head counts match and are non-zero
-	headCountK := llm.KV().EmbeddingHeadCountK()
-	headCountV := llm.KV().EmbeddingHeadCountV()
+	headCountK := ggml.KV().EmbeddingHeadCountK()
+	headCountV := ggml.KV().EmbeddingHeadCountV()
 	return headCountK != 0 && headCountV != 0 && headCountK == headCountV
 }
 

llm/ggml_test.go

@@ -0,0 +1 @@
+package llm

llm/gguf.go

@@ -1,4 +1,4 @@
-package ggml
+package llm
 
 import (
 	"bytes"
@@ -8,9 +8,10 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
-	"maps"
 	"slices"
 	"strings"
+
+	"golang.org/x/exp/maps"
 )
 
 type containerGGUF struct {
@@ -109,9 +110,9 @@ func (llm *gguf) KV() KV {
 	return llm.kv
 }
 
-func (llm *gguf) Tensors() Tensors {
-	return Tensors{
-		items:  llm.tensors,
+func (llm *gguf) Tensors() *Tensors {
+	return &Tensors{
+		Items:  llm.tensors,
 		Offset: llm.tensorOffset,
 	}
 }
@@ -522,7 +523,7 @@ func WriteGGUF(ws io.WriteSeeker, kv KV, ts []Tensor) error {
 		return err
 	}
 
-	keys := slices.Collect(maps.Keys(kv))
+	keys := maps.Keys(kv)
 	slices.Sort(keys)
 
 	for _, key := range keys {

llm/memory.go

@@ -11,19 +11,18 @@ import (
 	"github.com/ollama/ollama/discover"
 	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
-	"github.com/ollama/ollama/fs/ggml"
 )
 
 // This algorithm looks for a complete fit to determine if we need to unload other models
-func PredictServerFit(allGpus discover.GpuInfoList, f *ggml.GGML, adapters, projectors []string, opts api.Options) (bool, uint64) {
+func PredictServerFit(allGpus discover.GpuInfoList, ggml *GGML, adapters, projectors []string, opts api.Options) (bool, uint64) {
 	// Split up the GPUs by type and try them
 	var estimatedVRAM uint64
 	for _, gpus := range allGpus.ByLibrary() {
 		var layerCount int
-		estimate := EstimateGPULayers(gpus, f, projectors, opts)
+		estimate := EstimateGPULayers(gpus, ggml, projectors, opts)
 		layerCount, estimatedVRAM = estimate.Layers, estimate.VRAMSize
 		if opts.NumGPU < 0 {
-			if layerCount > 0 && layerCount >= int(f.KV().BlockCount()+1) {
+			if layerCount > 0 && layerCount >= int(ggml.KV().BlockCount()+1) {
 				return true, estimatedVRAM
 			}
 		} else {
@@ -71,7 +70,7 @@ type MemoryEstimate struct {
 
 // Given a model and one or more GPU targets, predict how many layers and bytes we can load, and the total size
 // The GPUs provided must all be the same Library
-func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []string, opts api.Options) MemoryEstimate {
+func EstimateGPULayers(gpus []discover.GpuInfo, ggml *GGML, projectors []string, opts api.Options) MemoryEstimate {
 	// Graph size for a partial offload, applies to all GPUs
 	var graphPartialOffload uint64
 
@@ -116,31 +115,33 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
 		opts.NumCtx = max(opts.NumCtx, 2048)
 	}
 
-	layers := f.Tensors().Layers()
+	layers := ggml.Tensors().Layers()
 	// add one layer worth of memory as a buffer
 	if blk0, ok := layers["blk.0"]; ok {
-		layerSize = blk0.Size()
+		layerSize = blk0.size()
 	} else {
 		slog.Warn("model missing blk.0 layer size")
 	}
 
-	var kvct string
-	if envconfig.FlashAttention() &&
+	fa := envconfig.FlashAttention() &&
 		discover.GetGPUInfo().FlashAttentionSupported() &&
-		f.SupportsFlashAttention() {
+		ggml.SupportsFlashAttention()
+
+	var kvct string
+	if fa {
 		requested := strings.ToLower(envconfig.KvCacheType())
-		if requested != "" && f.SupportsKVCacheType(requested) {
+		if requested != "" && ggml.SupportsKVCacheType(requested) {
 			kvct = requested
 		}
 	}
 
-	kv, graphPartialOffload, graphFullOffload := f.GraphSize(uint64(opts.NumCtx), uint64(min(opts.NumCtx, opts.NumBatch)), kvct)
+	kv, graphPartialOffload, graphFullOffload := ggml.GraphSize(uint64(opts.NumCtx), uint64(min(opts.NumCtx, opts.NumBatch)), kvct)
 
 	// KV is proportional to the number of layers
-	layerSize += kv / f.KV().BlockCount()
+	layerSize += kv / ggml.KV().BlockCount()
 
 	if graphPartialOffload == 0 {
-		graphPartialOffload = f.KV().GQA() * kv / 6
+		graphPartialOffload = ggml.KV().GQA() * kv / 6
 	}
 	if graphFullOffload == 0 {
 		graphFullOffload = graphPartialOffload
@@ -155,12 +156,12 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
 	}
 
 	if layer, ok := layers["output_norm"]; ok {
-		memoryLayerOutput += layer.Size()
+		memoryLayerOutput += layer.size()
 	}
 	if layer, ok := layers["output"]; ok {
-		memoryLayerOutput += layer.Size()
+		memoryLayerOutput += layer.size()
 	} else if layer, ok := layers["token_embd"]; ok {
-		memoryLayerOutput += layer.Size()
+		memoryLayerOutput += layer.size()
 	}
 
 	// Output layer handled at the end if we have space
@@ -210,11 +211,11 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
 	}
 
 	// For all the layers, find where they can fit on the GPU(s)
-	for i := range int(f.KV().BlockCount()) {
+	for i := range int(ggml.KV().BlockCount()) {
 		// Some models have inconsistent layer sizes
 		if blk, ok := layers[fmt.Sprintf("blk.%d", i)]; ok {
-			layerSize = blk.Size()
-			layerSize += kv / f.KV().BlockCount()
+			layerSize = blk.size()
+			layerSize += kv / ggml.KV().BlockCount()
 		}
 		memoryWeights += layerSize
 
@@ -237,10 +238,10 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
 			}
 		}
 	}
-	if layerCount >= int(f.KV().BlockCount()) {
+	if layerCount >= int(ggml.KV().BlockCount()) {
 		fullyLoaded = true
 	} else {
-		for i := layerCount; i < int(f.KV().BlockCount()); i++ {
+		for i := layerCount; i < int(ggml.KV().BlockCount()); i++ {
 			overflow += layerSize
 		}
 	}
@@ -258,7 +259,7 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
 			}
 		}
 
-		if layerCount < int(f.KV().BlockCount())+1 {
+		if layerCount < int(ggml.KV().BlockCount())+1 {
 			fullyLoaded = false
 			overflow += memoryLayerOutput
 		}
@@ -310,7 +311,7 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
 
 		inferenceLibrary:    gpus[0].Library,
 		layersRequested:     opts.NumGPU,
-		layersModel:         int(f.KV().BlockCount()) + 1,
+		layersModel:         int(ggml.KV().BlockCount()) + 1,
 		availableList:       availableList,
 		kv:                  kv,
 		allocationsList:     allocationsList,
@@ -338,9 +339,22 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
 	return estimate
 }
 
-func (m MemoryEstimate) LogValue() slog.Value {
-	attrs := []slog.Attr{
-		slog.String("library", m.inferenceLibrary),
+func (m MemoryEstimate) log() {
+	overhead := envconfig.GpuOverhead()
+
+	log := slog.With()
+	if m.projectorWeights > 0 {
+		log = log.With(
+			slog.Group(
+				"projector",
+				"weights", format.HumanBytes2(m.projectorWeights),
+				"graph", format.HumanBytes2(m.projectorGraph),
+			),
+		)
+	}
+
+	log.Info(
+		"offload to "+m.inferenceLibrary,
 		slog.Group(
 			"layers",
 			// requested number of layers to offload
@@ -356,7 +370,7 @@ func (m MemoryEstimate) LogValue() slog.Value {
 			"memory",
 			// memory available by GPU for offloading
 			"available", m.availableList,
-			"gpu_overhead", format.HumanBytes2(envconfig.GpuOverhead()),
+			"gpu_overhead", format.HumanBytes2(overhead),
 			slog.Group(
 				"required",
 				// memory required for full offloading
@@ -385,17 +399,7 @@ func (m MemoryEstimate) LogValue() slog.Value {
 				"partial", format.HumanBytes2(m.graphPartialOffload),
 			),
 		),
-	}
-
-	if m.projectorWeights > 0 {
-		attrs = append(attrs, slog.Group(
-			"projector",
-			"weights", format.HumanBytes2(m.projectorWeights),
-			"graph", format.HumanBytes2(m.projectorGraph),
-		))
-	}
-
-	return slog.GroupValue(attrs...)
+	)
 }
 
 func projectorMemoryRequirements(filename string) (weights, graphSize uint64) {
@@ -405,13 +409,13 @@ func projectorMemoryRequirements(filename string) (weights, graphSize uint64) {
 	}
 	defer file.Close()
 
-	ggml, _, err := ggml.Decode(file, 0)
+	ggml, _, err := DecodeGGML(file, 0)
 	if err != nil {
 		return 0, 0
 	}
 
 	for _, layer := range ggml.Tensors().Layers() {
-		weights += layer.Size()
+		weights += layer.size()
 	}
 
 	switch arch := ggml.KV().Architecture(); arch {

llm/memory_test.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/ollama/ollama/api"
 	"github.com/ollama/ollama/discover"
-	"github.com/ollama/ollama/fs/ggml"
 )
 
 func TestEstimateGPULayers(t *testing.T) {
@@ -24,7 +23,7 @@ func TestEstimateGPULayers(t *testing.T) {
 	defer f.Close()
 	inputLayerCount := 5
 
-	tensors := []ggml.Tensor{
+	tensors := []Tensor{
 		{Name: "blk.0.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
 		{Name: "blk.1.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
 		{Name: "blk.2.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
@@ -33,7 +32,7 @@ func TestEstimateGPULayers(t *testing.T) {
 		{Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
 	}
 	assert.Len(t, tensors, inputLayerCount+1)
-	err = ggml.WriteGGUF(f, ggml.KV{
+	err = WriteGGUF(f, KV{
 		"general.architecture":          "llama",
 		"llama.context_length":          uint32(32),
 		"llama.embedding_length":        uint32(4096),

llm/server.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ollama/ollama/discover"
 	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
-	"github.com/ollama/ollama/fs/ggml"
 	"github.com/ollama/ollama/llama"
 )
 
@@ -72,7 +71,7 @@ type llmServer struct {
 // It collects array values for arrays with a size less than or equal to
 // maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If
 // the maxArraySize is negative, all arrays are collected.
-func LoadModel(model string, maxArraySize int) (*ggml.GGML, error) {
+func LoadModel(model string, maxArraySize int) (*GGML, error) {
 	if _, err := os.Stat(model); err != nil {
 		return nil, err
 	}
@@ -83,17 +82,21 @@ func LoadModel(model string, maxArraySize int) (*ggml.GGML, error) {
 	}
 	defer f.Close()
 
-	ggml, _, err := ggml.Decode(f, maxArraySize)
+	ggml, _, err := DecodeGGML(f, maxArraySize)
 	return ggml, err
 }
 
 // NewLlamaServer will run a server for the given GPUs
 // The gpu list must be a single family.
-func NewLlamaServer(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error) {
+func NewLlamaServer(gpus discover.GpuInfoList, model string, ggml *GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error) {
+	var systemTotalMemory uint64
+	var systemFreeMemory uint64
+	var systemSwapFreeMemory uint64
+
 	systemInfo := discover.GetSystemInfo()
-	systemTotalMemory := systemInfo.System.TotalMemory
-	systemFreeMemory := systemInfo.System.FreeMemory
-	systemSwapFreeMemory := systemInfo.System.FreeSwap
+	systemTotalMemory = systemInfo.System.TotalMemory
+	systemFreeMemory = systemInfo.System.FreeMemory
+	systemSwapFreeMemory = systemInfo.System.FreeSwap
 	slog.Info("system memory", "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "free_swap", format.HumanBytes2(systemSwapFreeMemory))
 
 	// If the user wants zero GPU layers, reset the gpu list to be CPU/system ram info
@@ -101,12 +104,8 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapt
 		gpus = discover.GetCPUInfo()
 	}
 
-	var estimate MemoryEstimate
-	if len(gpus) == 1 && gpus[0].Library == "cpu" {
-		estimate = EstimateGPULayers(gpus, f, projectors, opts)
-	} else {
-		estimate = EstimateGPULayers(gpus, f, projectors, opts)
-
+	estimate := EstimateGPULayers(gpus, ggml, projectors, opts)
+	if len(gpus) > 1 || gpus[0].Library != "cpu" {
 		switch {
 		case gpus[0].Library == "metal" && estimate.VRAMSize > systemTotalMemory:
 			// disable partial offloading when model is greater than total system memory as this
@@ -131,7 +130,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapt
 		}
 	}
 
-	slog.Info("offload", "", estimate)
+	estimate.log()
 
 	params := []string{
 		"--model", model,
@@ -175,7 +174,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapt
 		fa = false
 	}
 
-	if fa && !f.SupportsFlashAttention() {
+	if fa && !ggml.SupportsFlashAttention() {
 		slog.Warn("flash attention enabled but not supported by model")
 		fa = false
 	}
@@ -188,7 +187,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapt
 
 		// Flash Attention also supports kv cache quantization
 		// Enable if the requested and kv cache type is supported by the model
-		if kvct != "" && f.SupportsKVCacheType(kvct) {
+		if kvct != "" && ggml.SupportsKVCacheType(kvct) {
 			params = append(params, "--kv-cache-type", kvct)
 		} else {
 			slog.Warn("kv cache type not supported by model", "type", kvct)
@@ -201,7 +200,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapt
 	for _, g := range gpus {
 		if g.Library == "metal" &&
 			uint64(opts.NumGPU) > 0 &&
-			uint64(opts.NumGPU) < f.KV().BlockCount()+1 {
+			uint64(opts.NumGPU) < ggml.KV().BlockCount()+1 {
 			opts.UseMMap = new(bool)
 			*opts.UseMMap = false
 		}
@@ -233,152 +232,205 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, f *ggml.GGML, adapt
 		params = append(params, "--multiuser-cache")
 	}
 
-	exe, err := os.Executable()
-	if err != nil {
-		return nil, err
-	}
-
-	// Find an availableServers  port, retry on each iteration in case the failure was a port conflict race
-	port := 0
-	if a, err := net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {
-		var l *net.TCPListener
-		if l, err = net.ListenTCP("tcp", a); err == nil {
-			port = l.Addr().(*net.TCPAddr).Port
-			l.Close()
+	libs := make(map[string]string)
+	if entries, err := os.ReadDir(discover.LibOllamaPath); err == nil {
+		for _, entry := range entries {
+			libs[entry.Name()] = filepath.Join(discover.LibOllamaPath, entry.Name())
 		}
 	}
-	if port == 0 {
-		slog.Debug("ResolveTCPAddr failed ", "error", err)
-		port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range
-	}
-	finalParams := []string{"runner"}
-	if envconfig.NewRunners() {
-		finalParams = append(finalParams, "--new-runner")
-	}
-	finalParams = append(finalParams, params...)
-	finalParams = append(finalParams, "--port", strconv.Itoa(port))
 
-	pathEnv := "LD_LIBRARY_PATH"
-	if runtime.GOOS == "windows" {
-		pathEnv = "PATH"
-	}
-	// Start with the server directory for the LD_LIBRARY_PATH/PATH
-	libraryPaths := []string{filepath.Dir(exe)}
-
-	if libraryPath, ok := os.LookupEnv(pathEnv); ok {
-		// favor our bundled library dependencies over system libraries
-		libraryPaths = append(libraryPaths, filepath.SplitList(libraryPath)...)
+	lib := gpus[0].RunnerName()
+	requested := envconfig.LLMLibrary()
+	if libs[requested] != "" {
+		slog.Info("using requested gpu library", "requested", requested)
+		lib = requested
 	}
 
-	// Note: we always put the dependency path first
-	// since this was the exact version we compiled/linked against
-	if gpus[0].DependencyPath != nil {
-		// assume gpus from the same library have the same dependency path
-		libraryPaths = append(gpus[0].DependencyPath, libraryPaths...)
+	var compatible []string
+	for k := range libs {
+		// exact match first
+		if k == lib {
+			compatible = append([]string{k}, compatible...)
+			continue
+		}
+
+		// then match the family (e.g. 'cuda')
+		if strings.Split(k, "_")[0] == strings.Split(lib, "_")[0] {
+			compatible = append(compatible, k)
+		}
 	}
+	slog.Debug("compatible gpu libraries", "compatible", compatible)
 
-	// TODO - once fully switched to the Go runner, load the model here for tokenize/detokenize cgo access
-	s := &llmServer{
-		port:        port,
-		cmd:         exec.Command(exe, finalParams...),
-		status:      NewStatusWriter(os.Stderr),
-		options:     opts,
-		modelPath:   model,
-		estimate:    estimate,
-		numParallel: numParallel,
-		sem:         semaphore.NewWeighted(int64(numParallel)),
-		totalLayers: f.KV().BlockCount() + 1,
-		gpus:        gpus,
-		done:        make(chan error, 1),
-	}
+	// iterate through compatible GPU libraries such as 'cuda_v12', 'cuda_v11', 'rocm', etc.
+	// adding each library's respective path to the LD_LIBRARY_PATH, until finally running
+	// without any LD_LIBRARY_PATH flags
+	for {
+		port := 0
+		if a, err := net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {
+			var l *net.TCPListener
+			if l, err = net.ListenTCP("tcp", a); err == nil {
+				port = l.Addr().(*net.TCPAddr).Port
+				l.Close()
+			}
+		}
+		if port == 0 {
+			slog.Debug("ResolveTCPAddr failed, using random port")
+			port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range
+		}
+		finalParams := []string{"runner"}
+		finalParams = append(finalParams, params...)
+		finalParams = append(finalParams, "--port", strconv.Itoa(port))
 
-	s.cmd.Env = os.Environ()
-	s.cmd.Stdout = os.Stdout
-	s.cmd.Stderr = s.status
-	s.cmd.SysProcAttr = LlamaServerSysProcAttr
+		var pathEnv string
+		switch runtime.GOOS {
+		case "windows":
+			pathEnv = "PATH"
+		case "darwin":
+			pathEnv = "DYLD_LIBRARY_PATH"
+		default:
+			pathEnv = "LD_LIBRARY_PATH"
+		}
 
-	envWorkarounds := [][2]string{}
-	for _, gpu := range gpus {
-		envWorkarounds = append(envWorkarounds, gpu.EnvWorkarounds...)
-	}
-	visibleDevicesEnv, visibleDevicesEnvVal := gpus.GetVisibleDevicesEnv()
-	pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))
+		var libraryPaths []string
+		if libraryPath, ok := os.LookupEnv(pathEnv); ok {
+			libraryPaths = append(libraryPaths, filepath.SplitList(libraryPath)...)
+		}
 
-	// Update or add the path and visible devices variable with our adjusted version
-	pathNeeded := true
-	devicesNeeded := visibleDevicesEnv != ""
-	for i := range s.cmd.Env {
-		cmp := strings.SplitN(s.cmd.Env[i], "=", 2)
-		if strings.EqualFold(cmp[0], pathEnv) {
-			s.cmd.Env[i] = pathEnv + "=" + pathEnvVal
-			pathNeeded = false
-		} else if devicesNeeded && strings.EqualFold(cmp[0], visibleDevicesEnv) {
-			s.cmd.Env[i] = visibleDevicesEnv + "=" + visibleDevicesEnvVal
-			devicesNeeded = false
-		} else if len(envWorkarounds) != 0 {
-			for _, kv := range envWorkarounds {
-				if strings.EqualFold(cmp[0], kv[0]) {
-					s.cmd.Env[i] = kv[0] + "=" + kv[1]
+		if len(compatible) > 0 {
+			c := compatible[0]
+			if libpath, ok := libs[c]; ok {
+				slog.Debug("adding gpu library", "path", libpath)
+				libraryPaths = append(libraryPaths, libpath)
+			}
+		}
+
+		// Note: we always put the dependency path first
+		// since this was the exact version we compiled/linked against
+		if gpus[0].DependencyPath != nil {
+			slog.Debug("adding gpu dependency paths", "paths", gpus[0].DependencyPath)
+			// assume gpus from the same library have the same dependency path
+			libraryPaths = append(gpus[0].DependencyPath, libraryPaths...)
+		}
+
+		// finally, add the root library path
+		libraryPaths = append(libraryPaths, discover.LibOllamaPath)
+
+		exe, err := os.Executable()
+		if err != nil {
+			return nil, fmt.Errorf("unable to lookup executable path: %w", err)
+		}
+
+		exe, err = filepath.EvalSymlinks(exe)
+		if err != nil {
+			return nil, fmt.Errorf("unable to evaluate symlinks for executable path: %w", err)
+		}
+
+		// TODO - once fully switched to the Go runner, load the model here for tokenize/detokenize cgo access
+		s := &llmServer{
+			port:        port,
+			cmd:         exec.Command(exe, finalParams...),
+			status:      NewStatusWriter(os.Stderr),
+			options:     opts,
+			modelPath:   model,
+			estimate:    estimate,
+			numParallel: numParallel,
+			sem:         semaphore.NewWeighted(int64(numParallel)),
+			totalLayers: ggml.KV().BlockCount() + 1,
+			gpus:        gpus,
+			done:        make(chan error, 1),
+		}
+
+		s.cmd.Env = os.Environ()
+		s.cmd.Stdout = os.Stdout
+		s.cmd.Stderr = s.status
+		s.cmd.SysProcAttr = LlamaServerSysProcAttr
+
+		envWorkarounds := [][2]string{}
+		for _, gpu := range gpus {
+			envWorkarounds = append(envWorkarounds, gpu.EnvWorkarounds...)
+		}
+		visibleDevicesEnv, visibleDevicesEnvVal := gpus.GetVisibleDevicesEnv()
+		pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))
+
+		// Update or add the path and visible devices variable with our adjusted version
+		pathNeeded := true
+		devicesNeeded := visibleDevicesEnv != ""
+		for i := range s.cmd.Env {
+			cmp := strings.SplitN(s.cmd.Env[i], "=", 2)
+			if strings.EqualFold(cmp[0], pathEnv) {
+				s.cmd.Env[i] = pathEnv + "=" + pathEnvVal
+				pathNeeded = false
+			} else if devicesNeeded && strings.EqualFold(cmp[0], visibleDevicesEnv) {
+				s.cmd.Env[i] = visibleDevicesEnv + "=" + visibleDevicesEnvVal
+				devicesNeeded = false
+			} else if len(envWorkarounds) != 0 {
+				for _, kv := range envWorkarounds {
+					if strings.EqualFold(cmp[0], kv[0]) {
+						s.cmd.Env[i] = kv[0] + "=" + kv[1]
+					}
 				}
 			}
 		}
-	}
-	if pathNeeded {
-		s.cmd.Env = append(s.cmd.Env, pathEnv+"="+pathEnvVal)
-	}
-	if devicesNeeded {
-		s.cmd.Env = append(s.cmd.Env, visibleDevicesEnv+"="+visibleDevicesEnvVal)
-	}
+		if pathNeeded {
+			s.cmd.Env = append(s.cmd.Env, pathEnv+"="+pathEnvVal)
+		}
+		if devicesNeeded {
+			s.cmd.Env = append(s.cmd.Env, visibleDevicesEnv+"="+visibleDevicesEnvVal)
+		}
 
-	slog.Info("starting llama server", "cmd", s.cmd.String())
-	if envconfig.Debug() {
-		filteredEnv := []string{}
-		for _, ev := range s.cmd.Env {
-			if strings.HasPrefix(ev, "CUDA_") ||
-				strings.HasPrefix(ev, "ROCR_") ||
-				strings.HasPrefix(ev, "ROCM_") ||
-				strings.HasPrefix(ev, "HIP_") ||
-				strings.HasPrefix(ev, "GPU_") ||
-				strings.HasPrefix(ev, "HSA_") ||
-				strings.HasPrefix(ev, "GGML_") ||
-				strings.HasPrefix(ev, "PATH=") ||
-				strings.HasPrefix(ev, "LD_LIBRARY_PATH=") {
-				filteredEnv = append(filteredEnv, ev)
+		slog.Info("starting llama server", "cmd", s.cmd.String())
+		if envconfig.Debug() {
+			filteredEnv := []string{}
+			for _, ev := range s.cmd.Env {
+				if strings.HasPrefix(ev, "CUDA_") ||
+					strings.HasPrefix(ev, "ROCR_") ||
+					strings.HasPrefix(ev, "ROCM_") ||
+					strings.HasPrefix(ev, "HIP_") ||
+					strings.HasPrefix(ev, "GPU_") ||
+					strings.HasPrefix(ev, "HSA_") ||
+					strings.HasPrefix(ev, "GGML_") ||
+					strings.HasPrefix(ev, "PATH=") ||
+					strings.HasPrefix(ev, "LD_LIBRARY_PATH=") ||
+					strings.HasPrefix(ev, "DYLD_LIBRARY_PATH=") {
+					filteredEnv = append(filteredEnv, ev)
+				}
 			}
-		}
-		// Log at debug as the environment is inherited and might contain sensitive information
-		slog.Debug("subprocess", "environment", filteredEnv)
-	}
-
-	if err = s.cmd.Start(); err != nil {
-		// Detect permission denied and augment the message about noexec
-		if errors.Is(err, os.ErrPermission) {
-			return nil, fmt.Errorf("unable to start server %w.  %s may have noexec set.  Set OLLAMA_TMPDIR for server to a writable executable directory", err, exe)
+			// Log at debug as the environment is inherited and might contain sensitive information
+			slog.Debug("subprocess", "environment", filteredEnv)
 		}
 
-		msg := ""
-		if s.status != nil && s.status.LastErrMsg != "" {
-			msg = s.status.LastErrMsg
-		}
-		return nil, fmt.Errorf("error starting the external llama server: %v %s", err, msg)
-	}
-
-	// reap subprocess when it exits
-	go func() {
-		err := s.cmd.Wait()
-		// Favor a more detailed message over the process exit status
-		if err != nil && s.status != nil && s.status.LastErrMsg != "" {
-			slog.Debug("llama runner terminated", "error", err)
-			if strings.Contains(s.status.LastErrMsg, "unknown model") {
-				s.status.LastErrMsg = "this model is not supported by your version of Ollama. You may need to upgrade"
+		if err = s.cmd.Start(); err != nil {
+			var msg string
+			if s.status != nil && s.status.LastErrMsg != "" {
+				msg = s.status.LastErrMsg
+			}
+			err := fmt.Errorf("error starting runner: %v %s", err, msg)
+			if len(compatible) == 0 {
+				return nil, err
 			}
-			s.done <- errors.New(s.status.LastErrMsg)
-		} else {
-			s.done <- err
-		}
-	}()
 
-	return s, nil
+			slog.Warn("unable to start runner with compatible gpu", "error", err, "compatible", compatible)
+			compatible = compatible[1:]
+			continue
+		}
+
+		// reap subprocess when it exits
+		go func() {
+			err := s.cmd.Wait()
+			// Favor a more detailed message over the process exit status
+			if err != nil && s.status != nil && s.status.LastErrMsg != "" {
+				slog.Error("llama runner terminated", "error", err)
+				if strings.Contains(s.status.LastErrMsg, "unknown model") {
+					s.status.LastErrMsg = "this model is not supported by your version of Ollama. You may need to upgrade"
+				}
+				s.done <- errors.New(s.status.LastErrMsg)
+			} else {
+				s.done <- err
+			}
+		}()
+
+		return s, nil
+	}
 }
 
 type ServerStatus int
@@ -592,12 +644,22 @@ type ImageData struct {
 	AspectRatioID int    `json:"aspect_ratio_id"`
 }
 
+// TokenProbs represents probability information for a token
+type TokenProbs struct {
+	TokenID int     `json:"id"`
+	Logit   float32 `json:"logit"`
+	Prob    float32 `json:"prob"`
+	LogProb float32 `json:"logprob"`
+	Token   string  `json:"token"`
+}
+
 type completion struct {
-	Content      string `json:"content"`
-	Model        string `json:"model"`
-	Prompt       string `json:"prompt"`
-	Stop         bool   `json:"stop"`
-	StoppedLimit bool   `json:"stopped_limit"`
+	Content      string       `json:"content"`
+	Model        string       `json:"model"`
+	Prompt       string       `json:"prompt"`
+	Stop         bool         `json:"stop"`
+	StoppedLimit bool         `json:"stopped_limit"`
+	LogProbs     []TokenProbs `json:"logprobs"`
 
 	Timings struct {
 		PredictedN  int     `json:"predicted_n"`
@@ -608,14 +670,16 @@ type completion struct {
 }
 
 type CompletionRequest struct {
-	Prompt  string
-	Format  json.RawMessage
-	Images  []ImageData
-	Options *api.Options
+	Prompt   string
+	Format   json.RawMessage
+	Images   []ImageData
+	LogProbs int
+	Options  *api.Options
 }
 
 type CompletionResponse struct {
 	Content            string
+	LogProbs           []TokenProbs
 	DoneReason         string
 	Done               bool
 	PromptEvalCount    int
@@ -646,9 +710,12 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu
 		"seed":              req.Options.Seed,
 		"stop":              req.Options.Stop,
 		"image_data":        req.Images,
+		"logprobs":          req.LogProbs,
 		"cache_prompt":      true,
 	}
 
+	fmt.Println("completion request:", request)
+
 	if len(req.Format) > 0 {
 		switch string(req.Format) {
 		case `null`, `""`:
@@ -744,7 +811,6 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu
 				continue
 			}
 
-			// slog.Debug("got line", "line", string(line))
 			evt, ok := bytes.CutPrefix(line, []byte("data: "))
 			if !ok {
 				evt = line
@@ -770,7 +836,8 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu
 
 			if c.Content != "" {
 				fn(CompletionResponse{
-					Content: c.Content,
+					Content:  c.Content,
+					LogProbs: c.LogProbs,
 				})
 			}
 
@@ -787,6 +854,7 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu
 					PromptEvalDuration: parseDurationMs(c.Timings.PromptMS),
 					EvalCount:          c.Timings.PredictedN,
 					EvalDuration:       parseDurationMs(c.Timings.PredictedMS),
+					LogProbs:           c.LogProbs,
 				})
 				return nil
 			}

macapp/README.md

@@ -6,14 +6,14 @@ This app builds upon Ollama to provide a desktop experience for running models.
 
 First, build the `ollama` binary:
 
-```
+```shell
 cd ..
 go build .
 ```
 
 Then run the desktop app with `npm start`:
 
-```
+```shell
 cd macapp
 npm install
 npm start

macapp/forge.config.ts

@@ -18,8 +18,8 @@ const config: ForgeConfig = {
     asar: true,
     icon: './assets/icon.icns',
     extraResource: [
-      '../dist/ollama',
-      '../dist/darwin-amd64/lib',
+      path.join(__dirname, '../dist/darwin/ollama'),
+      ...fs.readdirSync(path.join(__dirname, '../dist/darwin-amd64/lib/ollama')).map(f => path.join(__dirname, '../dist/darwin-amd64/lib/ollama', f)),
       path.join(__dirname, './assets/iconTemplate.png'),
       path.join(__dirname, './assets/iconTemplate@2x.png'),
       path.join(__dirname, './assets/iconUpdateTemplate.png'),
@@ -43,7 +43,7 @@ const config: ForgeConfig = {
         }
       : {}),
     osxUniversal: {
-      x64ArchFiles: '**/ollama*',
+      x64ArchFiles: '*',
     },
   },
   rebuildConfig: {},

ml/backend.go

@@ -1,192 +0,0 @@
-package ml
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"os"
-	"strings"
-)
-
-type Config interface {
-	Architecture() string
-	String(string, ...string) string
-	Uint(string, ...uint32) uint32
-	Float(string, ...float32) float32
-
-	Strings(string, ...[]string) []string
-	Uints(string, ...[]uint32) []uint32
-}
-
-type Backend interface {
-	Config() Config
-	Get(name string) Tensor
-	NewContext() Context
-}
-
-var backends = make(map[string]func(*os.File) (Backend, error))
-
-func RegisterBackend(name string, f func(*os.File) (Backend, error)) {
-	if _, ok := backends[name]; ok {
-		panic("backend: backend already registered")
-	}
-
-	backends[name] = f
-}
-
-func NewBackend(f *os.File) (Backend, error) {
-	if backend, ok := backends["ggml"]; ok {
-		return backend(f)
-	}
-
-	return nil, fmt.Errorf("unsupported backend")
-}
-
-type Context interface {
-	Zeros(dtype DType, shape ...int64) Tensor
-	FromFloatSlice(s []float32, shape ...int) (Tensor, error)
-	FromIntSlice(s []int32, shape ...int) (Tensor, error)
-
-	Forward(Tensor)
-	Compute(Tensor) Tensor
-	MaxTensors() int
-	Close() error
-}
-
-type Tensor interface {
-	Dim(n int) int64
-	Stride(n int) int64
-
-	Shape() []int64
-	DType() DType
-
-	Bytes() []byte
-	Floats() []float32
-
-	Add(ctx Context, t2 Tensor) Tensor
-	Mul(ctx Context, t2 Tensor) Tensor
-	Mulmat(ctx Context, t2 Tensor) Tensor
-
-	Softmax(ctx Context) Tensor
-	LayerNorm(ctx Context, weight, bias Tensor, eps float32) Tensor
-	RMSNorm(ctx Context, weight Tensor, eps float32) Tensor
-	Scale(ctx Context, s float64) Tensor
-
-	Conv2D(ctx Context, weight Tensor, s0, s1, p0, p1, d0, d1 int) Tensor
-	RoPE(ctx Context, positionIDs, ropeFactors Tensor, dim uint32, base, scale float32) Tensor
-
-	Tanh(ctx Context) Tensor
-	GELU(ctx Context) Tensor
-	SILU(ctx Context) Tensor
-
-	Reshape(ctx Context, shape ...int64) Tensor
-	View(ctx Context, offset int, shape ...int) Tensor
-	Permute(ctx Context, shape ...int) Tensor
-	Contiguous(ctx Context) Tensor
-
-	Pad(ctx Context, shape ...int64) Tensor
-	Unpad(ctx Context, shape ...int64) Tensor
-
-	Stack(ctx Context, dim int, s ...Tensor) Tensor
-	Concat(ctx Context, t2 Tensor, dim int) Tensor
-	Rows(ctx Context, t2 Tensor) Tensor
-	Copy(ctx Context, t2 Tensor) Tensor
-}
-
-type number interface {
-	~int | ~int8 | ~int16 | ~int32 | ~int64 |
-		~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 |
-		~float32 | ~float64 |
-		~complex64 | ~complex128
-}
-
-func mul[T number](s ...T) T {
-	p := T(1)
-	for _, v := range s {
-		p *= v
-	}
-
-	return p
-}
-
-type DumpOptions struct {
-	// Items is the number of elements to print at the beginning and end of each dimension.
-	Items int64
-
-	// Precision is the number of decimal places to print. Applies to float32 and float64.
-	Precision int
-}
-
-func Dump(t Tensor, opts ...DumpOptions) string {
-	if len(opts) < 1 {
-		opts = append(opts, DumpOptions{
-			Items:     3,
-			Precision: 4,
-		})
-	}
-
-	switch t.DType() {
-	case DTypeF32:
-		return dump[[]float32](t, opts[0])
-	case DTypeI32:
-		return dump[[]int32](t, opts[0])
-	default:
-		return "<unsupported>"
-	}
-}
-
-func dump[S ~[]E, E number](t Tensor, opts DumpOptions) string {
-	bts := t.Bytes()
-	if bts == nil {
-		return "<nil>"
-	}
-
-	s := make(S, mul(t.Shape()...))
-	if err := binary.Read(bytes.NewBuffer(t.Bytes()), binary.LittleEndian, &s); err != nil {
-		panic(err)
-	}
-
-	shape := t.Shape()
-
-	var sb strings.Builder
-	var f func([]int64, int64)
-	f = func(dims []int64, stride int64) {
-		prefix := strings.Repeat(" ", len(shape)-len(dims)+1)
-		fmt.Fprint(&sb, "[")
-		defer func() { fmt.Fprint(&sb, "]") }()
-		for i := int64(0); i < dims[0]; i++ {
-			if i >= opts.Items && i < dims[0]-opts.Items {
-				fmt.Fprint(&sb, "..., ")
-				// skip to next printable element
-				skip := dims[0] - 2*opts.Items
-				if len(dims) > 1 {
-					stride += mul(append(dims[1:], skip)...)
-					fmt.Fprint(&sb, strings.Repeat("\n", len(dims)-1), prefix)
-				}
-				i += skip - 1
-			} else if len(dims) > 1 {
-				f(dims[1:], stride)
-				stride += mul(dims[1:]...)
-				if i < dims[0]-1 {
-					fmt.Fprint(&sb, ",", strings.Repeat("\n", len(dims)-1), prefix)
-				}
-			} else {
-				fmt.Fprint(&sb, s[stride+i])
-				if i < dims[0]-1 {
-					fmt.Fprint(&sb, ", ")
-				}
-			}
-		}
-	}
-	f(shape, 0)
-
-	return sb.String()
-}
-
-type DType int
-
-const (
-	DTypeF32 DType = iota
-	DTypeI32
-	DTypeOther
-)

ml/backend/backend.go

@@ -1,5 +0,0 @@
-package backend
-
-import (
-	_ "github.com/ollama/ollama/ml/backend/ggml"
-)

ml/backend/ggml/ggml.go

@@ -1,592 +0,0 @@
-package ggml
-
-// #cgo CPPFLAGS: -I${SRCDIR}/ggml/include
-// #include <stdlib.h>
-// #include <stdint.h>
-// #include "ggml.h"
-// #include "ggml-cpu.h"
-// #include "ggml-backend.h"
-import "C"
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"io"
-	"log/slog"
-	"os"
-	"sync"
-	"unsafe"
-
-	"github.com/ollama/ollama/format"
-	fs "github.com/ollama/ollama/fs/ggml"
-	"github.com/ollama/ollama/ml"
-	"golang.org/x/sync/errgroup"
-
-	ggml "github.com/ollama/ollama/ml/backend/ggml/ggml/src"
-)
-
-type device struct {
-	d *C.struct_ggml_backend_device
-}
-
-func (d device) LogValue() slog.Value {
-	var free, total uint64
-	C.ggml_backend_dev_memory(d.d, (*C.size_t)(&free), (*C.size_t)(&total))
-
-	kind := "unknown"
-	switch C.ggml_backend_dev_type(d.d) {
-	case C.GGML_BACKEND_DEVICE_TYPE_CPU:
-		kind = "cpu"
-	case C.GGML_BACKEND_DEVICE_TYPE_GPU:
-		kind = "gpu"
-	case C.GGML_BACKEND_DEVICE_TYPE_ACCEL:
-		kind = "accel"
-	}
-
-	return slog.GroupValue(
-		slog.String("name", C.GoString(C.ggml_backend_dev_name(d.d))),
-		slog.String("description", C.GoString(C.ggml_backend_dev_description(d.d))),
-		slog.String("kind", kind),
-		slog.String("free", format.HumanBytes2(free)),
-		slog.String("total", format.HumanBytes2(total)),
-	)
-}
-
-var devices = sync.OnceValue(func() []device {
-	ggml.OnceLoad()
-
-	s := make([]device, C.ggml_backend_dev_count())
-	for i := range s {
-		s[i] = device{C.ggml_backend_dev_get(C.size_t(i))}
-	}
-
-	return s
-})
-
-type Backend struct {
-	meta       *fs.GGML
-	cpus, gpus []Context
-	tensors    map[string]*Context
-}
-
-func New(r *os.File) (ml.Backend, error) {
-	meta, n, err := fs.Decode(r, -1)
-	if err != nil {
-		return nil, err
-	}
-
-	slog.Info(
-		"",
-		"architecture", meta.KV().Architecture(),
-		"file_type", meta.KV().FileType(),
-		"name", meta.KV().String("general.name"),
-		"description", meta.KV().String("general.description"),
-		"num_tensors", len(meta.Tensors().Items()),
-		"num_key_values", len(meta.KV()),
-	)
-
-	var cpus, gpus []Context
-	for _, d := range devices() {
-		switch C.ggml_backend_dev_type(d.d) {
-		case C.GGML_BACKEND_DEVICE_TYPE_CPU,
-			C.GGML_BACKEND_DEVICE_TYPE_ACCEL:
-			slog.Info("cpu", "device", d)
-			cpus = append(cpus, Context{
-				ctx: C.ggml_init(C.struct_ggml_init_params{
-					mem_size: C.size_t(int(C.ggml_tensor_overhead()) * (len(meta.Tensors().Items()) + 1 + int(meta.KV().BlockCount())*2)),
-					no_alloc: true,
-				}),
-				backend: C.ggml_backend_dev_init(d.d, nil),
-			})
-		case C.GGML_BACKEND_DEVICE_TYPE_GPU:
-			slog.Info("gpu", "device", d)
-			gpus = append(gpus, Context{
-				ctx: C.ggml_init(C.struct_ggml_init_params{
-					mem_size: C.size_t(int(C.ggml_tensor_overhead()) * (len(meta.Tensors().Items()) + 1 + int(meta.KV().BlockCount())*2)),
-					no_alloc: true,
-				}),
-				backend: C.ggml_backend_dev_init(d.d, nil),
-			})
-		}
-	}
-
-	ctxFunc := func(s []Context) (*Context, error) {
-		for _, e := range s {
-			return &e, nil
-		}
-
-		return nil, fmt.Errorf("no devices available")
-	}
-
-	tensors := make(map[*fs.Tensor]*Context, len(meta.Tensors().Items()))
-	for _, t := range meta.Tensors().Items() {
-		c, err := ctxFunc(append(gpus, cpus...))
-		if err != nil {
-			return nil, err
-		}
-
-		func() {
-			tt := C.ggml_new_tensor(c.ctx, t.Kind, C.int(len(t.Shape)), (*C.int64_t)(unsafe.Pointer(&t.Shape[0])))
-
-			cname := C.CString(t.Name)
-			defer C.free(unsafe.Pointer(cname))
-			C.ggml_set_name(tt, cname)
-
-			tensors[t] = c
-		}()
-	}
-
-	for _, b := range append(gpus, cpus...) {
-		C.ggml_backend_alloc_ctx_tensors(b.ctx, b.backend)
-	}
-
-	sr := io.NewSectionReader(r, int64(meta.Tensors().Offset), n-int64(meta.Tensors().Offset))
-
-	var g errgroup.Group
-	for t, c := range tensors {
-		g.Go(func() error {
-			bts := make([]byte, t.Size())
-			n, err := io.ReadFull(io.NewSectionReader(sr, int64(t.Offset), int64(t.Size())), bts)
-			if err != nil {
-				return err
-			}
-
-			if n != int(t.Size()) {
-				return fmt.Errorf("expected %d bytes, got %d", t.Size(), n)
-			}
-
-			cname := C.CString(t.Name)
-			defer C.free(unsafe.Pointer(cname))
-
-			C.ggml_backend_tensor_set(C.ggml_get_tensor(c.ctx, cname), unsafe.Pointer(&bts[0]), 0, C.size_t(n))
-			return nil
-		})
-	}
-
-	if err := g.Wait(); err != nil {
-		return nil, err
-	}
-
-	return &Backend{
-		meta: meta,
-		cpus: cpus,
-		gpus: gpus,
-	}, nil
-}
-
-func init() {
-	ml.RegisterBackend("ggml", New)
-}
-
-func (b *Backend) Config() ml.Config {
-	return b.meta.KV()
-}
-
-func (b *Backend) Get(name string) ml.Tensor {
-	cname := C.CString(name)
-	defer C.free(unsafe.Pointer(cname))
-
-	for _, c := range append(b.gpus, b.cpus...) {
-		if t := C.ggml_get_tensor(c.ctx, cname); t != nil {
-			return &Tensor{t: t}
-		}
-	}
-
-	return nil
-}
-
-func (b *Backend) NewContext() ml.Context {
-	nodes := max(8192, len(b.meta.Tensors().Items())*5)
-	c := C.ggml_init(C.struct_ggml_init_params{
-		mem_buffer: nil,
-		mem_size:   C.size_t(nodes)*C.ggml_tensor_overhead() + C.ggml_graph_overhead_custom(C.size_t(nodes), false),
-		no_alloc:   true,
-	})
-
-	backends := make([]*C.struct_ggml_backend, len(b.gpus)+len(b.cpus))
-	bufts := make([]*C.struct_ggml_backend_buffer_type, len(b.gpus)+len(b.cpus))
-	for i, c := range append(b.gpus, b.cpus...) {
-		backends[i] = c.backend
-		bufts[i] = C.ggml_backend_get_default_buffer_type(c.backend)
-	}
-
-	return &Context{
-		ctx:     c,
-		backend: backends[0],
-		nodes:   nodes,
-		sched: C.ggml_backend_sched_new(
-			(*C.ggml_backend_t)(unsafe.Pointer(&backends[0])),
-			(*C.ggml_backend_buffer_type_t)(unsafe.Pointer(&bufts[0])),
-			C.int(len(backends)),
-			C.size_t(nodes),
-			true,
-		),
-	}
-}
-
-type Context struct {
-	ctx     *C.struct_ggml_context
-	backend *C.struct_ggml_backend
-
-	sched *C.struct_ggml_backend_sched
-	graph *C.struct_ggml_cgraph
-	nodes int
-}
-
-func (c *Context) Forward(t ml.Tensor) {
-	if c.graph == nil {
-		c.graph = C.ggml_new_graph_custom(c.ctx, C.size_t(c.nodes), false)
-	}
-
-	C.ggml_build_forward_expand(c.graph, t.(*Tensor).t)
-}
-
-func (c *Context) Compute(t ml.Tensor) ml.Tensor {
-	C.ggml_backend_sched_graph_compute_async(c.sched, c.graph)
-
-	if t != nil && C.ggml_nbytes(t.(*Tensor).t) != 0 {
-		backend := C.ggml_backend_sched_get_tensor_backend(c.sched, t.(*Tensor).t)
-
-		t.(*Tensor).data = make([]byte, C.ggml_nbytes(t.(*Tensor).t))
-		C.ggml_backend_tensor_get_async(backend, t.(*Tensor).t, unsafe.Pointer(&t.(*Tensor).data[0]), 0, C.ggml_nbytes(t.(*Tensor).t))
-	}
-
-	return t
-}
-
-func (c *Context) MaxTensors() int {
-	return c.nodes
-}
-
-func (c Context) Zeros(dtype ml.DType, shape ...int64) ml.Tensor {
-	if len(shape) < 1 || len(shape) > 4 {
-		panic("unsupported number of dimensions")
-	}
-
-	for _, dim := range shape {
-		if dim < 1 {
-			panic("invalid shape")
-		}
-	}
-
-	var t *C.struct_ggml_tensor
-	switch dtype {
-	case ml.DTypeF32:
-		t = C.ggml_new_tensor(c.ctx, C.GGML_TYPE_F32, C.int(len(shape)), (*C.int64_t)(unsafe.Pointer(&shape[0])))
-	case ml.DTypeI32:
-		t = C.ggml_new_tensor(c.ctx, C.GGML_TYPE_I32, C.int(len(shape)), (*C.int64_t)(unsafe.Pointer(&shape[0])))
-	default:
-		panic("unsupported dtype")
-	}
-
-	b := C.ggml_backend_alloc_buffer(c.backend, C.ggml_nbytes(t))
-	C.ggml_backend_tensor_alloc(b, t, C.ggml_backend_buffer_get_base(b))
-	C.ggml_set_zero(t)
-	return &Tensor{t: t}
-}
-
-func fromSlice[S ~[]E, E float32 | int32](ctx Context, s S, shape []int, dtype uint32) (ml.Tensor, error) {
-	n := len(s)
-
-	if n == 0 {
-		shape := 0
-		t := C.ggml_new_tensor(ctx.ctx, dtype, 1, (*C.int64_t)(unsafe.Pointer(&shape)))
-		return &Tensor{t: t}, nil
-	}
-
-	for _, v := range shape {
-		n /= v
-	}
-
-	if n != 1 {
-		return nil, fmt.Errorf("invalid shape %v for %d elements", shape, len(s))
-	}
-
-	t := C.ggml_new_tensor(ctx.ctx, dtype, C.int(len(shape)), (*C.int64_t)(unsafe.Pointer(&shape[0])))
-	b := C.ggml_backend_alloc_buffer(ctx.backend, C.ggml_nbytes(t))
-	C.ggml_backend_tensor_alloc(b, t, C.ggml_backend_buffer_get_base(b))
-	C.ggml_backend_tensor_set(t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t))
-	return &Tensor{t: t}, nil
-}
-
-func (c Context) FromFloatSlice(s []float32, shape ...int) (ml.Tensor, error) {
-	return fromSlice(c, s, shape, C.GGML_TYPE_F32)
-}
-
-func (c Context) FromIntSlice(s []int32, shape ...int) (ml.Tensor, error) {
-	return fromSlice(c, s, shape, C.GGML_TYPE_I32)
-}
-
-func (c *Context) Close() error {
-	C.ggml_backend_sched_free(c.sched)
-	C.ggml_free(c.ctx)
-	return nil
-}
-
-type Tensor struct {
-	t    *C.struct_ggml_tensor
-	data []byte
-}
-
-func (t *Tensor) LogValue() slog.Value {
-	return slog.GroupValue(
-		slog.String("name", C.GoString(C.ggml_get_name(t.t))),
-		slog.String("type", C.GoString(C.ggml_type_name(t.t._type))),
-		slog.Any("shape", t.Shape()),
-	)
-}
-
-func (t *Tensor) Dim(n int) int64 {
-	return int64(t.t.ne[n])
-}
-
-func (t *Tensor) Stride(n int) int64 {
-	return int64(t.t.nb[n])
-}
-
-func (t *Tensor) Shape() []int64 {
-	shape := make([]int64, C.ggml_n_dims(t.t))
-	for i := range shape {
-		shape[i] = t.Dim(i)
-	}
-
-	return shape
-}
-
-func (t *Tensor) Bytes() []byte {
-	if bts := C.ggml_get_data(t.t); bts != nil {
-		return C.GoBytes(bts, C.int(C.ggml_nbytes(t.t)))
-	}
-
-	return nil
-}
-
-func (t *Tensor) Floats() (f32s []float32) {
-	if t.data != nil {
-		f32s = make([]float32, C.ggml_nelements(t.t))
-		_ = binary.Read(bytes.NewReader(t.data), binary.LittleEndian, f32s)
-	}
-
-	return
-}
-
-func (t *Tensor) DType() ml.DType {
-	switch t.t._type {
-	case C.GGML_TYPE_F32:
-		return ml.DTypeF32
-	case C.GGML_TYPE_I32:
-		return ml.DTypeI32
-	default:
-		return ml.DTypeOther
-	}
-}
-
-func (t *Tensor) Add(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_add(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
-	}
-}
-
-func (t *Tensor) Stack(ctx ml.Context, dim int, s ...ml.Tensor) ml.Tensor {
-	if len(s) > 0 {
-		return t.Concat(ctx, s[0].Stack(ctx, dim, s[1:]...), dim)
-	}
-
-	return t
-}
-
-func (t *Tensor) Concat(ctx ml.Context, t2 ml.Tensor, dim int) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_concat(ctx.(*Context).ctx, t.t, t2.(*Tensor).t, C.int(dim)),
-	}
-}
-
-func (t *Tensor) Contiguous(ctx ml.Context) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_cont(ctx.(*Context).ctx, t.t),
-	}
-}
-
-func (t *Tensor) Mul(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_mul(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
-	}
-}
-
-func (t *Tensor) Mulmat(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_mul_mat(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
-	}
-}
-
-func (t *Tensor) LayerNorm(ctx ml.Context, w, b ml.Tensor, eps float32) ml.Tensor {
-	tt := (&Tensor{t: C.ggml_norm(ctx.(*Context).ctx, t.t, C.float(eps))}).Mul(ctx, w)
-	if b != nil {
-		tt = tt.Add(ctx, b)
-	}
-
-	return tt
-}
-
-func (t *Tensor) RMSNorm(ctx ml.Context, w ml.Tensor, eps float32) ml.Tensor {
-	return (&Tensor{t: C.ggml_norm(ctx.(*Context).ctx, t.t, C.float(eps))}).Mul(ctx, w)
-}
-
-func (t *Tensor) Pad(ctx ml.Context, shape ...int64) ml.Tensor {
-	if len(shape) != 4 {
-		panic("expected 4 dimensions")
-	}
-
-	return &Tensor{
-		t: C.ggml_pad(ctx.(*Context).ctx, t.t, C.int(shape[0]), C.int(shape[1]), C.int(shape[2]), C.int(shape[3])),
-	}
-}
-
-func (t *Tensor) Permute(ctx ml.Context, shape ...int) ml.Tensor {
-	if len(shape) != 4 {
-		panic("expected 4 dimensions")
-	}
-
-	return &Tensor{
-		t: C.ggml_permute(ctx.(*Context).ctx, t.t, C.int(shape[0]), C.int(shape[1]), C.int(shape[2]), C.int(shape[3])),
-	}
-}
-
-func (t *Tensor) Rows(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_get_rows(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
-	}
-}
-
-func (t *Tensor) Copy(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_cpy(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
-	}
-}
-
-func (t *Tensor) Reshape(ctx ml.Context, shape ...int64) ml.Tensor {
-	switch len(shape) {
-	case 1:
-		return &Tensor{
-			t: C.ggml_reshape_1d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0])),
-		}
-	case 2:
-		return &Tensor{
-			t: C.ggml_reshape_2d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.int64_t(shape[1])),
-		}
-	case 3:
-		return &Tensor{
-			t: C.ggml_reshape_3d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.int64_t(shape[1]), C.int64_t(shape[2])),
-		}
-	case 4:
-		return &Tensor{
-			t: C.ggml_reshape_4d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.int64_t(shape[1]), C.int64_t(shape[2]), C.int64_t(shape[3])),
-		}
-	default:
-		panic("unsupported number of dimensions")
-	}
-}
-
-func (t *Tensor) Scale(ctx ml.Context, s float64) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_scale(ctx.(*Context).ctx, t.t, (C.float)(s)),
-	}
-}
-
-func (t *Tensor) Softmax(ctx ml.Context) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_soft_max(ctx.(*Context).ctx, t.t),
-	}
-}
-
-func (t *Tensor) Tanh(ctx ml.Context) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_tanh_inplace(ctx.(*Context).ctx, t.t),
-	}
-}
-
-func (t *Tensor) Unpad(ctx ml.Context, shape ...int64) ml.Tensor {
-	if len(shape) != 4 {
-		panic("expected 4 dimensions")
-	}
-
-	return &Tensor{
-		t: C.ggml_unpad(ctx.(*Context).ctx, t.t, C.int(shape[0]), C.int(shape[1]), C.int(shape[2]), C.int(shape[3])),
-	}
-}
-
-func (t *Tensor) View(ctx ml.Context, offset int, shape ...int) ml.Tensor {
-	switch len(shape) {
-	case 1:
-		return &Tensor{
-			t: C.ggml_view_1d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.size_t(offset)),
-		}
-	case 3:
-		return &Tensor{
-			t: C.ggml_view_2d(ctx.(*Context).ctx, t.t,
-				C.int64_t(shape[0]), C.int64_t(shape[2]),
-				C.size_t(shape[1]),
-				C.size_t(offset)),
-		}
-	case 5:
-		return &Tensor{
-			t: C.ggml_view_3d(ctx.(*Context).ctx, t.t,
-				C.int64_t(shape[0]), C.int64_t(shape[2]), C.int64_t(shape[4]),
-				C.size_t(shape[1]), C.size_t(shape[3]),
-				C.size_t(offset)),
-		}
-	case 7:
-		return &Tensor{
-			t: C.ggml_view_4d(ctx.(*Context).ctx, t.t,
-				C.int64_t(shape[0]), C.int64_t(shape[2]), C.int64_t(shape[4]), C.int64_t(shape[6]),
-				C.size_t(shape[1]), C.size_t(shape[3]), C.size_t(shape[5]),
-				C.size_t(offset)),
-		}
-	default:
-		panic("unsupported number of dimensions")
-	}
-}
-
-const (
-	ropeTypeNorm C.int = iota
-)
-
-func (t *Tensor) RoPE(ctx ml.Context, positionIDs, ropeFactors ml.Tensor, ropeDim uint32, ropeBase, ropeScale float32) ml.Tensor {
-	if ropeFactors == nil {
-		ropeFactors = &Tensor{}
-	}
-
-	return &Tensor{
-		t: C.ggml_rope_ext(
-			ctx.(*Context).ctx, t.t, positionIDs.(*Tensor).t, ropeFactors.(*Tensor).t,
-			C.int(ropeDim),
-			131072,       // YaRN n_ctx_train
-			ropeTypeNorm, // ROPE_TYPE_NORM
-			C.float(ropeBase),
-			C.float(ropeScale),
-			0.,  // YaRN ext_factor
-			1.,  // YaRN attn_factor
-			32., // YaRN beta_fast
-			1.,  // YaRN beta_slow
-		),
-	}
-}
-
-func (t *Tensor) GELU(ctx ml.Context) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_gelu_inplace(ctx.(*Context).ctx, t.t),
-	}
-}
-
-func (t *Tensor) SILU(ctx ml.Context) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_silu_inplace(ctx.(*Context).ctx, t.t),
-	}
-}
-
-func (t *Tensor) Conv2D(ctx ml.Context, t2 ml.Tensor, s0, s1, p0, p1, d0, d1 int) ml.Tensor {
-	return &Tensor{
-		t: C.ggml_conv_2d(ctx.(*Context).ctx, t.t, t2.(*Tensor).t, C.int(s0), C.int(s1), C.int(p0), C.int(p1), C.int(d0), C.int(d1)),
-	}
-}

ml/backend/ggml/ggml/.rsync-filter

@@ -1,7 +1,9 @@
-protect **/*.go
-protect **/*-embed.*
+protect *.go
+protect *-embed.*
 include include/
 include src/
+include src/CMakeLists.txt
+include src/**/CMakeLists.txt
 include src/ggml-blas/
 include src/ggml-cpu/
 include src/ggml-cpu/amx/
@@ -10,12 +12,11 @@ include src/ggml-cuda/
 include src/ggml-cuda/template-instances/
 include src/ggml-hip/
 include src/ggml-metal/
-include **/CMakeLists.txt
-include **/*.c
-include **/*.h
-include **/*.cpp
-include **/*.cu
-include **/*.cuh
-include **/*.m
-include **/*.metal
+include *.c
+include *.h
+include *.cpp
+include *.cu
+include *.cuh
+include *.m
+include *.metal
 exclude *

ml/backend/ggml/ggml/CMakeLists.txt

@@ -1,262 +0,0 @@
-cmake_minimum_required(VERSION 3.14) # for add_link_options and implicit target directories.
-project("ggml" C CXX)
-include(CheckIncludeFileCXX)
-
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-
-if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
-    set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
-    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
-endif()
-
-if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
-    set(GGML_STANDALONE ON)
-
-    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
-
-    # configure project version
-    # TODO
-else()
-    set(GGML_STANDALONE OFF)
-endif()
-
-if (EMSCRIPTEN)
-    set(BUILD_SHARED_LIBS_DEFAULT OFF)
-
-    option(GGML_WASM_SINGLE_FILE "ggml: embed WASM inside the generated ggml.js" ON)
-else()
-    if (MINGW)
-        set(BUILD_SHARED_LIBS_DEFAULT OFF)
-    else()
-        set(BUILD_SHARED_LIBS_DEFAULT ON)
-    endif()
-endif()
-
-# remove the lib prefix on win32 mingw
-if (WIN32)
-    set(CMAKE_STATIC_LIBRARY_PREFIX "")
-    set(CMAKE_SHARED_LIBRARY_PREFIX "")
-    set(CMAKE_SHARED_MODULE_PREFIX  "")
-endif()
-
-option(BUILD_SHARED_LIBS "ggml: build shared libraries" ${BUILD_SHARED_LIBS_DEFAULT})
-option(GGML_BACKEND_DL   "ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)" OFF)
-
-#
-# option list
-#
-
-# TODO: mark all options as advanced when not GGML_STANDALONE
-
-if (APPLE)
-    set(GGML_METAL_DEFAULT ON)
-    set(GGML_BLAS_DEFAULT ON)
-    set(GGML_BLAS_VENDOR_DEFAULT "Apple")
-else()
-    set(GGML_METAL_DEFAULT OFF)
-    set(GGML_BLAS_DEFAULT OFF)
-    set(GGML_BLAS_VENDOR_DEFAULT "Generic")
-endif()
-
-if (CMAKE_CROSSCOMPILING)
-    set(GGML_NATIVE_DEFAULT OFF)
-else()
-    set(GGML_NATIVE_DEFAULT ON)
-endif()
-
-# defaults
-if (NOT GGML_LLAMAFILE_DEFAULT)
-    set(GGML_LLAMAFILE_DEFAULT OFF)
-endif()
-
-if (NOT GGML_CUDA_GRAPHS_DEFAULT)
-    set(GGML_CUDA_GRAPHS_DEFAULT OFF)
-endif()
-
-# general
-option(GGML_STATIC "ggml: static link libraries"                     OFF)
-option(GGML_NATIVE "ggml: optimize the build for the current system" ${GGML_NATIVE_DEFAULT})
-option(GGML_LTO    "ggml: enable link time optimization"             OFF)
-option(GGML_CCACHE "ggml: use ccache if available"                   ON)
-
-# debug
-option(GGML_ALL_WARNINGS           "ggml: enable all compiler warnings"                   ON)
-option(GGML_ALL_WARNINGS_3RD_PARTY "ggml: enable all compiler warnings in 3rd party libs" OFF)
-option(GGML_GPROF                  "ggml: enable gprof"                                   OFF)
-
-# build
-option(GGML_FATAL_WARNINGS    "ggml: enable -Werror flag"    OFF)
-
-# sanitizers
-option(GGML_SANITIZE_THREAD    "ggml: enable thread sanitizer"    OFF)
-option(GGML_SANITIZE_ADDRESS   "ggml: enable address sanitizer"   OFF)
-option(GGML_SANITIZE_UNDEFINED "ggml: enable undefined sanitizer" OFF)
-
-# instruction set specific
-if (GGML_NATIVE OR NOT GGML_NATIVE_DEFAULT)
-    set(INS_ENB OFF)
-else()
-    set(INS_ENB ON)
-endif()
-
-option(GGML_CPU_HBM          "ggml: use memkind for CPU HBM" OFF)
-option(GGML_CPU_AARCH64      "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON)
-option(GGML_AVX              "ggml: enable AVX"              ${INS_ENB})
-option(GGML_AVX_VNNI         "ggml: enable AVX-VNNI"         OFF)
-option(GGML_AVX2             "ggml: enable AVX2"             ${INS_ENB})
-option(GGML_AVX512           "ggml: enable AVX512F"          OFF)
-option(GGML_AVX512_VBMI      "ggml: enable AVX512-VBMI"      OFF)
-option(GGML_AVX512_VNNI      "ggml: enable AVX512-VNNI"      OFF)
-option(GGML_AVX512_BF16      "ggml: enable AVX512-BF16"      OFF)
-if (NOT MSVC)
-    # in MSVC F16C and FMA is implied with AVX2/AVX512
-    option(GGML_FMA          "ggml: enable FMA"              ${INS_ENB})
-    option(GGML_F16C         "ggml: enable F16C"             ${INS_ENB})
-    # MSVC does not seem to support AMX
-    option(GGML_AMX_TILE     "ggml: enable AMX-TILE"         OFF)
-    option(GGML_AMX_INT8     "ggml: enable AMX-INT8"         OFF)
-    option(GGML_AMX_BF16     "ggml: enable AMX-BF16"         OFF)
-endif()
-option(GGML_LASX             "ggml: enable lasx"             ON)
-option(GGML_LSX              "ggml: enable lsx"              ON)
-option(GGML_RVV              "ggml: enable rvv"              ON)
-
-option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF)
-set(GGML_CPU_ARM_ARCH "" CACHE STRING "ggml: CPU architecture for ARM")
-
-
-if (WIN32)
-    set(GGML_WIN_VER "0x602" CACHE STRING   "ggml: Windows version")
-endif()
-
-# ggml core
-set(GGML_SCHED_MAX_COPIES  "4" CACHE STRING "ggml: max input copies for pipeline parallelism")
-option(GGML_CPU                             "ggml: enable CPU backend"                        ON)
-
-# 3rd party libs / backends
-option(GGML_ACCELERATE                      "ggml: enable Accelerate framework"               ON)
-option(GGML_BLAS                            "ggml: use BLAS"                                  ${GGML_BLAS_DEFAULT})
-set(GGML_BLAS_VENDOR ${GGML_BLAS_VENDOR_DEFAULT} CACHE STRING
-                                            "ggml: BLAS library vendor")
-option(GGML_LLAMAFILE                       "ggml: use LLAMAFILE"                             ${GGML_LLAMAFILE_DEFAULT})
-
-option(GGML_CUDA                            "ggml: use CUDA"                                  OFF)
-option(GGML_MUSA                            "ggml: use MUSA"                                  OFF)
-option(GGML_CUDA_FORCE_MMQ                  "ggml: use mmq kernels instead of cuBLAS"         OFF)
-option(GGML_CUDA_FORCE_CUBLAS               "ggml: always use cuBLAS instead of mmq kernels"  OFF)
-option(GGML_CUDA_F16                        "ggml: use 16 bit floats for some calculations"   OFF)
-set   (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
-                                            "ggml: max. batch size for using peer access")
-option(GGML_CUDA_NO_PEER_COPY               "ggml: do not use peer to peer copies"            OFF)
-option(GGML_CUDA_NO_VMM                     "ggml: do not try to use CUDA VMM"                OFF)
-option(GGML_CUDA_FA_ALL_QUANTS              "ggml: compile all quants for FlashAttention"     OFF)
-option(GGML_CUDA_GRAPHS                     "ggml: use CUDA graphs (llama.cpp only)"          ${GGML_CUDA_GRAPHS_DEFAULT})
-
-option(GGML_HIP                             "ggml: use HIP"                                   OFF)
-option(GGML_HIP_UMA                         "ggml: use HIP unified memory architecture"       OFF)
-option(GGML_VULKAN                          "ggml: use Vulkan"                                OFF)
-option(GGML_VULKAN_CHECK_RESULTS            "ggml: run Vulkan op checks"                      OFF)
-option(GGML_VULKAN_DEBUG                    "ggml: enable Vulkan debug output"                OFF)
-option(GGML_VULKAN_MEMORY_DEBUG             "ggml: enable Vulkan memory debug output"         OFF)
-option(GGML_VULKAN_SHADER_DEBUG_INFO        "ggml: enable Vulkan shader debug info"           OFF)
-option(GGML_VULKAN_PERF                     "ggml: enable Vulkan perf output"                 OFF)
-option(GGML_VULKAN_VALIDATE                 "ggml: enable Vulkan validation"                  OFF)
-option(GGML_VULKAN_RUN_TESTS                "ggml: run Vulkan tests"                          OFF)
-option(GGML_KOMPUTE                         "ggml: use Kompute"                               OFF)
-option(GGML_METAL                           "ggml: use Metal"                                 ${GGML_METAL_DEFAULT})
-option(GGML_METAL_USE_BF16                  "ggml: use bfloat if available"                   OFF)
-option(GGML_METAL_NDEBUG                    "ggml: disable Metal debugging"                   OFF)
-option(GGML_METAL_SHADER_DEBUG              "ggml: compile Metal with -fno-fast-math"         OFF)
-option(GGML_METAL_EMBED_LIBRARY             "ggml: embed Metal library"                       ${GGML_METAL})
-set   (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING
-                                            "ggml: metal minimum macOS version")
-set   (GGML_METAL_STD "" CACHE STRING       "ggml: metal standard version (-std flag)")
-option(GGML_OPENMP                          "ggml: use OpenMP"                                ON)
-option(GGML_RPC                             "ggml: use RPC"                                   OFF)
-option(GGML_SYCL                            "ggml: use SYCL"                                  OFF)
-option(GGML_SYCL_F16                        "ggml: use 16 bit floats for sycl calculations"   OFF)
-set   (GGML_SYCL_TARGET "INTEL" CACHE STRING
-                                            "ggml: sycl target device")
-set   (GGML_SYCL_DEVICE_ARCH "" CACHE STRING
-                                            "ggml: sycl device architecture")
-
-option(GGML_OPENCL                          "ggml: use OpenCL"                                OFF)
-option(GGML_OPENCL_PROFILING                "ggml: use OpenCL profiling (increases overhead)" OFF)
-option(GGML_OPENCL_EMBED_KERNELS            "ggml: embed kernels"                             ON)
-option(GGML_OPENCL_USE_ADRENO_KERNELS       "ggml: use optimized kernels for Adreno"          ON)
-
-# extra artifacts
-option(GGML_BUILD_TESTS    "ggml: build tests"    ${GGML_STANDALONE})
-option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
-
-#
-# dependencies
-#
-
-set(CMAKE_C_STANDARD 11)
-set(CMAKE_C_STANDARD_REQUIRED true)
-
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED true)
-
-set(THREADS_PREFER_PTHREAD_FLAG ON)
-
-find_package(Threads REQUIRED)
-
-#
-# build the library
-#
-
-add_subdirectory(src)
-
-#
-# tests and examples
-#
-
-if (GGML_BUILD_TESTS)
-    enable_testing()
-    add_subdirectory(tests)
-endif ()
-
-if (GGML_BUILD_EXAMPLES)
-    add_subdirectory(examples)
-endif ()
-
-#
-# install
-#
-
-include(GNUInstallDirs)
-include(CMakePackageConfigHelpers)
-
-# all public headers
-set(GGML_PUBLIC_HEADERS
-    include/ggml.h
-    include/ggml-cpu.h
-    include/ggml-alloc.h
-    include/ggml-backend.h
-    include/ggml-blas.h
-    include/ggml-cann.h
-    include/ggml-cuda.h
-    include/ggml-kompute.h
-    include/ggml-opt.h
-    include/ggml-metal.h
-    include/ggml-rpc.h
-    include/ggml-sycl.h
-    include/ggml-vulkan.h)
-
-set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")
-#if (GGML_METAL)
-#    set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal")
-#endif()
-install(TARGETS ggml LIBRARY PUBLIC_HEADER)
-install(TARGETS ggml-base LIBRARY)
-
-if (GGML_STANDALONE)
-    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in
-        ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc
-        @ONLY)
-
-    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc
-        DESTINATION share/pkgconfig)
-endif()

ml/backend/ggml/ggml/src/ggml-backend-reg.cpp

@@ -512,32 +512,33 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
         }
         fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);
         for (const auto & entry : dir_it) {
-            if (entry.is_regular_file()) {
-                std::wstring filename = entry.path().filename().wstring();
-                std::wstring ext = entry.path().extension().wstring();
-                if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) {
-                    dl_handle_ptr handle { dl_load_library(entry.path().wstring()) };
-                    if (!handle && !silent) {
-                        GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
-                    }
-                    if (handle) {
+            try {
+                if (entry.is_regular_file()) {
+                    std::wstring filename = entry.path().filename().wstring();
+                    std::wstring ext = entry.path().extension().wstring();
+                    if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) {
+                        dl_handle_ptr handle { dl_load_library(entry.path().wstring()) };
+                        if (!handle) {
+                            GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
+                            continue;
+                        }
+
                         auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");
-                        if (score_fn) {
-                            int s = score_fn();
-#ifndef NDEBUG
-                            GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s);
-#endif
-                            if (s > best_score) {
-                                best_score = s;
-                                best_path = entry.path().wstring();
-                            }
-                        } else {
-                            if (!silent) {
-                                GGML_LOG_INFO("%s: failed to find ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
-                            }
+                        if (!score_fn) {
+                            GGML_LOG_DEBUG("%s: failed to find ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
+                            continue;
+                        }
+
+                        int s = score_fn();
+                        GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s);
+                        if (s > best_score) {
+                            best_score = s;
+                            best_path = entry.path().wstring();
                         }
                     }
                 }
+            } catch (const std::exception & e) {
+                GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), e.what());
             }
         }
     }

ml/backend/ggml/ggml/src/ggml-cpu/cpu.go

@@ -1,5 +1,6 @@
 package cpu
 
+// #cgo CFLAGS: -O3 -Wno-implicit-function-declaration
 // #cgo CXXFLAGS: -std=c++17
 // #cgo CPPFLAGS: -I${SRCDIR}/amx -I${SRCDIR}/llamafile -I${SRCDIR}/.. -I${SRCDIR}/../../include
 // #cgo CPPFLAGS: -DGGML_USE_LLAMAFILE

ml/backend/ggml/ggml/src/ggml-cpu/llamafile/sgemm.cpp

@@ -279,14 +279,6 @@ template <> inline __m256bh load(const float *p) {
 }
 #endif
 
-////////////////////////////////////////////////////////////////////////////////////////////////////
-// CONSTANTS
-
-#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
-static const int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
-static const __m128i iq4nlt = _mm_loadu_si128((const __m128i *) kvalues_iq4nl);
-#endif
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 // FLOATING POINT MATRIX MULTIPLICATION
 
@@ -613,6 +605,14 @@ class tinyBLAS_Q0_AVX {
                     TC *C, int64_t ldc,
                     int ith, int nth)
         : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+        const int8_t kvalues_iq4nl[16] = {
+            -127, -104, -83, -65,
+            -49,  -35,  -22, -10,
+              1,   13,   25,  38,
+             53,   69,   89, 113
+        };
+
+        iq4nlt = _mm_loadu_si128((const __m128i *)kvalues_iq4nl);
     }
 
     void matmul(int64_t m, int64_t n) {
@@ -1037,6 +1037,7 @@ class tinyBLAS_Q0_AVX {
     const int64_t ldc;
     const int ith;
     const int nth;
+    __m128i iq4nlt;
 };
 #endif // __AVX__
 

ml/backend/ggml/ggml/src/ggml-cuda/template-instances/generate_cu_files.py

@@ -1,77 +0,0 @@
-#!/usr/bin/env python3
-
-from glob import glob
-import os
-
-TYPES_KV = ["GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_F16"]
-
-SOURCE_FATTN_VEC = """// This file has been autogenerated by generate_cu_files.py, do not edit manually.
-
-#include "../fattn-vec-f{vkq_size}.cuh"
-
-DECL_FATTN_VEC_F{vkq_size}_CASE({head_size}, {type_k}, {type_v});
-"""
-
-SOURCE_FATTN_WMMA_START = """// This file has been autogenerated by generate_cu_files.py, do not edit manually.
-
-#include "../fattn-wmma-f16.cuh"
-
-"""
-
-SOURCE_FATTN_WMMA_CASE = "DECL_FATTN_WMMA_F16_CASE({head_size}, {cols_per_block}, {kq_acc_t});\n"
-
-TYPES_MMQ = [
-    "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0",
-    "GGML_TYPE_Q2_K", "GGML_TYPE_Q3_K", "GGML_TYPE_Q4_K", "GGML_TYPE_Q5_K", "GGML_TYPE_Q6_K",
-    "GGML_TYPE_IQ2_XXS", "GGML_TYPE_IQ2_XS", "GGML_TYPE_IQ2_S", "GGML_TYPE_IQ3_XXS", "GGML_TYPE_IQ3_S",
-    "GGML_TYPE_IQ1_S", "GGML_TYPE_IQ4_NL", "GGML_TYPE_IQ4_XS"
-]
-
-SOURCE_MMQ = """// This file has been autogenerated by generate_cu_files.py, do not edit manually.
-
-#include "../mmq.cuh"
-
-DECL_MMQ_CASE({type});
-"""
-
-
-def get_short_name(long_quant_name):
-    return long_quant_name.replace("GGML_TYPE_", "").lower()
-
-
-def get_head_sizes(type_k, type_v):
-    if type_k == "GGML_TYPE_F16" and type_v == "GGML_TYPE_F16":
-        return [64, 128, 256]
-    if type_k == "GGML_TYPE_F16":
-        return [64, 128]
-    return [128]
-
-
-for filename in glob("*.cu"):
-    os.remove(filename)
-
-for vkq_size in [16, 32]:
-    for type_k in TYPES_KV:
-        for type_v in TYPES_KV:
-            for head_size in get_head_sizes(type_k, type_v):
-                with open(f"fattn-vec-f{vkq_size}-instance-hs{head_size}-{get_short_name(type_k)}-{get_short_name(type_v)}.cu", "w") as f:
-                    f.write(SOURCE_FATTN_VEC.format(vkq_size=vkq_size, head_size=head_size, type_k=type_k, type_v=type_v))
-
-for kq_acc_t in ["half", "float"]:
-    for cols_per_block in [8, 16, 32]:
-        if kq_acc_t == "float" and cols_per_block == 8:
-            continue
-
-        with open(f"fattn-wmma-f16-instance-kq{kq_acc_t}-cpb{cols_per_block}.cu", "w") as f:
-            f.write(SOURCE_FATTN_WMMA_START)
-
-            for head_size in [64, 80, 96, 112, 128, 256]:
-                if cols_per_block == 8 and head_size % 32 != 0: # wmma fragment is 8x32
-                    continue
-                if kq_acc_t == "float" and cols_per_block == 32 and head_size == 256: # register spilling, bad performance
-                    continue
-                f.write(SOURCE_FATTN_WMMA_CASE.format(kq_acc_t=kq_acc_t, cols_per_block=cols_per_block, head_size=head_size))
-
-for type in TYPES_MMQ:
-    with open(f"mmq-instance-{get_short_name(type)}.cu", "w") as f:
-        f.write(SOURCE_MMQ.format(type=type))

ml/backend/ggml/ggml/src/ggml.go

@@ -41,28 +41,58 @@ func sink(level C.int, text *C.char, _ unsafe.Pointer) {
 }
 
 var OnceLoad = sync.OnceFunc(func() {
-	var lib struct{ name, defaultValue string }
+	exe, err := os.Executable()
+	if err != nil {
+		slog.Warn("failed to get executable path", "error", err)
+		exe = "."
+	}
+
+	// PATH, LD_LIBRARY_PATH, and DYLD_LIBRARY_PATH are often
+	// set by the parent process, however, use a default value
+	// if the environment variable is not set.
+	var name, value string
 	switch runtime.GOOS {
-	case "darwin", "linux":
-		lib.name = "LD_LIBRARY_PATH"
-		lib.defaultValue = "/usr/local/lib:/usr/lib"
+	case "darwin":
+		// On macOS, DYLD_LIBRARY_PATH is often not set, so
+		// we use the directory of the executable as the default.
+		name = "DYLD_LIBRARY_PATH"
+		value = filepath.Dir(exe)
 	case "windows":
-		lib.name = "PATH"
-		lib.defaultValue = "."
+		name = "PATH"
+		value = filepath.Join(filepath.Dir(exe), "lib", "ollama")
 	default:
-		return
+		name = "LD_LIBRARY_PATH"
+		value = filepath.Join(filepath.Dir(exe), "..", "lib", "ollama")
 	}
 
-	paths, ok := os.LookupEnv(lib.name)
+	paths, ok := os.LookupEnv(name)
 	if !ok {
-		paths = lib.defaultValue
+		paths = value
 	}
 
-	for _, path := range filepath.SplitList(paths) {
-		func() {
-			cpath := C.CString(path)
-			defer C.free(unsafe.Pointer(cpath))
-			C.ggml_backend_load_all_from_path(cpath)
-		}()
+	split := filepath.SplitList(paths)
+	visited := make(map[string]struct{}, len(split))
+	for _, path := range split {
+		abspath, err := filepath.Abs(path)
+		if err != nil {
+			slog.Error("failed to get absolute path", "error", err)
+			continue
+		}
+
+		if abspath != filepath.Dir(exe) && !strings.Contains(abspath, filepath.FromSlash("lib/ollama")) {
+			slog.Debug("skipping path which is not part of ollama", "path", abspath)
+			continue
+		}
+
+		if _, ok := visited[abspath]; !ok {
+			func() {
+				slog.Debug("ggml backend load all from path", "path", abspath)
+				cpath := C.CString(abspath)
+				defer C.free(unsafe.Pointer(cpath))
+				C.ggml_backend_load_all_from_path(cpath)
+			}()
+
+			visited[abspath] = struct{}{}
+		}
 	}
 })

ml/nn/convolution.go

@@ -1,11 +0,0 @@
-package nn
-
-import "github.com/ollama/ollama/ml"
-
-type Conv2D struct {
-	Weight ml.Tensor `gguf:"weight"`
-}
-
-func (m *Conv2D) Forward(ctx ml.Context, t ml.Tensor, s0, s1, p0, p1, d0, d1 int) ml.Tensor {
-	return m.Weight.Conv2D(ctx, t, s0, s1, p0, p1, d0, d1)
-}

ml/nn/embedding.go

@@ -1,11 +0,0 @@
-package nn
-
-import "github.com/ollama/ollama/ml"
-
-type Embedding struct {
-	Weight ml.Tensor `gguf:"weight"`
-}
-
-func (m *Embedding) Forward(ctx ml.Context, hiddenState ml.Tensor) ml.Tensor {
-	return m.Weight.Rows(ctx, hiddenState)
-}

ml/nn/linear.go

@@ -1,17 +0,0 @@
-package nn
-
-import "github.com/ollama/ollama/ml"
-
-type Linear struct {
-	Weight ml.Tensor `gguf:"weight"`
-	Bias   ml.Tensor `gguf:"bias"`
-}
-
-func (m *Linear) Forward(ctx ml.Context, t ml.Tensor) ml.Tensor {
-	t = m.Weight.Mulmat(ctx, t)
-	if m.Bias != nil {
-		t = t.Add(ctx, m.Bias)
-	}
-
-	return t
-}

ml/nn/normalization.go

@@ -1,22 +0,0 @@
-package nn
-
-import (
-	"github.com/ollama/ollama/ml"
-)
-
-type LayerNorm struct {
-	Weight ml.Tensor `gguf:"weight"`
-	Bias   ml.Tensor `gguf:"bias"`
-}
-
-func (m *LayerNorm) Forward(ctx ml.Context, t ml.Tensor, eps float32) ml.Tensor {
-	return t.LayerNorm(ctx, m.Weight, m.Bias, eps)
-}
-
-type RMSNorm struct {
-	Weight ml.Tensor `gguf:"weight"`
-}
-
-func (m *RMSNorm) Forward(ctx ml.Context, t ml.Tensor, eps float32) ml.Tensor {
-	return t.RMSNorm(ctx, m.Weight, eps)
-}

model/llama/model.go

@@ -1,158 +0,0 @@
-package llama
-
-import (
-	"math"
-
-	"github.com/ollama/ollama/cache"
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-	"github.com/ollama/ollama/model"
-)
-
-type Options struct {
-	RopeFactors                      ml.Tensor `gguf:"rope_freqs.weight"`
-	hiddenSize, numHeads, numKVHeads int64
-	eps, ropeBase, ropeScale         float32
-	ropeDim                          uint32
-}
-
-type Model struct {
-	model.Base
-	model.BytePairEncoding
-
-	TokenEmbedding *nn.Embedding `gguf:"token_embd"`
-	Layers         []Layer       `gguf:"blk"`
-	OutputNorm     *nn.RMSNorm   `gguf:"output_norm"`
-	Output         *nn.Linear    `gguf:"output,alt:token_embd"`
-
-	*Options
-}
-
-func New(c ml.Config) (model.Model, error) {
-	return &Model{
-		BytePairEncoding: model.BytePairEncoding{
-			Pretokenizer: c.String("tokenizer.ggml.pretokenizer", `(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+`),
-			Vocabulary: &model.Vocabulary{
-				Values: c.Strings("tokenizer.ggml.tokens"),
-				Types:  c.Uints("tokenizer.ggml.token_type"),
-				Merges: c.Strings("tokenizer.ggml.merges"),
-				BOS:    c.Uint("tokenizer.ggml.bos_token_id"),
-				EOS:    c.Uint("tokenizer.ggml.eos_token_id"),
-			},
-		},
-		Layers: make([]Layer, c.Uint("block_count")),
-		Options: &Options{
-			hiddenSize: int64(c.Uint("embedding_length")),
-			numHeads:   int64(c.Uint("attention.head_count")),
-			numKVHeads: int64(c.Uint("attention.head_count_kv")),
-			eps:        c.Float("attention.layer_norm_rms_epsilon"),
-			ropeBase:   c.Float("rope.freq_base"),
-			ropeScale:  c.Float("rope.freq_scale", 1),
-			ropeDim:    c.Uint("rope.dimension_count"),
-		},
-	}, nil
-}
-
-type SelfAttention struct {
-	Query  *nn.Linear `gguf:"attn_q"`
-	Key    *nn.Linear `gguf:"attn_k"`
-	Value  *nn.Linear `gguf:"attn_v"`
-	Output *nn.Linear `gguf:"attn_output"`
-}
-
-func (sa *SelfAttention) Forward(ctx ml.Context, hiddenState, positionIDs ml.Tensor, cache cache.Cache, opts *Options) ml.Tensor {
-	batchSize := hiddenState.Dim(1)
-	headDim := opts.hiddenSize / opts.numHeads
-
-	q := sa.Query.Forward(ctx, hiddenState)
-	q = q.Reshape(ctx, headDim, opts.numHeads, batchSize)
-	q = q.RoPE(ctx, positionIDs, opts.RopeFactors, opts.ropeDim, opts.ropeBase, opts.ropeScale)
-
-	k := sa.Key.Forward(ctx, hiddenState)
-	k = k.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
-	k = k.RoPE(ctx, positionIDs, opts.RopeFactors, opts.ropeDim, opts.ropeBase, opts.ropeScale)
-
-	v := sa.Value.Forward(ctx, hiddenState)
-	v = v.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
-
-	cache.Put(ctx, k, v)
-	k, v, mask := cache.Get(ctx)
-
-	q = q.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	k = k.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	v = v.Permute(ctx, 1, 2, 0, 3).Contiguous(ctx)
-
-	kq := k.Mulmat(ctx, q)
-	kq = kq.Scale(ctx, 1.0/math.Sqrt(float64(headDim)))
-	kq = kq.Add(ctx, mask)
-	kq = kq.Softmax(ctx)
-
-	kqv := v.Mulmat(ctx, kq)
-	kqv = kqv.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	kqv = kqv.Reshape(ctx, opts.hiddenSize, batchSize)
-
-	return sa.Output.Forward(ctx, kqv)
-}
-
-type MLP struct {
-	Up   *nn.Linear `gguf:"ffn_up"`
-	Down *nn.Linear `gguf:"ffn_down"`
-	Gate *nn.Linear `gguf:"ffn_gate"`
-}
-
-func (mlp *MLP) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *Options) ml.Tensor {
-	hiddenState = mlp.Gate.Forward(ctx, hiddenState).SILU(ctx).Mul(ctx, mlp.Up.Forward(ctx, hiddenState))
-	return mlp.Down.Forward(ctx, hiddenState)
-}
-
-type Layer struct {
-	AttentionNorm *nn.RMSNorm `gguf:"attn_norm"`
-	SelfAttention *SelfAttention
-	MLPNorm       *nn.RMSNorm `gguf:"ffn_norm"`
-	MLP           *MLP
-}
-
-func (l *Layer) Forward(ctx ml.Context, hiddenState, positionIDs ml.Tensor, cache cache.Cache, opts *Options) ml.Tensor {
-	residual := hiddenState
-
-	hiddenState = l.AttentionNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = l.SelfAttention.Forward(ctx, hiddenState, positionIDs, cache, opts)
-	hiddenState = hiddenState.Add(ctx, residual)
-	residual = hiddenState
-
-	hiddenState = l.MLPNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = l.MLP.Forward(ctx, hiddenState, opts)
-	return hiddenState.Add(ctx, residual)
-}
-
-func (m *Model) Forward(ctx ml.Context, opts model.Options) (ml.Tensor, error) {
-	inputs, err := ctx.FromIntSlice(opts.Inputs(), len(opts.Inputs()))
-	if err != nil {
-		return nil, err
-	}
-
-	positions, err := ctx.FromIntSlice(opts.Positions(), len(opts.Positions()))
-	if err != nil {
-		return nil, err
-	}
-
-	hiddenState := m.TokenEmbedding.Forward(ctx, inputs)
-
-	for i, layer := range m.Layers {
-		hiddenState = layer.Forward(ctx, hiddenState, positions, opts.Cache.Sub(i), m.Options)
-	}
-
-	hiddenState = m.OutputNorm.Forward(ctx, hiddenState, m.eps)
-	hiddenState = m.Output.Forward(ctx, hiddenState)
-
-	outputs, err := ctx.FromIntSlice(opts.Outputs(), len(opts.Outputs()))
-	if err != nil {
-		return nil, err
-	}
-
-	return hiddenState.Rows(ctx, outputs), nil
-}
-
-func init() {
-	model.Register("llama", New)
-}

model/mllama/model.go

@@ -1,100 +0,0 @@
-package mllama
-
-import (
-	"sync"
-
-	"github.com/ollama/ollama/cache"
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-	"github.com/ollama/ollama/model"
-)
-
-type Model struct {
-	model.Base
-
-	*VisionModel `gguf:"v,vision"`
-	*TextModel
-
-	Projector *nn.Linear `gguf:"mm.0"`
-
-	ImageProcessor
-	TextProcessor
-
-	start  sync.Once
-	tCache *cache.TensorCache
-}
-
-func New(c ml.Config) (model.Model, error) {
-	return &Model{
-		ImageProcessor: newImageProcessor(c),
-		VisionModel:    newVisionModel(c),
-		TextProcessor:  newTextProcessor(c),
-		TextModel:      newTextModel(c),
-	}, nil
-}
-
-func (m *Model) Forward(ctx ml.Context, opts model.Options) (ml.Tensor, error) {
-	m.start.Do(func() {
-		m.tCache = cache.NewTensorCache(m.Backend())
-	})
-
-	var crossAttentionStates ml.Tensor
-	if opts.Images != nil {
-		f32s, aspectRatioID, err := m.ImageProcessor.ProcessImage(opts.Images[0])
-		if err != nil {
-			return nil, err
-		}
-
-		pixelValues, err := ctx.FromFloatSlice(f32s,
-			m.ImageProcessor.imageSize,
-			m.ImageProcessor.imageSize,
-			m.ImageProcessor.numChannels,
-			m.ImageProcessor.maxNumTiles,
-		)
-		if err != nil {
-			return nil, err
-		}
-
-		aspectRatio, err := ctx.FromIntSlice([]int32{int32(aspectRatioID)}, 1)
-		if err != nil {
-			return nil, err
-		}
-
-		positions := make([]int32, 1601)
-		for i := range positions {
-			positions[i] = int32(i)
-		}
-
-		positionIDs, err := ctx.FromIntSlice(positions, len(positions))
-		if err != nil {
-			return nil, err
-		}
-
-		crossAttentionStates = m.VisionModel.Forward(ctx, pixelValues, positionIDs, aspectRatio)
-		crossAttentionStates = m.Projector.Forward(ctx, crossAttentionStates)
-	}
-
-	inputs, err := ctx.FromIntSlice(opts.Inputs(), len(opts.Inputs()))
-	if err != nil {
-		return nil, err
-	}
-
-	positions, err := ctx.FromIntSlice(opts.Positions(), len(opts.Positions()))
-	if err != nil {
-		return nil, err
-	}
-
-	// TODO: attention mask, cross attention mask
-	hiddenState := m.TextModel.Forward(ctx, inputs, positions, nil, crossAttentionStates, nil, opts.Cache, m.tCache)
-
-	outputs, err := ctx.FromIntSlice(opts.Outputs(), len(opts.Outputs()))
-	if err != nil {
-		return nil, err
-	}
-
-	return hiddenState.Rows(ctx, outputs), nil
-}
-
-func init() {
-	model.Register("mllama", New)
-}

model/mllama/model_text.go

@@ -1,241 +0,0 @@
-package mllama
-
-import (
-	"math"
-	"slices"
-
-	"github.com/ollama/ollama/cache"
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-)
-
-type TextSelfAttention struct {
-	Query  *nn.Linear `gguf:"attn_q"`
-	Key    *nn.Linear `gguf:"attn_k"`
-	Value  *nn.Linear `gguf:"attn_v"`
-	Output *nn.Linear `gguf:"attn_output"`
-}
-
-func (sa *TextSelfAttention) Forward(ctx ml.Context, hiddenState, positions, _ ml.Tensor, cache cache.Cache, opts *TextModelOptions) ml.Tensor {
-	batchSize := hiddenState.Dim(1)
-	headDim := opts.hiddenSize / opts.numHeads
-
-	query := sa.Query.Forward(ctx, hiddenState)
-	query = query.Reshape(ctx, headDim, opts.numHeads, batchSize)
-	query = query.RoPE(ctx, positions, opts.RopeFactors, opts.ropeDim, opts.ropeBase, opts.ropeScale)
-
-	key := sa.Key.Forward(ctx, hiddenState)
-	key = key.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
-	key = key.RoPE(ctx, positions, opts.RopeFactors, opts.ropeDim, opts.ropeBase, opts.ropeScale)
-
-	value := sa.Value.Forward(ctx, hiddenState)
-	value = value.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
-
-	cache.Put(ctx, key, value)
-	key, value, mask := cache.Get(ctx)
-
-	query = query.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	key = key.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	value = value.Permute(ctx, 1, 2, 0, 3).Contiguous(ctx)
-
-	scores := key.Mulmat(ctx, query)
-	scores = scores.Scale(ctx, 1.0/math.Sqrt(float64(headDim)))
-	scores = scores.Add(ctx, mask)
-	scores = scores.Softmax(ctx)
-
-	attention := value.Mulmat(ctx, scores)
-	attention = attention.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	attention = attention.Reshape(ctx, opts.hiddenSize, batchSize)
-
-	return sa.Output.Forward(ctx, attention)
-}
-
-type TextMLP struct {
-	Up   *nn.Linear `gguf:"ffn_up"`
-	Down *nn.Linear `gguf:"ffn_down"`
-	Gate *nn.Linear `gguf:"ffn_gate"`
-}
-
-func (mlp *TextMLP) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *TextModelOptions) ml.Tensor {
-	hiddenState = mlp.Gate.Forward(ctx, hiddenState).SILU(ctx).Mul(ctx, mlp.Up.Forward(ctx, hiddenState))
-	return mlp.Down.Forward(ctx, hiddenState)
-}
-
-type TextSelfAttentionDecoderLayer struct {
-	AttentionNorm *nn.RMSNorm `gguf:"attn_norm"`
-	SelfAttention *TextSelfAttention
-
-	MLPNorm *nn.RMSNorm `gguf:"ffn_norm"`
-	MLP     *TextMLP
-}
-
-func (d *TextSelfAttentionDecoderLayer) Forward(ctx ml.Context, hiddenState, positions, mask, _, _ ml.Tensor, cache cache.Cache, _ *cache.TensorCache, opts *TextModelOptions) ml.Tensor {
-	residual := hiddenState
-
-	hiddenState = d.AttentionNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = d.SelfAttention.Forward(ctx, hiddenState, positions, mask, cache, opts)
-	hiddenState = hiddenState.Add(ctx, residual)
-	residual = hiddenState
-
-	hiddenState = d.MLPNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = d.MLP.Forward(ctx, hiddenState, opts)
-	return hiddenState.Add(ctx, residual)
-}
-
-func (d *TextSelfAttentionDecoderLayer) Run() bool {
-	return true
-}
-
-type TextCrossAttention struct {
-	QueryNorm *nn.RMSNorm `gguf:"cross_attn_q_norm"`
-	Query     *nn.Linear  `gguf:"cross_attn_q_proj"`
-	KeyNorm   *nn.RMSNorm `gguf:"cross_attn_k_norm"`
-	Key       *nn.Linear  `gguf:"cross_attn_k_proj"`
-	Value     *nn.Linear  `gguf:"cross_attn_v_proj"`
-	Output    *nn.Linear  `gguf:"cross_attn_o_proj"`
-}
-
-func (ca *TextCrossAttention) Forward(ctx ml.Context, hiddenState, crossAttentionStates ml.Tensor, _ cache.Cache, tCache *cache.TensorCache, opts *TextModelOptions) ml.Tensor {
-	batchSize := hiddenState.Dim(1)
-	headDim := opts.hiddenSize / opts.numHeads
-
-	query := ca.Query.Forward(ctx, hiddenState)
-	query = query.Reshape(ctx, headDim, opts.numHeads, batchSize)
-	query = ca.QueryNorm.Forward(ctx, query, opts.eps)
-
-	var key, value ml.Tensor
-	if crossAttentionStates != nil {
-		numVisionTokens, numTiles := crossAttentionStates.Dim(1), crossAttentionStates.Dim(2)
-
-		key = ca.Key.Forward(ctx, crossAttentionStates)
-		key = key.Reshape(ctx, headDim, opts.numKVHeads, numVisionTokens*numTiles)
-		key = ca.KeyNorm.Forward(ctx, key, opts.eps)
-
-		value = ca.Value.Forward(ctx, crossAttentionStates)
-		value = value.Reshape(ctx, headDim, opts.numKVHeads, numVisionTokens*numTiles)
-
-		tCache.Put(ctx, key, value)
-	} else {
-		key, value, _ = tCache.Get(ctx)
-	}
-
-	query = query.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	key = key.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	value = value.Permute(ctx, 1, 2, 0, 3).Contiguous(ctx)
-
-	scores := key.Mulmat(ctx, query)
-	scores = scores.Scale(ctx, 1.0/math.Sqrt(float64(headDim)))
-	scores = scores.Softmax(ctx)
-
-	attention := value.Mulmat(ctx, scores)
-	attention = attention.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	attention = attention.Reshape(ctx, opts.hiddenSize, batchSize)
-
-	return ca.Output.Forward(ctx, attention)
-}
-
-type TextCrossAttentionDecoderLayer struct {
-	AttentionNorm  *nn.RMSNorm `gguf:"attn_norm"`
-	CrossAttention *TextCrossAttention
-	AttentionGate  ml.Tensor `gguf:"cross_attn_attn_gate"`
-
-	MLPNorm *nn.RMSNorm `gguf:"ffn_norm"`
-	MLP     *TextMLP
-	MLPGate ml.Tensor `gguf:"cross_attn_mlp_gate"`
-
-	run bool
-}
-
-func (d *TextCrossAttentionDecoderLayer) Forward(ctx ml.Context, hiddenState, _, _, crossAttentionStates, crossAttentionMask ml.Tensor, cache cache.Cache, tCache *cache.TensorCache, opts *TextModelOptions) ml.Tensor {
-	d.run = true
-
-	residual := hiddenState
-
-	hiddenState = d.AttentionNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = d.CrossAttention.Forward(ctx, hiddenState, crossAttentionStates, cache, tCache, opts)
-	hiddenState = hiddenState.Mul(ctx, d.AttentionGate.Tanh(ctx))
-	hiddenState = hiddenState.Add(ctx, residual)
-	residual = hiddenState
-
-	hiddenState = d.MLPNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = d.MLP.Forward(ctx, hiddenState, opts)
-	hiddenState = hiddenState.Mul(ctx, d.MLPGate.Tanh(ctx))
-	return hiddenState.Add(ctx, residual)
-}
-
-func (d *TextCrossAttentionDecoderLayer) Run() bool {
-	return d.run
-}
-
-type TextDecoderLayer interface {
-	Forward(ctx ml.Context, hiddenState, positionIDs, mask, crossAttentionStates, crossAttentionMask ml.Tensor, cache cache.Cache, tCache *cache.TensorCache, opts *TextModelOptions) ml.Tensor
-	Run() bool
-}
-
-type TextDecoder struct {
-	Layers []TextDecoderLayer
-}
-
-func (d *TextDecoder) Forward(ctx ml.Context, hiddenState, positionIDs, mask, crossAttentionStates, crossAttentionMask ml.Tensor, cache cache.Cache, tCache *cache.TensorCache, opts *TextModelOptions) ml.Tensor {
-	for i, layer := range d.Layers {
-		if layer.Run() || crossAttentionStates != nil {
-			hiddenState = layer.Forward(ctx, hiddenState, positionIDs, mask, crossAttentionStates, crossAttentionMask, cache.Sub(i), tCache.Sub(i), opts)
-		}
-	}
-
-	return hiddenState
-}
-
-type TextModelOptions struct {
-	RopeFactors ml.Tensor `gguf:"rope_freqs.weight"`
-
-	hiddenSize, numHeads, numKVHeads int64
-	eps, ropeBase, ropeScale         float32
-	ropeDim                          uint32
-
-	crossAttentionLayers []uint32
-}
-
-type TextModel struct {
-	TokenEmbedding *nn.Embedding `gguf:"token_embd"`
-	Transformer    *TextDecoder  `gguf:"blk"`
-	OutputNorm     *nn.RMSNorm   `gguf:"output_norm"`
-	Output         *nn.Linear    `gguf:"output"`
-
-	*TextModelOptions
-}
-
-func (m *TextModel) Forward(ctx ml.Context, inputIDs, positionIDs, mask, crossAttentionStates, crossAttentionMask ml.Tensor, cache cache.Cache, tCache *cache.TensorCache) ml.Tensor {
-	hiddenState := m.TokenEmbedding.Forward(ctx, inputIDs)
-	hiddenState = m.Transformer.Forward(ctx, hiddenState, positionIDs, mask, crossAttentionStates, crossAttentionMask, cache, tCache, m.TextModelOptions)
-	hiddenState = m.OutputNorm.Forward(ctx, hiddenState, m.eps)
-	return m.Output.Forward(ctx, hiddenState)
-}
-
-func newTextModel(c ml.Config) *TextModel {
-	var decoderLayers []TextDecoderLayer
-	for i := range c.Uint("block_count") {
-		var textDecoderLayer TextDecoderLayer
-		if slices.Contains(c.Uints("attention.cross_attention_layers"), i) {
-			textDecoderLayer = &TextCrossAttentionDecoderLayer{}
-		} else {
-			textDecoderLayer = &TextSelfAttentionDecoderLayer{}
-		}
-
-		decoderLayers = append(decoderLayers, textDecoderLayer)
-	}
-
-	return &TextModel{
-		Transformer: &TextDecoder{Layers: decoderLayers},
-		TextModelOptions: &TextModelOptions{
-			hiddenSize:           int64(c.Uint("embedding_length")),
-			numHeads:             int64(c.Uint("attention.head_count")),
-			numKVHeads:           int64(c.Uint("attention.head_count_kv")),
-			eps:                  c.Float("attention.layer_norm_rms_epsilon"),
-			ropeBase:             c.Float("rope.freq_base"),
-			ropeScale:            c.Float("rope.freq_scale", 1),
-			ropeDim:              c.Uint("rope.dimension_count"),
-			crossAttentionLayers: c.Uints("attention.cross_attention_layers"),
-		},
-	}
-}