update tests

.gitattributes

@@ -15,12 +15,8 @@ ml/backend/**/*.cu linguist-vendored
 ml/backend/**/*.cuh linguist-vendored
 ml/backend/**/*.m linguist-vendored
 ml/backend/**/*.metal linguist-vendored
-ml/backend/**/*.comp linguist-vendored
-ml/backend/**/*.glsl linguist-vendored
 ml/backend/**/CMakeLists.txt linguist-vendored
 
-app/webview linguist-vendored
-
 llama/build-info.cpp linguist-generated
 ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-embed.s linguist-generated
 

.github/workflows/release.yaml

@@ -16,15 +16,13 @@ jobs:
     outputs:
       GOFLAGS: ${{ steps.goflags.outputs.GOFLAGS }}
       VERSION: ${{ steps.goflags.outputs.VERSION }}
-      vendorsha: ${{ steps.changes.outputs.vendorsha }}
     steps:
       - uses: actions/checkout@v4
       - name: Set environment
         id: goflags
         run: |
-          echo GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=${GITHUB_REF_NAME#v}\" \"-X=github.com/ollama/ollama/server.mode=release\"'" | tee -a $GITHUB_OUTPUT
+          echo GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=${GITHUB_REF_NAME#v}\" \"-X=github.com/ollama/ollama/server.mode=release\"'" >>$GITHUB_OUTPUT
-          echo VERSION="${GITHUB_REF_NAME#v}" | tee -a $GITHUB_OUTPUT
+          echo VERSION="${GITHUB_REF_NAME#v}" >>$GITHUB_OUTPUT
-          echo vendorsha=$(make -f Makefile.sync print-base) | tee -a $GITHUB_OUTPUT
 
   darwin-build:
     runs-on: macos-14-xlarge
@@ -55,9 +53,6 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
-          cache-dependency-path: |
-            go.sum
-            Makefile.sync
       - run: |
           ./scripts/build_darwin.sh
       - name: Log build results
@@ -190,7 +185,7 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}\.ccache
-          key: ccache-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.preset }}-${{ needs.setup-environment.outputs.vendorsha }}
+          key: ccache-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.preset }}
       - name: Build target "${{ matrix.preset }}"
         run: |
           Import-Module 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
@@ -254,9 +249,6 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
-          cache-dependency-path: |
-            go.sum
-            Makefile.sync
       - name: Verify gcc is actually clang
         run: |
           $ErrorActionPreference='Continue'
@@ -310,9 +302,6 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
-          cache-dependency-path: |
-            go.sum
-            Makefile.sync
       - uses: actions/download-artifact@v4
         with:
           pattern: depends-windows*
@@ -377,7 +366,6 @@ jobs:
               bin/ollama)                echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;
               lib/ollama/*.so*)          echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;
               lib/ollama/cuda_v*)        echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;
-              lib/ollama/vulkan*)        echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}.tar.in ;;
               lib/ollama/cuda_jetpack5)  echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}-jetpack5.tar.in ;;
               lib/ollama/cuda_jetpack6)  echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}-jetpack6.tar.in ;;
               lib/ollama/rocm)           echo $COMPONENT >>ollama-${{ matrix.os }}-${{ matrix.arch }}-rocm.tar.in ;;

.github/workflows/test.yaml

@@ -22,7 +22,6 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       changed: ${{ steps.changes.outputs.changed }}
-      vendorsha: ${{ steps.changes.outputs.vendorsha }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -38,7 +37,6 @@ jobs:
           }
 
           echo changed=$(changed 'llama/llama.cpp/**/*' 'ml/backend/ggml/ggml/**/*') | tee -a $GITHUB_OUTPUT
-          echo vendorsha=$(make -f Makefile.sync print-base) | tee -a $GITHUB_OUTPUT
 
   linux:
     needs: [changes]
@@ -85,7 +83,7 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: /github/home/.cache/ccache
-          key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.preset }}-${{ needs.changes.outputs.vendorsha }}
+          key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.preset }}
       - run: |
           cmake --preset ${{ matrix.preset }} ${{ matrix.flags }}
           cmake --build --preset ${{ matrix.preset }} --parallel
@@ -180,7 +178,7 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}\.ccache
-          key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.preset }}-${{ needs.changes.outputs.vendorsha }}
+          key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.preset }}
       - run: |
           Import-Module 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
           Enter-VsDevShell -VsInstallPath 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise' -SkipAutomaticLocation  -DevCmdArguments '-arch=x64 -no_logo'
@@ -208,9 +206,6 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version-file: 'go.mod'
-          cache-dependency-path: |
-            go.sum
-            Makefile.sync
       - uses: actions/setup-node@v4
         with:
           node-version: '20'
@@ -231,9 +226,12 @@ jobs:
         if: always()
         run: go test -count=1 -benchtime=1x ./...
 
-      - uses: golangci/golangci-lint-action@v9
+      # TODO(bmizerany): replace this heavy tool with just the
+      # tools/checks/binaries we want and then make them all run in parallel
+      # across jobs, not on a single tiny vm on Github Actions.
+      - uses: golangci/golangci-lint-action@v6
         with:
-          only-new-issues: true
+          args: --timeout 10m0s -v
 
   patches:
     runs-on: ubuntu-latest
@@ -242,4 +240,4 @@ jobs:
       - name: Verify patches apply cleanly and do not change files
         run: |
           make -f Makefile.sync clean checkout apply-patches sync
-          git diff --compact-summary --exit-code
+          git diff --compact-summary --exit-code

.golangci.yaml

@@ -1,4 +1,5 @@
-version: "2"
+run:
+  timeout: 5m
 linters:
   enable:
     - asasalint
@@ -6,46 +7,35 @@ linters:
     - bodyclose
     - containedctx
     - gocheckcompilerdirectives
+    - gofmt
+    - gofumpt
+    - gosimple
+    - govet
+    - ineffassign
     - intrange
     - makezero
     - misspell
     - nilerr
     - nolintlint
     - nosprintfhostport
+    - staticcheck
     - unconvert
     - usetesting
     - wastedassign
     - whitespace
   disable:
-    - errcheck
     - usestdlibvars
-  settings:
+    - errcheck
-    govet:
+linters-settings:
-      disable:
+  staticcheck:
-        - unusedresult
+    checks:
-    staticcheck:
+      - all
-      checks:
+      - -SA1019 # omit Deprecated check
-        - all
-        - -QF* # disable quick fix suggestions
-        - -SA1019
-        - -ST1000 # package comment format
-        - -ST1003 # underscores in package names
-        - -ST1005 # error strings should not be capitalized
-        - -ST1012 # error var naming (ErrFoo)
-        - -ST1016 # receiver name consistency
-        - -ST1020 # comment on exported function format
-        - -ST1021 # comment on exported type format
-        - -ST1022 # comment on exported var format
-        - -ST1023 # omit type from declaration
 severity:
-  default: error
+  default-severity: error
   rules:
     - linters:
         - gofmt
         - goimports
         - intrange
       severity: info
-formatters:
-  enable:
-    - gofmt
-    - gofumpt

CMakeLists.txt

@@ -54,13 +54,6 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ml/backend/ggml/ggml/src/ggml-cp
 
 add_compile_definitions(NDEBUG GGML_VERSION=0x0 GGML_COMMIT=0x0)
 
-# Define GGML version variables for shared library SOVERSION
-# These are required by ggml/src/CMakeLists.txt for proper library versioning
-set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 0)
-set(GGML_VERSION_PATCH 0)
-set(GGML_VERSION "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
-
 set(GGML_CPU ON)
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ml/backend/ggml/ggml/src)
 set_property(TARGET ggml PROPERTY EXCLUDE_FROM_ALL TRUE)

CONTRIBUTING.md

@@ -16,7 +16,7 @@ See the [development documentation](./docs/development.md) for instructions on h
 
 * New features: new features (e.g. API fields, environment variables) add surface area to Ollama and make it harder to maintain in the long run as they cannot be removed without potentially breaking users in the future.
 * Refactoring: large code improvements are important, but can be harder or take longer to review and merge.
-* Documentation: small updates to fill in or correct missing documentation are helpful, however large documentation additions can be hard to maintain over time.
+* Documentation: small updates to fill in or correct missing documentation is helpful, however large documentation additions can be hard to maintain over time.
 
 ### Issues that may not be accepted
 
@@ -43,7 +43,7 @@ Tips for proposals:
 * Explain how the change will be tested.
 
 Additionally, for bonus points: Provide draft documentation you would expect to
-see if the changes were accepted.
+see if the change were accepted.
 
 ## Pull requests
 
@@ -66,6 +66,7 @@ Examples:
 
       llm/backend/mlx: support the llama architecture
       CONTRIBUTING: provide clarity on good commit messages, and bad
+      docs: simplify manual installation with shorter curl commands
 
 Bad Examples:
 

Dockerfile

@@ -39,14 +39,14 @@ ENV CC=clang CXX=clang++
 FROM base-${TARGETARCH} AS base
 ARG CMAKEVERSION
 RUN curl -fsSL https://github.com/Kitware/CMake/releases/download/v${CMAKEVERSION}/cmake-${CMAKEVERSION}-linux-$(uname -m).tar.gz | tar xz -C /usr/local --strip-components 1
+COPY CMakeLists.txt CMakePresets.json .
+COPY ml/backend/ggml/ggml ml/backend/ggml/ggml
 ENV LDFLAGS=-s
 
 FROM base AS cpu
 RUN dnf install -y gcc-toolset-11-gcc gcc-toolset-11-gcc-c++
 ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
 ARG PARALLEL
-COPY CMakeLists.txt CMakePresets.json .
-COPY ml/backend/ggml/ggml ml/backend/ggml/ggml
 RUN --mount=type=cache,target=/root/.ccache \
     cmake --preset 'CPU' \
         && cmake --build --parallel ${PARALLEL} --preset 'CPU' \
@@ -57,8 +57,6 @@ ARG CUDA11VERSION=11.8
 RUN dnf install -y cuda-toolkit-${CUDA11VERSION//./-}
 ENV PATH=/usr/local/cuda-11/bin:$PATH
 ARG PARALLEL
-COPY CMakeLists.txt CMakePresets.json .
-COPY ml/backend/ggml/ggml ml/backend/ggml/ggml
 RUN --mount=type=cache,target=/root/.ccache \
     cmake --preset 'CUDA 11' \
         && cmake --build --parallel ${PARALLEL} --preset 'CUDA 11' \
@@ -69,8 +67,6 @@ ARG CUDA12VERSION=12.8
 RUN dnf install -y cuda-toolkit-${CUDA12VERSION//./-}
 ENV PATH=/usr/local/cuda-12/bin:$PATH
 ARG PARALLEL
-COPY CMakeLists.txt CMakePresets.json .
-COPY ml/backend/ggml/ggml ml/backend/ggml/ggml
 RUN --mount=type=cache,target=/root/.ccache \
     cmake --preset 'CUDA 12' \
         && cmake --build --parallel ${PARALLEL} --preset 'CUDA 12' \
@@ -82,8 +78,6 @@ ARG CUDA13VERSION=13.0
 RUN dnf install -y cuda-toolkit-${CUDA13VERSION//./-}
 ENV PATH=/usr/local/cuda-13/bin:$PATH
 ARG PARALLEL
-COPY CMakeLists.txt CMakePresets.json .
-COPY ml/backend/ggml/ggml ml/backend/ggml/ggml
 RUN --mount=type=cache,target=/root/.ccache \
     cmake --preset 'CUDA 13' \
         && cmake --build --parallel ${PARALLEL} --preset 'CUDA 13' \
@@ -93,8 +87,6 @@ RUN --mount=type=cache,target=/root/.ccache \
 FROM base AS rocm-6
 ENV PATH=/opt/rocm/hcc/bin:/opt/rocm/hip/bin:/opt/rocm/bin:/opt/rocm/hcc/bin:$PATH
 ARG PARALLEL
-COPY CMakeLists.txt CMakePresets.json .
-COPY ml/backend/ggml/ggml ml/backend/ggml/ggml
 RUN --mount=type=cache,target=/root/.ccache \
     cmake --preset 'ROCm 6' \
         && cmake --build --parallel ${PARALLEL} --preset 'ROCm 6' \
@@ -126,8 +118,6 @@ RUN --mount=type=cache,target=/root/.ccache \
         && cmake --install build --component CUDA --strip --parallel ${PARALLEL}
 
 FROM base AS vulkan
-COPY CMakeLists.txt CMakePresets.json .
-COPY ml/backend/ggml/ggml ml/backend/ggml/ggml
 RUN --mount=type=cache,target=/root/.ccache \
     cmake --preset 'Vulkan' \
         && cmake --build --parallel --preset 'Vulkan' \

Makefile.sync

@@ -1,6 +1,6 @@
 UPSTREAM=https://github.com/ggml-org/llama.cpp.git
 WORKDIR=llama/vendor
-FETCH_HEAD=ec98e2002
+FETCH_HEAD=3cfa9c3f125763305b4226bc032f1954f08990dc
 
 .PHONY: help
 help:
@@ -57,7 +57,7 @@ checkout: $(WORKDIR)
 $(WORKDIR):
 	git clone $(UPSTREAM) $(WORKDIR)
 
-.PHONY: format-patches
+.PHONE: format-patches
 format-patches: llama/patches
 	git -C $(WORKDIR) format-patch \
 		--no-signature \
@@ -66,11 +66,7 @@ format-patches: llama/patches
 		-o $(realpath $<) \
 		$(FETCH_HEAD)
 
-.PHONY: clean
+.PHONE: clean
 clean: checkout
 	@git -C $(WORKDIR) am --abort || true
 	$(RM) llama/patches/.*.patched
-
-.PHONY: print-base
-print-base:
-	@echo $(FETCH_HEAD)

NO_AVX_GUIDE.md

@@ -1,38 +0,0 @@
-# Instrukcja budowania dla Intel Xeon (bez AVX) + NVIDIA GPU (MX Linux)
-
-Ten build naprawia błąd `Illegal instruction` na starszych procesorach i wymusza użycie CUDA.
-
-## Wymagania
-* Zainstalowane `cuda-toolkit` (bez sterowników, jeśli już są w systemie).
-* Pobrane repozytorium z `git submodule update --init --recursive`.
-
-## 1. Symlinki (Naprawa ścieżek MX Linux)
-MX Linux trzyma CUDA w niestandardowym miejscu. Wykonaj raz:
-```bash
-sudo mkdir -p /usr/local/cuda
-sudo ln -sFn /usr/lib/cuda/include /usr/local/cuda/include
-sudo ln -sFn /usr/lib/x86_64-linux-gnu/nvidia/current /usr/local/cuda/lib64
-```
-
-```bash
-# Wyczyść stare
-rm -rf build
-
-# Konfiguracja
-cmake -B build \
-    -DOLLAMA_CUDA=ON \
-    -DOLLAMA_VULKAN=OFF \
-    -DGGML_VULKAN=OFF \
-    -DCMAKE_DISABLE_FIND_PACKAGE_Vulkan=TRUE
-
-# Kompilacja (1 wątek dla stabilności przy OC)
-cmake --build build -j1
-
-# Zbudowanie pliku wykonywalnego
-go build .
-```
-
-```bash
-sudo mv ollama /usr/bin/ollama
-sudo chmod +x /usr/bin/ollama
-```

README.md

@@ -6,9 +6,6 @@
 
 # Ollama
 
-W przypadku xeon 5675 przeczytaj plik NO_AVX_GUIDE.md!!
-Możesz zastosować też build_custom.sh dla automatycznego FIX
-
 Get up and running with large language models.
 
 ### macOS
@@ -370,7 +367,6 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [Ollama4j Web UI](https://github.com/ollama4j/ollama4j-web-ui) - Java-based Web UI for Ollama built with Vaadin, Spring Boot, and Ollama4j
 - [PyOllaMx](https://github.com/kspviswa/pyOllaMx) - macOS application capable of chatting with both Ollama and Apple MLX models.
 - [Cline](https://github.com/cline/cline) - Formerly known as Claude Dev is a VS Code extension for multi-file/whole-repo coding
-- [Void](https://github.com/voideditor/void) (Open source AI code editor and Cursor alternative)
 - [Cherry Studio](https://github.com/kangfenmao/cherry-studio) (Desktop client with Ollama support)
 - [ConfiChat](https://github.com/1runeberg/confichat) (Lightweight, standalone, multi-platform, and privacy-focused LLM chat interface with optional encryption)
 - [Archyve](https://github.com/nickthecook/archyve) (RAG-enabling document library)
@@ -431,7 +427,6 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [Mayan EDMS](https://gitlab.com/mayan-edms/mayan-edms) (Open source document management system to organize, tag, search, and automate your files with powerful Ollama driven workflows.)
 - [Serene Pub](https://github.com/doolijb/serene-pub) (Beginner friendly, open source AI Roleplaying App for Windows, Mac OS and Linux. Search, download and use models with Ollama all inside the app.)
 - [Andes](https://github.com/aqerd/andes) (A Visual Studio Code extension that provides a local UI interface for Ollama models)
-- [KDeps](https://github.com/kdeps/kdeps) (Kdeps is an offline-first AI framework for building Dockerized full-stack AI applications declaratively using Apple PKL and integrates APIs with Ollama on the backend.)
 - [Clueless](https://github.com/KashyapTan/clueless) (Open Source & Local Cluely: A desktop application LLM assistant to help you talk to anything on your screen using locally served Ollama models. Also undetectable to screenshare)
 - [ollama-co2](https://github.com/carbonatedWaterOrg/ollama-co2) (FastAPI web interface for monitoring and managing local and remote Ollama servers with real-time model monitoring and concurrent downloads)
 - [Hillnote](https://hillnote.com) (A Markdown-first workspace designed to supercharge your AI workflow. Create documents ready to integrate with Claude, ChatGPT, Gemini, Cursor, and more - all while keeping your work on your device.)
@@ -558,7 +553,7 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [Parakeet](https://github.com/parakeet-nest/parakeet) is a GoLang library, made to simplify the development of small generative AI applications with Ollama.
 - [Haverscript](https://github.com/andygill/haverscript) with [examples](https://github.com/andygill/haverscript/tree/main/examples)
 - [Ollama for Swift](https://github.com/mattt/ollama-swift)
-- [Swollama for Swift](https://github.com/guitaripod/Swollama) with [DocC](https://guitaripod.github.io/Swollama/documentation/swollama)
+- [Swollama for Swift](https://github.com/marcusziade/Swollama) with [DocC](https://marcusziade.github.io/Swollama/documentation/swollama/)
 - [GoLamify](https://github.com/prasad89/golamify)
 - [Ollama for Haskell](https://github.com/tusharad/ollama-haskell)
 - [multi-llm-ts](https://github.com/nbonamy/multi-llm-ts) (A Typescript/JavaScript library allowing access to different LLM in a unified API)
@@ -621,7 +616,7 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [LSP-AI](https://github.com/SilasMarvin/lsp-ai) (Open-source language server for AI-powered functionality)
 - [QodeAssist](https://github.com/Palm1r/QodeAssist) (AI-powered coding assistant plugin for Qt Creator)
 - [Obsidian Quiz Generator plugin](https://github.com/ECuiDev/obsidian-quiz-generator)
-- [AI Summary Helper plugin](https://github.com/philffm/ai-summary-helper)
+- [AI Summmary Helper plugin](https://github.com/philffm/ai-summary-helper)
 - [TextCraft](https://github.com/suncloudsmoon/TextCraft) (Copilot in Word alternative using Ollama)
 - [Alfred Ollama](https://github.com/zeitlings/alfred-ollama) (Alfred Workflow)
 - [TextLLaMA](https://github.com/adarshM84/TextLLaMA) A Chrome Extension that helps you write emails, correct grammar, and translate into any language
@@ -629,7 +624,7 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [LLM Telegram Bot](https://github.com/innightwolfsleep/llm_telegram_bot) (telegram bot, primary for RP. Oobabooga-like buttons, [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui) API integration e.t.c)
 - [mcp-llm](https://github.com/sammcj/mcp-llm) (MCP Server to allow LLMs to call other LLMs)
 - [SimpleOllamaUnity](https://github.com/HardCodeDev777/SimpleOllamaUnity) (Unity Engine extension for communicating with Ollama in a few lines of code. Also works at runtime)
-- [UnityCodeLama](https://github.com/HardCodeDev777/UnityCodeLama) (Unity Editor tool to analyze scripts via Ollama)
+- [UnityCodeLama](https://github.com/HardCodeDev777/UnityCodeLama) (Unity Edtior tool to analyze scripts via Ollama)
 - [NativeMind](https://github.com/NativeMindBrowser/NativeMindExtension) (Private, on-device AI Assistant, no cloud dependencies)
 - [GMAI - Gradle Managed AI](https://gmai.premex.se/) (Gradle plugin for automated Ollama lifecycle management during build phases)
 - [NOMYO Router](https://github.com/nomyo-ai/nomyo-router) (A transparent Ollama proxy with model deployment aware routing which auto-manages multiple Ollama instances in a given network)
@@ -639,7 +634,7 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [llama.cpp](https://github.com/ggml-org/llama.cpp) project founded by Georgi Gerganov.
 
 ### Observability
-- [Opik](https://www.comet.com/docs/opik/cookbook/ollama) is an open-source platform to debug, evaluate, and monitor your LLM applications, RAG systems, and agentic workflows with comprehensive tracing, automated evaluations, and production-ready dashboards. Opik supports native integration to Ollama.
+- [Opik](https://www.comet.com/docs/opik/cookbook/ollama) is an open-source platform to debug, evaluate, and monitor your LLM applications, RAG systems, and agentic workflows with comprehensive tracing, automated evaluations, and production-ready dashboards. Opik supports native intergration to Ollama.
 - [Lunary](https://lunary.ai/docs/integrations/ollama) is the leading open-source LLM observability platform. It provides a variety of enterprise-grade features such as real-time analytics, prompt templates management, PII masking, and comprehensive agent tracing.
 - [OpenLIT](https://github.com/openlit/openlit) is an OpenTelemetry-native tool for monitoring Ollama Applications & GPUs using traces and metrics.
 - [HoneyHive](https://docs.honeyhive.ai/integrations/ollama) is an AI observability and evaluation platform for AI agents. Use HoneyHive to evaluate agent performance, interrogate failures, and monitor quality in production.

SECURITY.md

@@ -14,7 +14,7 @@ Please include the following details in your report:
 
 ## Security best practices
 
-While the maintainer team does its best to secure Ollama, users are encouraged to implement their own security best practices, such as:
+While the maintainer team does their best to secure Ollama, users are encouraged to implement their own security best practices, such as:
 
 - Regularly updating to the latest version of Ollama
 - Securing access to hosted instances of Ollama

api/client.go

@@ -226,14 +226,7 @@ func (c *Client) stream(ctx context.Context, method, path string, data any, fn f
 
 		bts := scanner.Bytes()
 		if err := json.Unmarshal(bts, &errorResponse); err != nil {
-			if response.StatusCode >= http.StatusBadRequest {
+			return fmt.Errorf("unmarshal: %w", err)
-				return StatusError{
-					StatusCode:   response.StatusCode,
-					Status:       response.Status,
-					ErrorMessage: string(bts),
-				}
-			}
-			return errors.New(string(bts))
 		}
 
 		if response.StatusCode == http.StatusUnauthorized {
@@ -347,7 +340,7 @@ type CreateProgressFunc func(ProgressResponse) error
 // Create creates a model from a [Modelfile]. fn is a progress function that
 // behaves similarly to other methods (see [Client.Pull]).
 //
-// [Modelfile]: https://github.com/ollama/ollama/blob/main/docs/modelfile.mdx
+// [Modelfile]: https://github.com/ollama/ollama/blob/main/docs/modelfile.md
 func (c *Client) Create(ctx context.Context, req *CreateRequest, fn CreateProgressFunc) error {
 	return c.stream(ctx, http.MethodPost, "/api/create", req, func(bts []byte) error {
 		var resp ProgressResponse

api/client_test.go

@@ -55,7 +55,6 @@ func TestClientFromEnvironment(t *testing.T) {
 type testError struct {
 	message    string
 	statusCode int
-	raw        bool // if true, write message as-is instead of JSON encoding
 }
 
 func (e testError) Error() string {
@@ -112,20 +111,6 @@ func TestClientStream(t *testing.T) {
 				},
 			},
 		},
-		{
-			name: "plain text error response",
-			responses: []any{
-				"internal server error",
-			},
-			wantErr: "internal server error",
-		},
-		{
-			name: "HTML error page",
-			responses: []any{
-				"<html><body>404 Not Found</body></html>",
-			},
-			wantErr: "404 Not Found",
-		},
 	}
 
 	for _, tc := range testCases {
@@ -150,12 +135,6 @@ func TestClientStream(t *testing.T) {
 						return
 					}
 
-					if str, ok := resp.(string); ok {
-						fmt.Fprintln(w, str)
-						flusher.Flush()
-						continue
-					}
-
 					if err := json.NewEncoder(w).Encode(resp); err != nil {
 						t.Fatalf("failed to encode response: %v", err)
 					}
@@ -194,10 +173,9 @@ func TestClientStream(t *testing.T) {
 
 func TestClientDo(t *testing.T) {
 	testCases := []struct {
-		name           string
+		name     string
-		response       any
+		response any
-		wantErr        string
+		wantErr  string
-		wantStatusCode int
 	}{
 		{
 			name: "immediate error response",
@@ -205,8 +183,7 @@ func TestClientDo(t *testing.T) {
 				message:    "test error message",
 				statusCode: http.StatusBadRequest,
 			},
-			wantErr:        "test error message",
+			wantErr: "test error message",
-			wantStatusCode: http.StatusBadRequest,
 		},
 		{
 			name: "server error response",
@@ -214,8 +191,7 @@ func TestClientDo(t *testing.T) {
 				message:    "internal error",
 				statusCode: http.StatusInternalServerError,
 			},
-			wantErr:        "internal error",
+			wantErr: "internal error",
-			wantStatusCode: http.StatusInternalServerError,
 		},
 		{
 			name: "successful response",
@@ -227,26 +203,6 @@ func TestClientDo(t *testing.T) {
 				Success: true,
 			},
 		},
-		{
-			name: "plain text error response",
-			response: testError{
-				message:    "internal server error",
-				statusCode: http.StatusInternalServerError,
-				raw:        true,
-			},
-			wantErr:        "internal server error",
-			wantStatusCode: http.StatusInternalServerError,
-		},
-		{
-			name: "HTML error page",
-			response: testError{
-				message:    "<html><body>404 Not Found</body></html>",
-				statusCode: http.StatusNotFound,
-				raw:        true,
-			},
-			wantErr:        "<html><body>404 Not Found</body></html>",
-			wantStatusCode: http.StatusNotFound,
-		},
 	}
 
 	for _, tc := range testCases {
@@ -254,16 +210,11 @@ func TestClientDo(t *testing.T) {
 			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				if errResp, ok := tc.response.(testError); ok {
 					w.WriteHeader(errResp.statusCode)
-					if !errResp.raw {
+					err := json.NewEncoder(w).Encode(map[string]string{
-						err := json.NewEncoder(w).Encode(map[string]string{
+						"error": errResp.message,
-							"error": errResp.message,
+					})
-						})
+					if err != nil {
-						if err != nil {
+						t.Fatal("failed to encode error response:", err)
-							t.Fatal("failed to encode error response:", err)
-						}
-					} else {
-						// Write raw message (simulates non-JSON error responses)
-						fmt.Fprint(w, errResp.message)
 					}
 					return
 				}
@@ -290,15 +241,6 @@ func TestClientDo(t *testing.T) {
 				if err.Error() != tc.wantErr {
 					t.Errorf("error message mismatch: got %q, want %q", err.Error(), tc.wantErr)
 				}
-				if tc.wantStatusCode != 0 {
-					if statusErr, ok := err.(StatusError); ok {
-						if statusErr.StatusCode != tc.wantStatusCode {
-							t.Errorf("status code mismatch: got %d, want %d", statusErr.StatusCode, tc.wantStatusCode)
-						}
-					} else {
-						t.Errorf("expected StatusError, got %T", err)
-					}
-				}
 				return
 			}
 

api/examples/chat/main.go

@@ -15,19 +15,19 @@ func main() {
 	}
 
 	messages := []api.Message{
-		{
+		api.Message{
 			Role:    "system",
 			Content: "Provide very brief, concise responses",
 		},
-		{
+		api.Message{
 			Role:    "user",
 			Content: "Name some unusual animals",
 		},
-		{
+		api.Message{
 			Role:    "assistant",
 			Content: "Monotreme, platypus, echidna",
 		},
-		{
+		api.Message{
 			Role:    "user",
 			Content: "which of these is the most dangerous?",
 		},

api/types.go

@@ -3,7 +3,6 @@ package api
 import (
 	"encoding/json"
 	"fmt"
-	"iter"
 	"log/slog"
 	"math"
 	"os"
@@ -15,7 +14,6 @@ import (
 	"github.com/google/uuid"
 
 	"github.com/ollama/ollama/envconfig"
-	"github.com/ollama/ollama/internal/orderedmap"
 	"github.com/ollama/ollama/types/model"
 )
 
@@ -229,79 +227,13 @@ type ToolCallFunction struct {
 	Arguments ToolCallFunctionArguments `json:"arguments"`
 }
 
-// ToolCallFunctionArguments holds tool call arguments in insertion order.
+type ToolCallFunctionArguments map[string]any
-type ToolCallFunctionArguments struct {
-	om *orderedmap.Map[string, any]
-}
-
-// NewToolCallFunctionArguments creates a new empty ToolCallFunctionArguments.
-func NewToolCallFunctionArguments() ToolCallFunctionArguments {
-	return ToolCallFunctionArguments{om: orderedmap.New[string, any]()}
-}
-
-// Get retrieves a value by key.
-func (t *ToolCallFunctionArguments) Get(key string) (any, bool) {
-	if t == nil || t.om == nil {
-		return nil, false
-	}
-	return t.om.Get(key)
-}
-
-// Set sets a key-value pair, preserving insertion order.
-func (t *ToolCallFunctionArguments) Set(key string, value any) {
-	if t == nil {
-		return
-	}
-	if t.om == nil {
-		t.om = orderedmap.New[string, any]()
-	}
-	t.om.Set(key, value)
-}
-
-// Len returns the number of arguments.
-func (t *ToolCallFunctionArguments) Len() int {
-	if t == nil || t.om == nil {
-		return 0
-	}
-	return t.om.Len()
-}
-
-// All returns an iterator over all key-value pairs in insertion order.
-func (t *ToolCallFunctionArguments) All() iter.Seq2[string, any] {
-	if t == nil || t.om == nil {
-		return func(yield func(string, any) bool) {}
-	}
-	return t.om.All()
-}
-
-// ToMap returns a regular map (order not preserved).
-func (t *ToolCallFunctionArguments) ToMap() map[string]any {
-	if t == nil || t.om == nil {
-		return nil
-	}
-	return t.om.ToMap()
-}
 
 func (t *ToolCallFunctionArguments) String() string {
-	if t == nil || t.om == nil {
+	bts, _ := json.Marshal(t)
-		return "{}"
-	}
-	bts, _ := json.Marshal(t.om)
 	return string(bts)
 }
 
-func (t *ToolCallFunctionArguments) UnmarshalJSON(data []byte) error {
-	t.om = orderedmap.New[string, any]()
-	return json.Unmarshal(data, t.om)
-}
-
-func (t ToolCallFunctionArguments) MarshalJSON() ([]byte, error) {
-	if t.om == nil {
-		return []byte("{}"), nil
-	}
-	return json.Marshal(t.om)
-}
-
 type Tool struct {
 	Type     string       `json:"type"`
 	Items    any          `json:"items,omitempty"`
@@ -350,78 +282,12 @@ func (pt PropertyType) String() string {
 	return fmt.Sprintf("%v", []string(pt))
 }
 
-// ToolPropertiesMap holds tool properties in insertion order.
-type ToolPropertiesMap struct {
-	om *orderedmap.Map[string, ToolProperty]
-}
-
-// NewToolPropertiesMap creates a new empty ToolPropertiesMap.
-func NewToolPropertiesMap() *ToolPropertiesMap {
-	return &ToolPropertiesMap{om: orderedmap.New[string, ToolProperty]()}
-}
-
-// Get retrieves a property by name.
-func (t *ToolPropertiesMap) Get(key string) (ToolProperty, bool) {
-	if t == nil || t.om == nil {
-		return ToolProperty{}, false
-	}
-	return t.om.Get(key)
-}
-
-// Set sets a property, preserving insertion order.
-func (t *ToolPropertiesMap) Set(key string, value ToolProperty) {
-	if t == nil {
-		return
-	}
-	if t.om == nil {
-		t.om = orderedmap.New[string, ToolProperty]()
-	}
-	t.om.Set(key, value)
-}
-
-// Len returns the number of properties.
-func (t *ToolPropertiesMap) Len() int {
-	if t == nil || t.om == nil {
-		return 0
-	}
-	return t.om.Len()
-}
-
-// All returns an iterator over all properties in insertion order.
-func (t *ToolPropertiesMap) All() iter.Seq2[string, ToolProperty] {
-	if t == nil || t.om == nil {
-		return func(yield func(string, ToolProperty) bool) {}
-	}
-	return t.om.All()
-}
-
-// ToMap returns a regular map (order not preserved).
-func (t *ToolPropertiesMap) ToMap() map[string]ToolProperty {
-	if t == nil || t.om == nil {
-		return nil
-	}
-	return t.om.ToMap()
-}
-
-func (t ToolPropertiesMap) MarshalJSON() ([]byte, error) {
-	if t.om == nil {
-		return []byte("null"), nil
-	}
-	return json.Marshal(t.om)
-}
-
-func (t *ToolPropertiesMap) UnmarshalJSON(data []byte) error {
-	t.om = orderedmap.New[string, ToolProperty]()
-	return json.Unmarshal(data, t.om)
-}
-
 type ToolProperty struct {
-	AnyOf       []ToolProperty     `json:"anyOf,omitempty"`
+	AnyOf       []ToolProperty `json:"anyOf,omitempty"`
-	Type        PropertyType       `json:"type,omitempty"`
+	Type        PropertyType   `json:"type,omitempty"`
-	Items       any                `json:"items,omitempty"`
+	Items       any            `json:"items,omitempty"`
-	Description string             `json:"description,omitempty"`
+	Description string         `json:"description,omitempty"`
-	Enum        []any              `json:"enum,omitempty"`
+	Enum        []any          `json:"enum,omitempty"`
-	Properties  *ToolPropertiesMap `json:"properties,omitempty"`
 }
 
 // ToTypeScriptType converts a ToolProperty to a TypeScript type string
@@ -470,11 +336,11 @@ func mapToTypeScriptType(jsonType string) string {
 }
 
 type ToolFunctionParameters struct {
-	Type       string             `json:"type"`
+	Type       string                  `json:"type"`
-	Defs       any                `json:"$defs,omitempty"`
+	Defs       any                     `json:"$defs,omitempty"`
-	Items      any                `json:"items,omitempty"`
+	Items      any                     `json:"items,omitempty"`
-	Required   []string           `json:"required,omitempty"`
+	Required   []string                `json:"required,omitempty"`
-	Properties *ToolPropertiesMap `json:"properties"`
+	Properties map[string]ToolProperty `json:"properties"`
 }
 
 func (t *ToolFunctionParameters) String() string {
@@ -687,9 +553,6 @@ type CreateRequest struct {
 	Renderer string `json:"renderer,omitempty"`
 	Parser   string `json:"parser,omitempty"`
 
-	// Requires is the minimum version of Ollama required by the model.
-	Requires string `json:"requires,omitempty"`
-
 	// Info is a map of additional information for the model
 	Info map[string]any `json:"info,omitempty"`
 
@@ -740,7 +603,6 @@ type ShowResponse struct {
 	Tensors       []Tensor           `json:"tensors,omitempty"`
 	Capabilities  []model.Capability `json:"capabilities,omitempty"`
 	ModifiedAt    time.Time          `json:"modified_at,omitempty"`
-	Requires      string             `json:"requires,omitempty"`
 }
 
 // CopyRequest is the request passed to [Client.Copy].

api/types_test.go

@@ -11,24 +11,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// testPropsMap creates a ToolPropertiesMap from a map (convenience function for tests, order not preserved)
-func testPropsMap(m map[string]ToolProperty) *ToolPropertiesMap {
-	props := NewToolPropertiesMap()
-	for k, v := range m {
-		props.Set(k, v)
-	}
-	return props
-}
-
-// testArgs creates ToolCallFunctionArguments from a map (convenience function for tests, order not preserved)
-func testArgs(m map[string]any) ToolCallFunctionArguments {
-	args := NewToolCallFunctionArguments()
-	for k, v := range m {
-		args.Set(k, v)
-	}
-	return args
-}
-
 func TestKeepAliveParsingFromJSON(t *testing.T) {
 	tests := []struct {
 		name string
@@ -327,9 +309,9 @@ func TestToolFunctionParameters_MarshalJSON(t *testing.T) {
 			input: ToolFunctionParameters{
 				Type:     "object",
 				Required: []string{"name"},
-				Properties: testPropsMap(map[string]ToolProperty{
+				Properties: map[string]ToolProperty{
 					"name": {Type: PropertyType{"string"}},
-				}),
+				},
 			},
 			expected: `{"type":"object","required":["name"],"properties":{"name":{"type":"string"}}}`,
 		},
@@ -337,9 +319,9 @@ func TestToolFunctionParameters_MarshalJSON(t *testing.T) {
 			name: "no required",
 			input: ToolFunctionParameters{
 				Type: "object",
-				Properties: testPropsMap(map[string]ToolProperty{
+				Properties: map[string]ToolProperty{
 					"name": {Type: PropertyType{"string"}},
-				}),
+				},
 			},
 			expected: `{"type":"object","properties":{"name":{"type":"string"}}}`,
 		},
@@ -357,7 +339,7 @@ func TestToolFunctionParameters_MarshalJSON(t *testing.T) {
 func TestToolCallFunction_IndexAlwaysMarshals(t *testing.T) {
 	fn := ToolCallFunction{
 		Name:      "echo",
-		Arguments: testArgs(map[string]any{"message": "hi"}),
+		Arguments: ToolCallFunctionArguments{"message": "hi"},
 	}
 
 	data, err := json.Marshal(fn)
@@ -522,116 +504,6 @@ func TestThinking_UnmarshalJSON(t *testing.T) {
 	}
 }
 
-func TestToolPropertyNestedProperties(t *testing.T) {
-	tests := []struct {
-		name     string
-		input    string
-		expected ToolProperty
-	}{
-		{
-			name: "nested object properties",
-			input: `{
-				"type": "object",
-				"description": "Location details",
-				"properties": {
-					"address": {
-						"type": "string",
-						"description": "Street address"
-					},
-					"city": {
-						"type": "string",
-						"description": "City name"
-					}
-				}
-			}`,
-			expected: ToolProperty{
-				Type:        PropertyType{"object"},
-				Description: "Location details",
-				Properties: testPropsMap(map[string]ToolProperty{
-					"address": {
-						Type:        PropertyType{"string"},
-						Description: "Street address",
-					},
-					"city": {
-						Type:        PropertyType{"string"},
-						Description: "City name",
-					},
-				}),
-			},
-		},
-		{
-			name: "deeply nested properties",
-			input: `{
-				"type": "object",
-				"description": "Event",
-				"properties": {
-					"location": {
-						"type": "object",
-						"description": "Location",
-						"properties": {
-							"coordinates": {
-								"type": "object",
-								"description": "GPS coordinates",
-								"properties": {
-									"lat": {"type": "number", "description": "Latitude"},
-									"lng": {"type": "number", "description": "Longitude"}
-								}
-							}
-						}
-					}
-				}
-			}`,
-			expected: ToolProperty{
-				Type:        PropertyType{"object"},
-				Description: "Event",
-				Properties: testPropsMap(map[string]ToolProperty{
-					"location": {
-						Type:        PropertyType{"object"},
-						Description: "Location",
-						Properties: testPropsMap(map[string]ToolProperty{
-							"coordinates": {
-								Type:        PropertyType{"object"},
-								Description: "GPS coordinates",
-								Properties: testPropsMap(map[string]ToolProperty{
-									"lat": {Type: PropertyType{"number"}, Description: "Latitude"},
-									"lng": {Type: PropertyType{"number"}, Description: "Longitude"},
-								}),
-							},
-						}),
-					},
-				}),
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			var prop ToolProperty
-			err := json.Unmarshal([]byte(tt.input), &prop)
-			require.NoError(t, err)
-
-			// Compare JSON representations since pointer comparison doesn't work
-			expectedJSON, err := json.Marshal(tt.expected)
-			require.NoError(t, err)
-			actualJSON, err := json.Marshal(prop)
-			require.NoError(t, err)
-			assert.JSONEq(t, string(expectedJSON), string(actualJSON))
-
-			// Round-trip test: marshal and unmarshal again
-			data, err := json.Marshal(prop)
-			require.NoError(t, err)
-
-			var prop2 ToolProperty
-			err = json.Unmarshal(data, &prop2)
-			require.NoError(t, err)
-
-			prop2JSON, err := json.Marshal(prop2)
-			require.NoError(t, err)
-			assert.JSONEq(t, string(expectedJSON), string(prop2JSON))
-		})
-	}
-}
-
 func TestToolFunctionParameters_String(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -643,12 +515,12 @@ func TestToolFunctionParameters_String(t *testing.T) {
 			params: ToolFunctionParameters{
 				Type:     "object",
 				Required: []string{"name"},
-				Properties: testPropsMap(map[string]ToolProperty{
+				Properties: map[string]ToolProperty{
 					"name": {
 						Type:        PropertyType{"string"},
 						Description: "The name of the person",
 					},
-				}),
+				},
 			},
 			expected: `{"type":"object","required":["name"],"properties":{"name":{"type":"string","description":"The name of the person"}}}`,
 		},
@@ -665,7 +537,7 @@ func TestToolFunctionParameters_String(t *testing.T) {
 					s.Self = s
 					return s
 				}(),
-				Properties: testPropsMap(map[string]ToolProperty{}),
+				Properties: map[string]ToolProperty{},
 			},
 			expected: "",
 		},
@@ -678,235 +550,3 @@ func TestToolFunctionParameters_String(t *testing.T) {
 		})
 	}
 }
-
-func TestToolCallFunctionArguments_OrderPreservation(t *testing.T) {
-	t.Run("marshal preserves insertion order", func(t *testing.T) {
-		args := NewToolCallFunctionArguments()
-		args.Set("zebra", "z")
-		args.Set("apple", "a")
-		args.Set("mango", "m")
-
-		data, err := json.Marshal(args)
-		require.NoError(t, err)
-
-		// Should preserve insertion order, not alphabetical
-		assert.Equal(t, `{"zebra":"z","apple":"a","mango":"m"}`, string(data))
-	})
-
-	t.Run("unmarshal preserves JSON order", func(t *testing.T) {
-		jsonData := `{"zebra":"z","apple":"a","mango":"m"}`
-
-		var args ToolCallFunctionArguments
-		err := json.Unmarshal([]byte(jsonData), &args)
-		require.NoError(t, err)
-
-		// Verify iteration order matches JSON order
-		var keys []string
-		for k := range args.All() {
-			keys = append(keys, k)
-		}
-		assert.Equal(t, []string{"zebra", "apple", "mango"}, keys)
-	})
-
-	t.Run("round trip preserves order", func(t *testing.T) {
-		original := `{"z":1,"a":2,"m":3,"b":4}`
-
-		var args ToolCallFunctionArguments
-		err := json.Unmarshal([]byte(original), &args)
-		require.NoError(t, err)
-
-		data, err := json.Marshal(args)
-		require.NoError(t, err)
-
-		assert.Equal(t, original, string(data))
-	})
-
-	t.Run("String method returns ordered JSON", func(t *testing.T) {
-		args := NewToolCallFunctionArguments()
-		args.Set("c", 3)
-		args.Set("a", 1)
-		args.Set("b", 2)
-
-		assert.Equal(t, `{"c":3,"a":1,"b":2}`, args.String())
-	})
-
-	t.Run("Get retrieves correct values", func(t *testing.T) {
-		args := NewToolCallFunctionArguments()
-		args.Set("key1", "value1")
-		args.Set("key2", 42)
-
-		v, ok := args.Get("key1")
-		assert.True(t, ok)
-		assert.Equal(t, "value1", v)
-
-		v, ok = args.Get("key2")
-		assert.True(t, ok)
-		assert.Equal(t, 42, v)
-
-		_, ok = args.Get("nonexistent")
-		assert.False(t, ok)
-	})
-
-	t.Run("Len returns correct count", func(t *testing.T) {
-		args := NewToolCallFunctionArguments()
-		assert.Equal(t, 0, args.Len())
-
-		args.Set("a", 1)
-		assert.Equal(t, 1, args.Len())
-
-		args.Set("b", 2)
-		assert.Equal(t, 2, args.Len())
-	})
-
-	t.Run("empty args marshal to empty object", func(t *testing.T) {
-		args := NewToolCallFunctionArguments()
-		data, err := json.Marshal(args)
-		require.NoError(t, err)
-		assert.Equal(t, `{}`, string(data))
-	})
-
-	t.Run("zero value args marshal to empty object", func(t *testing.T) {
-		var args ToolCallFunctionArguments
-		assert.Equal(t, "{}", args.String())
-	})
-}
-
-func TestToolPropertiesMap_OrderPreservation(t *testing.T) {
-	t.Run("marshal preserves insertion order", func(t *testing.T) {
-		props := NewToolPropertiesMap()
-		props.Set("zebra", ToolProperty{Type: PropertyType{"string"}})
-		props.Set("apple", ToolProperty{Type: PropertyType{"number"}})
-		props.Set("mango", ToolProperty{Type: PropertyType{"boolean"}})
-
-		data, err := json.Marshal(props)
-		require.NoError(t, err)
-
-		// Should preserve insertion order, not alphabetical
-		expected := `{"zebra":{"type":"string"},"apple":{"type":"number"},"mango":{"type":"boolean"}}`
-		assert.Equal(t, expected, string(data))
-	})
-
-	t.Run("unmarshal preserves JSON order", func(t *testing.T) {
-		jsonData := `{"zebra":{"type":"string"},"apple":{"type":"number"},"mango":{"type":"boolean"}}`
-
-		var props ToolPropertiesMap
-		err := json.Unmarshal([]byte(jsonData), &props)
-		require.NoError(t, err)
-
-		// Verify iteration order matches JSON order
-		var keys []string
-		for k := range props.All() {
-			keys = append(keys, k)
-		}
-		assert.Equal(t, []string{"zebra", "apple", "mango"}, keys)
-	})
-
-	t.Run("round trip preserves order", func(t *testing.T) {
-		original := `{"z":{"type":"string"},"a":{"type":"number"},"m":{"type":"boolean"}}`
-
-		var props ToolPropertiesMap
-		err := json.Unmarshal([]byte(original), &props)
-		require.NoError(t, err)
-
-		data, err := json.Marshal(props)
-		require.NoError(t, err)
-
-		assert.Equal(t, original, string(data))
-	})
-
-	t.Run("Get retrieves correct values", func(t *testing.T) {
-		props := NewToolPropertiesMap()
-		props.Set("name", ToolProperty{Type: PropertyType{"string"}, Description: "The name"})
-		props.Set("age", ToolProperty{Type: PropertyType{"integer"}, Description: "The age"})
-
-		v, ok := props.Get("name")
-		assert.True(t, ok)
-		assert.Equal(t, "The name", v.Description)
-
-		v, ok = props.Get("age")
-		assert.True(t, ok)
-		assert.Equal(t, "The age", v.Description)
-
-		_, ok = props.Get("nonexistent")
-		assert.False(t, ok)
-	})
-
-	t.Run("Len returns correct count", func(t *testing.T) {
-		props := NewToolPropertiesMap()
-		assert.Equal(t, 0, props.Len())
-
-		props.Set("a", ToolProperty{})
-		assert.Equal(t, 1, props.Len())
-
-		props.Set("b", ToolProperty{})
-		assert.Equal(t, 2, props.Len())
-	})
-
-	t.Run("nil props marshal to null", func(t *testing.T) {
-		var props *ToolPropertiesMap
-		data, err := json.Marshal(props)
-		require.NoError(t, err)
-		assert.Equal(t, `null`, string(data))
-	})
-
-	t.Run("ToMap returns regular map", func(t *testing.T) {
-		props := NewToolPropertiesMap()
-		props.Set("a", ToolProperty{Type: PropertyType{"string"}})
-		props.Set("b", ToolProperty{Type: PropertyType{"number"}})
-
-		m := props.ToMap()
-		assert.Equal(t, 2, len(m))
-		assert.Equal(t, PropertyType{"string"}, m["a"].Type)
-		assert.Equal(t, PropertyType{"number"}, m["b"].Type)
-	})
-}
-
-func TestToolCallFunctionArguments_ComplexValues(t *testing.T) {
-	t.Run("nested objects preserve order", func(t *testing.T) {
-		jsonData := `{"outer":{"z":1,"a":2},"simple":"value"}`
-
-		var args ToolCallFunctionArguments
-		err := json.Unmarshal([]byte(jsonData), &args)
-		require.NoError(t, err)
-
-		// Outer keys should be in order
-		var keys []string
-		for k := range args.All() {
-			keys = append(keys, k)
-		}
-		assert.Equal(t, []string{"outer", "simple"}, keys)
-	})
-
-	t.Run("arrays as values", func(t *testing.T) {
-		args := NewToolCallFunctionArguments()
-		args.Set("items", []string{"a", "b", "c"})
-		args.Set("numbers", []int{1, 2, 3})
-
-		data, err := json.Marshal(args)
-		require.NoError(t, err)
-
-		assert.Equal(t, `{"items":["a","b","c"],"numbers":[1,2,3]}`, string(data))
-	})
-}
-
-func TestToolPropertiesMap_NestedProperties(t *testing.T) {
-	t.Run("nested properties preserve order", func(t *testing.T) {
-		props := NewToolPropertiesMap()
-
-		nestedProps := NewToolPropertiesMap()
-		nestedProps.Set("z_field", ToolProperty{Type: PropertyType{"string"}})
-		nestedProps.Set("a_field", ToolProperty{Type: PropertyType{"number"}})
-
-		props.Set("outer", ToolProperty{
-			Type:       PropertyType{"object"},
-			Properties: nestedProps,
-		})
-
-		data, err := json.Marshal(props)
-		require.NoError(t, err)
-
-		// Both outer and inner should preserve order
-		expected := `{"outer":{"type":"object","properties":{"z_field":{"type":"string"},"a_field":{"type":"number"}}}}`
-		assert.Equal(t, expected, string(data))
-	})
-}

app/cmd/app/app.go

@@ -273,6 +273,10 @@ func main() {
 		Handler: uiServer.Handler(),
 	}
 
+	if _, err := uiServer.UserData(ctx); err != nil {
+		slog.Warn("failed to load user data", "error", err)
+	}
+
 	// Start the UI server
 	slog.Info("starting ui server", "port", port)
 	go func() {
@@ -316,17 +320,6 @@ func main() {
 		slog.Debug("no URL scheme request to handle")
 	}
 
-	go func() {
-		slog.Debug("waiting for ollama server to be ready")
-		if err := ui.WaitForServer(ctx, 10*time.Second); err != nil {
-			slog.Warn("ollama server not ready, continuing anyway", "error", err)
-		}
-
-		if _, err := uiServer.UserData(ctx); err != nil {
-			slog.Warn("failed to load user data", "error", err)
-		}
-	}()
-
 	osRun(cancel, hasCompletedFirstRun, startHidden)
 
 	slog.Info("shutting down desktop server")
@@ -368,7 +361,7 @@ func checkUserLoggedIn(uiServerPort int) bool {
 		return false
 	}
 
-	resp, err := http.Post(fmt.Sprintf("http://127.0.0.1:%d/api/me", uiServerPort), "application/json", nil)
+	resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/api/v1/me", uiServerPort))
 	if err != nil {
 		slog.Debug("failed to call local auth endpoint", "error", err)
 		return false
@@ -404,8 +397,8 @@ func checkUserLoggedIn(uiServerPort int) bool {
 // handleConnectURLScheme fetches the connect URL and opens it in the browser
 func handleConnectURLScheme() {
 	if checkUserLoggedIn(uiServerPort) {
-		slog.Info("user is already logged in, opening app instead")
+		slog.Info("user is already logged in, opening settings instead")
-		showWindow(wv.webview.Window())
+		sendUIRequestMessage("/")
 		return
 	}
 
@@ -441,30 +434,37 @@ func openInBrowser(url string) {
 	}
 }
 
-// parseURLScheme parses an ollama:// URL and validates it
+// parseURLScheme parses an ollama:// URL and returns whether it's a connect URL and the UI path
-// Supports: ollama:// (open app) and ollama://connect (OAuth)
+func parseURLScheme(urlSchemeRequest string) (isConnect bool, uiPath string, err error) {
-func parseURLScheme(urlSchemeRequest string) (isConnect bool, err error) {
 	parsedURL, err := url.Parse(urlSchemeRequest)
 	if err != nil {
-		return false, fmt.Errorf("invalid URL: %w", err)
+		return false, "", err
 	}
 
 	// Check if this is a connect URL
 	if parsedURL.Host == "connect" || strings.TrimPrefix(parsedURL.Path, "/") == "connect" {
-		return true, nil
+		return true, "", nil
 	}
 
-	// Allow bare ollama:// or ollama:/// to open the app
+	// Extract the UI path
-	if (parsedURL.Host == "" && parsedURL.Path == "") || parsedURL.Path == "/" {
+	path := "/"
-		return false, nil
+	if parsedURL.Path != "" && parsedURL.Path != "/" {
+		// For URLs like ollama:///settings, use the path directly
+		path = parsedURL.Path
+	} else if parsedURL.Host != "" {
+		// For URLs like ollama://settings (without triple slash),
+		// the "settings" part is parsed as the host, not the path.
+		// We need to convert it to a path by prepending "/"
+		// This also handles ollama://settings/ where Windows adds a trailing slash
+		path = "/" + parsedURL.Host
 	}
 
-	return false, fmt.Errorf("unsupported ollama:// URL path: %s", urlSchemeRequest)
+	return false, path, nil
 }
 
 // handleURLSchemeInCurrentInstance processes URL scheme requests in the current instance
 func handleURLSchemeInCurrentInstance(urlSchemeRequest string) {
-	isConnect, err := parseURLScheme(urlSchemeRequest)
+	isConnect, uiPath, err := parseURLScheme(urlSchemeRequest)
 	if err != nil {
 		slog.Error("failed to parse URL scheme request", "url", urlSchemeRequest, "error", err)
 		return
@@ -473,8 +473,6 @@ func handleURLSchemeInCurrentInstance(urlSchemeRequest string) {
 	if isConnect {
 		handleConnectURLScheme()
 	} else {
-		if wv.webview != nil {
+		sendUIRequestMessage(uiPath)
-			showWindow(wv.webview.Window())
-		}
 	}
 }

app/cmd/app/app_darwin.go

@@ -191,6 +191,13 @@ func LaunchNewApp() {
 	C.launchApp(appName)
 }
 
+// Send a request to the main app thread to load a UI page
+func sendUIRequestMessage(path string) {
+	p := C.CString(path)
+	defer C.free(unsafe.Pointer(p))
+	C.uiRequest(p)
+}
+
 func registerLaunchAgent(hasCompletedFirstRun bool) {
 	// Remove any stale Login Item registrations
 	C.unregisterSelfFromLoginItem()

app/cmd/app/app_darwin.m

@@ -24,14 +24,27 @@ bool firstTimeRun,startHidden; // Set in run before initialization
     for (NSURL *url in urls) {
         if ([url.scheme isEqualToString:@"ollama"]) {
             NSString *path = url.path;
-
+            if (!path || [path isEqualToString:@""]) {
-            if (path && ([path isEqualToString:@"/connect"] || [url.host isEqualToString:@"connect"])) {
+                // For URLs like ollama://settings (without triple slash),
+                // the "settings" part is parsed as the host, not the path.
+                // We need to convert it to a path by prepending "/"
+                if (url.host && ![url.host isEqualToString:@""]) {
+                    path = [@"/" stringByAppendingString:url.host];
+                } else {
+                    path = @"/";
+                }
+            }
+            
+            if ([path isEqualToString:@"/connect"] || [url.host isEqualToString:@"connect"]) {
                 // Special case: handle connect by opening browser instead of app
                 handleConnectURL();
             } else {
                 // Set app to be active and visible
                 [NSApp setActivationPolicy:NSApplicationActivationPolicyRegular];
                 [NSApp activateIgnoringOtherApps:YES];
+                
+                // Open the path with the UI
+                [self uiRequest:path];
             }
             
             break;
@@ -247,7 +260,7 @@ bool firstTimeRun,startHidden; // Set in run before initialization
 }
 
 - (void)openHelp:(id)sender {
-    NSURL *url = [NSURL URLWithString:@"https://docs.ollama.com/"];
+    NSURL *url = [NSURL URLWithString:@"https://github.com/ollama/ollama/tree/main/docs"];
     [[NSWorkspace sharedWorkspace] openURL:url];
 }
 

app/cmd/app/app_windows.go

@@ -138,7 +138,7 @@ func (app *appCallbacks) HandleURLScheme(urlScheme string) {
 
 // handleURLSchemeRequest processes URL scheme requests from other instances
 func handleURLSchemeRequest(urlScheme string) {
-	isConnect, err := parseURLScheme(urlScheme)
+	isConnect, uiPath, err := parseURLScheme(urlScheme)
 	if err != nil {
 		slog.Error("failed to parse URL scheme request", "url", urlScheme, "error", err)
 		return
@@ -147,9 +147,7 @@ func handleURLSchemeRequest(urlScheme string) {
 	if isConnect {
 		handleConnectURLScheme()
 	} else {
-		if wv.webview != nil {
+		sendUIRequestMessage(uiPath)
-			showWindow(wv.webview.Window())
-		}
 	}
 }
 
@@ -263,6 +261,11 @@ func createLoginShortcut() error {
 	return nil
 }
 
+// Send a request to the main app thread to load a UI page
+func sendUIRequestMessage(path string) {
+	wintray.SendUIRequestMessage(path)
+}
+
 func LaunchNewApp() {
 }
 

app/dialog/cocoa/dlg.m

@@ -169,47 +169,37 @@ DlgResult fileDlg(FileDlgParams* params) {
 	}
 	
 	NSArray* urls = [panel URLs];
-	if([urls count] == 0) {
+	if(self->params->allowMultiple && [urls count] >= 1) {
-		return DLG_CANCEL;
-	}
-	
-	if(self->params->allowMultiple) {
 		// For multiple files, we need to return all paths separated by null bytes
 		char* bufPtr = self->params->buf;
 		int remainingBuf = self->params->nbuf;
 		
-		// Calculate total required buffer size first
+  // Calculate total required buffer size first
-		int totalSize = 0;
+  int totalSize = 0;
-		for(NSURL* url in urls) {
+  for(NSURL* url in urls) {
-			char tempBuf[PATH_MAX];
+      char tempBuf[PATH_MAX];
-			if(![url getFileSystemRepresentation:tempBuf maxLength:PATH_MAX]) {
+      if(![url getFileSystemRepresentation:tempBuf maxLength:PATH_MAX]) {
-				return DLG_URLFAIL;
+          return DLG_URLFAIL;
-			}
+      }
-			totalSize += strlen(tempBuf) + 1; // +1 for null terminator
+      totalSize += strlen(tempBuf) + 1; // +1 for null terminator
-		}
+  }
-		totalSize += 1; // Final null terminator
+  totalSize += 1; // Final null terminator
 
-		if(totalSize > self->params->nbuf) {
+  if(totalSize > self->params->nbuf) {
-			// Not enough buffer space
+      // Not enough buffer space
-			return DLG_URLFAIL;
+      return DLG_URLFAIL;
-		}
+  }
 
-		// Now actually copy the paths (we know we have space)
+  // Now actually copy the paths (we know we have space)
-		bufPtr = self->params->buf;
+  bufPtr = self->params->buf;
-		for(NSURL* url in urls) {
+  for(NSURL* url in urls) {
-			char tempBuf[PATH_MAX];
+      char tempBuf[PATH_MAX];
-			[url getFileSystemRepresentation:tempBuf maxLength:PATH_MAX];
+      [url getFileSystemRepresentation:tempBuf maxLength:PATH_MAX];
-			int pathLen = strlen(tempBuf);
+      int pathLen = strlen(tempBuf);
-			strcpy(bufPtr, tempBuf);
+      strcpy(bufPtr, tempBuf);
-			bufPtr += pathLen + 1;
+      bufPtr += pathLen + 1;
-		}
+  }
-		*bufPtr = '\0'; // Final null terminator
+  *bufPtr = '\0'; // Final null terminator
-	} else {
-		// Single file/directory selection - write path to buffer
-		NSURL* url = [urls firstObject];
-		if(![url getFileSystemRepresentation:self->params->buf maxLength:self->params->nbuf]) {
-			return DLG_URLFAIL;
-		}
 	}
 	
 	return DLG_OK;

app/dialog/dlgs_windows.go

@@ -15,7 +15,7 @@ const multiFileBufferSize = w32.MAX_PATH * 10
 type WinDlgError int
 
 func (e WinDlgError) Error() string {
-	return fmt.Sprintf("CommDlgExtendedError: %#x", int(e))
+	return fmt.Sprintf("CommDlgExtendedError: %#x", e)
 }
 
 func err() error {

app/server/server.go

@@ -224,7 +224,9 @@ func (s *Server) cmd(ctx context.Context) (*exec.Cmd, error) {
 		if _, err := os.Stat(settings.Models); err == nil {
 			env["OLLAMA_MODELS"] = settings.Models
 		} else {
-			slog.Warn("models path not accessible, using default", "path", settings.Models, "err", err)
+			slog.Warn("models path not accessible, clearing models setting", "path", settings.Models, "err", err)
+			settings.Models = ""
+			s.store.SetSettings(settings)
 		}
 	}
 	if settings.ContextLength > 0 {

app/ui/app/codegen/gotypes.gen.ts

@@ -469,24 +469,26 @@ export class HealthResponse {
 }
 export class User {
     id: string;
-    email: string;
     name: string;
-    bio?: string;
+    email: string;
-    avatarurl?: string;
+    avatarURL: string;
-    firstname?: string;
+    plan: string;
-    lastname?: string;
+    bio: string;
-    plan?: string;
+    firstName: string;
+    lastName: string;
+    overThreshold: boolean;
 
     constructor(source: any = {}) {
         if ('string' === typeof source) source = JSON.parse(source);
         this.id = source["id"];
-        this.email = source["email"];
         this.name = source["name"];
-        this.bio = source["bio"];
+        this.email = source["email"];
-        this.avatarurl = source["avatarurl"];
+        this.avatarURL = source["avatarURL"];
-        this.firstname = source["firstname"];
-        this.lastname = source["lastname"];
         this.plan = source["plan"];
+        this.bio = source["bio"];
+        this.firstName = source["firstName"];
+        this.lastName = source["lastName"];
+        this.overThreshold = source["overThreshold"];
     }
 }
 export class Attachment {

app/ui/app/src/api.ts

@@ -15,7 +15,6 @@ import {
 import { parseJsonlFromResponse } from "./util/jsonl-parsing";
 import { ollamaClient as ollama } from "./lib/ollama-client";
 import type { ModelResponse } from "ollama/browser";
-import { API_BASE, OLLAMA_DOT_COM } from "./lib/config";
 
 // Extend Model class with utility methods
 declare module "@/gotypes" {
@@ -27,6 +26,9 @@ declare module "@/gotypes" {
 Model.prototype.isCloud = function (): boolean {
   return this.model.endsWith("cloud");
 };
+
+const API_BASE = import.meta.env.DEV ? "http://127.0.0.1:3001" : "";
+
 // Helper function to convert Uint8Array to base64
 function uint8ArrayToBase64(uint8Array: Uint8Array): string {
   const chunkSize = 0x8000; // 32KB chunks to avoid stack overflow
@@ -41,50 +43,44 @@ function uint8ArrayToBase64(uint8Array: Uint8Array): string {
 }
 
 export async function fetchUser(): Promise<User | null> {
-  const response = await fetch(`${API_BASE}/api/me`, {
+  try {
-    method: "POST",
+    const response = await fetch(`${API_BASE}/api/v1/me`, {
-    headers: {
+      method: "GET",
-      "Content-Type": "application/json",
+      headers: {
-    },
+        "Content-Type": "application/json",
-  });
+      },
+    });
 
-  if (response.ok) {
+    if (response.ok) {
-    const userData: User = await response.json();
+      const userData: User = await response.json();
-
+      return userData;
-    if (userData.avatarurl && !userData.avatarurl.startsWith("http")) {
-      userData.avatarurl = `${OLLAMA_DOT_COM}${userData.avatarurl}`;
     }
 
-    return userData;
+    return null;
-  }
+  } catch (error) {
-
+    console.error("Error fetching user:", error);
-  if (response.status === 401 || response.status === 403) {
     return null;
   }
-
-  throw new Error(`Failed to fetch user: ${response.status}`);
 }
 
 export async function fetchConnectUrl(): Promise<string> {
-  const response = await fetch(`${API_BASE}/api/me`, {
+  const response = await fetch(`${API_BASE}/api/v1/connect`, {
-    method: "POST",
+    method: "GET",
     headers: {
       "Content-Type": "application/json",
     },
   });
 
-  if (response.status === 401) {
+  if (!response.ok) {
-    const data = await response.json();
+    throw new Error("Failed to fetch connect URL");
-    if (data.signin_url) {
-      return data.signin_url;
-    }
   }
 
-  throw new Error("Failed to fetch connect URL");
+  const data = await response.json();
+  return data.connect_url;
 }
 
 export async function disconnectUser(): Promise<void> {
-  const response = await fetch(`${API_BASE}/api/signout`, {
+  const response = await fetch(`${API_BASE}/api/v1/disconnect`, {
     method: "POST",
     headers: {
       "Content-Type": "application/json",
@@ -209,10 +205,12 @@ export async function* sendMessage(
     data: uint8ArrayToBase64(att.data),
   }));
 
-  // Send think parameter when it's explicitly set (true, false, or a non-empty string).
+  // Only send think parameter when actually requesting thinking
+  // Don't send false as it causes issues with some providers
   const shouldSendThink =
     think !== undefined &&
-    (typeof think === "boolean" || (typeof think === "string" && think !== ""));
+    ((typeof think === "boolean" && think) ||
+      (typeof think === "string" && think !== ""));
 
   const response = await fetch(`${API_BASE}/api/v1/chat/${chatId}`, {
     method: "POST",
@@ -394,8 +392,7 @@ export async function getInferenceCompute(): Promise<InferenceCompute[]> {
 
 export async function fetchHealth(): Promise<boolean> {
   try {
-    // Use the /api/version endpoint as a health check
+    const response = await fetch(`${API_BASE}/api/v1/health`, {
-    const response = await fetch(`${API_BASE}/api/version`, {
       method: "GET",
       headers: {
         "Content-Type": "application/json",
@@ -404,8 +401,7 @@ export async function fetchHealth(): Promise<boolean> {
 
     if (response.ok) {
       const data = await response.json();
-      // If we get a version back, the server is healthy
+      return data.healthy || false;
-      return !!data.version;
     }
 
     return false;

app/ui/app/src/components/Settings.tsx

@@ -299,9 +299,9 @@ export default function Settings() {
                         </Button>
                       </div>
                     </div>
-                    {user?.avatarurl && (
+                    {user?.avatarURL && (
                       <img
-                        src={user.avatarurl}
+                        src={user.avatarURL}
                         alt={user?.name}
                         className="h-10 w-10 rounded-full bg-neutral-200 dark:bg-neutral-700 flex-shrink-0"
                         onError={(e) => {

app/ui/app/src/components/Thinking.tsx

@@ -50,33 +50,21 @@ export default function Thinking({
   // Position content to show bottom when collapsed
   useEffect(() => {
     if (isCollapsed && contentRef.current && wrapperRef.current) {
-      requestAnimationFrame(() => {
+      const contentHeight = contentRef.current.scrollHeight;
-        if (!contentRef.current || !wrapperRef.current) return;
+      const wrapperHeight = wrapperRef.current.clientHeight;
-
+      if (contentHeight > wrapperHeight) {
-        const contentHeight = contentRef.current.scrollHeight;
+        const translateY = -(contentHeight - wrapperHeight);
-        const wrapperHeight = wrapperRef.current.clientHeight;
+        contentRef.current.style.transform = `translateY(${translateY}px)`;
-        if (contentHeight > wrapperHeight) {
+        setHasOverflow(true);
-          const translateY = -(contentHeight - wrapperHeight);
+      } else {
-          contentRef.current.style.transform = `translateY(${translateY}px)`;
+        setHasOverflow(false);
-          setHasOverflow(true);
+      }
-        } else {
-          contentRef.current.style.transform = "translateY(0)";
-          setHasOverflow(false);
-        }
-      });
     } else if (contentRef.current) {
       contentRef.current.style.transform = "translateY(0)";
       setHasOverflow(false);
     }
   }, [thinking, isCollapsed]);
 
-  useEffect(() => {
-    if (activelyThinking && wrapperRef.current && !isCollapsed) {
-      // When expanded and actively thinking, scroll to bottom
-      wrapperRef.current.scrollTop = wrapperRef.current.scrollHeight;
-    }
-  }, [thinking, activelyThinking, isCollapsed]);
-
   const handleToggle = () => {
     setIsCollapsed(!isCollapsed);
     setHasUserInteracted(true);

app/ui/app/src/hooks/useChats.ts

@@ -7,7 +7,6 @@ import { createQueryBatcher } from "./useQueryBatcher";
 import { useRefetchModels } from "./useModels";
 import { useStreamingContext } from "@/contexts/StreamingContext";
 import { useSettings } from "./useSettings";
-import { getModelCapabilities } from "@/api";
 
 export const useChats = () => {
   return useQuery({
@@ -607,24 +606,6 @@ export const useSendMessage = (chatId: string) => {
               queryClient.setQueryData(["staleModels"], newStaleMap);
 
               queryClient.invalidateQueries({ queryKey: ["models"] });
-
-              // Fetch fresh capabilities for the downloaded model
-              getModelCapabilities(selectedModel.model)
-                .then((capabilities) => {
-                  queryClient.setQueryData(
-                    ["modelCapabilities", selectedModel.model],
-                    capabilities,
-                  );
-                })
-                .catch((error) => {
-                  console.error(
-                    "Failed to fetch capabilities after download:",
-                    error,
-                  );
-                  queryClient.invalidateQueries({
-                    queryKey: ["modelCapabilities", selectedModel.model],
-                  });
-                });
             }
             break;
           }

app/ui/app/src/hooks/useDownloadModel.ts

@@ -0,0 +1,114 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+import { useState } from "react";
+import { pullModel } from "@/api";
+import { useSelectedModel } from "./useSelectedModel";
+import { useSettings } from "./useSettings";
+
+interface DownloadProgress {
+  status: string;
+  digest?: string;
+  total?: number;
+  completed?: number;
+  done?: boolean;
+}
+
+export function useDownloadModel(chatId?: string) {
+  const queryClient = useQueryClient();
+  const { selectedModel } = useSelectedModel(chatId);
+  const { setSettings } = useSettings();
+  const [downloadProgress, setDownloadProgress] =
+    useState<DownloadProgress | null>(null);
+  const [abortController, setAbortController] =
+    useState<AbortController | null>(null);
+  const [downloadingChatIds, setDownloadingChatIds] = useState<Set<string>>(
+    new Set(),
+  );
+
+  const mutation = useMutation({
+    mutationFn: async (modelName: string) => {
+      const controller = new AbortController();
+      setAbortController(controller);
+      setDownloadProgress({ status: "Starting download..." });
+      if (chatId) {
+        setDownloadingChatIds((prev) => new Set(prev).add(chatId));
+      }
+
+      try {
+        for await (const progress of pullModel(modelName, controller.signal)) {
+          setDownloadProgress(progress);
+
+          if (progress.status === "success") {
+            // Update selected model to indicate it's now available locally
+            if (selectedModel && selectedModel.model === modelName) {
+              setSettings({ SelectedModel: modelName });
+            }
+            // Invalidate models query to refresh the list
+            await queryClient.invalidateQueries({ queryKey: ["models"] });
+            break;
+          }
+        }
+      } finally {
+        setAbortController(null);
+        if (chatId) {
+          setDownloadingChatIds((prev) => {
+            const newSet = new Set(prev);
+            newSet.delete(chatId);
+            return newSet;
+          });
+        }
+      }
+    },
+    onSuccess: () => {
+      setDownloadProgress(null);
+      if (chatId) {
+        setDownloadingChatIds((prev) => {
+          const newSet = new Set(prev);
+          newSet.delete(chatId);
+          return newSet;
+        });
+      }
+    },
+    onError: (error: Error) => {
+      const status =
+        error.name === "AbortError" ? "Download cancelled" : "Download failed";
+      setDownloadProgress({ status, done: true });
+
+      // Clear error message after delay
+      const delay = error.name === "AbortError" ? 1500 : 3000;
+      setTimeout(() => {
+        setDownloadProgress(null);
+        if (chatId) {
+          setDownloadingChatIds((prev) => {
+            const newSet = new Set(prev);
+            newSet.delete(chatId);
+            return newSet;
+          });
+        }
+      }, delay);
+    },
+  });
+
+  const cancelDownload = () => {
+    if (abortController) {
+      abortController.abort();
+      setAbortController(null);
+      if (chatId) {
+        setDownloadingChatIds((prev) => {
+          const newSet = new Set(prev);
+          newSet.delete(chatId);
+          return newSet;
+        });
+      }
+    }
+  };
+
+  return {
+    downloadModel: mutation.mutate,
+    isDownloading:
+      mutation.isPending && chatId ? downloadingChatIds.has(chatId) : false,
+    downloadProgress:
+      chatId && downloadingChatIds.has(chatId) ? downloadProgress : null,
+    error: mutation.error,
+    cancelDownload,
+  };
+}

app/ui/app/src/hooks/useUser.ts

@@ -1,20 +1,29 @@
 import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
+import { useEffect, useState } from "react";
 import { fetchUser, fetchConnectUrl, disconnectUser } from "@/api";
 
 export function useUser() {
   const queryClient = useQueryClient();
+  const [initialDataLoaded, setInitialDataLoaded] = useState(false);
+
+  // Wait for initial data to be loaded
+  useEffect(() => {
+    const initialPromise = window.__initialUserDataPromise;
+    if (initialPromise) {
+      initialPromise.finally(() => {
+        setInitialDataLoaded(true);
+      });
+    } else {
+      setInitialDataLoaded(true);
+    }
+  }, []);
 
   const userQuery = useQuery({
     queryKey: ["user"],
-    queryFn: async () => {
+    queryFn: () => fetchUser(),
-      const result = await fetchUser();
-      return result;
-    },
     staleTime: 5 * 60 * 1000, // Consider data stale after 5 minutes
     gcTime: 10 * 60 * 1000, // Keep in cache for 10 minutes
-    retry: 10,
+    initialData: null, // Start with null to prevent flashing
-    retryDelay: (attemptIndex) => Math.min(500 * attemptIndex, 2000),
-    refetchOnMount: true, // Always fetch when component mounts
   });
 
   // Mutation to refresh user data
@@ -40,15 +49,14 @@ export function useUser() {
     },
   });
 
-  const isLoading = userQuery.isLoading || userQuery.isFetching;
-  const isAuthenticated = Boolean(userQuery.data?.name);
-
   return {
     user: userQuery.data,
-    isLoading,
+    isLoading:
+      !initialDataLoaded ||
+      (userQuery.isLoading && userQuery.data === undefined), // Show loading until initial data is loaded
     isError: userQuery.isError,
     error: userQuery.error,
-    isAuthenticated,
+    isAuthenticated: Boolean(userQuery.data?.name),
     refreshUser: refreshUser.mutate,
     isRefreshing: refreshUser.isPending,
     refetchUser: userQuery.refetch,

app/ui/app/src/lib/config.ts

@@ -1,13 +0,0 @@
-// API configuration
-const DEV_API_URL = "http://127.0.0.1:3001";
-
-// Base URL for fetch API calls (can be relative in production)
-export const API_BASE = import.meta.env.DEV ? DEV_API_URL : "";
-
-// Full host URL for Ollama client (needs full origin in production)
-export const OLLAMA_HOST = import.meta.env.DEV
-  ? DEV_API_URL
-  : window.location.origin;
-
-export const OLLAMA_DOT_COM =
-  import.meta.env.VITE_OLLAMA_DOT_COM_URL || "https://ollama.com";

app/ui/app/src/lib/highlighter.ts

@@ -147,7 +147,6 @@ export const highlighterPromise = createHighlighter({
     "c",
     "cpp",
     "sql",
-    "swift",
     "yaml",
     "markdown",
   ],

app/ui/app/src/lib/ollama-client.ts

@@ -1,5 +1,4 @@
 import { Ollama } from "ollama/browser";
-import { OLLAMA_HOST } from "./config";
 
 let _ollamaClient: Ollama | null = null;
 
@@ -7,7 +6,7 @@ export const ollamaClient = new Proxy({} as Ollama, {
   get(_target, prop) {
     if (!_ollamaClient) {
       _ollamaClient = new Ollama({
-        host: OLLAMA_HOST,
+        host: window.location.origin,
       });
     }
     const value = _ollamaClient[prop as keyof Ollama];

app/ui/app/src/main.tsx

@@ -5,6 +5,13 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
 import { routeTree } from "./routeTree.gen";
 import { fetchUser } from "./api";
 import { StreamingProvider } from "./contexts/StreamingContext";
+import { User } from "@/gotypes";
+
+declare global {
+  interface Window {
+    __initialUserDataPromise?: Promise<User | null>;
+  }
+}
 
 const queryClient = new QueryClient({
   defaultOptions: {
@@ -17,11 +24,27 @@ const queryClient = new QueryClient({
   },
 });
 
-fetchUser().then((userData) => {
+// Track initial user data fetch
-  if (userData) {
+let initialUserDataPromise: Promise<User | null> | null = null;
+
+// Initialize user data on app startup
+const initializeUserData = async () => {
+  try {
+    const userData = await fetchUser();
     queryClient.setQueryData(["user"], userData);
+    return userData;
+  } catch (error) {
+    console.error("Error initializing user data:", error);
+    queryClient.setQueryData(["user"], null);
+    return null;
   }
-});
+};
+
+// Start initialization immediately and track the promise
+initialUserDataPromise = initializeUserData();
+
+// Export the promise so hooks can await it
+window.__initialUserDataPromise = initialUserDataPromise;
 
 const router = createRouter({
   routeTree,

app/ui/responses/types.go

@@ -101,14 +101,15 @@ type HealthResponse struct {
 }
 
 type User struct {
-	ID        string `json:"id"`
+	ID            string `json:"id"`
-	Email     string `json:"email"`
+	Name          string `json:"name"`
-	Name      string `json:"name"`
+	Email         string `json:"email"`
-	Bio       string `json:"bio,omitempty"`
+	AvatarURL     string `json:"avatarURL"`
-	AvatarURL string `json:"avatarurl,omitempty"`
+	Plan          string `json:"plan"`
-	FirstName string `json:"firstname,omitempty"`
+	Bio           string `json:"bio"`
-	LastName  string `json:"lastname,omitempty"`
+	FirstName     string `json:"firstName"`
-	Plan      string `json:"plan,omitempty"`
+	LastName      string `json:"lastName"`
+	OverThreshold bool   `json:"overThreshold"`
 }
 
 type Attachment struct {

app/ui/ui.go

@@ -12,17 +12,18 @@ import (
 	"log/slog"
 	"net/http"
 	"net/http/httputil"
+	"net/url"
 	"os"
 	"runtime"
 	"runtime/debug"
 	"slices"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/ollama/ollama/api"
+	"github.com/ollama/ollama/app/auth"
 	"github.com/ollama/ollama/app/server"
 	"github.com/ollama/ollama/app/store"
 	"github.com/ollama/ollama/app/tools"
@@ -117,66 +118,40 @@ func (s *Server) log() *slog.Logger {
 
 // ollamaProxy creates a reverse proxy handler to the Ollama server
 func (s *Server) ollamaProxy() http.Handler {
-	var (
+	ollamaHost := os.Getenv("OLLAMA_HOST")
-		proxy   http.Handler
+	if ollamaHost == "" {
-		proxyMu sync.Mutex
+		ollamaHost = "http://127.0.0.1:11434"
-	)
+	}
 
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	if !strings.HasPrefix(ollamaHost, "http://") && !strings.HasPrefix(ollamaHost, "https://") {
-		proxyMu.Lock()
+		ollamaHost = "http://" + ollamaHost
-		p := proxy
+	}
-		proxyMu.Unlock()
 
-		if p == nil {
+	target, err := url.Parse(ollamaHost)
-			proxyMu.Lock()
+	if err != nil {
-			if proxy == nil {
+		s.log().Error("failed to parse OLLAMA_HOST", "error", err, "host", ollamaHost)
-				var err error
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				for i := range 2 {
+			http.Error(w, "failed to configure proxy", http.StatusInternalServerError)
-					if i > 0 {
+		})
-						s.log().Warn("ollama server not ready, retrying", "attempt", i+1)
+	}
-						time.Sleep(1 * time.Second)
-					}
 
-					err = WaitForServer(context.Background(), 10*time.Second)
+	s.log().Info("configuring ollama proxy", "target", target.String())
-					if err == nil {
-						break
-					}
-				}
 
-				if err != nil {
+	proxy := httputil.NewSingleHostReverseProxy(target)
-					proxyMu.Unlock()
-					s.log().Error("ollama server not ready after retries", "error", err)
-					http.Error(w, "Ollama server is not ready", http.StatusServiceUnavailable)
-					return
-				}
 
-				target := envconfig.Host()
+	originalDirector := proxy.Director
-				s.log().Info("configuring ollama proxy", "target", target.String())
+	proxy.Director = func(req *http.Request) {
+		originalDirector(req)
+		req.Host = target.Host
+		s.log().Debug("proxying request", "method", req.Method, "path", req.URL.Path, "target", target.Host)
+	}
 
-				newProxy := httputil.NewSingleHostReverseProxy(target)
+	proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
+		s.log().Error("proxy error", "error", err, "path", r.URL.Path, "target", target.String())
+		http.Error(w, "proxy error: "+err.Error(), http.StatusBadGateway)
+	}
 
-				originalDirector := newProxy.Director
+	return proxy
-				newProxy.Director = func(req *http.Request) {
-					originalDirector(req)
-					req.Host = target.Host
-					s.log().Debug("proxying request", "method", req.Method, "path", req.URL.Path, "target", target.Host)
-				}
-
-				newProxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
-					s.log().Error("proxy error", "error", err, "path", r.URL.Path, "target", target.String())
-					http.Error(w, "proxy error: "+err.Error(), http.StatusBadGateway)
-				}
-
-				proxy = newProxy
-				p = newProxy
-			} else {
-				p = proxy
-			}
-			proxyMu.Unlock()
-		}
-
-		p.ServeHTTP(w, r)
-	})
 }
 
 type errHandlerFunc func(http.ResponseWriter, *http.Request) error
@@ -289,10 +264,11 @@ func (s *Server) Handler() http.Handler {
 	ollamaProxy := s.ollamaProxy()
 	mux.Handle("GET /api/tags", ollamaProxy)
 	mux.Handle("POST /api/show", ollamaProxy)
-	mux.Handle("GET /api/version", ollamaProxy)
+
-	mux.Handle("HEAD /api/version", ollamaProxy)
+	mux.Handle("GET /api/v1/me", handle(s.me))
-	mux.Handle("POST /api/me", ollamaProxy)
+	mux.Handle("POST /api/v1/disconnect", handle(s.disconnect))
-	mux.Handle("POST /api/signout", ollamaProxy)
+	mux.Handle("GET /api/v1/connect", handle(s.connectURL))
+	mux.Handle("GET /api/v1/health", handle(s.health))
 
 	// React app - catch all non-API routes and serve the React app
 	mux.Handle("GET /", s.appHandler())
@@ -362,7 +338,7 @@ func (s *Server) doSelfSigned(ctx context.Context, method, path string) (*http.R
 }
 
 // UserData fetches user data from ollama.com API for the current ollama key
-func (s *Server) UserData(ctx context.Context) (*api.UserResponse, error) {
+func (s *Server) UserData(ctx context.Context) (*responses.User, error) {
 	resp, err := s.doSelfSigned(ctx, http.MethodPost, "/api/me")
 	if err != nil {
 		return nil, fmt.Errorf("failed to call ollama.com/api/me: %w", err)
@@ -373,7 +349,7 @@ func (s *Server) UserData(ctx context.Context) (*api.UserResponse, error) {
 		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 
-	var user api.UserResponse
+	var user responses.User
 	if err := json.NewDecoder(resp.Body).Decode(&user); err != nil {
 		return nil, fmt.Errorf("failed to parse user response: %w", err)
 	}
@@ -392,27 +368,29 @@ func (s *Server) UserData(ctx context.Context) (*api.UserResponse, error) {
 	return &user, nil
 }
 
-// WaitForServer waits for the Ollama server to be ready
+func waitForServer(ctx context.Context) error {
-func WaitForServer(ctx context.Context, timeout time.Duration) error {
+	timeout := time.Now().Add(10 * time.Second)
-	deadline := time.Now().Add(timeout)
+	// TODO: this avoids an error on first load of the app
-	for time.Now().Before(deadline) {
+	// however we should either show a loading state or
+	// wait for the Ollama server to be ready before redirecting
+	for {
 		c, err := api.ClientFromEnvironment()
 		if err != nil {
 			return err
 		}
 		if _, err := c.Version(ctx); err == nil {
-			slog.Debug("ollama server is ready")
+			break
-			return nil
+		}
+		if time.Now().After(timeout) {
+			return fmt.Errorf("timeout waiting for Ollama server to be ready")
 		}
 		time.Sleep(10 * time.Millisecond)
 	}
-	return errors.New("timeout waiting for Ollama server to be ready")
+	return nil
 }
 
 func (s *Server) createChat(w http.ResponseWriter, r *http.Request) error {
-	if err := WaitForServer(r.Context(), 10*time.Second); err != nil {
+	waitForServer(r.Context())
-		return err
-	}
 
 	id, err := uuid.NewV7()
 	if err != nil {
@@ -997,7 +975,7 @@ func (s *Server) chat(w http.ResponseWriter, r *http.Request) error {
 				for _, toolCall := range res.Message.ToolCalls {
 					// continues loop as tools were executed
 					toolsExecuted = true
-					result, content, err := registry.Execute(ctx, toolCall.Function.Name, toolCall.Function.Arguments.ToMap())
+					result, content, err := registry.Execute(ctx, toolCall.Function.Name, toolCall.Function.Arguments)
 					if err != nil {
 						errContent := fmt.Sprintf("Error: %v", err)
 						toolErrMsg := store.NewMessage("tool", errContent, nil)
@@ -1460,6 +1438,129 @@ func (s *Server) settings(w http.ResponseWriter, r *http.Request) error {
 	})
 }
 
+func (s *Server) me(w http.ResponseWriter, r *http.Request) error {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return nil
+	}
+
+	user, err := s.UserData(r.Context())
+	if err != nil {
+		// If fetching from API fails, try to return cached user data if available
+		if cachedUser, cacheErr := s.Store.User(); cacheErr == nil && cachedUser != nil {
+			s.log().Info("API request failed, returning cached user data", "error", err)
+			responseUser := &responses.User{
+				Name:  cachedUser.Name,
+				Email: cachedUser.Email,
+				Plan:  cachedUser.Plan,
+			}
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			return json.NewEncoder(w).Encode(responseUser)
+		}
+
+		s.log().Error("failed to get user data", "error", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return json.NewEncoder(w).Encode(responses.Error{
+			Error: "failed to get user data",
+		})
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	return json.NewEncoder(w).Encode(user)
+}
+
+func (s *Server) disconnect(w http.ResponseWriter, r *http.Request) error {
+	if r.Method != http.MethodPost {
+		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return nil
+	}
+
+	if err := s.Store.ClearUser(); err != nil {
+		s.log().Warn("failed to clear cached user data", "error", err)
+	}
+
+	// Get the SSH public key to encode for the delete request
+	pubKey, err := ollamaAuth.GetPublicKey()
+	if err != nil {
+		s.log().Error("failed to get public key", "error", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return json.NewEncoder(w).Encode(responses.Error{
+			Error: "failed to get public key",
+		})
+	}
+
+	// Encode the key using base64 URL encoding
+	encodedKey := base64.RawURLEncoding.EncodeToString([]byte(pubKey))
+
+	// Call the /api/user/keys/{encodedKey} endpoint with DELETE
+	resp, err := s.doSelfSigned(r.Context(), http.MethodDelete, fmt.Sprintf("/api/user/keys/%s", encodedKey))
+	if err != nil {
+		s.log().Error("failed to call ollama.com/api/user/keys", "error", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return json.NewEncoder(w).Encode(responses.Error{
+			Error: "failed to disconnect from ollama.com",
+		})
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		s.log().Error("disconnect request failed", "status", resp.StatusCode)
+		w.WriteHeader(http.StatusInternalServerError)
+		return json.NewEncoder(w).Encode(responses.Error{
+			Error: "failed to disconnect from ollama.com",
+		})
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	return json.NewEncoder(w).Encode(map[string]string{"status": "disconnected"})
+}
+
+func (s *Server) connectURL(w http.ResponseWriter, r *http.Request) error {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return nil
+	}
+
+	connectURL, err := auth.BuildConnectURL(OllamaDotCom)
+	if err != nil {
+		s.log().Error("failed to build connect URL", "error", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return json.NewEncoder(w).Encode(responses.Error{
+			Error: "failed to build connect URL",
+		})
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	return json.NewEncoder(w).Encode(map[string]string{
+		"connect_url": connectURL,
+	})
+}
+
+func (s *Server) health(w http.ResponseWriter, r *http.Request) error {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return nil
+	}
+
+	healthy := false
+	c, err := api.ClientFromEnvironment()
+	if err == nil {
+		if _, err := c.Version(r.Context()); err == nil {
+			healthy = true
+		}
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	return json.NewEncoder(w).Encode(responses.HealthResponse{
+		Healthy: healthy,
+	})
+}
+
 func (s *Server) getInferenceCompute(w http.ResponseWriter, r *http.Request) error {
 	ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond)
 	defer cancel()
@@ -1558,13 +1659,13 @@ func convertToOllamaTool(toolSchema map[string]any) api.Tool {
 
 	tool.Function.Parameters.Type = "object"
 	tool.Function.Parameters.Required = []string{}
-	tool.Function.Parameters.Properties = api.NewToolPropertiesMap()
+	tool.Function.Parameters.Properties = make(map[string]api.ToolProperty)
 
 	if schemaProps, ok := toolSchema["schema"].(map[string]any); ok {
 		tool.Function.Parameters.Type = getStringFromMap(schemaProps, "type", "object")
 
 		if props, ok := schemaProps["properties"].(map[string]any); ok {
-			tool.Function.Parameters.Properties = api.NewToolPropertiesMap()
+			tool.Function.Parameters.Properties = make(map[string]api.ToolProperty)
 
 			for propName, propDef := range props {
 				if propMap, ok := propDef.(map[string]any); ok {
@@ -1572,7 +1673,7 @@ func convertToOllamaTool(toolSchema map[string]any) api.Tool {
 						Type:        api.PropertyType{getStringFromMap(propMap, "type", "string")},
 						Description: getStringFromMap(propMap, "description", ""),
 					}
-					tool.Function.Parameters.Properties.Set(propName, prop)
+					tool.Function.Parameters.Properties[propName] = prop
 				}
 			}
 		}

app/wintray/eventloop.go

@@ -158,16 +158,16 @@ func (t *winTray) wndProc(hWnd windows.Handle, message uint32, wParam, lParam ui
 	case uint32(UI_REQUEST_MSG_ID):
 		// Requests for the UI must always come from the main event thread
 		l := int(wParam)
-		path := unsafe.String((*byte)(unsafe.Pointer(lParam)), l) //nolint:govet,gosec
+		path := unsafe.String((*byte)(unsafe.Pointer(lParam)), l)
 		t.app.UIRun(path)
 	case WM_COPYDATA:
 		// Handle URL scheme requests from other instances
 		if lParam != 0 {
-			cds := (*COPYDATASTRUCT)(unsafe.Pointer(lParam)) //nolint:govet,gosec
+			cds := (*COPYDATASTRUCT)(unsafe.Pointer(lParam))
-			if cds.DwData == 1 {                             // Our identifier for URL scheme messages
+			if cds.DwData == 1 { // Our identifier for URL scheme messages
 				// Convert the data back to string
 				data := make([]byte, cds.CbData)
-				copy(data, (*[1 << 30]byte)(unsafe.Pointer(cds.LpData))[:cds.CbData:cds.CbData]) //nolint:govet,gosec
+				copy(data, (*[1 << 30]byte)(unsafe.Pointer(cds.LpData))[:cds.CbData:cds.CbData])
 				urlScheme := string(data)
 				handleURLSchemeRequest(urlScheme)
 				lResult = 1 // Return non-zero to indicate success

build_custom.sh

@@ -1,44 +0,0 @@
----
-
-### 2. `build_custom.sh` (Skrypt automatyzujący)
-To jest opcja "Pro". Zamiast wklepywać te komendy ręcznie, tworzysz skrypt bashowy. Jak będziesz chciał zaktualizować Ollamę za pół roku, po prostu odpalisz `./build_custom.sh` i pójdziesz na kawę.
-
-**Zawartość pliku:**
-```bash
-#!/bin/bash
-
-# Skrypt budowania Ollama dla Xeon X5675 (No AVX) + GTX 1070
-# Uruchom to w głównym katalogu repozytorium
-
-echo "--- [1/4] Czyszczenie poprzedniego builda ---"
-rm -rf build
-go clean -cache
-
-echo "--- [2/4] Konfiguracja CMake (CUDA ON, Vulkan OFF) ---"
-# Flagi kluczowe dla Twojego systemu
-cmake -B build \
-    -DOLLAMA_CUDA=ON \
-    -DOLLAMA_VULKAN=OFF \
-    -DGGML_VULKAN=OFF \
-    -DCMAKE_DISABLE_FIND_PACKAGE_Vulkan=TRUE
-
-if [ $? -ne 0 ]; then
-    echo "Błąd konfiguracji CMake!"
-    exit 1
-fi
-
-echo "--- [3/4] Kompilacja silnika (Tryb bezpieczny -j1) ---"
-# Używamy -j1 bo przy OC Twój Xeon może być niestabilny przy kompilacji
-cmake --build build -j1
-
-if [ $? -ne 0 ]; then
-    echo "Błąd kompilacji!"
-    exit 1
-fi
-
-echo "--- [4/4] Budowanie pliku binarnego Go ---"
-go build .
-
-echo "--- GOTOWE! ---"
-echo "Twój plik 'ollama' jest gotowy."
-echo "Aby zainstalować wpisz: sudo mv ollama /usr/bin/ollama"

cmd/bench/README.md

@@ -1,115 +0,0 @@
-Ollama Benchmark Tool
----------------------
-
-A Go-based command-line tool for benchmarking Ollama models with configurable parameters and multiple output formats.
-
-## Features
-
- * Benchmark multiple models in a single run
- * Support for both text and image prompts
- * Configurable generation parameters (temperature, max tokens, seed, etc.)
- * Supports benchstat and CSV output formats
- * Detailed performance metrics (prefill, generate, load, total durations)
-
-## Building from Source
-
-```
-go build -o ollama-bench bench.go
-./ollama-bench -model gpt-oss:20b -epochs 6 -format csv
-```
-
-Using Go Run (without building)
-
-```
-go run bench.go -model gpt-oss:20b -epochs 3
-```
-
-## Usage
-
-### Basic Example
-
-```
-./ollama-bench -model gemma3 -epochs 6
-```
-
-### Benchmark Multiple Models
-
-```
-./ollama-bench -model gemma3,gemma3n -epochs 6 -max-tokens 100 -p "Write me a short story" | tee gemma.bench
-benchstat -col /name gemma.bench
-```
-
-### With Image Prompt
-
-```
-./ollama-bench -model qwen3-vl -image photo.jpg -epochs 6 -max-tokens 100 -p "Describe this image"
-```
-
-### Advanced Example
-
-```
-./ollama-bench -model llama3 -epochs 10 -temperature 0.7 -max-tokens 500 -seed 42 -format csv -output results.csv
-```
-
-## Command Line Options
-
-| Option  	| Description | Default |
-|----------|-------------|---------|
-| -model	| Comma-separated list of models to benchmark	| (required)		|
-| -epochs	| Number of iterations per model		| 1			|
-| -max-tokens	| Maximum tokens for model response		| 0 (unlimited)		|
-| -temperature	| Temperature parameter				| 0.0			|
-| -seed		| Random seed					| 0 (random)		|
-| -timeout	| Timeout in seconds				| 300			|
-| -p		| Prompt text					| "Write a long story."	|
-| -image	| Image file to include in prompt		| 			|
-| -k		| Keep-alive duration in seconds		| 0			|
-| -format	| Output format (benchstat, csv)		| benchstat		|
-| -output	| Output file for results			| "" (stdout)		|
-| -v		| Verbose mode					| false			|
-| -debug	| Show debug information			| false			|
-
-## Output Formats
-
-### Markdown Format
-
-The default markdown format is suitable for copying and pasting into a GitHub issue and will look like:
-```
- Model | Step | Count | Duration | nsPerToken | tokensPerSec |
-|-------|------|-------|----------|------------|--------------|
-| gpt-oss:20b | prefill | 124 | 30.006458ms | 241987.56 | 4132.44 |
-| gpt-oss:20b | generate | 200 | 2.646843954s | 13234219.77 | 75.56 |
-| gpt-oss:20b | load | 1 | 121.674208ms | - | - |
-| gpt-oss:20b | total | 1 | 2.861047625s | - | - |
-```
-
-### Benchstat Format
-
-Compatible with Go's benchstat tool for statistical analysis:
-
-```
-BenchmarkModel/name=gpt-oss:20b/step=prefill 128 78125.00 ns/token 12800.00 token/sec
-BenchmarkModel/name=gpt-oss:20b/step=generate 512 19531.25 ns/token 51200.00 token/sec
-BenchmarkModel/name=gpt-oss:20b/step=load 1 1500000000 ns/request
-```
-
-### CSV Format
-
-Machine-readable comma-separated values:
-
-```
-NAME,STEP,COUNT,NS_PER_COUNT,TOKEN_PER_SEC
-gpt-oss:20b,prefill,128,78125.00,12800.00
-gpt-oss:20b,generate,512,19531.25,51200.00
-gpt-oss:20b,load,1,1500000000,0
-```
-
-## Metrics Explained
-
-The tool reports four types of metrics for each model:
-
- * prefill: Time spent processing the prompt
- * generate: Time spent generating the response
- * load: Model loading time (one-time cost)
- * total: Total request duration
-

cmd/bench/bench.go

@@ -1,321 +0,0 @@
-package main
-
-import (
-	"cmp"
-	"context"
-	"flag"
-	"fmt"
-	"io"
-	"os"
-	"runtime"
-	"slices"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/ollama/ollama/api"
-)
-
-type flagOptions struct {
-	models      *string
-	epochs      *int
-	maxTokens   *int
-	temperature *float64
-	seed        *int
-	timeout     *int
-	prompt      *string
-	imageFile   *string
-	keepAlive   *float64
-	format      *string
-	outputFile  *string
-	debug       *bool
-	verbose     *bool
-}
-
-type Metrics struct {
-	Model    string
-	Step     string
-	Count    int
-	Duration time.Duration
-}
-
-var once sync.Once
-
-const DefaultPrompt = `Please write a descriptive story about a llama named Alonso who grows up to be President of the Land of Llamas. Include details about Alonso's childhood, adolescent years, and how he grew up to be a political mover and shaker. Write the story with a sense of whimsy.`
-
-func OutputMetrics(w io.Writer, format string, metrics []Metrics, verbose bool) {
-	switch format {
-	case "benchstat":
-		if verbose {
-			printHeader := func() {
-				fmt.Fprintf(w, "sysname: %s\n", runtime.GOOS)
-				fmt.Fprintf(w, "machine: %s\n", runtime.GOARCH)
-			}
-			once.Do(printHeader)
-		}
-		for _, m := range metrics {
-			if m.Step == "generate" || m.Step == "prefill" {
-				if m.Count > 0 {
-					nsPerToken := float64(m.Duration.Nanoseconds()) / float64(m.Count)
-					tokensPerSec := float64(m.Count) / (float64(m.Duration.Nanoseconds()) + 1e-12) * 1e9
-
-					fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s %d %.2f ns/token %.2f token/sec\n",
-						m.Model, m.Step, m.Count, nsPerToken, tokensPerSec)
-				} else {
-					fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s %d 0 ns/token 0 token/sec\n",
-						m.Model, m.Step, m.Count)
-				}
-			} else {
-				var suffix string
-				if m.Step == "load" {
-					suffix = "/step=load"
-				}
-				fmt.Fprintf(w, "BenchmarkModel/name=%s%s 1 %d ns/request\n",
-					m.Model, suffix, m.Duration.Nanoseconds())
-			}
-		}
-	case "csv":
-		printHeader := func() {
-			headings := []string{"NAME", "STEP", "COUNT", "NS_PER_COUNT", "TOKEN_PER_SEC"}
-			fmt.Fprintln(w, strings.Join(headings, ","))
-		}
-		once.Do(printHeader)
-
-		for _, m := range metrics {
-			if m.Step == "generate" || m.Step == "prefill" {
-				var nsPerToken float64
-				var tokensPerSec float64
-				if m.Count > 0 {
-					nsPerToken = float64(m.Duration.Nanoseconds()) / float64(m.Count)
-					tokensPerSec = float64(m.Count) / (float64(m.Duration.Nanoseconds()) + 1e-12) * 1e9
-				}
-				fmt.Fprintf(w, "%s,%s,%d,%.2f,%.2f\n", m.Model, m.Step, m.Count, nsPerToken, tokensPerSec)
-			} else {
-				fmt.Fprintf(w, "%s,%s,1,%d,0\n", m.Model, m.Step, m.Duration.Nanoseconds())
-			}
-		}
-	case "markdown":
-		printHeader := func() {
-			fmt.Fprintln(w, "| Model | Step | Count | Duration | nsPerToken | tokensPerSec |")
-			fmt.Fprintln(w, "|-------|------|-------|----------|------------|--------------|")
-		}
-		once.Do(printHeader)
-
-		for _, m := range metrics {
-			var nsPerToken, tokensPerSec float64
-			var nsPerTokenStr, tokensPerSecStr string
-
-			if m.Step == "generate" || m.Step == "prefill" {
-				nsPerToken = float64(m.Duration.Nanoseconds()) / float64(m.Count)
-				tokensPerSec = float64(m.Count) / (float64(m.Duration.Nanoseconds()) + 1e-12) * 1e9
-				nsPerTokenStr = fmt.Sprintf("%.2f", nsPerToken)
-				tokensPerSecStr = fmt.Sprintf("%.2f", tokensPerSec)
-			} else {
-				nsPerTokenStr = "-"
-				tokensPerSecStr = "-"
-			}
-
-			fmt.Fprintf(w, "| %s | %s | %d | %v | %s | %s |\n",
-				m.Model, m.Step, m.Count, m.Duration, nsPerTokenStr, tokensPerSecStr)
-		}
-	default:
-		fmt.Fprintf(os.Stderr, "Unknown output format '%s'\n", format)
-	}
-}
-
-func BenchmarkChat(fOpt flagOptions) error {
-	models := strings.Split(*fOpt.models, ",")
-
-	// todo - add multi-image support
-	var imgData api.ImageData
-	var err error
-	if *fOpt.imageFile != "" {
-		imgData, err = readImage(*fOpt.imageFile)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "ERROR: Couldn't read image '%s': %v\n", *fOpt.imageFile, err)
-			return err
-		}
-	}
-
-	if *fOpt.debug && imgData != nil {
-		fmt.Fprintf(os.Stderr, "Read file '%s'\n", *fOpt.imageFile)
-	}
-
-	client, err := api.ClientFromEnvironment()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "ERROR: Couldn't create ollama client: %v\n", err)
-		return err
-	}
-
-	var out io.Writer = os.Stdout
-	if fOpt.outputFile != nil && *fOpt.outputFile != "" {
-		f, err := os.OpenFile(*fOpt.outputFile, os.O_CREATE|os.O_WRONLY, 0o644)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "ERROR: cannot open output file %s: %v\n", *fOpt.outputFile, err)
-			return err
-		}
-		defer f.Close()
-		out = f
-	}
-
-	for _, model := range models {
-		for range *fOpt.epochs {
-			options := make(map[string]interface{})
-			if *fOpt.maxTokens > 0 {
-				options["num_predict"] = *fOpt.maxTokens
-			}
-			options["temperature"] = *fOpt.temperature
-			if fOpt.seed != nil && *fOpt.seed > 0 {
-				options["seed"] = *fOpt.seed
-			}
-
-			var keepAliveDuration *api.Duration
-			if *fOpt.keepAlive > 0 {
-				duration := api.Duration{Duration: time.Duration(*fOpt.keepAlive * float64(time.Second))}
-				keepAliveDuration = &duration
-			}
-
-			req := &api.ChatRequest{
-				Model: model,
-				Messages: []api.Message{
-					{
-						Role:    "user",
-						Content: *fOpt.prompt,
-					},
-				},
-				Options:   options,
-				KeepAlive: keepAliveDuration,
-			}
-
-			if imgData != nil {
-				req.Messages[0].Images = []api.ImageData{imgData}
-			}
-
-			var responseMetrics *api.Metrics
-
-			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*fOpt.timeout)*time.Second)
-			defer cancel()
-
-			err = client.Chat(ctx, req, func(resp api.ChatResponse) error {
-				if *fOpt.debug {
-					fmt.Fprintf(os.Stderr, "%s", cmp.Or(resp.Message.Thinking, resp.Message.Content))
-				}
-
-				if resp.Done {
-					responseMetrics = &resp.Metrics
-				}
-				return nil
-			})
-
-			if *fOpt.debug {
-				fmt.Fprintln(os.Stderr)
-			}
-
-			if err != nil {
-				if ctx.Err() == context.DeadlineExceeded {
-					fmt.Fprintf(os.Stderr, "ERROR: Chat request timed out with model '%s' after %vs\n", model, 1)
-					continue
-				}
-				fmt.Fprintf(os.Stderr, "ERROR: Couldn't chat with model '%s': %v\n", model, err)
-				continue
-			}
-
-			if responseMetrics == nil {
-				fmt.Fprintf(os.Stderr, "ERROR: No metrics received for model '%s'\n", model)
-				continue
-			}
-
-			metrics := []Metrics{
-				{
-					Model:    model,
-					Step:     "prefill",
-					Count:    responseMetrics.PromptEvalCount,
-					Duration: responseMetrics.PromptEvalDuration,
-				},
-				{
-					Model:    model,
-					Step:     "generate",
-					Count:    responseMetrics.EvalCount,
-					Duration: responseMetrics.EvalDuration,
-				},
-				{
-					Model:    model,
-					Step:     "load",
-					Count:    1,
-					Duration: responseMetrics.LoadDuration,
-				},
-				{
-					Model:    model,
-					Step:     "total",
-					Count:    1,
-					Duration: responseMetrics.TotalDuration,
-				},
-			}
-
-			OutputMetrics(out, *fOpt.format, metrics, *fOpt.verbose)
-
-			if *fOpt.keepAlive > 0 {
-				time.Sleep(time.Duration(*fOpt.keepAlive*float64(time.Second)) + 200*time.Millisecond)
-			}
-		}
-	}
-
-	return nil
-}
-
-func readImage(filePath string) (api.ImageData, error) {
-	file, err := os.Open(filePath)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	data, err := io.ReadAll(file)
-	if err != nil {
-		return nil, err
-	}
-
-	return api.ImageData(data), nil
-}
-
-func main() {
-	fOpt := flagOptions{
-		models:      flag.String("model", "", "Model to benchmark"),
-		epochs:      flag.Int("epochs", 6, "Number of epochs (iterations) per model"),
-		maxTokens:   flag.Int("max-tokens", 200, "Maximum tokens for model response"),
-		temperature: flag.Float64("temperature", 0, "Temperature parameter"),
-		seed:        flag.Int("seed", 0, "Random seed"),
-		timeout:     flag.Int("timeout", 60*5, "Timeout in seconds (default 300s)"),
-		prompt:      flag.String("p", DefaultPrompt, "Prompt to use"),
-		imageFile:   flag.String("image", "", "Filename for an image to include"),
-		keepAlive:   flag.Float64("k", 0, "Keep alive duration in seconds"),
-		format:      flag.String("format", "markdown", "Output format [benchstat|csv] (default benchstat)"),
-		outputFile:  flag.String("output", "", "Output file for results (stdout if empty)"),
-		verbose:     flag.Bool("v", false, "Show system information"),
-		debug:       flag.Bool("debug", false, "Show debug information"),
-	}
-
-	flag.Usage = func() {
-		fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n\n", os.Args[0])
-		fmt.Fprintf(os.Stderr, "Description:\n")
-		fmt.Fprintf(os.Stderr, "  Model benchmarking tool with configurable parameters\n\n")
-		fmt.Fprintf(os.Stderr, "Options:\n")
-		flag.PrintDefaults()
-		fmt.Fprintf(os.Stderr, "\nExamples:\n")
-		fmt.Fprintf(os.Stderr, "  bench -model gpt-oss:20b -epochs 3 -temperature 0.7\n")
-	}
-	flag.Parse()
-
-	if !slices.Contains([]string{"markdown", "benchstat", "csv"}, *fOpt.format) {
-		fmt.Fprintf(os.Stderr, "ERROR: Unknown format '%s'\n", *fOpt.format)
-		os.Exit(1)
-	}
-
-	if len(*fOpt.models) == 0 {
-		fmt.Fprintf(os.Stderr, "ERROR: No model(s) specified to benchmark.\n")
-		flag.Usage()
-		return
-	}
-
-	BenchmarkChat(fOpt)
-}

cmd/bench/bench_test.go

@@ -1,463 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"crypto/rand"
-	"encoding/json"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/ollama/ollama/api"
-)
-
-func createTestFlagOptions() flagOptions {
-	models := "test-model"
-	format := "benchstat"
-	epochs := 1
-	maxTokens := 100
-	temperature := 0.7
-	seed := 42
-	timeout := 30
-	prompt := "test prompt"
-	imageFile := ""
-	keepAlive := 5.0
-	verbose := false
-	debug := false
-
-	return flagOptions{
-		models:      &models,
-		format:      &format,
-		epochs:      &epochs,
-		maxTokens:   &maxTokens,
-		temperature: &temperature,
-		seed:        &seed,
-		timeout:     &timeout,
-		prompt:      &prompt,
-		imageFile:   &imageFile,
-		keepAlive:   &keepAlive,
-		verbose:     &verbose,
-		debug:       &debug,
-	}
-}
-
-func captureOutput(f func()) string {
-	oldStdout := os.Stdout
-	oldStderr := os.Stderr
-	defer func() {
-		os.Stdout = oldStdout
-		os.Stderr = oldStderr
-	}()
-
-	r, w, _ := os.Pipe()
-	os.Stdout = w
-	os.Stderr = w
-
-	f()
-
-	w.Close()
-	var buf bytes.Buffer
-	io.Copy(&buf, r)
-	return buf.String()
-}
-
-func createMockOllamaServer(t *testing.T, responses []api.ChatResponse) *httptest.Server {
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path != "/api/chat" {
-			t.Errorf("Expected path /api/chat, got %s", r.URL.Path)
-			http.Error(w, "Not found", http.StatusNotFound)
-			return
-		}
-
-		if r.Method != "POST" {
-			t.Errorf("Expected POST method, got %s", r.Method)
-			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-			return
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(http.StatusOK)
-
-		for _, resp := range responses {
-			jsonData, err := json.Marshal(resp)
-			if err != nil {
-				t.Errorf("Failed to marshal response: %v", err)
-				return
-			}
-			w.Write(jsonData)
-			w.Write([]byte("\n"))
-			if f, ok := w.(http.Flusher); ok {
-				f.Flush()
-			}
-			time.Sleep(10 * time.Millisecond) // Simulate some delay
-		}
-	}))
-}
-
-func TestBenchmarkChat_Success(t *testing.T) {
-	fOpt := createTestFlagOptions()
-
-	mockResponses := []api.ChatResponse{
-		{
-			Model: "test-model",
-			Message: api.Message{
-				Role:    "assistant",
-				Content: "test response part 1",
-			},
-			Done: false,
-		},
-		{
-			Model: "test-model",
-			Message: api.Message{
-				Role:    "assistant",
-				Content: "test response part 2",
-			},
-			Done: true,
-			Metrics: api.Metrics{
-				PromptEvalCount:    10,
-				PromptEvalDuration: 100 * time.Millisecond,
-				EvalCount:          50,
-				EvalDuration:       500 * time.Millisecond,
-				TotalDuration:      600 * time.Millisecond,
-				LoadDuration:       50 * time.Millisecond,
-			},
-		},
-	}
-
-	server := createMockOllamaServer(t, mockResponses)
-	defer server.Close()
-
-	t.Setenv("OLLAMA_HOST", server.URL)
-
-	output := captureOutput(func() {
-		err := BenchmarkChat(fOpt)
-		if err != nil {
-			t.Errorf("Expected no error, got %v", err)
-		}
-	})
-
-	if !strings.Contains(output, "BenchmarkModel/name=test-model/step=prefill") {
-		t.Errorf("Expected output to contain prefill metrics, got: %s", output)
-	}
-	if !strings.Contains(output, "BenchmarkModel/name=test-model/step=generate") {
-		t.Errorf("Expected output to contain generate metrics, got: %s", output)
-	}
-	if !strings.Contains(output, "ns/token") {
-		t.Errorf("Expected output to contain ns/token metric, got: %s", output)
-	}
-}
-
-func TestBenchmarkChat_ServerError(t *testing.T) {
-	fOpt := createTestFlagOptions()
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		http.Error(w, "Internal server error", http.StatusInternalServerError)
-	}))
-	defer server.Close()
-
-	t.Setenv("OLLAMA_HOST", server.URL)
-
-	output := captureOutput(func() {
-		err := BenchmarkChat(fOpt)
-		if err != nil {
-			t.Errorf("Expected error to be handled internally, got returned error: %v", err)
-		}
-	})
-
-	if !strings.Contains(output, "ERROR: Couldn't chat with model") {
-		t.Errorf("Expected error message about chat failure, got: %s", output)
-	}
-}
-
-func TestBenchmarkChat_Timeout(t *testing.T) {
-	fOpt := createTestFlagOptions()
-	shortTimeout := 1 // Very short timeout
-	fOpt.timeout = &shortTimeout
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Simulate a long delay that will cause timeout
-		time.Sleep(2 * time.Second)
-
-		w.Header().Set("Content-Type", "application/json")
-		response := api.ChatResponse{
-			Model: "test-model",
-			Message: api.Message{
-				Role:    "assistant",
-				Content: "test response",
-			},
-			Done: true,
-			Metrics: api.Metrics{
-				PromptEvalCount:    10,
-				PromptEvalDuration: 100 * time.Millisecond,
-				EvalCount:          50,
-				EvalDuration:       500 * time.Millisecond,
-				TotalDuration:      600 * time.Millisecond,
-				LoadDuration:       50 * time.Millisecond,
-			},
-		}
-		jsonData, _ := json.Marshal(response)
-		w.Write(jsonData)
-	}))
-	defer server.Close()
-
-	t.Setenv("OLLAMA_HOST", server.URL)
-
-	output := captureOutput(func() {
-		err := BenchmarkChat(fOpt)
-		if err != nil {
-			t.Errorf("Expected timeout to be handled internally, got returned error: %v", err)
-		}
-	})
-
-	if !strings.Contains(output, "ERROR: Chat request timed out") {
-		t.Errorf("Expected timeout error message, got: %s", output)
-	}
-}
-
-func TestBenchmarkChat_NoMetrics(t *testing.T) {
-	fOpt := createTestFlagOptions()
-
-	mockResponses := []api.ChatResponse{
-		{
-			Model: "test-model",
-			Message: api.Message{
-				Role:    "assistant",
-				Content: "test response",
-			},
-			Done: false, // Never sends Done=true
-		},
-	}
-
-	server := createMockOllamaServer(t, mockResponses)
-	defer server.Close()
-
-	t.Setenv("OLLAMA_HOST", server.URL)
-
-	output := captureOutput(func() {
-		err := BenchmarkChat(fOpt)
-		if err != nil {
-			t.Errorf("Expected no error, got %v", err)
-		}
-	})
-
-	if !strings.Contains(output, "ERROR: No metrics received") {
-		t.Errorf("Expected no metrics error message, got: %s", output)
-	}
-}
-
-func TestBenchmarkChat_MultipleModels(t *testing.T) {
-	fOpt := createTestFlagOptions()
-	models := "model1,model2"
-	epochs := 2
-	fOpt.models = &models
-	fOpt.epochs = &epochs
-
-	callCount := 0
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		callCount++
-
-		w.Header().Set("Content-Type", "application/json")
-
-		var req api.ChatRequest
-		body, _ := io.ReadAll(r.Body)
-		json.Unmarshal(body, &req)
-
-		response := api.ChatResponse{
-			Model: req.Model,
-			Message: api.Message{
-				Role:    "assistant",
-				Content: "test response for " + req.Model,
-			},
-			Done: true,
-			Metrics: api.Metrics{
-				PromptEvalCount:    10,
-				PromptEvalDuration: 100 * time.Millisecond,
-				EvalCount:          50,
-				EvalDuration:       500 * time.Millisecond,
-				TotalDuration:      600 * time.Millisecond,
-				LoadDuration:       50 * time.Millisecond,
-			},
-		}
-		jsonData, _ := json.Marshal(response)
-		w.Write(jsonData)
-	}))
-	defer server.Close()
-
-	t.Setenv("OLLAMA_HOST", server.URL)
-
-	output := captureOutput(func() {
-		err := BenchmarkChat(fOpt)
-		if err != nil {
-			t.Errorf("Expected no error, got %v", err)
-		}
-	})
-
-	// Should be called 4 times (2 models × 2 epochs)
-	if callCount != 4 {
-		t.Errorf("Expected 4 API calls, got %d", callCount)
-	}
-
-	if !strings.Contains(output, "BenchmarkModel/name=model1") || !strings.Contains(output, "BenchmarkModel/name=model2") {
-		t.Errorf("Expected output for both models, got: %s", output)
-	}
-}
-
-func TestBenchmarkChat_WithImage(t *testing.T) {
-	fOpt := createTestFlagOptions()
-
-	tmpfile, err := os.CreateTemp(t.TempDir(), "testimage")
-	if err != nil {
-		t.Fatalf("Failed to create temp file: %v", err)
-	}
-	defer os.Remove(tmpfile.Name())
-
-	content := []byte("fake image data")
-	if _, err := tmpfile.Write(content); err != nil {
-		t.Fatalf("Failed to write to temp file: %v", err)
-	}
-	tmpfile.Close()
-
-	tmpfileName := tmpfile.Name()
-	fOpt.imageFile = &tmpfileName
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Verify the request contains image data
-		var req api.ChatRequest
-		body, _ := io.ReadAll(r.Body)
-		json.Unmarshal(body, &req)
-
-		if len(req.Messages) == 0 || len(req.Messages[0].Images) == 0 {
-			t.Error("Expected request to contain images")
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-		response := api.ChatResponse{
-			Model: "test-model",
-			Message: api.Message{
-				Role:    "assistant",
-				Content: "test response with image",
-			},
-			Done: true,
-			Metrics: api.Metrics{
-				PromptEvalCount:    10,
-				PromptEvalDuration: 100 * time.Millisecond,
-				EvalCount:          50,
-				EvalDuration:       500 * time.Millisecond,
-				TotalDuration:      600 * time.Millisecond,
-				LoadDuration:       50 * time.Millisecond,
-			},
-		}
-		jsonData, _ := json.Marshal(response)
-		w.Write(jsonData)
-	}))
-	defer server.Close()
-
-	t.Setenv("OLLAMA_HOST", server.URL)
-
-	output := captureOutput(func() {
-		err := BenchmarkChat(fOpt)
-		if err != nil {
-			t.Errorf("Expected no error, got %v", err)
-		}
-	})
-
-	if !strings.Contains(output, "BenchmarkModel/name=test-model") {
-		t.Errorf("Expected benchmark output, got: %s", output)
-	}
-}
-
-func TestBenchmarkChat_ImageError(t *testing.T) {
-	randFileName := func() string {
-		const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
-		const length = 8
-
-		result := make([]byte, length)
-		rand.Read(result) // Fill with random bytes
-
-		for i := range result {
-			result[i] = charset[result[i]%byte(len(charset))]
-		}
-
-		return string(result) + ".txt"
-	}
-
-	fOpt := createTestFlagOptions()
-	imageFile := randFileName()
-	fOpt.imageFile = &imageFile
-
-	output := captureOutput(func() {
-		err := BenchmarkChat(fOpt)
-		if err == nil {
-			t.Error("Expected error from image reading, got nil")
-		}
-	})
-
-	if !strings.Contains(output, "ERROR: Couldn't read image") {
-		t.Errorf("Expected image read error message, got: %s", output)
-	}
-}
-
-func TestReadImage_Success(t *testing.T) {
-	tmpfile, err := os.CreateTemp(t.TempDir(), "testimage")
-	if err != nil {
-		t.Fatalf("Failed to create temp file: %v", err)
-	}
-	defer os.Remove(tmpfile.Name())
-
-	content := []byte("fake image data")
-	if _, err := tmpfile.Write(content); err != nil {
-		t.Fatalf("Failed to write to temp file: %v", err)
-	}
-	tmpfile.Close()
-
-	imgData, err := readImage(tmpfile.Name())
-	if err != nil {
-		t.Errorf("Expected no error, got %v", err)
-	}
-
-	if imgData == nil {
-		t.Error("Expected image data, got nil")
-	}
-
-	expected := api.ImageData(content)
-	if string(imgData) != string(expected) {
-		t.Errorf("Expected image data %v, got %v", expected, imgData)
-	}
-}
-
-func TestReadImage_FileNotFound(t *testing.T) {
-	imgData, err := readImage("nonexistentfile.jpg")
-	if err == nil {
-		t.Error("Expected error for non-existent file, got nil")
-	}
-	if imgData != nil {
-		t.Error("Expected nil image data for non-existent file")
-	}
-}
-
-func TestOptionsMapCreation(t *testing.T) {
-	fOpt := createTestFlagOptions()
-
-	options := make(map[string]interface{})
-	if *fOpt.maxTokens > 0 {
-		options["num_predict"] = *fOpt.maxTokens
-	}
-	options["temperature"] = *fOpt.temperature
-	if fOpt.seed != nil && *fOpt.seed > 0 {
-		options["seed"] = *fOpt.seed
-	}
-
-	if options["num_predict"] != *fOpt.maxTokens {
-		t.Errorf("Expected num_predict %d, got %v", *fOpt.maxTokens, options["num_predict"])
-	}
-	if options["temperature"] != *fOpt.temperature {
-		t.Errorf("Expected temperature %f, got %v", *fOpt.temperature, options["temperature"])
-	}
-	if options["seed"] != *fOpt.seed {
-		t.Errorf("Expected seed %d, got %v", *fOpt.seed, options["seed"])
-	}
-}

cmd/cmd.go

@@ -45,7 +45,6 @@ import (
 	"github.com/ollama/ollama/types/model"
 	"github.com/ollama/ollama/types/syncmap"
 	"github.com/ollama/ollama/version"
-	xcmd "github.com/ollama/ollama/x/cmd"
 )
 
 const ConnectInstructions = "To sign in, navigate to:\n    %s\n\n"
@@ -518,9 +517,6 @@ func RunHandler(cmd *cobra.Command, args []string) error {
 		return generateEmbedding(cmd, name, opts.Prompt, opts.KeepAlive, truncate, dimensions)
 	}
 
-	// Check for experimental flag
-	isExperimental, _ := cmd.Flags().GetBool("experimental")
-
 	if interactive {
 		if err := loadOrUnloadModel(cmd, &opts); err != nil {
 			var sErr api.AuthorizationError
@@ -547,11 +543,6 @@ func RunHandler(cmd *cobra.Command, args []string) error {
 			}
 		}
 
-		// Use experimental agent loop with
-		if isExperimental {
-			return xcmd.GenerateInteractive(cmd, opts.Model, opts.WordWrap, opts.Options, opts.Think, opts.HideThinking, opts.KeepAlive)
-		}
-
 		return generateInteractive(cmd, opts)
 	}
 	return generate(cmd, opts)
@@ -952,9 +943,6 @@ func showInfo(resp *api.ShowResponse, verbose bool, w io.Writer) error {
 			rows = append(rows, []string{"", "parameters", resp.Details.ParameterSize})
 		}
 		rows = append(rows, []string{"", "quantization", resp.Details.QuantizationLevel})
-		if resp.Requires != "" {
-			rows = append(rows, []string{"", "requires", resp.Requires})
-		}
 		return
 	})
 
@@ -1442,7 +1430,7 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) {
 		latest.Summary()
 	}
 
-	return &api.Message{Role: role, Thinking: thinkingContent.String(), Content: fullResponse.String()}, nil
+	return &api.Message{Role: role, Content: fullResponse.String()}, nil
 }
 
 func generate(cmd *cobra.Command, opts runOptions) error {
@@ -1763,7 +1751,6 @@ func NewCLI() *cobra.Command {
 	runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)")
 	runCmd.Flags().Bool("truncate", false, "For embedding models: truncate inputs exceeding context length (default: true). Set --truncate=false to error instead")
 	runCmd.Flags().Int("dimensions", 0, "Truncate output embeddings to specified dimension (embedding models only)")
-	runCmd.Flags().Bool("experimental", false, "Enable experimental agent loop with tools")
 
 	stopCmd := &cobra.Command{
 		Use:     "stop MODEL",

cmd/cmd_test.go

@@ -291,31 +291,6 @@ Weigh anchor!
 			t.Errorf("unexpected output (-want +got):\n%s", diff)
 		}
 	})
-
-	t.Run("min version", func(t *testing.T) {
-		var b bytes.Buffer
-		if err := showInfo(&api.ShowResponse{
-			Details: api.ModelDetails{
-				Family:            "test",
-				ParameterSize:     "7B",
-				QuantizationLevel: "FP16",
-			},
-			Requires: "0.14.0",
-		}, false, &b); err != nil {
-			t.Fatal(err)
-		}
-
-		expect := `  Model
-    architecture    test      
-    parameters      7B        
-    quantization    FP16      
-    requires        0.14.0    
-
-`
-		if diff := cmp.Diff(expect, b.String()); diff != "" {
-			t.Errorf("unexpected output (-want +got):\n%s", diff)
-		}
-	})
 }
 
 func TestDeleteHandler(t *testing.T) {

cmd/interactive.go

@@ -40,7 +40,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
 		fmt.Fprintln(os.Stderr, "  /bye            Exit")
 		fmt.Fprintln(os.Stderr, "  /?, /help       Help for a command")
 		fmt.Fprintln(os.Stderr, "  /? shortcuts    Help for keyboard shortcuts")
-
 		fmt.Fprintln(os.Stderr, "")
 		fmt.Fprintln(os.Stderr, "Use \"\"\" to begin a multi-line message.")
 

convert/convert.go

@@ -182,8 +182,6 @@ func ConvertModel(fsys fs.FS, f *os.File) error {
 		conv = &llama4Model{}
 	case "Mistral3ForConditionalGeneration":
 		conv = &mistral3Model{}
-	case "Ministral3ForCausalLM":
-		conv = &mistral3CausalModel{}
 	case "MixtralForCausalLM":
 		conv = &mixtralModel{}
 	case "GemmaForCausalLM":
@@ -202,20 +200,12 @@ func ConvertModel(fsys fs.FS, f *os.File) error {
 		conv = &qwen25VLModel{}
 	case "Qwen3VLForConditionalGeneration", "Qwen3VLMoeForConditionalGeneration":
 		conv = &qwen3VLModel{}
-	case "Olmo3ForCausalLM":
-		conv = &olmoModel{}
 	case "BertModel":
 		conv = &bertModel{}
-	case "NomicBertModel", "NomicBertMoEModel":
-		conv = &nomicbertModel{}
 	case "CohereForCausalLM":
 		conv = &commandrModel{}
 	case "GptOssForCausalLM":
 		conv = &gptossModel{}
-	case "DeepseekOCRForCausalLM":
-		conv = &deepseekocr{}
-	case "DeepseekV3ForCausalLM":
-		conv = &deepseek2Model{}
 	default:
 		return fmt.Errorf("unsupported architecture %q", p.Architectures[0])
 	}

convert/convert_deepseek2.go

@@ -1,173 +0,0 @@
-package convert
-
-import (
-	"cmp"
-	"fmt"
-	"log/slog"
-	"regexp"
-	"strconv"
-
-	"github.com/ollama/ollama/fs/ggml"
-)
-
-type deepseek2Model struct {
-	ModelParameters               // architectures, vocab_size
-	MaxPositionEmbeddings uint32  `json:"max_position_embeddings"`
-	HiddenSize            uint32  `json:"hidden_size"`
-	HiddenLayers          uint32  `json:"num_hidden_layers"`
-	IntermediateSize      uint32  `json:"intermediate_size"`
-	NumAttentionHeads     uint32  `json:"num_attention_heads"`
-	NumKeyValueHeads      uint32  `json:"num_key_value_heads"`
-	RMSNormEPS            float32 `json:"rms_norm_eps"`
-
-	RopeTheta     float32 `json:"rope_theta"`
-	QKNopeHeadDim uint32  `json:"qk_nope_head_dim"`
-	QKRopeHeadDim uint32  `json:"qk_rope_head_dim"`
-	KVLoraRank    uint32  `json:"kv_lora_rank"`
-	QLoraRank     uint32  `json:"q_lora_rank"`
-	VHeadDim      uint32  `json:"v_head_dim"`
-
-	ExpertCount            uint32  `json:"n_routed_experts"`
-	ExpertSharedCount      uint32  `json:"n_shared_experts"`
-	ExpertIntermediateSize uint32  `json:"moe_intermediate_size"`
-	ExpertUsedCount        uint32  `json:"num_experts_per_tok"`
-	ExpertWeightsNorm      bool    `json:"norm_topk_prob"`
-	ExpertWeightsScale     float32 `json:"routed_scaling_factor"`
-
-	ScoringFunc            string `json:"scoring_func"`
-	LeadingDenseBlockCount uint32 `json:"first_k_dense_replace"`
-
-	RopeScaling struct {
-		Factor                        float32 `json:"factor"`
-		OriginalMaxPositionEmbeddings uint32  `json:"original_max_position_embeddings"`
-		Type                          string  `json:"type"`
-		MScaleAllDim                  float32 `json:"mscale_all_dim"`
-	} `json:"rope_scaling"`
-
-	Architecture string
-}
-
-func (p *deepseek2Model) KV(t *Tokenizer) ggml.KV {
-	kv := p.ModelParameters.KV(t)
-	kv["general.architecture"] = "deepseek2"
-	kv["general.type"] = "model"
-	kv["deepseek2.block_count"] = p.HiddenLayers
-
-	numHeads := p.NumAttentionHeads
-	numKVHeads := p.NumKeyValueHeads
-
-	kv["deepseek2.attention.head_count"] = numHeads
-	kv["deepseek2.attention.head_count_kv"] = numKVHeads
-	kv["deepseek2.attention.key_length"] = p.QKNopeHeadDim + p.QKRopeHeadDim
-	kv["deepseek2.attention.kv_lora_rank"] = p.KVLoraRank
-	kv["deepseek2.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS
-	kv["deepseek2.attention.q_lora_rank"] = p.QLoraRank
-	kv["deepseek2.attention.value_length"] = p.VHeadDim
-	kv["deepseek2.context_length"] = p.MaxPositionEmbeddings
-	kv["deepseek2.embedding_length"] = p.HiddenSize
-	kv["deepseek2.expert_count"] = p.ExpertCount
-	kv["deepseek2.expert_feed_forward_length"] = p.ExpertIntermediateSize
-	kv["deepseek2.expert_shared_count"] = p.ExpertSharedCount
-
-	var scoringFunc uint32
-	switch p.ScoringFunc {
-	case "softmax":
-		// not currently supported in the model, but needed for Deepseek-OCR
-		scoringFunc = 1
-	case "sigmoid":
-		scoringFunc = 2
-	}
-	kv["deepseek2.expert_gating_func"] = scoringFunc
-	kv["deepseek2.expert_used_count"] = p.ExpertUsedCount
-	kv["deepseek2.expert_weights_norm"] = p.ExpertWeightsNorm
-	kv["deepseek2.expert_weights_scale"] = p.ExpertWeightsScale
-	kv["deepseek2.feed_forward_length"] = p.IntermediateSize
-	kv["deepseek2.leading_dense_block_count"] = p.LeadingDenseBlockCount
-
-	kv["deepseek2.rope.dimension_count"] = p.QKRopeHeadDim
-	kv["deepseek2.rope.freq_base"] = cmp.Or(p.RopeTheta, 10000.0)
-	kv["deepseek2.rope.scaling.factor"] = p.RopeScaling.Factor
-	kv["deepseek2.rope.scaling.original_context_length"] = p.RopeScaling.OriginalMaxPositionEmbeddings
-	kv["deepseek2.rope.scaling.type"] = p.RopeScaling.Type
-	kv["deepseek2.rope.scaling.yarn_log_multiplier"] = 0.1 * p.RopeScaling.MScaleAllDim
-
-	kv["tokenizer.ggml.pre"] = "deepseek-v3"
-
-	return kv
-}
-
-func (p *deepseek2Model) Replacements() []string {
-	return []string{
-		"lm_head", "output",
-		"model.embed_tokens", "token_embd",
-		"model.norm", "output_norm",
-		"language_model.", "",
-		"model.layers", "blk",
-		"input_layernorm", "attn_norm",
-		"self_attn.kv_a_proj_with_mqa", "attn_kv_a_mqa",
-		"self_attn.kv_a_layernorm", "attn_kv_a_norm",
-		"self_attn.kv_b_proj", "attn_kv_b",
-		"self_attn.q_a_proj", "attn_q_a",
-		"self_attn.q_a_layernorm", "attn_q_a_norm",
-		"self_attn.q_b_proj", "attn_q_b",
-		"self_attn.o_proj", "attn_output",
-		"post_attention_layernorm", "ffn_norm",
-		"mlp.shared_experts.down_proj", "ffn_down_shexp",
-		"mlp.shared_experts.gate_proj", "ffn_gate_shexp",
-		"mlp.shared_experts.up_proj", "ffn_up_shexp",
-		"mlp.gate_proj", "ffn_gate",
-		"mlp.down_proj", "ffn_down",
-		"mlp.up_proj", "ffn_up",
-		"mlp.gate.e_score_correction_bias", "exp_probs_b.bias",
-		"mlp.gate", "ffn_gate_inp",
-	}
-}
-
-func (p *deepseek2Model) Tensors(s []Tensor) (out []*ggml.Tensor) {
-	merges := make([]merge, p.HiddenLayers*3)
-	for i := range p.HiddenLayers {
-		merges[i*3+0] = merge{
-			fmt.Sprintf("blk.%d.mlp.experts.*.gate_proj.weight", i),
-			fmt.Sprintf("blk.%d.ffn_gate_exps.weight", i),
-		}
-		merges[i*3+1] = merge{
-			fmt.Sprintf("blk.%d.mlp.experts.*.up_proj.weight", i),
-			fmt.Sprintf("blk.%d.ffn_up_exps.weight", i),
-		}
-		merges[i*3+2] = merge{
-			fmt.Sprintf("blk.%d.mlp.experts.*.down_proj.weight", i),
-			fmt.Sprintf("blk.%d.ffn_down_exps.weight", i),
-		}
-	}
-
-	skipLayer := func(n string, minValue uint32) bool {
-		re := regexp.MustCompile(`^blk\.(\d+)`)
-		matches := re.FindStringSubmatch(n)
-		if matches == nil {
-			return false
-		}
-
-		blkNum, err := strconv.Atoi(matches[1])
-		if err != nil {
-			return false
-		}
-
-		return uint32(blkNum) >= minValue
-	}
-
-	out, s = mergeTensors(s, merges...)
-	for _, t := range s {
-		// skip any additional layers (such as the Multi-Token Prediction layer)
-		if skipLayer(t.Name(), p.HiddenLayers) {
-			slog.Debug("skipping layer", "name", t.Name())
-			continue
-		}
-		out = append(out, &ggml.Tensor{
-			Name:     t.Name(),
-			Kind:     t.Kind(),
-			Shape:    t.Shape(),
-			WriterTo: t,
-		})
-	}
-	return out
-}

convert/convert_deepseekocr.go

@@ -1,136 +0,0 @@
-package convert
-
-import (
-	"fmt"
-
-	"github.com/ollama/ollama/fs/ggml"
-)
-
-type deepseekocr struct {
-	ModelParameters
-	LanguageConfig struct {
-		MaxPositionEmbeddings uint32 `json:"max_position_embeddings"`
-		HiddenSize            uint32 `json:"hidden_size"`
-		HiddenLayers          uint32 `json:"num_hidden_layers"`
-		IntermediateSize      uint32 `json:"intermediate_size"`
-		NumAttentionHeads     uint32 `json:"num_attention_heads"`
-		NumKeyValueHeads      uint32 `json:"num_key_value_heads"`
-		NumRoutedExperts      uint32 `json:"n_routed_experts"`
-		NumSharedExperts      uint32 `json:"n_shared_experts"`
-		NumExpertsPerToken    uint32 `json:"num_experts_per_tok"`
-		FirstKDenseReplace    uint32 `json:"first_k_dense_replace"`
-	} `json:"language_config"`
-
-	VisionConfig struct {
-		ImageSize uint32 `json:"image_size"`
-		Width     struct {
-			Vision struct {
-				Heads     uint32 `json:"heads"`
-				ImageSize uint32 `json:"image_size"`
-				Layers    uint32 `json:"layers"`
-				PatchSize uint32 `json:"patch_size"`
-				Width     uint32 `json:"width"`
-			} `json:"clip-l-14-224"`
-			Sam struct {
-				GlobalAttentionIndexes []int32 `json:"global_attn_indexes"`
-				Heads                  uint32  `json:"heads"`
-				Layers                 uint32  `json:"layers"`
-				Width                  uint32  `json:"width"`
-			} `json:"sam_vit_b"`
-		}
-	} `json:"vision_config"`
-}
-
-func (m *deepseekocr) KV(t *Tokenizer) ggml.KV {
-	kv := m.ModelParameters.KV(t)
-	kv["general.architecture"] = "deepseekocr"
-	kv["block_count"] = m.LanguageConfig.HiddenLayers
-	kv["context_length"] = m.LanguageConfig.MaxPositionEmbeddings
-	kv["embedding_length"] = m.LanguageConfig.HiddenSize
-	kv["feed_forward_length"] = m.LanguageConfig.IntermediateSize
-	kv["attention.head_count"] = m.LanguageConfig.NumAttentionHeads
-	kv["attention.head_count_kv"] = m.LanguageConfig.NumKeyValueHeads
-	kv["expert_count"] = m.LanguageConfig.NumRoutedExperts
-	kv["expert_used_count"] = m.LanguageConfig.NumExpertsPerToken
-	kv["leading_dense_block_count"] = m.LanguageConfig.FirstKDenseReplace
-
-	kv["vision.block_count"] = m.VisionConfig.Width.Vision.Layers
-	kv["vision.embedding_length"] = m.VisionConfig.Width.Vision.Width
-	kv["vision.head_count"] = m.VisionConfig.Width.Vision.Heads
-	kv["vision.image_size"] = m.VisionConfig.Width.Vision.ImageSize
-	kv["vision.patch_size"] = m.VisionConfig.Width.Vision.PatchSize
-
-	kv["sam.block_count"] = m.VisionConfig.Width.Sam.Layers
-	kv["sam.embedding_length"] = m.VisionConfig.Width.Sam.Width
-	kv["sam.head_count"] = m.VisionConfig.Width.Sam.Heads
-	kv["sam.global_attention_indexes"] = m.VisionConfig.Width.Sam.GlobalAttentionIndexes
-	return kv
-}
-
-func (m *deepseekocr) Tensors(s []Tensor) (out []*ggml.Tensor) {
-	merges := make([]merge, m.LanguageConfig.HiddenLayers*3)
-	for i := range m.LanguageConfig.HiddenLayers {
-		merges[i*3+0] = merge{
-			fmt.Sprintf("blk.%d.mlp.experts.*.gate_proj.weight", i),
-			fmt.Sprintf("blk.%d.ffn_gate_exps.weight", i),
-		}
-		merges[i*3+1] = merge{
-			fmt.Sprintf("blk.%d.mlp.experts.*.up_proj.weight", i),
-			fmt.Sprintf("blk.%d.ffn_up_exps.weight", i),
-		}
-		merges[i*3+2] = merge{
-			fmt.Sprintf("blk.%d.mlp.experts.*.down_proj.weight", i),
-			fmt.Sprintf("blk.%d.ffn_down_exps.weight", i),
-		}
-	}
-
-	out, s = mergeTensors(s, merges...)
-	for _, t := range s {
-		out = append(out, &ggml.Tensor{
-			Name:     t.Name(),
-			Kind:     t.Kind(),
-			Shape:    t.Shape(),
-			WriterTo: t,
-		})
-	}
-	return out
-}
-
-func (m *deepseekocr) Replacements() []string {
-	return []string{
-		"model.embed_tokens", "token_embd",
-		"model.layers", "blk",
-		"input_layernorm", "attn_norm",
-		"self_attn.q_proj", "attn_q",
-		"self_attn.k_proj", "attn_k",
-		"self_attn.v_proj", "attn_v",
-		"self_attn.o_proj", "attn_output",
-		"post_attention_layernorm", "ffn_norm",
-		"mlp.gate_proj", "ffn_gate",
-		"mlp.up_proj", "ffn_up",
-		"mlp.down_proj", "ffn_down",
-		"mlp.gate", "ffn_gate_inp",
-		"mlp.shared_experts.gate_proj", "ffn_gate_shexp",
-		"mlp.shared_experts.up_proj", "ffn_up_shexp",
-		"mlp.shared_experts.down_proj", "ffn_down_shexp",
-		"model.norm", "output_norm",
-		"lm_head", "output",
-
-		"model.vision_model", "v",
-		"embeddings.patch_embedding", "patch_embd",
-		"embeddings.class_embedding", "class_embd",
-		"embeddings.position_embedding", "position_embd",
-		"transformer.layers", "blk",
-
-		"model.projector", "mm",
-		"model.image_newline", "mm.image_newline",
-		//nolint:misspell // this misspelling is upstream. fixing it breaks the model
-		"model.view_seperator", "mm.view_seperator",
-
-		"model.sam_model.patch_embed.proj", "s.patch_embd",
-		"model.sam_model.pos_embed", "s.position_embd",
-		"model.sam_model.blocks", "s.blk",
-		"model.sam_model.neck", "s.neck",
-		"model.sam_model.net_", "s.net_",
-	}
-}

convert/convert_gemma3.go

@@ -2,7 +2,6 @@ package convert
 
 import (
 	"cmp"
-	"slices"
 
 	"github.com/ollama/ollama/fs/ggml"
 )
@@ -27,26 +26,16 @@ type gemma3Model struct {
 		NumChannels       uint32  `json:"num_channels"`        // num_channels 3
 		PatchSize         uint32  `json:"patch_size"`          // patch_size 14
 	} `json:"vision_config"`
-	MaxPositionEmbeddings    uint32   `json:"max_position_embeddings"`
+	MaxPositionEmbeddings    uint32  `json:"max_position_embeddings"`
-	NumAttentionHeads        uint32   `json:"num_attention_heads"`
+	NumAttentionHeads        uint32  `json:"num_attention_heads"`
-	NumKeyValueHeads         uint32   `json:"num_key_value_heads"`
+	NumKeyValueHeads         uint32  `json:"num_key_value_heads"`
-	RMSNormEPS               float32  `json:"rms_norm_eps"`
+	RMSNormEPS               float32 `json:"rms_norm_eps"`
-	HeadDim                  uint32   `json:"head_dim"`
+	HeadDim                  uint32  `json:"head_dim"`
-	FinalLogitSoftcap        float32  `json:"final_logit_softcapping"`
+	FinalLogitSoftcap        float32 `json:"final_logit_softcapping"`
-	RopeLocalTheta           float32  `json:"rope_local_base_freq"`
+	RopeLocalTheta           float32 `json:"rope_local_base_freq"`
-	RopeTheta                float32  `json:"rope_theta"`
+	RopeGlobalTheta          float32 `json:"rope_global_base_freq"`
-	SlidingWindow            uint32   `json:"sliding_window"`
+	SlidingWindow            uint32  `json:"sliding_window"`
-	SlidingWindowPattern     *uint32  `json:"sliding_window_pattern"`
+	MultiModalTokensPerImage uint32  `json:"mm_tokens_per_image"`
-	LayerTypes               []string `json:"layer_types"`
-	MultiModalTokensPerImage uint32   `json:"mm_tokens_per_image"`
-	RopeScaling              *struct {
-		Type                          string  `json:"rope_type"`
-		Factor                        float32 `json:"factor"`
-		OriginalMaxPositionEmbeddings uint32  `json:"original_max_position_embeddings"`
-		ExtrapolationFactor           float32 `json:"extrapolation_factor"`
-		BetaFast                      float32 `json:"beta_fast"`
-		BetaSlow                      float32 `json:"beta_slow"`
-	} `json:"rope_scaling"`
 }
 
 const (
@@ -92,38 +81,9 @@ func (p *gemma3Model) KV(t *Tokenizer) ggml.KV {
 		kv["gemma3.attention.key_length"] = p.HeadDim
 		kv["gemma3.attention.value_length"] = p.HeadDim
 		kv["gemma3.attention.sliding_window"] = p.SlidingWindow
-
+		kv["gemma3.final_logit_softcapping"] = cmp.Or(p.FinalLogitSoftcap, 30)
-		// The sliding window pattern is either provided as the sliding_window_pattern
-		// key (an int) or as the layer_types key (a list of strings).
-		if p.SlidingWindowPattern != nil || len(p.LayerTypes) > 0 {
-			kv["gemma3.attention.sliding_window_pattern"] = slices.Collect(func(yield func(bool) bool) {
-				for i := range numBlocks {
-					var isLocal bool
-					if len(p.LayerTypes) > 0 && int(i) < len(p.LayerTypes) {
-						isLocal = p.LayerTypes[i] == "sliding_attention"
-					} else if p.SlidingWindowPattern != nil && *p.SlidingWindowPattern > 0 {
-						isLocal = (i+1)%*p.SlidingWindowPattern != 0
-					}
-					if !yield(isLocal) {
-						break
-					}
-				}
-			})
-		}
-		if p.FinalLogitSoftcap > 0 {
-			kv["gemma3.final_logit_softcapping"] = p.FinalLogitSoftcap
-		}
 		kv["gemma3.rope.local.freq_base"] = cmp.Or(p.RopeLocalTheta, 10000.0)
-		kv["gemma3.rope.freq_base"] = cmp.Or(p.RopeTheta, 1000000.0)
+		kv["gemma3.rope.global.freq_base"] = cmp.Or(p.RopeGlobalTheta, 1000000.0)
-		if p.RopeScaling != nil && p.RopeScaling.Type == "yarn" && p.RopeScaling.Factor > 0 {
-			kv["gemma3.rope.scaling.type"] = "yarn"
-			kv["gemma3.rope.scaling.factor"] = p.RopeScaling.Factor
-			kv["gemma3.rope.scaling.original_context_length"] = p.RopeScaling.OriginalMaxPositionEmbeddings
-			kv["gemma3.rope.scaling.extrapolation_factor"] = cmp.Or(p.RopeScaling.ExtrapolationFactor, float32(1.0))
-			kv["gemma3.rope.scaling.beta_fast"] = cmp.Or(p.RopeScaling.BetaFast, float32(64.0))
-			kv["gemma3.rope.scaling.beta_slow"] = cmp.Or(p.RopeScaling.BetaSlow, float32(1.0))
-		}
-
 		kv["gemma3.embedding_length"] = p.HiddenSize
 		kv["gemma3.feed_forward_length"] = p.IntermediateSize
 	default:

convert/convert_mistral.go

@@ -29,17 +29,6 @@ type mistral3Model struct {
 		SlidingWindow         *uint32 `json:"sliding_window"`
 		HiddenAct             string  `json:"hidden_act"`
 		VocabSize             uint32  `json:"vocab_size"`
-		RopeParameters        struct {
-			BetaFast                  float32  `json:"beta_fast"`
-			BetaSlow                  float32  `json:"beta_slow"`
-			Factor                    float32  `json:"factor"`
-			Llama4ScalingBeta         *float32 `json:"llama_4_scaling_beta"`
-			OrigMaxPositionEmbeddings uint32   `json:"original_max_position_embeddings"`
-			RopeType                  string   `json:"rope_type"`
-			RopeTheta                 float32  `json:"rope_theta"`
-			Mscale                    *float32 `json:"mscale"`
-			MscaleAllDim              *float32 `json:"mscale_all_dim"`
-		} `json:"rope_parameters"`
 	} `json:"text_config"`
 	VisionModel struct {
 		NumAttentionHeads uint32  `json:"num_attention_heads"`
@@ -52,9 +41,6 @@ type mistral3Model struct {
 		HeadDim           uint32  `json:"head_dim"`
 		HiddenAct         string  `json:"hidden_act"`
 		RopeTheta         float32 `json:"rope_theta"`
-		RopeParameters    struct {
-			RopeTheta float32 `json:"rope_theta"`
-		} `json:"rope_parameters"`
 	} `json:"vision_config"`
 	MultiModalProjectorBias bool   `json:"multimodal_projector_bias"`
 	ProjectorHiddenAct      string `json:"projector_hidden_act"`
@@ -75,25 +61,8 @@ func (p *mistral3Model) KV(t *Tokenizer) ggml.KV {
 	kv["mistral3.attention.layer_norm_rms_epsilon"] = p.TextModel.RMSNormEPS
 	kv["mistral3.attention.key_length"] = p.TextModel.HeadDim
 	kv["mistral3.attention.value_length"] = p.TextModel.HeadDim
-	kv["mistral3.rope.dimension_count"] = cmp.Or(p.TextModel.HeadDim, p.TextModel.HiddenSize/p.TextModel.NumAttentionHeads)
+	kv["mistral3.rope.dimension_count"] = p.TextModel.HiddenSize / p.TextModel.NumHiddenLayers
-	kv["mistral3.rope.freq_base"] = cmp.Or(p.TextModel.RopeTheta, p.TextModel.RopeParameters.RopeTheta)
+	kv["mistral3.rope.freq_base"] = p.TextModel.RopeTheta
-	kv["mistral3.rope.scaling.factor"] = p.TextModel.RopeParameters.Factor
-	kv["mistral3.rope.scaling.type"] = p.TextModel.RopeParameters.RopeType
-	kv["mistral3.rope.scaling.beta_fast"] = p.TextModel.RopeParameters.BetaFast
-	kv["mistral3.rope.scaling.beta_slow"] = p.TextModel.RopeParameters.BetaSlow
-
-	if p.TextModel.RopeParameters.Mscale != nil {
-		kv["mistral3.rope.scaling.mscale"] = *p.TextModel.RopeParameters.Mscale
-	}
-	if p.TextModel.RopeParameters.MscaleAllDim != nil {
-		kv["mistral3.rope.scaling.mscale_all_dim"] = *p.TextModel.RopeParameters.MscaleAllDim
-	}
-	if p.TextModel.RopeParameters.OrigMaxPositionEmbeddings > 0 {
-		kv["mistral3.rope.scaling.original_context_length"] = p.TextModel.RopeParameters.OrigMaxPositionEmbeddings
-	}
-	if p.TextModel.RopeParameters.Llama4ScalingBeta != nil {
-		kv["mistral3.rope.scaling_beta"] = *p.TextModel.RopeParameters.Llama4ScalingBeta
-	}
 
 	// Vision configuration
 	kv["mistral3.vision.block_count"] = p.VisionModel.NumHiddenLayers
@@ -105,7 +74,7 @@ func (p *mistral3Model) KV(t *Tokenizer) ggml.KV {
 	kv["mistral3.vision.patch_size"] = p.VisionModel.PatchSize
 	kv["mistral3.vision.num_channels"] = p.VisionModel.NumChannels
 	// kv["mistral3.vision.attention.layer_norm_epsilon"] = 1e-05 // Default value
-	kv["mistral3.vision.rope.freq_base"] = cmp.Or(p.VisionModel.RopeTheta, p.VisionModel.RopeParameters.RopeTheta)
+	kv["mistral3.vision.rope.freq_base"] = p.VisionModel.RopeTheta
 
 	// Multimodal configuration
 	kv["mistral3.image_token_index"] = p.ImageTokenIndex

convert/convert_mistral_causal.go

@@ -1,181 +0,0 @@
-package convert
-
-import (
-	"cmp"
-	"fmt"
-	"strings"
-
-	"github.com/pdevine/tensor"
-	"github.com/pdevine/tensor/native"
-
-	"github.com/ollama/ollama/fs/ggml"
-)
-
-type mistral3CausalModel struct {
-	ModelParameters
-
-	NumHiddenLayers       uint32  `json:"num_hidden_layers"`
-	MaxPositionEmbeddings uint32  `json:"max_position_embeddings"`
-	HiddenSize            uint32  `json:"hidden_size"`
-	IntermediateSize      uint32  `json:"intermediate_size"`
-	NumAttentionHeads     uint32  `json:"num_attention_heads"`
-	NumKeyValueHeads      uint32  `json:"num_key_value_heads"`
-	RopeTheta             float32 `json:"rope_theta"`
-	RMSNormEPS            float32 `json:"rms_norm_eps"`
-	HeadDim               uint32  `json:"head_dim"`
-	SlidingWindow         *uint32 `json:"sliding_window"`
-	HiddenAct             string  `json:"hidden_act"`
-	VocabSize             uint32  `json:"vocab_size"`
-	RopeParameters        struct {
-		BetaFast                  float32  `json:"beta_fast"`
-		BetaSlow                  float32  `json:"beta_slow"`
-		Factor                    float32  `json:"factor"`
-		Llama4ScalingBeta         *float32 `json:"llama_4_scaling_beta"`
-		OrigMaxPositionEmbeddings uint32   `json:"original_max_position_embeddings"`
-		RopeType                  string   `json:"rope_type"`
-		RopeTheta                 float32  `json:"rope_theta"`
-		Mscale                    *float32 `json:"mscale"`
-		MscaleAllDim              *float32 `json:"mscale_all_dim"`
-	} `json:"rope_parameters"`
-}
-
-func (p *mistral3CausalModel) KV(t *Tokenizer) ggml.KV {
-	kv := p.ModelParameters.KV(t)
-	kv["general.architecture"] = "mistral3"
-	kv["mistral3.vocab_size"] = p.VocabSize
-
-	// Text configuration
-	kv["mistral3.block_count"] = p.NumHiddenLayers
-	kv["mistral3.context_length"] = p.MaxPositionEmbeddings
-	kv["mistral3.embedding_length"] = p.HiddenSize
-	kv["mistral3.feed_forward_length"] = p.IntermediateSize
-	kv["mistral3.attention.head_count"] = p.NumAttentionHeads
-	kv["mistral3.attention.head_count_kv"] = p.NumKeyValueHeads
-	kv["mistral3.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS
-	kv["mistral3.attention.key_length"] = p.HeadDim
-	kv["mistral3.attention.value_length"] = p.HeadDim
-	kv["mistral3.rope.dimension_count"] = cmp.Or(p.HeadDim, p.HiddenSize/p.NumAttentionHeads)
-	kv["mistral3.rope.freq_base"] = cmp.Or(p.RopeTheta, p.RopeParameters.RopeTheta)
-	kv["mistral3.rope.scaling.factor"] = p.RopeParameters.Factor
-	kv["mistral3.rope.scaling.type"] = p.RopeParameters.RopeType
-	kv["mistral3.rope.scaling.beta_fast"] = p.RopeParameters.BetaFast
-	kv["mistral3.rope.scaling.beta_slow"] = p.RopeParameters.BetaSlow
-
-	if p.RopeParameters.Mscale != nil {
-		kv["mistral3.rope.scaling.mscale"] = *p.RopeParameters.Mscale
-	}
-
-	if p.RopeParameters.MscaleAllDim != nil {
-		kv["mistral3.rope.scaling.mscale_all_dim"] = *p.RopeParameters.MscaleAllDim
-	}
-
-	if p.RopeParameters.OrigMaxPositionEmbeddings > 0 {
-		kv["mistral3.rope.scaling.original_context_length"] = p.RopeParameters.OrigMaxPositionEmbeddings
-		kv["mistral3.rope.scaling_beta"] = *p.RopeParameters.Llama4ScalingBeta
-	}
-
-	if p.RopeParameters.Llama4ScalingBeta != nil {
-		kv["mistral3.rope.scaling_beta"] = *p.RopeParameters.Llama4ScalingBeta
-	}
-
-	return kv
-}
-
-func (p *mistral3CausalModel) Tensors(ts []Tensor) []*ggml.Tensor {
-	var out []*ggml.Tensor
-
-	for _, t := range ts {
-		if !strings.HasPrefix(t.Name(), "v.") {
-			if strings.HasSuffix(t.Name(), ".attn_q.weight") ||
-				strings.HasSuffix(t.Name(), ".attn_k.weight") {
-				t.SetRepacker(p.repack)
-			}
-		}
-
-		out = append(out, &ggml.Tensor{
-			Name:     t.Name(),
-			Kind:     t.Kind(),
-			Shape:    t.Shape(),
-			WriterTo: t,
-		})
-	}
-
-	return out
-}
-
-func (p *mistral3CausalModel) Replacements() []string {
-	return []string{
-		"model.norm", "output_norm",
-		"model.", "",
-		"layers", "blk",
-		"transformer.layers", "blk",
-		"vision_tower", "v",
-		"ln_pre", "encoder_norm",
-		"input_layernorm", "attn_norm",
-		"post_attention_layernorm", "ffn_norm",
-		"embed_tokens", "token_embd",
-		"self_attn.q_proj", "attn_q",
-		"self_attn.k_proj", "attn_k",
-		"self_attn.v_proj", "attn_v",
-		"self_attn.o_proj", "attn_output",
-		"mlp.down_proj", "ffn_down",
-		"mlp.gate_proj", "ffn_gate",
-		"mlp.up_proj", "ffn_up",
-		"attention.q_proj", "attn_q",
-		"attention.k_proj", "attn_k",
-		"attention.v_proj", "attn_v",
-		"attention.o_proj", "attn_output",
-		"attention_norm", "attn_norm",
-		"feed_forward.gate_proj", "ffn_gate",
-		"feed_forward.down_proj", "ffn_down",
-		"feed_forward.up_proj", "ffn_up",
-		"multi_modal_projector", "mm",
-		"ffn_norm", "ffn_norm",
-		"lm_head", "output",
-	}
-}
-
-func (p *mistral3CausalModel) repack(name string, data []float32, shape []uint64) ([]float32, error) {
-	var dims []int
-	for _, dim := range shape {
-		dims = append(dims, int(dim))
-	}
-
-	var heads uint32
-	if strings.HasSuffix(name, ".attn_q.weight") {
-		heads = p.NumAttentionHeads
-	} else if strings.HasSuffix(name, ".attn_k.weight") {
-		heads = cmp.Or(p.NumKeyValueHeads, p.NumAttentionHeads)
-	} else {
-		return nil, fmt.Errorf("unknown tensor for repack: %s", name)
-	}
-
-	n := tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data))
-	if err := n.Reshape(append([]int{int(heads), 2, dims[0] / int(heads) / 2}, dims[1:]...)...); err != nil {
-		return nil, err
-	}
-
-	if err := n.T(0, 2, 1, 3); err != nil {
-		return nil, err
-	}
-
-	if err := n.Reshape(dims...); err != nil {
-		return nil, err
-	}
-
-	if err := n.Transpose(); err != nil {
-		return nil, err
-	}
-
-	ts, err := native.SelectF32(n, 1)
-	if err != nil {
-		return nil, err
-	}
-
-	var f32s []float32
-	for _, t := range ts {
-		f32s = append(f32s, t...)
-	}
-
-	return f32s, nil
-}

convert/convert_nomicbert.go

@@ -1,213 +0,0 @@
-package convert
-
-import (
-	"cmp"
-	"encoding/json"
-	"io/fs"
-	"path/filepath"
-	"slices"
-	"strings"
-
-	"github.com/ollama/ollama/fs/ggml"
-)
-
-type nomicbertModel struct {
-	ModelParameters
-	NLayers               uint32  `json:"n_layers"`
-	NumHiddenLayers       uint32  `json:"num_hidden_layers"`
-	MaxPositionEmbeddings uint32  `json:"max_position_embeddings"`
-	HiddenSize            uint32  `json:"hidden_size"`
-	IntermediateSize      uint32  `json:"intermediate_size"`
-	NumAttentionHeads     uint32  `json:"num_attention_heads"`
-	NumKeyValueHeads      uint32  `json:"num_key_value_heads"`
-	LayerNormEPS          float32 `json:"layer_norm_eps"`
-	LayerNormEpsilon      float32 `json:"layer_norm_epsilon"`
-	RopeFreqBase          float32 `json:"rope_theta"`
-	normalizeEmbeddings   bool
-	PoolingType           uint32
-
-	// MoE parameters (only present in v2 models)
-	NumExperts      uint32 `json:"num_local_experts"`
-	NumExpertsUsed  uint32 `json:"num_experts_per_tok"`
-	MoEEveryNLayers uint32 `json:"moe_every_n_layers"`
-}
-
-var (
-	_ ModelConverter = (*nomicbertModel)(nil)
-	_ moreParser     = (*nomicbertModel)(nil)
-)
-
-func (p *nomicbertModel) parseMore(fsys fs.FS) error {
-	bts, err := fs.ReadFile(fsys, "modules.json")
-	if err != nil {
-		return err
-	}
-
-	var modules []struct {
-		Type string `json:"type"`
-		Path string `json:"path"`
-	}
-
-	if err := json.Unmarshal(bts, &modules); err != nil {
-		return err
-	}
-
-	var pooling string
-	for _, m := range modules {
-		switch m.Type {
-		case "sentence_transformers.models.Pooling":
-			pooling = m.Path
-		case "sentence_transformers.models.Normalize":
-			p.normalizeEmbeddings = true
-		}
-	}
-
-	if pooling != "" {
-		bts, err := fs.ReadFile(fsys, filepath.Join(pooling, "config.json"))
-		if err != nil {
-			return err
-		}
-
-		var pc struct {
-			PoolingModeCLSToken   bool `json:"pooling_mode_cls_token"`
-			PoolingModeMeanTokens bool `json:"pooling_mode_mean_tokens"`
-		}
-
-		if err := json.Unmarshal(bts, &pc); err != nil {
-			return err
-		}
-
-		if pc.PoolingModeMeanTokens {
-			p.PoolingType = 1
-		} else if pc.PoolingModeCLSToken {
-			p.PoolingType = 2
-		}
-	}
-
-	return nil
-}
-
-func (p *nomicbertModel) KV(t *Tokenizer) ggml.KV {
-	kv := p.ModelParameters.KV(t)
-
-	// Determine architecture based on MoE parameters (following qwen3 pattern)
-	arch := "nomic-bert"
-	if p.MoEEveryNLayers > 0 {
-		arch += "-moe"
-	}
-
-	kv["general.architecture"] = arch
-	kv["attention.causal"] = false
-	kv["pooling_type"] = p.PoolingType
-	kv["normalize_embeddings"] = p.normalizeEmbeddings
-
-	kv["block_count"] = cmp.Or(p.NLayers, p.NumHiddenLayers)
-
-	if contextLength := p.MaxPositionEmbeddings; contextLength > 0 {
-		kv["context_length"] = contextLength
-	}
-
-	if embeddingLength := p.HiddenSize; embeddingLength > 0 {
-		kv["embedding_length"] = p.HiddenSize
-	}
-
-	if feedForwardLength := p.IntermediateSize; feedForwardLength > 0 {
-		kv["feed_forward_length"] = p.IntermediateSize
-	}
-
-	if headCount := p.NumAttentionHeads; headCount > 0 {
-		kv["attention.head_count"] = p.NumAttentionHeads
-	}
-
-	if kvHeadCount := p.NumKeyValueHeads; kvHeadCount > 0 {
-		kv["attention.head_count_kv"] = p.NumKeyValueHeads
-	}
-
-	if layerNormEpsilon := cmp.Or(p.LayerNormEPS, p.LayerNormEpsilon); layerNormEpsilon > 0 {
-		kv["attention.layer_norm_epsilon"] = layerNormEpsilon
-	}
-
-	if p.RopeFreqBase > 0 {
-		kv["rope.freq_base"] = p.RopeFreqBase
-	}
-
-	// MoE specific parameters (only if MoE is enabled)
-	if p.NumExperts > 0 {
-		kv["expert_count"] = p.NumExperts
-	}
-
-	if p.NumExpertsUsed > 0 {
-		kv["expert_used_count"] = p.NumExpertsUsed
-	}
-
-	if p.MoEEveryNLayers > 0 {
-		kv["moe_every_n_layers"] = p.MoEEveryNLayers
-	}
-
-	kv["tokenizer.ggml.model"] = "bert"
-	kv["tokenizer.ggml.token_type_count"] = uint32(2)
-
-	// convert to phantom space tokens
-	for i, e := range t.Tokens {
-		switch {
-		case strings.HasPrefix(e, "[") && strings.HasSuffix(e, "]"):
-			// noop - keep special tokens as-is
-		case strings.HasPrefix(e, "##"):
-			t.Tokens[i] = e[2:]
-		default:
-			t.Tokens[i] = "\u2581" + e
-		}
-	}
-
-	kv["tokenizer.ggml.tokens"] = t.Tokens
-
-	return kv
-}
-
-func (p *nomicbertModel) Tensors(ts []Tensor) []*ggml.Tensor {
-	out := make([]*ggml.Tensor, 0, len(ts))
-	for _, t := range ts {
-		if slices.Contains([]string{
-			"embeddings.position_ids",
-			"pooler.dense.weight",
-			"pooler.dense.bias",
-		}, t.Name()) {
-			continue
-		}
-
-		out = append(out, &ggml.Tensor{
-			Name:     t.Name(),
-			Kind:     t.Kind(),
-			Shape:    t.Shape(),
-			WriterTo: t,
-		})
-	}
-
-	return out
-}
-
-func (nomicbertModel) Replacements() []string {
-	return []string{
-		"encoder.layer", "blk",
-		"encoder.layers", "blk",
-		"embeddings.word_embeddings", "token_embd",
-		"embeddings.token_type_embeddings", "token_types",
-		"embeddings.LayerNorm", "token_embd_norm",
-
-		"attention.self.qkv", "attn_qkv",
-
-		"attention.output.dense", "attn_output",
-		"attention.output.LayerNorm", "attn_output_norm",
-
-		"mlp.up", "ffn_up",
-		"mlp.down", "ffn_down",
-
-		"mlp.router", "ffn_gate_inp",
-		"mlp.experts.up", "ffn_up_exps",
-		"mlp.experts.down", "ffn_down_exps",
-
-		"intermediate.dense", "ffn_up",
-		"output.dense", "ffn_down",
-		"output.LayerNorm", "layer_output_norm",
-	}
-}

convert/convert_olmo.go

@@ -1,117 +0,0 @@
-package convert
-
-import (
-	"cmp"
-
-	"github.com/ollama/ollama/fs/ggml"
-)
-
-type ropeScaling struct {
-	Factor                    float32 `json:"factor"`
-	OriginalMaxPositionEmbeds uint32  `json:"original_max_position_embeddings"`
-	AttentionFactor           float32 `json:"attention_factor"`
-	BetaFast                  float32 `json:"beta_fast"`
-	BetaSlow                  float32 `json:"beta_slow"`
-	RopeType                  string  `json:"rope_type"`
-	ExtrapolationFactor       float32 `json:"extrapolation_factor"`
-}
-
-type olmoModel struct {
-	ModelParameters
-
-	HiddenSize            uint32       `json:"hidden_size"`
-	NumHiddenLayers       uint32       `json:"num_hidden_layers"`
-	IntermediateSize      uint32       `json:"intermediate_size"`
-	NumAttentionHeads     uint32       `json:"num_attention_heads"`
-	NumKeyValueHeads      uint32       `json:"num_key_value_heads"`
-	MaxPositionEmbeddings uint32       `json:"max_position_embeddings"`
-	RMSNormEPS            float32      `json:"rms_norm_eps"`
-	RopeTheta             float32      `json:"rope_theta"`
-	RopeScaling           *ropeScaling `json:"rope_scaling"`
-	SlidingWindow         uint32       `json:"sliding_window"`
-	LayerTypes            []string     `json:"layer_types"`
-}
-
-var _ ModelConverter = (*olmoModel)(nil)
-
-func (p *olmoModel) KV(t *Tokenizer) ggml.KV {
-	kv := p.ModelParameters.KV(t)
-	kv["general.architecture"] = "olmo3"
-	kv["olmo3.block_count"] = p.NumHiddenLayers
-	kv["olmo3.context_length"] = p.MaxPositionEmbeddings
-	kv["olmo3.embedding_length"] = p.HiddenSize
-	kv["olmo3.feed_forward_length"] = p.IntermediateSize
-	kv["olmo3.attention.head_count"] = p.NumAttentionHeads
-	kv["olmo3.attention.head_count_kv"] = cmp.Or(p.NumKeyValueHeads, p.NumAttentionHeads)
-
-	if p.RopeTheta > 0 {
-		kv["olmo3.rope.freq_base"] = p.RopeTheta
-	}
-
-	if p.RopeScaling != nil {
-		if p.RopeScaling.Factor > 0 {
-			kv["olmo3.rope.scaling.factor"] = p.RopeScaling.Factor
-		}
-		if p.RopeScaling.OriginalMaxPositionEmbeds > 0 {
-			kv["olmo3.rope.scaling.original_context_length"] = p.RopeScaling.OriginalMaxPositionEmbeds
-		}
-		if p.RopeScaling.AttentionFactor > 0 {
-			kv["olmo3.rope.scaling.attn_factor"] = p.RopeScaling.AttentionFactor
-		}
-		if p.RopeScaling.RopeType != "" {
-			kv["olmo3.rope.scaling.type"] = p.RopeScaling.RopeType
-		}
-	}
-
-	if p.RMSNormEPS > 0 {
-		kv["olmo3.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS
-	}
-
-	if p.SlidingWindow > 0 {
-		kv["olmo3.attention.sliding_window"] = p.SlidingWindow
-	}
-
-	if len(p.LayerTypes) > 0 {
-		slidingPattern := make([]bool, len(p.LayerTypes))
-		for i, layerType := range p.LayerTypes {
-			slidingPattern[i] = (layerType == "sliding_attention")
-		}
-		kv["olmo3.attention.sliding_window_pattern"] = slidingPattern
-	}
-
-	return kv
-}
-
-func (p *olmoModel) Tensors(ts []Tensor) []*ggml.Tensor {
-	out := make([]*ggml.Tensor, 0, len(ts))
-	for _, t := range ts {
-		out = append(out, &ggml.Tensor{
-			Name:     t.Name(),
-			Kind:     t.Kind(),
-			Shape:    t.Shape(),
-			WriterTo: t,
-		})
-	}
-
-	return out
-}
-
-func (p *olmoModel) Replacements() []string {
-	return []string{
-		"lm_head", "output",
-		"model.embed_tokens", "token_embd",
-		"model.layers", "blk",
-		"model.norm", "output_norm",
-		"self_attn.q_proj", "attn_q",
-		"self_attn.k_proj", "attn_k",
-		"self_attn.v_proj", "attn_v",
-		"self_attn.o_proj", "attn_output",
-		"self_attn.q_norm", "attn_q_norm",
-		"self_attn.k_norm", "attn_k_norm",
-		"post_attention_layernorm", "post_attention_norm",
-		"post_feedforward_layernorm", "post_ffw_norm",
-		"mlp.gate_proj", "ffn_gate",
-		"mlp.down_proj", "ffn_down",
-		"mlp.up_proj", "ffn_up",
-	}
-}

convert/reader.go

@@ -44,10 +44,7 @@ func (t tensorBase) Kind() uint32 {
 		t.name == "v.positional_embedding_vlm" ||
 		t.name == "v.tile_position_embd.weight" ||
 		t.name == "v.pre_tile_position_embd.weight" ||
-		t.name == "v.post_tile_position_embd.weight" ||
+		t.name == "v.post_tile_position_embd.weight" {
-		t.name == "s.position_embd" ||
-		strings.HasSuffix(t.name, "rel_pos_h") ||
-		strings.HasSuffix(t.name, "rel_pos_w") {
 		// these tensors are always F32
 		return tensorKindFP32
 	}

convert/reader_safetensors.go

@@ -96,10 +96,7 @@ type safetensor struct {
 
 func (st safetensor) Kind() uint32 {
 	kind := st.tensorBase.Kind()
-	if st.dtype == "BF16" &&
+	if !strings.HasPrefix(st.name, "v.") && st.dtype == "BF16" && kind != tensorKindFP32 {
-		!strings.HasPrefix(st.name, "v.") &&
-		!strings.HasPrefix(st.name, "s.") &&
-		kind != tensorKindFP32 {
 		kind = tensorKindBF16
 	}
 

convert/tokenizer_spm.go

@@ -49,8 +49,7 @@ func parseSentencePiece(fsys fs.FS) (*Vocabulary, error) {
 			tt := int32(sentencepiece.ModelProto_SentencePiece_NORMAL)
 
 			// temporary fix to handle gemma3 broken configs
-			// TODO(parthsareen): allow reading of tokenizer.json to allow managing special tokens when using spm
+			if slices.Contains([]string{"<end_of_turn>", "<start_of_turn>"}, piece.GetPiece()) {
-			if slices.Contains([]string{"<end_of_turn>", "<start_of_turn>", "<start_function_declaration>", "<end_function_declaration>", "<start_function_call>", "<end_function_call>", "<start_function_response>", "<end_function_response>", "<escape>"}, piece.GetPiece()) {
 				tt = int32(sentencepiece.ModelProto_SentencePiece_CONTROL)
 			}
 

discover/cpu_linux.go

@@ -2,7 +2,6 @@ package discover
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
 	"io"
 	"log/slog"
@@ -11,21 +10,12 @@ import (
 	"reflect"
 	"regexp"
 	"sort"
-	"strconv"
 	"strings"
 
 	"github.com/ollama/ollama/format"
 )
 
 func GetCPUMem() (memInfo, error) {
-	mem, err := getCPUMem()
-	if err != nil {
-		return memInfo{}, err
-	}
-	return getCPUMemByCgroups(mem), nil
-}
-
-func getCPUMem() (memInfo, error) {
 	var mem memInfo
 	var total, available, free, buffers, cached, freeSwap uint64
 	f, err := os.Open("/proc/meminfo")
@@ -66,32 +56,6 @@ func getCPUMem() (memInfo, error) {
 	return mem, nil
 }
 
-func getCPUMemByCgroups(mem memInfo) memInfo {
-	total, err := getUint64ValueFromFile("/sys/fs/cgroup/memory.max")
-	if err == nil {
-		mem.TotalMemory = total
-	}
-	used, err := getUint64ValueFromFile("/sys/fs/cgroup/memory.current")
-	if err == nil {
-		mem.FreeMemory = mem.TotalMemory - used
-	}
-	return mem
-}
-
-func getUint64ValueFromFile(path string) (uint64, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return 0, err
-	}
-	defer f.Close()
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		line := s.Text()
-		return strconv.ParseUint(line, 10, 64)
-	}
-	return 0, errors.New("empty file content")
-}
-
 const CpuInfoFilename = "/proc/cpuinfo"
 
 type linuxCpuInfo struct {
@@ -110,41 +74,7 @@ func GetCPUDetails() []CPU {
 		return nil
 	}
 	defer file.Close()
-	cpus := linuxCPUDetails(file)
+	return linuxCPUDetails(file)
-	return overwriteThreadCountByLinuxCgroups(cpus)
-}
-
-func overwriteThreadCountByLinuxCgroups(cpus []CPU) []CPU {
-	file, err := os.Open("/sys/fs/cgroup/cpu.max")
-	if err != nil {
-		return cpus
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		line := scanner.Text()
-		if sl := strings.Split(line, " "); len(sl) == 2 {
-			allowdUs, err := strconv.ParseInt(sl[0], 10, 64)
-			if err != nil {
-				slog.Warn("failed to parse CPU allowed micro secs", "error", err)
-				return cpus
-			}
-			unitUs, err := strconv.ParseInt(sl[1], 10, 64)
-			if err != nil {
-				slog.Warn("failed to parse CPU unit micro secs", "error", err)
-				return cpus
-			}
-
-			threads := int(max(allowdUs/unitUs, 1))
-
-			cpu := cpus[0]
-			cpu.CoreCount = threads
-			cpu.ThreadCount = threads
-			return []CPU{cpu}
-		}
-	}
-	return cpus
 }
 
 func linuxCPUDetails(file io.Reader) []CPU {

discover/runner.go

@@ -65,11 +65,6 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
 		}
 
 		slog.Info("discovering available GPUs...")
-		detectIncompatibleLibraries()
-
-		// Warn if any user-overrides are set which could lead to incorrect GPU discovery
-		overrideWarnings()
-
 		requested := envconfig.LLMLibrary()
 		jetpack := cudaJetpack()
 
@@ -95,13 +90,10 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
 			var dirs []string
 			if dir != "" {
 				if requested != "" && filepath.Base(dir) != requested {
-					slog.Debug("skipping available library at user's request", "requested", requested, "libDir", dir)
+					slog.Debug("skipping available library at users request", "requested", requested, "libDir", dir)
 					continue
 				} else if jetpack != "" && filepath.Base(dir) != "cuda_"+jetpack {
 					continue
-				} else if jetpack == "" && strings.Contains(filepath.Base(dir), "cuda_jetpack") {
-					slog.Debug("jetpack not detected (set JETSON_JETPACK or OLLAMA_LLM_LIBRARY to override), skipping", "libDir", dir)
-					continue
 				} else if !envconfig.EnableVulkan() && strings.Contains(filepath.Base(dir), "vulkan") {
 					slog.Info("experimental Vulkan support disabled.  To enable, set OLLAMA_VULKAN=1")
 					continue
@@ -121,7 +113,7 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
 		// In the second pass, we more deeply initialize the GPUs to weed out devices that
 		// aren't supported by a given library.  We run this phase in parallel to speed up discovery.
 		// Only devices that need verification are included in this pass
-		slog.Debug("evaluating which, if any, devices to filter out", "initial_count", len(devices))
+		slog.Debug("evluating which if any devices to filter out", "initial_count", len(devices))
 		ctx2ndPass, cancel := context.WithTimeout(ctx, 30*time.Second)
 		defer cancel()
 		var wg sync.WaitGroup
@@ -129,25 +121,15 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
 		supportedMu := sync.Mutex{}
 		supported := make(map[string]map[string]map[string]int) // [Library][libDir][ID] = pre-deletion devices index
 		for i := range devices {
-			libDir := devices[i].LibraryPath[len(devices[i].LibraryPath)-1]
 			if !devices[i].NeedsInitValidation() {
-				// No need to validate, add to the supported map
-				supportedMu.Lock()
-				if _, ok := supported[devices[i].Library]; !ok {
-					supported[devices[i].Library] = make(map[string]map[string]int)
-				}
-				if _, ok := supported[devices[i].Library][libDir]; !ok {
-					supported[devices[i].Library][libDir] = make(map[string]int)
-				}
-				supported[devices[i].Library][libDir][devices[i].ID] = i
-				supportedMu.Unlock()
 				continue
 			}
-			slog.Debug("verifying if device is supported", "library", libDir, "description", devices[i].Description, "compute", devices[i].Compute(), "id", devices[i].ID, "pci_id", devices[i].PCIID)
+			libDir := devices[i].LibraryPath[len(devices[i].LibraryPath)-1]
+			slog.Debug("verifying device is supported", "library", libDir, "description", devices[i].Description, "compute", devices[i].Compute(), "id", devices[i].ID, "pci_id", devices[i].PCIID)
 			wg.Add(1)
 			go func(i int) {
 				defer wg.Done()
-				extraEnvs := ml.GetVisibleDevicesEnv(devices[i:i+1], true)
+				extraEnvs := ml.GetVisibleDevicesEnv(devices[i : i+1])
 				devices[i].AddInitValidation(extraEnvs)
 				if len(bootstrapDevices(ctx2ndPass, devices[i].LibraryPath, extraEnvs)) == 0 {
 					slog.Debug("filtering device which didn't fully initialize",
@@ -333,8 +315,7 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
 			defer cancel()
 
 			// Apply any dev filters to avoid re-discovering unsupported devices, and get IDs correct
-			// We avoid CUDA filters here to keep ROCm from failing to discover GPUs in a mixed environment
+			devFilter := ml.GetVisibleDevicesEnv(devices)
-			devFilter := ml.GetVisibleDevicesEnv(devices, false)
 
 			for dir := range libDirs {
 				updatedDevices := bootstrapDevices(ctx, []string{ml.LibOllamaPath, dir}, devFilter)
@@ -468,37 +449,3 @@ func bootstrapDevices(ctx context.Context, ollamaLibDirs []string, extraEnvs map
 
 	return devices
 }
-
-func overrideWarnings() {
-	anyFound := false
-	m := envconfig.AsMap()
-	for _, k := range []string{
-		"CUDA_VISIBLE_DEVICES",
-		"HIP_VISIBLE_DEVICES",
-		"ROCR_VISIBLE_DEVICES",
-		"GGML_VK_VISIBLE_DEVICES",
-		"GPU_DEVICE_ORDINAL",
-		"HSA_OVERRIDE_GFX_VERSION",
-	} {
-		if e, found := m[k]; found && e.Value != "" {
-			anyFound = true
-			slog.Warn("user overrode visible devices", k, e.Value)
-		}
-	}
-	if anyFound {
-		slog.Warn("if GPUs are not correctly discovered, unset and try again")
-	}
-}
-
-func detectIncompatibleLibraries() {
-	if runtime.GOOS != "windows" {
-		return
-	}
-	basePath, err := exec.LookPath("ggml-base.dll")
-	if err != nil || basePath == "" {
-		return
-	}
-	if !strings.HasPrefix(basePath, ml.LibOllamaPath) {
-		slog.Warn("potentially incompatible library detected in PATH", "location", basePath)
-	}
-}

docs/api.md

@@ -50,7 +50,7 @@ Generate a response for a given prompt with a provided model. This is a streamin
 Advanced parameters (optional):
 
 - `format`: the format to return a response in. Format can be `json` or a JSON schema
-- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.mdx#valid-parameters-and-values) such as `temperature`
+- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
 - `system`: system message to (overrides what is defined in the `Modelfile`)
 - `template`: the prompt template to use (overrides what is defined in the `Modelfile`)
 - `stream`: if `false` the response will be returned as a single response object, rather than a stream of objects
@@ -507,7 +507,7 @@ The `message` object has the following fields:
 Advanced parameters (optional):
 
 - `format`: the format to return a response in. Format can be `json` or a JSON schema.
-- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.mdx#valid-parameters-and-values) such as `temperature`
+- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
 - `stream`: if `false` the response will be returned as a single response object, rather than a stream of objects
 - `keep_alive`: controls how long the model will stay loaded into memory following the request (default: `5m`)
 
@@ -895,11 +895,11 @@ curl http://localhost:11434/api/chat -d '{
       "tool_calls": [
         {
           "function": {
-            "name": "get_weather",
+            "name": "get_temperature",
             "arguments": {
               "city": "Toronto"
             }
-          }
+          },
         }
       ]
     },
@@ -907,7 +907,7 @@ curl http://localhost:11434/api/chat -d '{
     {
       "role": "tool",
       "content": "11 degrees celsius",
-      "tool_name": "get_weather"
+      "tool_name": "get_temperature",
     }
   ],
   "stream": false,
@@ -1189,7 +1189,7 @@ If you are creating a model from a safetensors directory or from a GGUF file, yo
 - `template`: (optional) the prompt template for the model
 - `license`: (optional) a string or list of strings containing the license or licenses for the model
 - `system`: (optional) a string containing the system prompt for the model
-- `parameters`: (optional) a dictionary of parameters for the model (see [Modelfile](./modelfile.mdx#valid-parameters-and-values) for a list of parameters)
+- `parameters`: (optional) a dictionary of parameters for the model (see [Modelfile](./modelfile.md#valid-parameters-and-values) for a list of parameters)
 - `messages`: (optional) a list of message objects used to create a conversation
 - `stream`: (optional) if `false` the response will be returned as a single response object, rather than a stream of objects
 - `quantize` (optional): quantize a non-quantized (e.g. float16) model
@@ -1698,7 +1698,7 @@ Generate embeddings from a model
 Advanced parameters:
 
 - `truncate`: truncates the end of each input to fit within context length. Returns error if `false` and context length is exceeded. Defaults to `true`
-- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.mdx#valid-parameters-and-values) such as `temperature`
+- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
 - `keep_alive`: controls how long the model will stay loaded into memory following the request (default: `5m`)
 - `dimensions`: number of dimensions for the embedding
 
@@ -1817,7 +1817,7 @@ Generate embeddings from a model
 
 Advanced parameters:
 
-- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.mdx#valid-parameters-and-values) such as `temperature`
+- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
 - `keep_alive`: controls how long the model will stay loaded into memory following the request (default: `5m`)
 
 ### Examples

docs/capabilities/tool-calling.mdx

@@ -15,7 +15,7 @@ Also known as "single-shot" tool calling.
     ```shell
     curl -s http://localhost:11434/api/chat -H "Content-Type: application/json" -d '{
       "model": "qwen3",
-      "messages": [{"role": "user", "content": "What is the temperature in New York?"}],
+      "messages": [{"role": "user", "content": "What's the temperature in New York?"}],
       "stream": false,
       "tools": [
         {
@@ -41,7 +41,7 @@ Also known as "single-shot" tool calling.
     curl -s http://localhost:11434/api/chat -H "Content-Type: application/json" -d '{
       "model": "qwen3",
       "messages": [
-        {"role": "user", "content": "What is the temperature in New York?"},
+        {"role": "user", "content": "What's the temperature in New York?"},
         {
           "role": "assistant",
           "tool_calls": [
@@ -90,7 +90,7 @@ Also known as "single-shot" tool calling.
       }
       return temperatures.get(city, "Unknown")
 
-    messages = [{"role": "user", "content": "What is the temperature in New York?"}]
+    messages = [{"role": "user", "content": "What's the temperature in New York?"}]
 
     # pass functions directly as tools in the tools list or as a JSON schema
     response = chat(model="qwen3", messages=messages, tools=[get_temperature], think=True)
@@ -146,7 +146,7 @@ Also known as "single-shot" tool calling.
       },
     ]
 
-    const messages = [{ role: 'user', content: "What is the temperature in New York?" }]
+    const messages = [{ role: 'user', content: "What's the temperature in New York?" }]
 
     const response = await ollama.chat({
       model: 'qwen3',
@@ -609,7 +609,7 @@ def get_temperature(city: str) -> str:
   return temperatures.get(city, 'Unknown')
 
 
-messages = [{'role': 'user', 'content': "What is the temperature in New York?"}]
+messages = [{'role': 'user', 'content': "What's the temperature in New York?"}]
 
 while True:
   stream = chat(
@@ -684,7 +684,7 @@ const getTemperatureTool = {
 }
 
 async function agentLoop() {
-  const messages = [{ role: 'user', content: "What is the temperature in New York?" }]
+  const messages = [{ role: 'user', content: "What's the temperature in New York?" }]
 
   while (true) {
     const stream = await ollama.chat({

docs/capabilities/vision.mdx

@@ -36,6 +36,7 @@ Provide an `images` array. SDKs accept file paths, URLs or raw bytes while the R
         }],
         "stream": false
     }'
+    "
     ```
   </Tab>
   <Tab title="Python">

docs/cloud.mdx

@@ -9,9 +9,15 @@ sidebarTitle: Cloud
 
 Ollama's cloud models are a new kind of model in Ollama that can run without a powerful GPU. Instead, cloud models are automatically offloaded to Ollama's cloud service while offering the same capabilities as local models, making it possible to keep using your local tools while running larger models that wouldn't fit on a personal computer.
 
-### Supported models
+Ollama currently supports the following cloud models, with more coming soon:
 
-For a list of supported models, see Ollama's [model library](https://ollama.com/search?c=cloud).
+- `deepseek-v3.1:671b-cloud`
+- `gpt-oss:20b-cloud`
+- `gpt-oss:120b-cloud`
+- `kimi-k2:1t-cloud`
+- `qwen3-coder:480b-cloud`
+- `glm-4.6:cloud`
+- `minimax-m2:cloud`
 
 ### Running Cloud models
 

docs/development.md

@@ -49,8 +49,6 @@ Install prerequisites:
     - [Ninja](https://github.com/ninja-build/ninja/releases)
 - (Optional) NVIDIA GPU support
     - [CUDA SDK](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=11&target_type=exe_network)
-- (Optional) VULKAN GPU support
-    - [VULKAN SDK](https://vulkan.lunarg.com/sdk/home) - useful for AMD/Intel GPUs
 
 Then, configure and build the project:
 
@@ -59,17 +57,6 @@ cmake -B build
 cmake --build build --config Release
 ```
 
-> Building for Vulkan requires VULKAN_SDK environment variable:
-> 
-> PowerShell
-> ```powershell
-> $env:VULKAN_SDK="C:\VulkanSDK\<version>"
-> ```
-> CMD
-> ```cmd
-> set VULKAN_SDK=C:\VulkanSDK\<version>
-> ```
-
 > [!IMPORTANT]
 > Building for ROCm requires additional flags:
 > ```
@@ -78,7 +65,6 @@ cmake --build build --config Release
 > ```
 
 
-
 Lastly, run Ollama:
 
 ```shell
@@ -98,9 +84,7 @@ Install prerequisites:
     - [ROCm](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/quick-start.html)
 - (Optional) NVIDIA GPU support
     - [CUDA SDK](https://developer.nvidia.com/cuda-downloads)
-- (Optional) VULKAN GPU support
+
-    - [VULKAN SDK](https://vulkan.lunarg.com/sdk/home) - useful for AMD/Intel GPUs
-    - Or install via package manager: `sudo apt install vulkan-sdk` (Ubuntu/Debian) or `sudo dnf install vulkan-sdk` (Fedora/CentOS)
 > [!IMPORTANT]
 > Ensure prerequisites are in `PATH` before running CMake.
 

docs/faq.mdx

@@ -14,11 +14,11 @@ curl -fsSL https://ollama.com/install.sh | sh
 
 ## How can I view the logs?
 
-Review the [Troubleshooting](./troubleshooting) docs for more about using logs.
+Review the [Troubleshooting](./troubleshooting.md) docs for more about using logs.
 
 ## Is my GPU compatible with Ollama?
 
-Please refer to the [GPU docs](./gpu).
+Please refer to the [GPU docs](./gpu.md).
 
 ## How can I specify the context window size?
 
@@ -57,13 +57,8 @@ ollama ps
 ```
 
 <Info>
-
+  **Output**: ``` NAME ID SIZE PROCESSOR UNTIL llama3:70b bcfb190ca3a7 42 GB
-**Output**:
+  100% GPU 4 minutes from now ```
-
-```
-NAME        ID            SIZE    PROCESSOR   UNTIL
-llama3:70b  bcfb190ca3a7  42 GB   100% GPU    4 minutes from now
-```
 </Info>
 
 The `Processor` column will show which memory the model was loaded in to:
@@ -390,4 +385,4 @@ Ollama for Windows and macOS register as a login item during installation.  You
 - In `Task Manager` go to the `Startup apps` tab, search for `ollama` then click `Disable`
 
 **MacOS**
-- Open `Settings` and search for "Login Items", find the `Ollama` entry under "Allow in the Background`, then click the slider to disable.
+- Open `Settings` and search for "Login Items", find the `Ollama` entry under "Allow in the Background`, then click the slider to disable.

docs/gpu.mdx

@@ -33,7 +33,7 @@ Check your compute compatibility to see if your card is supported:
 | 5.0                | GeForce GTX         | `GTX 750 Ti` `GTX 750` `NVS 810`                                                                                               |
 |                    | Quadro              | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M`                     |
 
-For building locally to support older GPUs, see [developer](./development#linux-cuda-nvidia)
+For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia)
 
 ### GPU Selection
 
@@ -54,7 +54,7 @@ sudo modprobe nvidia_uvm`
 
 Ollama supports the following AMD GPUs via the ROCm library:
 
-> **NOTE:**
+> [!NOTE]
 > Additional AMD GPU support is provided by the Vulkan Library - see below.
 
 
@@ -132,9 +132,9 @@ Ollama supports GPU acceleration on Apple devices via the Metal API.
 
 ## Vulkan GPU Support
 
-> **NOTE:**
+> [!NOTE]
 > Vulkan is currently an Experimental feature.  To enable, you must set OLLAMA_VULKAN=1 for the Ollama server as
-described in the [FAQ](faq#how-do-i-configure-ollama-server)
+described in the [FAQ](faq.md#how-do-i-configure-ollama-server)
 
 Additional GPU support on Windows and Linux is provided via
 [Vulkan](https://www.vulkan.org/). On Windows most GPU vendors drivers come
@@ -161,6 +161,6 @@ sudo setcap cap_perfmon+ep /usr/local/bin/ollama
 
 To select specific Vulkan GPU(s), you can set the environment variable
 `GGML_VK_VISIBLE_DEVICES` to one or more numeric IDs on the Ollama server as
-described in the [FAQ](faq#how-do-i-configure-ollama-server). If you
+described in the [FAQ](faq.md#how-do-i-configure-ollama-server). If you
 encounter any problems with Vulkan based GPUs, you can disable all Vulkan GPUs
 by setting `GGML_VK_VISIBLE_DEVICES=-1` 

docs/integrations/vscode.mdx

@@ -1,34 +1,34 @@
 ---
-title: VS Code
+title: VS Code 
 ---
 
 ## Install
 
-Install [VS Code](https://code.visualstudio.com/download).
+Install [VS Code](https://code.visualstudio.com/download). 
 
-## Usage with Ollama
+## Usage with Ollama 
 
 1. Open Copilot side bar found in top right window
-   <div style={{ display: "flex", justifyContent: "center" }}>
+<div style={{ display: 'flex', justifyContent: 'center' }}>
-     <img
+  <img 
-       src="/images/vscode-sidebar.png"
+    src="/images/vscode-sidebar.png" 
-       alt="VS Code chat Sidebar"
+    alt="VS Code chat Sidebar"
-       width="75%"
+    width="75%"
-     />
+  />
-   </div>
+</div>
-2. Select the model dropdown > **Manage models**
+2. Select the model drowpdown > **Manage models**
-   <div style={{ display: "flex", justifyContent: "center" }}>
+<div style={{ display: 'flex', justifyContent: 'center' }}>
-     <img
+  <img 
-       src="/images/vscode-models.png"
+    src="/images/vscode-models.png" 
-       alt="VS Code model picker"
+    alt="VS Code model picker"
-       width="75%"
+    width="75%"
-     />
+  />
-   </div>
+</div>
 3. Enter **Ollama** under **Provider Dropdown** and select desired models (e.g `qwen3, qwen3-coder:480b-cloud`)
-   <div style={{ display: "flex", justifyContent: "center" }}>
+<div style={{ display: 'flex', justifyContent: 'center' }}>
-     <img
+  <img 
-       src="/images/vscode-model-options.png"
+    src="/images/vscode-model-options.png" 
-       alt="VS Code model options dropdown"
+    alt="VS Code model options dropdown"
-       width="75%"
+    width="75%"
-     />
+  />
-   </div>
+</div>

docs/modelfile.mdx

@@ -41,7 +41,6 @@ INSTRUCTION arguments
 | [`ADAPTER`](#adapter)               | Defines the (Q)LoRA adapters to apply to the model.            |
 | [`LICENSE`](#license)               | Specifies the legal license.                                   |
 | [`MESSAGE`](#message)               | Specify message history.                                       |
-| [`REQUIRES`](#requires)             | Specify the minimum version of Ollama required by the model.   |
 
 ## Examples
 
@@ -150,6 +149,9 @@ PARAMETER <parameter> <parametervalue>
 
 | Parameter      | Description                                                                                                                                                                                                                                                                                                                                                                     | Value Type | Example Usage        |
 | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------- |
+| mirostat       | Enable Mirostat sampling for controlling perplexity. (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)                                                                                                                                                                                                                                                                 | int        | mirostat 0           |
+| mirostat_eta   | Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. (Default: 0.1)                                                                                                                                                | float      | mirostat_eta 0.1     |
+| mirostat_tau   | Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. (Default: 5.0)                                                                                                                                                                                                                                 | float      | mirostat_tau 5.0     |
 | num_ctx        | Sets the size of the context window used to generate the next token. (Default: 2048)                                                                                                                                                                                                                                                                                            | int        | num_ctx 4096         |
 | repeat_last_n  | Sets how far back for the model to look back to prevent repetition. (Default: 64, 0 = disabled, -1 = num_ctx)                                                                                                                                                                                                                                                                   | int        | repeat_last_n 64     |
 | repeat_penalty | Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. (Default: 1.1)                                                                                                                                                                                             | float      | repeat_penalty 1.1   |
@@ -249,16 +251,6 @@ MESSAGE user Is Ontario in Canada?
 MESSAGE assistant yes
 ```
 
-### REQUIRES
-
-The `REQUIRES` instruction allows you to specify the minimum version of Ollama required by the model.
-
-```
-REQUIRES <version>
-```
-
-The version should be a valid Ollama version (e.g. 0.14.0).
-
 ## Notes
 
 - the **`Modelfile` is not case sensitive**. In the examples, uppercase instructions are used to make it easier to distinguish it from arguments.

docs/openapi.yaml

@@ -111,12 +111,6 @@ components:
           description: Model keep-alive duration (for example `5m` or `0` to unload immediately)
         options:
           $ref: "#/components/schemas/ModelOptions"
-        logprobs:
-          type: boolean
-          description: Whether to return log probabilities of the output tokens
-        top_logprobs:
-          type: integer
-          description: Number of most likely tokens to return at each token position when logprobs are enabled
     GenerateResponse:
       type: object
       properties:
@@ -156,11 +150,6 @@ components:
         eval_duration:
           type: integer
           description: Time spent generating tokens in nanoseconds
-        logprobs:
-          type: array
-          items:
-            $ref: "#/components/schemas/Logprob"
-          description: Log probability information for the generated tokens when logprobs are enabled
     GenerateStreamEvent:
       type: object
       properties:
@@ -298,12 +287,6 @@ components:
             - type: string
             - type: number
           description: Model keep-alive duration (for example `5m` or `0` to unload immediately)
-        logprobs:
-          type: boolean
-          description: Whether to return log probabilities of the output tokens
-        top_logprobs:
-          type: integer
-          description: Number of most likely tokens to return at each token position when logprobs are enabled
     ChatResponse:
       type: object
       properties:
@@ -361,11 +344,6 @@ components:
         eval_duration:
           type: integer
           description: Time spent generating tokens in nanoseconds
-        logprobs:
-          type: array
-          items:
-            $ref: "#/components/schemas/Logprob"
-          description: Log probability information for the generated tokens when logprobs are enabled
     ChatStreamEvent:
       type: object
       properties:
@@ -728,41 +706,6 @@ components:
         version:
           type: string
           description: Version of Ollama
-    TokenLogprob:
-      type: object
-      description: Log probability information for a single token alternative
-      properties:
-        token:
-          type: string
-          description: The text representation of the token
-        logprob:
-          type: number
-          description: The log probability of this token
-        bytes:
-          type: array
-          items:
-            type: integer
-          description: The raw byte representation of the token
-    Logprob:
-      type: object
-      description: Log probability information for a generated token
-      properties:
-        token:
-          type: string
-          description: The text representation of the token
-        logprob:
-          type: number
-          description: The log probability of this token
-        bytes:
-          type: array
-          items:
-            type: integer
-          description: The raw byte representation of the token
-        top_logprobs:
-          type: array
-          items:
-            $ref: "#/components/schemas/TokenLogprob"
-          description: Most likely tokens and their log probabilities at this position
     ErrorResponse:
       type: object
       properties:

docs/tools/extract-examples/README.md

@@ -1,46 +0,0 @@
-# extract-examples
-
-Extracts code examples from MDX files to a temp directory so you can run them.
-
-## Usage
-
-```shell
-go run docs/tools/extract-examples/main.go <mdx-file>
-```
-
-## Example
-
-```shell
-go run docs/tools/extract-examples/main.go docs/api/openai-compatibility.mdx
-```
-
-Output:
-
-```
-Extracting code examples to: /var/folders/vq/wfm2g6k917d3ldzpjdxc8ph00000gn/T/mdx-examples-3271754368
-
-  - 01_basic.py
-  - 01_basic.js
-  - 01_basic.sh
-  - 02_responses.py
-  - 02_responses.js
-  - 02_responses.sh
-  - 03_vision.py
-  - 03_vision.js
-  - 03_vision.sh
-
-Extracted 9 file(s) to /var/folders/vq/wfm2g6k917d3ldzpjdxc8ph00000gn/T/mdx-examples-3271754368
-
-To run examples:
-
-  cd /var/folders/vq/wfm2g6k917d3ldzpjdxc8ph00000gn/T/mdx-examples-3271754368
-  npm install   # for JS examples
-
-then run individual files with `node file.js`, `python file.py`, `bash file.sh`
-```
-
-## How it works
-
-- Parses MDX files looking for fenced code blocks with filenames (e.g., ` ```python basic.py `)
-- Groups examples by their `<CodeGroup>` and prefixes filenames with `01_`, `02_`, etc.
-- Writes all extracted files to a temp directory

docs/tools/extract-examples/main.go

@@ -1,137 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"fmt"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
-)
-
-func main() {
-	if len(os.Args) < 2 {
-		fmt.Fprintln(os.Stderr, "Usage: go run extract-examples.go <mdx-file>")
-		os.Exit(1)
-	}
-
-	mdxFile := os.Args[1]
-
-	f, err := os.Open(mdxFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
-		os.Exit(1)
-	}
-	defer f.Close()
-
-	// Create temp directory
-	tempDir, err := os.MkdirTemp("", "mdx-examples-*")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating temp dir: %v\n", err)
-		os.Exit(1)
-	}
-
-	fmt.Printf("Extracting code examples to: %s\n\n", tempDir)
-
-	// Patterns
-	codeBlockStart := regexp.MustCompile("^```([a-zA-Z0-9_-]+)\\s+([^\\s]+)$")
-	codeGroupStart := regexp.MustCompile("^<CodeGroup")
-	codeGroupEnd := regexp.MustCompile("^</CodeGroup>")
-
-	scanner := bufio.NewScanner(f)
-	inCodeBlock := false
-	inCodeGroup := false
-	var currentFile string
-	var content strings.Builder
-	count := 0
-	codeGroupNum := 0
-
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		// Track CodeGroup boundaries
-		if codeGroupStart.MatchString(line) {
-			inCodeGroup = true
-			codeGroupNum++
-			continue
-		}
-		if codeGroupEnd.MatchString(line) {
-			inCodeGroup = false
-			continue
-		}
-
-		if inCodeBlock {
-			if line == "```" {
-				// End of code block - write file
-				if currentFile != "" {
-					outPath := filepath.Join(tempDir, currentFile)
-					if err := os.WriteFile(outPath, []byte(content.String()), 0o644); err != nil {
-						fmt.Fprintf(os.Stderr, "Error writing %s: %v\n", currentFile, err)
-					} else {
-						fmt.Printf("  - %s\n", currentFile)
-						count++
-					}
-				}
-				inCodeBlock = false
-				currentFile = ""
-				content.Reset()
-			} else {
-				content.WriteString(line)
-				content.WriteString("\n")
-			}
-		} else {
-			if matches := codeBlockStart.FindStringSubmatch(line); matches != nil {
-				inCodeBlock = true
-				filename := matches[2]
-				// Prefix with CodeGroup number if inside a CodeGroup
-				if inCodeGroup {
-					currentFile = fmt.Sprintf("%02d_%s", codeGroupNum, filename)
-				} else {
-					currentFile = filename
-				}
-				content.Reset()
-			}
-		}
-	}
-
-	if err := scanner.Err(); err != nil {
-		fmt.Fprintf(os.Stderr, "Error reading file: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Write package.json for JavaScript dependencies
-	packageJSON := `{
-  "name": "mdx-examples",
-  "type": "module",
-  "dependencies": {
-    "openai": "^4",
-    "ollama": "^0.5"
-  }
-}
-`
-	if err := os.WriteFile(filepath.Join(tempDir, "package.json"), []byte(packageJSON), 0o644); err != nil {
-		fmt.Fprintf(os.Stderr, "Error writing package.json: %v\n", err)
-	}
-
-	// Write pyproject.toml for Python dependencies
-	pyprojectTOML := `[project]
-name = "mdx-examples"
-version = "0.0.0"
-dependencies = [
-    "openai",
-    "ollama",
-]
-`
-	if err := os.WriteFile(filepath.Join(tempDir, "pyproject.toml"), []byte(pyprojectTOML), 0o644); err != nil {
-		fmt.Fprintf(os.Stderr, "Error writing pyproject.toml: %v\n", err)
-	}
-
-	fmt.Printf("\n")
-	fmt.Printf("Extracted %d file(s) to %s\n", count, tempDir)
-	fmt.Printf("\n")
-	fmt.Printf("To run examples:\n")
-	fmt.Printf("\n")
-	fmt.Printf("  cd %s\n  npm install   # for JS examples\n", tempDir)
-	fmt.Printf("\n")
-	fmt.Printf("then run individual files with `node file.js`, `python file.py`, `bash file.sh`\n")
-}

docs/troubleshooting.mdx

@@ -87,7 +87,7 @@ When Ollama starts up, it takes inventory of the GPUs present in the system to d
 
 ### Linux NVIDIA Troubleshooting
 
-If you are using a container to run Ollama, make sure you've set up the container runtime first as described in [docker](./docker)
+If you are using a container to run Ollama, make sure you've set up the container runtime first as described in [docker.md](./docker.md)
 
 Sometimes the Ollama can have difficulties initializing the GPU. When you check the server logs, this can show up as various error codes, such as "3" (not initialized), "46" (device unavailable), "100" (no device), "999" (unknown), or others. The following troubleshooting techniques may help resolve the problem
 

fs/ggml/ggml.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/ollama/ollama/format"
 	"github.com/ollama/ollama/fs/util/bufioutil"
-	"github.com/ollama/ollama/ml"
 )
 
 type GGML struct {
@@ -241,17 +240,12 @@ func (kv KV) Bools(key string, defaultValue ...[]bool) []bool {
 
 func (kv KV) OllamaEngineRequired() bool {
 	return slices.Contains([]string{
-		"bert",
-		"deepseek2",
-		"deepseekocr",
 		"gemma3",
 		"gemma3n",
 		"gptoss", "gpt-oss",
 		"llama4",
 		"mistral3",
 		"mllama",
-		"nomic-bert",
-		"olmo3",
 		"qwen25vl",
 		"qwen3", "qwen3moe",
 		"qwen3vl", "qwen3vlmoe",
@@ -553,7 +547,7 @@ func Decode(rs io.ReadSeeker, maxArraySize int) (*GGML, error) {
 	}, nil
 }
 
-func (f GGML) GraphSize(context, batch uint64, numParallel int, kvCacheType string, useFlashAttention ml.FlashAttentionType) (kv []uint64, partialOffload, fullOffload uint64) {
+func (f GGML) GraphSize(context, batch uint64, numParallel int, kvCacheType string, useFlashAttention bool) (kv []uint64, partialOffload, fullOffload uint64) {
 	context *= uint64(numParallel)
 
 	embedding := f.KV().EmbeddingLength()
@@ -794,7 +788,7 @@ func (f GGML) GraphSize(context, batch uint64, numParallel int, kvCacheType stri
 		}
 
 		partialOffload = 2 * f.KV().HeadCountMax() / cmp.Or(f.KV().HeadCountKVMin(), 1) * kvTotal / 6
-		if useFlashAttention == ml.FlashAttentionEnabled {
+		if useFlashAttention {
 			// rough estimate of graph size with flash attention on
 			partialOffload = (4*uint64(numParallel) + context>>10 + 110) * format.MebiByte
 		}
@@ -812,14 +806,6 @@ func (f GGML) SupportsKVCacheType(cacheType string) bool {
 	return slices.Contains([]string{"q8_0", "q4_0"}, cacheType)
 }
 
-// KVCacheTypeIsQuantized checks if the requested cache type is a quantized type
-func (f GGML) KVCacheTypeIsQuantized(cacheType string) bool {
-	if cacheType == "" || cacheType == "f16" || cacheType == "f32" || cacheType == "bf16" {
-		return false
-	}
-	return true
-}
-
 // SupportsFlashAttention checks if the model supports flash attention
 func (f GGML) SupportsFlashAttention() bool {
 	_, isEmbedding := f.KV()[fmt.Sprintf("%s.pooling_type", f.KV().Architecture())]
@@ -840,11 +826,8 @@ func (f GGML) SupportsFlashAttention() bool {
 // FlashAttention checks if the model should enable flash attention
 func (f GGML) FlashAttention() bool {
 	return slices.Contains([]string{
-		"bert",
 		"gemma3",
 		"gptoss", "gpt-oss",
-		"mistral3",
-		"olmo3",
 		"qwen3", "qwen3moe",
 		"qwen3vl", "qwen3vlmoe",
 	}, f.KV().String("general.architecture"))

fs/ggml/gguf.go

@@ -305,7 +305,7 @@ func readGGUFV1StringsData(llm *gguf, r io.Reader, a *array[string]) (any, error
 
 			a.values[i] = e
 		} else {
-			_ = discardGGUFString(llm, r)
+			discardGGUFString(llm, r)
 		}
 	}
 
@@ -568,6 +568,7 @@ func WriteGGUF(f *os.File, kv KV, ts []*Tensor) error {
 	g.SetLimit(runtime.GOMAXPROCS(0))
 	// TODO consider reducing if tensors size * gomaxprocs is larger than free memory
 	for _, t := range ts {
+		t := t
 		w := io.NewOffsetWriter(f, offset+int64(t.Offset))
 		g.Go(func() error {
 			_, err := t.WriteTo(w)
@@ -597,10 +598,6 @@ func ggufWriteKV(ws io.WriteSeeker, arch, k string, v any) error {
 
 	var err error
 	switch v := v.(type) {
-	case int32:
-		err = writeGGUF(ws, ggufTypeInt32, v)
-	case int64:
-		err = writeGGUF(ws, ggufTypeInt64, v)
 	case uint32, FileType:
 		err = writeGGUF(ws, ggufTypeUint32, v)
 	case uint64:
@@ -615,10 +612,6 @@ func ggufWriteKV(ws io.WriteSeeker, arch, k string, v any) error {
 		err = writeGGUFArray(ws, ggufTypeInt32, v)
 	case *array[int32]:
 		err = writeGGUFArray(ws, ggufTypeInt32, v.values)
-	case []int64:
-		err = writeGGUFArray(ws, ggufTypeInt64, v)
-	case *array[int64]:
-		err = writeGGUFArray(ws, ggufTypeInt64, v.values)
 	case []uint32:
 		err = writeGGUFArray(ws, ggufTypeUint32, v)
 	case *array[uint32]:

fs/ggml/gguf_test.go

@@ -42,10 +42,6 @@ func TestWriteGGUF(t *testing.T) {
 				"general.architecture": "test",
 				"general.alignment":    uint32(16),
 				"test.key":             "value",
-				"test.int32_key":       int32(-42),
-				"test.int64_key":       int64(-9223372036854775808),
-				"test.int32_array":     []int32{-1, 0, 1, 2147483647, -2147483648},
-				"test.int64_array":     []int64{-1, 0, 1, 9223372036854775807, -9223372036854775808},
 				"attention.key":        "value2",
 				"tokenizer.key":        "value3",
 				"adapter.key":          "value4",
@@ -59,7 +55,7 @@ func TestWriteGGUF(t *testing.T) {
 			}
 			defer r.Close()
 
-			ff, err := Decode(r, -1)
+			ff, err := Decode(r, 0)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -69,19 +65,15 @@ func TestWriteGGUF(t *testing.T) {
 				"general.alignment":       uint32(16),
 				"general.parameter_count": uint64(54),
 				"test.key":                "value",
-				"test.int32_key":          int32(-42),
-				"test.int64_key":          int64(-9223372036854775808),
-				"test.int32_array":        &array[int32]{size: 5, values: []int32{-1, 0, 1, 2147483647, -2147483648}},
-				"test.int64_array":        &array[int64]{size: 5, values: []int64{-1, 0, 1, 9223372036854775807, -9223372036854775808}},
 				"test.attention.key":      "value2",
 				"tokenizer.key":           "value3",
 				"adapter.key":             "value4",
-			}, ff.KV(), cmp.AllowUnexported(array[int32]{}, array[int64]{})); diff != "" {
+			}, ff.KV()); diff != "" {
 				t.Errorf("Mismatch (-want +got):\n%s", diff)
 			}
 
 			if diff := cmp.Diff(Tensors{
-				Offset: 992,
+				Offset: 800,
 				items: []*Tensor{
 					{Name: "blk.0.attn_k.weight", Offset: 0, Shape: []uint64{2, 3}},
 					{Name: "blk.0.attn_norm.weight", Offset: 32, Shape: []uint64{2, 3}},

go.mod

@@ -15,8 +15,9 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.9.0
 	github.com/x448/float16 v0.8.4
-	golang.org/x/sync v0.17.0
+	golang.org/x/sync v0.12.0
-	golang.org/x/sys v0.37.0
+	golang.org/x/sys v0.36.0
+
 )
 
 require (
@@ -28,17 +29,13 @@ require (
 	github.com/nlpodyssey/gopickle v0.3.0
 	github.com/pdevine/tensor v0.0.0-20240510204454-f88f4562727c
 	github.com/tkrajina/typescriptify-golang-structs v0.2.0
-	github.com/wk8/go-ordered-map/v2 v2.1.8
 	golang.org/x/image v0.22.0
-	golang.org/x/mod v0.30.0
+	golang.org/x/tools v0.30.0
-	golang.org/x/tools v0.38.0
 	gonum.org/v1/gonum v0.15.0
 )
 
 require (
 	github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
-	github.com/bahlo/generic-list-go v0.2.0 // indirect
-	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/chewxy/hm v1.0.0 // indirect
 	github.com/chewxy/math32 v1.11.0 // indirect
@@ -48,7 +45,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/flatbuffers v24.3.25+incompatible // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
@@ -81,11 +77,11 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	golang.org/x/arch v0.8.0 // indirect
-	golang.org/x/crypto v0.43.0
+	golang.org/x/crypto v0.36.0
 	golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect
-	golang.org/x/net v0.46.0 // indirect
+	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/term v0.36.0
+	golang.org/x/term v0.30.0
-	golang.org/x/text v0.30.0
+	golang.org/x/text v0.23.0
 	google.golang.org/protobuf v1.34.1
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -14,11 +14,7 @@ github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6IC
 github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
-github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
-github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
-github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
 github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
@@ -127,7 +123,6 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
@@ -148,8 +143,6 @@ github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+
 github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
-github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -214,8 +207,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
-github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
-github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xtgo/set v1.0.0 h1:6BCNBRv3ORNDQ7fyoJXRv+tstJz3m1JVFQErfeZz2pY=
@@ -233,8 +224,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -264,8 +255,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
-golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -278,8 +267,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
+golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
-golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -289,8 +278,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -306,17 +295,17 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
+golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
-golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
+golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
-golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -330,8 +319,8 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
+golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

harmony/harmonyparser.go

@@ -388,9 +388,9 @@ func NewFunctionNameMap() *FunctionNameMap {
 	}
 }
 
-// Init initializes the handler with tools, optional last message, and think value
+// Init initializes the handler with tools and optional last message
 // Implements the Parser interface
-func (h *HarmonyMessageHandler) Init(tools []api.Tool, lastMessage *api.Message, thinkValue *api.ThinkValue) []api.Tool {
+func (h *HarmonyMessageHandler) Init(tools []api.Tool, lastMessage *api.Message) []api.Tool {
 	// Initialize the harmony parser
 	if h.HarmonyParser == nil {
 		h.HarmonyParser = &HarmonyParser{

integration/README.md

@@ -13,5 +13,3 @@ The integration tests have 2 modes of operating.
 > [!IMPORTANT]
 > Before running the tests locally without the "test existing" setting, compile ollama from the top of the source tree  `go build .` in addition to GPU support with cmake if applicable on your platform.  The integration tests expect to find an ollama binary at the top of the tree.
 
-
-Many tests use a default small model suitable to run on many systems.  You can override this default model by setting `OLLAMA_TEST_DEFAULT_MODEL`

integration/embed_test.go

@@ -4,9 +4,7 @@ package integration
 
 import (
 	"context"
-	"errors"
 	"math"
-	"strings"
 	"testing"
 	"time"
 
@@ -206,8 +204,8 @@ func TestAllMiniLMEmbed(t *testing.T) {
 		t.Fatalf("expected %v, got %v (similarity: %f)", expected[0:5], res.Embeddings[0][0:5], sim)
 	}
 
-	if res.PromptEvalCount != 8 {
+	if res.PromptEvalCount != 6 {
-		t.Fatalf("expected 8 prompt tokens, got %d", res.PromptEvalCount)
+		t.Fatalf("expected 6 prompt tokens, got %d", res.PromptEvalCount)
 	}
 }
 
@@ -253,8 +251,8 @@ func TestAllMiniLMBatchEmbed(t *testing.T) {
 		t.Fatalf("expected %v, got %v (similarity: %f)", expected[1][0:5], res.Embeddings[1][0:5], sim)
 	}
 
-	if res.PromptEvalCount != 16 {
+	if res.PromptEvalCount != 12 {
-		t.Fatalf("expected 16 prompt tokens, got %d", res.PromptEvalCount)
+		t.Fatalf("expected 12 prompt tokens, got %d", res.PromptEvalCount)
 	}
 }
 
@@ -277,7 +275,7 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
 	cases := []struct {
 		name    string
 		request api.EmbedRequest
-		check   func(*testing.T, *api.EmbedResponse, error)
+		check   func(*api.EmbedResponse, error)
 	}{
 		{
 			name: "target truncation",
@@ -285,7 +283,7 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
 				Model: "all-minilm",
 				Input: "why",
 			},
-			check: func(t *testing.T, got *api.EmbedResponse, err error) {
+			check: func(got *api.EmbedResponse, err error) {
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -302,11 +300,10 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
 				Input:   "why is the sky blue?",
 				Options: map[string]any{"num_ctx": 3},
 			},
-			check: func(t *testing.T, got *api.EmbedResponse, err error) {
+			check: func(got *api.EmbedResponse, err error) {
 				if err != nil {
 					t.Fatal(err)
 				}
-				t.Logf("PromptEvalCount: want=%d got=%d", want.PromptEvalCount, got.PromptEvalCount)
 				if diff := cmp.Diff(want.Embeddings[0], got.Embeddings[0]); diff != "" {
 					t.Errorf("embedding mismatch (-want +got):\n%s", diff)
 				}
@@ -320,11 +317,10 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
 				Truncate: &truncTrue,
 				Options:  map[string]any{"num_ctx": 3},
 			},
-			check: func(t *testing.T, got *api.EmbedResponse, err error) {
+			check: func(got *api.EmbedResponse, err error) {
 				if err != nil {
 					t.Fatal(err)
 				}
-				t.Logf("PromptEvalCount: want=%d got=%d", want.PromptEvalCount, got.PromptEvalCount)
 				if diff := cmp.Diff(want.Embeddings[0], got.Embeddings[0]); diff != "" {
 					t.Errorf("embedding mismatch (-want +got):\n%s", diff)
 				}
@@ -338,21 +334,21 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
 				Truncate: &truncFalse,
 				Options:  map[string]any{"num_ctx": 3},
 			},
-			check: func(t *testing.T, res *api.EmbedResponse, err error) {
+			check: func(res *api.EmbedResponse, err error) {
-				if err.Error() != "the input length exceeds the context length" {
+				if err.Error() != "input exceeds maximum context length" {
 					t.Fatalf("expected truncation error, got: %v", err)
 				}
 			},
 		},
 		{
-			name: "input after truncate error with context length of 1",
+			name: "input after truncate error",
 			request: api.EmbedRequest{
 				Model:    "all-minilm",
 				Input:    "why is the sky blue?",
 				Truncate: &truncTrue,
 				Options:  map[string]any{"num_ctx": 1},
 			},
-			check: func(t *testing.T, res *api.EmbedResponse, err error) {
+			check: func(res *api.EmbedResponse, err error) {
 				if err.Error() != "input after truncation exceeds maximum context length" {
 					t.Fatalf("expected truncation error, got: %v", err)
 				}
@@ -366,7 +362,7 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
 				Truncate: &truncTrue,
 				Options:  map[string]any{"num_ctx": 0},
 			},
-			check: func(t *testing.T, res *api.EmbedResponse, err error) {
+			check: func(res *api.EmbedResponse, err error) {
 				if err.Error() != "input after truncation exceeds maximum context length" {
 					t.Fatalf("expected truncation error, got: %v", err)
 				}
@@ -379,7 +375,7 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
 				Input:   "why is the sky blue? Why is the sky blue? hi there my",
 				Options: map[string]any{"num_ctx": 16},
 			},
-			check: func(t *testing.T, res *api.EmbedResponse, err error) {
+			check: func(res *api.EmbedResponse, err error) {
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -389,8 +385,7 @@ func TestAllMiniLMEmbedTruncate(t *testing.T) {
 
 	for _, req := range cases {
 		t.Run(req.name, func(t *testing.T) {
-			resp, err := embedTestHelper(ctx, client, t, req.request)
+			req.check(embedTestHelper(ctx, client, t, req.request))
-			req.check(t, resp, err)
 		})
 	}
 }
@@ -414,230 +409,3 @@ func embedTestHelper(ctx context.Context, client *api.Client, t *testing.T, req
 
 	return client.Embed(ctx, &req)
 }
-
-func TestEmbedTruncation(t *testing.T) {
-	// Use test deadline if set, otherwise default to 2 minutes
-	timeout := 2 * time.Minute
-	if deadline, ok := t.Deadline(); ok {
-		timeout = time.Until(deadline) - 10*time.Second // Reserve 10s buffer
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	client, _, cleanup := InitServerConnection(ctx, t)
-	defer cleanup()
-
-	for _, model := range libraryEmbedModels {
-		model := model
-		t.Run(model, func(t *testing.T) {
-			// Check if we're running out of time (reserve 20s for current model)
-			if deadline, ok := t.Deadline(); ok && time.Until(deadline) < 20*time.Second {
-				t.Skip("skipping remaining tests to avoid timeout")
-			}
-
-			// Give each model its own budget to account for first-time pulls/loads
-			mctx, mcancel := context.WithTimeout(ctx, 3*time.Minute)
-			defer mcancel()
-
-			t.Run("truncation batch", func(t *testing.T) {
-				truncTrue := true
-				req := api.EmbedRequest{
-					Model:    model,
-					Input:    []string{"short", strings.Repeat("long ", 100), "medium text"},
-					Truncate: &truncTrue,
-					Options:  map[string]any{"num_ctx": 30},
-				}
-
-				res, err := embedTestHelper(mctx, client, t, req)
-				if err != nil {
-					t.Fatal(err)
-				}
-
-				if len(res.Embeddings) != 3 {
-					t.Fatalf("expected 3 embeddings, got %d", len(res.Embeddings))
-				}
-
-				if res.PromptEvalCount > 90 {
-					t.Fatalf("expected tokens <= 90 (3 × 30 max), got %d", res.PromptEvalCount)
-				}
-			})
-
-			t.Run("runner token count accuracy", func(t *testing.T) {
-				baseline := api.EmbedRequest{Model: model, Input: "test"}
-				baseRes, err := embedTestHelper(mctx, client, t, baseline)
-				if err != nil {
-					t.Fatal(err)
-				}
-
-				batch := api.EmbedRequest{
-					Model: model,
-					Input: []string{"test", "test", "test"},
-				}
-				batchRes, err := embedTestHelper(mctx, client, t, batch)
-				if err != nil {
-					t.Fatal(err)
-				}
-
-				expectedCount := baseRes.PromptEvalCount * 3
-				if batchRes.PromptEvalCount < expectedCount-2 || batchRes.PromptEvalCount > expectedCount+2 {
-					t.Fatalf("expected ~%d tokens (3 × %d), got %d",
-						expectedCount, baseRes.PromptEvalCount, batchRes.PromptEvalCount)
-				}
-			})
-		})
-	}
-}
-
-// TestEmbedLargeInput tests that embedding models can handle large inputs that would exceed typical batch sizes.
-func TestEmbedLargeInput(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
-	defer cancel()
-	client, _, cleanup := InitServerConnection(ctx, t)
-	defer cleanup()
-
-	for _, model := range libraryEmbedModels {
-		model := model
-		t.Run(model, func(t *testing.T) {
-			mctx, mcancel := context.WithTimeout(ctx, 2*time.Minute)
-			defer mcancel()
-
-			// Test with progressively larger inputs
-			testCases := []struct {
-				name       string
-				inputWords int
-			}{
-				{"medium_input_256_words", 256},
-				{"large_input_512_words", 512},
-				{"very_large_input_800_words", 800},
-			}
-
-			for _, tc := range testCases {
-				t.Run(tc.name, func(t *testing.T) {
-					words := make([]string, tc.inputWords)
-					for i := range words {
-						words[i] = "word"
-					}
-					input := strings.Join(words, " ")
-
-					req := api.EmbedRequest{
-						Model:     model,
-						Input:     input,
-						KeepAlive: &api.Duration{Duration: 30 * time.Second},
-					}
-
-					res, err := embedTestHelper(mctx, client, t, req)
-					if err != nil {
-						t.Fatalf("embedding failed for %d words: %v", tc.inputWords, err)
-					}
-
-					if len(res.Embeddings) != 1 {
-						t.Fatalf("expected 1 embedding, got %d", len(res.Embeddings))
-					}
-
-					if len(res.Embeddings[0]) == 0 {
-						t.Fatal("expected non-empty embedding")
-					}
-
-					t.Logf("Successfully embedded %d words (%d tokens)", tc.inputWords, res.PromptEvalCount)
-				})
-			}
-		})
-	}
-}
-
-// TestEmbedStatusCode tests that errors from the embedding endpoint
-// properly preserve their HTTP status codes when returned to the client.
-// This test specifically checks the error handling path in EmbedHandler
-// where api.StatusError errors should maintain their original status code.
-func TestEmbedStatusCode(t *testing.T) {
-	// Use test deadline if set, otherwise default to 2 minutes
-	timeout := 2 * time.Minute
-	if deadline, ok := t.Deadline(); ok {
-		timeout = time.Until(deadline) - 10*time.Second // Reserve 10s buffer
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	client, _, cleanup := InitServerConnection(ctx, t)
-	defer cleanup()
-
-	for _, model := range libraryEmbedModels {
-		model := model
-		t.Run(model, func(t *testing.T) {
-			// Check if we're running out of time (reserve 20s for current model)
-			if deadline, ok := t.Deadline(); ok && time.Until(deadline) < 20*time.Second {
-				t.Skip("skipping remaining tests to avoid timeout")
-			}
-
-			mctx, mcancel := context.WithTimeout(ctx, 3*time.Minute)
-			defer mcancel()
-
-			// Pull the model if needed
-			if err := PullIfMissing(mctx, client, model); err != nil {
-				t.Fatal(err)
-			}
-
-			t.Run("truncation error status code", func(t *testing.T) {
-				truncFalse := false
-				longInput := strings.Repeat("word ", 100)
-
-				req := api.EmbedRequest{
-					Model:    model,
-					Input:    longInput,
-					Truncate: &truncFalse,
-					Options:  map[string]any{"num_ctx": 10},
-				}
-
-				_, err := embedTestHelper(mctx, client, t, req)
-				if err == nil {
-					t.Fatal("expected error when truncate=false with long input")
-				}
-
-				// Check that it's a StatusError with the correct status code
-				var statusErr api.StatusError
-				if !errors.As(err, &statusErr) {
-					t.Fatalf("expected api.StatusError, got %T: %v", err, err)
-				}
-
-				// The error should be a 4xx client error (likely 400 Bad Request)
-				// not a 500 Internal Server Error
-				if statusErr.StatusCode < 400 || statusErr.StatusCode >= 500 {
-					t.Errorf("expected 4xx status code, got %d", statusErr.StatusCode)
-				}
-
-				// Verify the error message is meaningful
-				if !strings.Contains(err.Error(), "context length") {
-					t.Errorf("expected error message to mention context length, got: %v", err)
-				}
-			})
-
-			t.Run("batch truncation error status code", func(t *testing.T) {
-				truncFalse := false
-				req := api.EmbedRequest{
-					Model: model,
-					Input: []string{
-						"short input",
-						strings.Repeat("very long input ", 100),
-						"another short input",
-					},
-					Truncate: &truncFalse,
-					Options:  map[string]any{"num_ctx": 10},
-				}
-
-				_, err := embedTestHelper(mctx, client, t, req)
-				if err == nil {
-					t.Fatal("expected error when one input exceeds context with truncate=false")
-				}
-
-				// Check that it's a StatusError with the correct status code
-				var statusErr api.StatusError
-				if !errors.As(err, &statusErr) {
-					t.Fatalf("expected api.StatusError, got %T: %v", err, err)
-				}
-
-				// The error should be a 4xx client error, not a 500 Internal Server Error
-				if statusErr.StatusCode < 400 || statusErr.StatusCode >= 500 {
-					t.Errorf("expected 4xx status code, got %d", statusErr.StatusCode)
-				}
-			})
-		})
-	}
-}

integration/llm_image_test.go

@@ -33,9 +33,6 @@ func TestVisionModels(t *testing.T) {
 			// Qwen 3 VL mixture of experts
 			model: "qwen3-vl:30b",
 		},
-		{
-			model: "ministral-3",
-		},
 	}
 
 	for _, v := range testCases {

integration/tools_test.go

@@ -3,140 +3,353 @@
 package integration
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/ollama/ollama/api"
+	"github.com/ollama/ollama/openai"
 )
 
-// testPropsMap creates a ToolPropertiesMap from a map (convenience function for tests)
+var libraryToolsModels = []string{
-func testPropsMap(m map[string]api.ToolProperty) *api.ToolPropertiesMap {
+	"qwen3-vl",
-	props := api.NewToolPropertiesMap()
+	"gpt-oss:20b",
-	for k, v := range m {
+	"gpt-oss:120b",
-		props.Set(k, v)
+	"qwen3",
-	}
+	"llama3.1",
-	return props
+	"llama3.2",
+	"mistral",
+	"qwen2.5",
+	"qwen2",
+	"mistral-nemo",
+	"mistral-small",
+	"mixtral:8x22b",
+	"qwq",
+	"granite3.3",
 }
 
-func TestAPIToolCalling(t *testing.T) {
+func float64Ptr(v float64) *float64 {
+	return &v
+}
+
+func sendOpenAIChatRequest(ctx context.Context, endpoint string, req openai.ChatCompletionRequest) (*openai.ChatCompletion, error) {
+	jsonData, err := json.Marshal(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, "POST", endpoint+"/v1/chat/completions", bytes.NewBuffer(jsonData))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{
+		Timeout: 10 * time.Minute,
+	}
+
+	resp, err := client.Do(httpReq)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("API error: status=%d, body=%s", resp.StatusCode, string(body))
+	}
+
+	var chatResp openai.ChatCompletion
+	if err := json.Unmarshal(body, &chatResp); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response: %w, body: %s", err, string(body))
+	}
+
+	return &chatResp, nil
+}
+
+func sendOpenAIChatStreamRequest(ctx context.Context, endpoint string, req openai.ChatCompletionRequest, fn func(openai.ChatCompletionChunk) error) error {
+	jsonData, err := json.Marshal(req)
+	if err != nil {
+		return fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, "POST", endpoint+"/v1/chat/completions", bytes.NewBuffer(jsonData))
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	httpReq.Header.Set("Content-Type", "application/json")
+	httpReq.Header.Set("Accept", "text/event-stream")
+
+	client := &http.Client{
+		Timeout: 0, // No timeout for streaming
+	}
+
+	resp, err := client.Do(httpReq)
+	if err != nil {
+		return fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("API error: status=%d, body=%s", resp.StatusCode, string(body))
+	}
+
+	decoder := resp.Body
+	reader := bytes.NewBuffer([]byte{})
+	buf := make([]byte, 4096)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			n, err := decoder.Read(buf)
+			if n > 0 {
+				reader.Write(buf[:n])
+
+				// Process complete lines
+				for {
+					line, err := reader.ReadString('\n')
+					if err != nil {
+						// Not a complete line yet
+						reader.WriteString(line)
+						break
+					}
+
+					line = strings.TrimSpace(line)
+					if strings.HasPrefix(line, "data: ") {
+						data := strings.TrimPrefix(line, "data: ")
+
+						if data == "[DONE]" {
+							return nil
+						}
+
+						var streamResp openai.ChatCompletionChunk
+						if err := json.Unmarshal([]byte(data), &streamResp); err != nil {
+							return fmt.Errorf("failed to unmarshal stream response: %w", err)
+						}
+
+						if err := fn(streamResp); err != nil {
+							return err
+						}
+					}
+				}
+			}
+
+			if err != nil {
+				if err != io.EOF {
+					return fmt.Errorf("error reading stream: %w", err)
+				}
+				break
+			}
+		}
+	}
+
+	return nil
+}
+
+// TestToolCallingAllAPIs tests both Ollama and OpenAI APIs with shared model loading
+func TestToolCallingAllAPIs(t *testing.T) {
 	initialTimeout := 60 * time.Second
 	streamTimeout := 60 * time.Second
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	defer cancel()
 
-	client, _, cleanup := InitServerConnection(ctx, t)
+	client, endpoint, cleanup := InitServerConnection(ctx, t)
 	defer cleanup()
 
-	minVRAM := map[string]uint64{
-		"qwen3-vl":      16,
-		"gpt-oss:20b":   16,
-		"gpt-oss:120b":  70,
-		"qwen3":         6,
-		"llama3.1":      8,
-		"llama3.2":      4,
-		"mistral":       6,
-		"qwen2.5":       6,
-		"qwen2":         6,
-		"ministral-3":   20,
-		"mistral-nemo":  9,
-		"mistral-small": 16,
-		"mixtral:8x22b": 80,
-		"qwq":           20,
-		"granite3.3":    7,
-	}
-
 	for _, model := range libraryToolsModels {
 		t.Run(model, func(t *testing.T) {
+			// Skip if insufficient VRAM
 			if v, ok := minVRAM[model]; ok {
 				skipUnderMinVRAM(t, v)
 			}
 
+			// Pull model if missing - only do this once per model
 			if err := PullIfMissing(ctx, client, model); err != nil {
 				t.Fatalf("pull failed %s", err)
 			}
 
-			tools := []api.Tool{
+			t.Run("OllamaAPI", func(t *testing.T) {
-				{
+				tools := []api.Tool{
-					Type: "function",
+					{
-					Function: api.ToolFunction{
+						Type: "function",
-						Name:        "get_weather",
+						Function: api.ToolFunction{
-						Description: "Get the current weather in a given location",
+							Name:        "get_weather",
-						Parameters: api.ToolFunctionParameters{
+							Description: "Get the current weather in a given location",
-							Type:     "object",
+							Parameters: api.ToolFunctionParameters{
-							Required: []string{"location"},
+								Type:     "object",
-							Properties: testPropsMap(map[string]api.ToolProperty{
+								Required: []string{"location"},
-								"location": {
+								Properties: map[string]api.ToolProperty{
-									Type:        api.PropertyType{"string"},
+									"location": {
-									Description: "The city and state, e.g. San Francisco, CA",
+										Type:        api.PropertyType{"string"},
+										Description: "The city and state, e.g. San Francisco, CA",
+									},
 								},
-							}),
+							},
 						},
 					},
-				},
+				}
-			}
 
-			req := api.ChatRequest{
+				req := api.ChatRequest{
-				Model: model,
+					Model: model,
-				Messages: []api.Message{
+					Messages: []api.Message{
-					{
+						{
-						Role:    "user",
+							Role:    "user",
-						Content: "Call get_weather with location set to San Francisco.",
+							Content: "Call get_weather with location set to San Francisco.",
+						},
+					},
+					Tools: tools,
+					Options: map[string]any{
+						"temperature": 0,
 					},
-				},
-				Tools: tools,
-				Options: map[string]any{
-					"temperature": 0,
-				},
-			}
-
-			stallTimer := time.NewTimer(initialTimeout)
-			var gotToolCall bool
-			var lastToolCall api.ToolCall
-
-			fn := func(response api.ChatResponse) error {
-				if len(response.Message.ToolCalls) > 0 {
-					gotToolCall = true
-					lastToolCall = response.Message.ToolCalls[len(response.Message.ToolCalls)-1]
-				}
-				if !stallTimer.Reset(streamTimeout) {
-					return fmt.Errorf("stall was detected while streaming response, aborting")
-				}
-				return nil
-			}
-
-			stream := true
-			req.Stream = &stream
-			done := make(chan int)
-			var genErr error
-			go func() {
-				genErr = client.Chat(ctx, &req, fn)
-				done <- 0
-			}()
-
-			select {
-			case <-stallTimer.C:
-				t.Errorf("tool-calling chat never started. Timed out after: %s", initialTimeout.String())
-			case <-done:
-				if genErr != nil {
-					t.Fatalf("chat failed: %v", genErr)
 				}
 
-				if !gotToolCall {
+				stallTimer := time.NewTimer(initialTimeout)
-					t.Fatalf("expected at least one tool call, got none")
+				var gotToolCall bool
+				var lastToolCall api.ToolCall
+
+				fn := func(response api.ChatResponse) error {
+					if len(response.Message.ToolCalls) > 0 {
+						gotToolCall = true
+						lastToolCall = response.Message.ToolCalls[len(response.Message.ToolCalls)-1]
+					}
+					if !stallTimer.Reset(streamTimeout) {
+						return fmt.Errorf("stall was detected while streaming response, aborting")
+					}
+					return nil
 				}
 
-				if lastToolCall.Function.Name != "get_weather" {
+				stream := true
-					t.Errorf("unexpected tool called: got %q want %q", lastToolCall.Function.Name, "get_weather")
+				req.Stream = &stream
+				done := make(chan int)
+				var genErr error
+				go func() {
+					genErr = client.Chat(ctx, &req, fn)
+					done <- 0
+				}()
+
+				select {
+				case <-stallTimer.C:
+					t.Errorf("tool-calling chat never started. Timed out after: %s", initialTimeout.String())
+				case <-done:
+					if genErr != nil {
+						t.Fatalf("chat failed: %v", genErr)
+					}
+
+					if !gotToolCall {
+						t.Fatalf("expected at least one tool call, got none")
+					}
+
+					if lastToolCall.Function.Name != "get_weather" {
+						t.Errorf("unexpected tool called: got %q want %q", lastToolCall.Function.Name, "get_weather")
+					}
+
+					if _, ok := lastToolCall.Function.Arguments["location"]; !ok {
+						t.Errorf("expected tool arguments to include 'location', got: %s", lastToolCall.Function.Arguments.String())
+					}
+				case <-ctx.Done():
+					t.Error("outer test context done while waiting for tool-calling chat")
+				}
+			})
+
+			t.Run("OpenAIAPI", func(t *testing.T) {
+				tools := []api.Tool{
+					{
+						Type: "function",
+						Function: api.ToolFunction{
+							Name:        "get_weather",
+							Description: "Get the current weather in a given location",
+							Parameters: api.ToolFunctionParameters{
+								Type:     "object",
+								Required: []string{"location"},
+								Properties: map[string]api.ToolProperty{
+									"location": {
+										Type:        api.PropertyType{"string"},
+										Description: "The city and state, e.g. San Francisco, CA",
+									},
+								},
+							},
+						},
+					},
 				}
 
-				if _, ok := lastToolCall.Function.Arguments["location"]; !ok {
+				req := openai.ChatCompletionRequest{
-					t.Errorf("expected tool arguments to include 'location', got: %s", lastToolCall.Function.Arguments.String())
+					Model: model,
+					Messages: []openai.Message{
+						{
+							Role:    "user",
+							Content: "Call get_weather with location set to San Francisco.",
+						},
+					},
+					Tools:       tools,
+					Stream:      true,
+					Temperature: float64Ptr(0),
 				}
-			case <-ctx.Done():
+
-				t.Error("outer test context done while waiting for tool-calling chat")
+				stallTimer := time.NewTimer(initialTimeout)
-			}
+				var gotToolCall bool
+				var lastToolCall openai.ToolCall
+
+				fn := func(response openai.ChatCompletionChunk) error {
+					if len(response.Choices) > 0 && len(response.Choices[0].Delta.ToolCalls) > 0 {
+						gotToolCall = true
+						toolCalls := response.Choices[0].Delta.ToolCalls
+						lastToolCall = toolCalls[len(toolCalls)-1]
+					}
+					if !stallTimer.Reset(streamTimeout) {
+						return fmt.Errorf("stall was detected while streaming response, aborting")
+					}
+					return nil
+				}
+
+				done := make(chan int)
+				var genErr error
+				go func() {
+					genErr = sendOpenAIChatStreamRequest(ctx, "http://"+endpoint, req, fn)
+					done <- 0
+				}()
+
+				select {
+				case <-stallTimer.C:
+					t.Errorf("tool-calling chat never started. Timed out after: %s", initialTimeout.String())
+				case <-done:
+					if genErr != nil {
+						t.Fatalf("chat failed: %v", genErr)
+					}
+
+					if !gotToolCall {
+						t.Fatalf("expected at least one tool call, got none")
+					}
+
+					if lastToolCall.Function.Name != "get_weather" {
+						t.Errorf("unexpected tool called: got %q want %q", lastToolCall.Function.Name, "get_weather")
+					}
+
+					if !strings.Contains(lastToolCall.Function.Arguments, "location") {
+						t.Errorf("expected tool arguments to include 'location', got: %s", lastToolCall.Function.Arguments)
+					}
+
+					if !strings.Contains(lastToolCall.Function.Arguments, "San Francisco") {
+						t.Errorf("expected tool arguments to include 'San Francisco', got: %s", lastToolCall.Function.Arguments)
+					}
+				case <-ctx.Done():
+					t.Error("outer test context done while waiting for tool-calling chat")
+				}
+			})
 		})
 	}
 }

integration/utils_test.go

@@ -38,7 +38,6 @@ var (
 
 	// Note: add newer models at the top of the list to test them first
 	ollamaEngineChatModels = []string{
-		"ministral-3",
 		"qwen3-coder:30b",
 		"gpt-oss:20b",
 		"gemma3n:e2b",
@@ -168,7 +167,6 @@ var (
 		"medllama2",
 		"megadolphin",
 		"minicpm-v",
-		"ministral-3",
 		"mistral-large",
 		"mistral-nemo",
 		"mistral-openorca",
@@ -262,23 +260,6 @@ var (
 		"snowflake-arctic-embed",
 		"snowflake-arctic-embed2",
 	}
-	libraryToolsModels = []string{
-		"qwen3-vl",
-		"gpt-oss:20b",
-		"gpt-oss:120b",
-		"qwen3",
-		"llama3.1",
-		"llama3.2",
-		"mistral",
-		"qwen2.5",
-		"qwen2",
-		"ministral-3",
-		"mistral-nemo",
-		"mistral-small",
-		"mixtral:8x22b",
-		"qwq",
-		"granite3.3",
-	}
 
 	blueSkyPrompt   = "why is the sky blue? Be brief but factual in your reply"
 	blueSkyExpected = []string{"rayleigh", "scatter", "atmosphere", "nitrogen", "oxygen", "wavelength", "interact"}
@@ -750,6 +731,23 @@ func skipUnderMinVRAM(t *testing.T, gb uint64) {
 	}
 }
 
+var minVRAM = map[string]uint64{
+	"qwen3-vl":      16,
+	"gpt-oss:20b":   16,
+	"gpt-oss:120b":  70,
+	"qwen3":         6,
+	"llama3.1":      8,
+	"llama3.2":      4,
+	"mistral":       6,
+	"qwen2.5":       6,
+	"qwen2":         6,
+	"mistral-nemo":  9,
+	"mistral-small": 16,
+	"mixtral:8x22b": 80,
+	"qwq":           20,
+	"granite3.3":    7,
+}
+
 // Skip if the target model isn't X% GPU loaded to avoid excessive runtime
 func skipIfNotGPULoaded(ctx context.Context, t *testing.T, client *api.Client, model string, minPercent int) {
 	gpuPercent := getGPUPercent(ctx, t, client, model)

internal/orderedmap/orderedmap.go

@@ -1,94 +0,0 @@
-// Package orderedmap provides a generic ordered map that maintains insertion order.
-// It wraps github.com/wk8/go-ordered-map/v2 to encapsulate the dependency.
-package orderedmap
-
-import (
-	"encoding/json"
-	"iter"
-
-	orderedmap "github.com/wk8/go-ordered-map/v2"
-)
-
-// Map is a generic ordered map that maintains insertion order.
-type Map[K comparable, V any] struct {
-	om *orderedmap.OrderedMap[K, V]
-}
-
-// New creates a new empty ordered map.
-func New[K comparable, V any]() *Map[K, V] {
-	return &Map[K, V]{
-		om: orderedmap.New[K, V](),
-	}
-}
-
-// Get retrieves a value by key.
-func (m *Map[K, V]) Get(key K) (V, bool) {
-	if m == nil || m.om == nil {
-		var zero V
-		return zero, false
-	}
-	return m.om.Get(key)
-}
-
-// Set sets a key-value pair. If the key already exists, its value is updated
-// but its position in the iteration order is preserved. If the key is new,
-// it is appended to the end.
-func (m *Map[K, V]) Set(key K, value V) {
-	if m == nil {
-		return
-	}
-	if m.om == nil {
-		m.om = orderedmap.New[K, V]()
-	}
-	m.om.Set(key, value)
-}
-
-// Len returns the number of entries.
-func (m *Map[K, V]) Len() int {
-	if m == nil || m.om == nil {
-		return 0
-	}
-	return m.om.Len()
-}
-
-// All returns an iterator over all key-value pairs in insertion order.
-func (m *Map[K, V]) All() iter.Seq2[K, V] {
-	return func(yield func(K, V) bool) {
-		if m == nil || m.om == nil {
-			return
-		}
-		for pair := m.om.Oldest(); pair != nil; pair = pair.Next() {
-			if !yield(pair.Key, pair.Value) {
-				return
-			}
-		}
-	}
-}
-
-// ToMap converts to a regular Go map.
-// Note: The resulting map does not preserve order.
-func (m *Map[K, V]) ToMap() map[K]V {
-	if m == nil || m.om == nil {
-		return nil
-	}
-	result := make(map[K]V, m.om.Len())
-	for pair := m.om.Oldest(); pair != nil; pair = pair.Next() {
-		result[pair.Key] = pair.Value
-	}
-	return result
-}
-
-// MarshalJSON implements json.Marshaler. The JSON output preserves key order.
-func (m *Map[K, V]) MarshalJSON() ([]byte, error) {
-	if m == nil || m.om == nil {
-		return []byte("null"), nil
-	}
-	return json.Marshal(m.om)
-}
-
-// UnmarshalJSON implements json.Unmarshaler. The insertion order matches the
-// order of keys in the JSON input.
-func (m *Map[K, V]) UnmarshalJSON(data []byte) error {
-	m.om = orderedmap.New[K, V]()
-	return json.Unmarshal(data, &m.om)
-}

internal/orderedmap/orderedmap_test.go

@@ -1,348 +0,0 @@
-package orderedmap
-
-import (
-	"encoding/json"
-	"slices"
-	"testing"
-)
-
-func TestMap_BasicOperations(t *testing.T) {
-	m := New[string, int]()
-
-	// Test empty map
-	if m.Len() != 0 {
-		t.Errorf("expected Len() = 0, got %d", m.Len())
-	}
-	v, ok := m.Get("a")
-	if ok {
-		t.Error("expected Get on empty map to return false")
-	}
-	if v != 0 {
-		t.Errorf("expected zero value, got %d", v)
-	}
-
-	// Test Set and Get
-	m.Set("a", 1)
-	m.Set("b", 2)
-	m.Set("c", 3)
-
-	if m.Len() != 3 {
-		t.Errorf("expected Len() = 3, got %d", m.Len())
-	}
-
-	v, ok = m.Get("a")
-	if !ok || v != 1 {
-		t.Errorf("expected Get(a) = (1, true), got (%d, %v)", v, ok)
-	}
-
-	v, ok = m.Get("b")
-	if !ok || v != 2 {
-		t.Errorf("expected Get(b) = (2, true), got (%d, %v)", v, ok)
-	}
-
-	v, ok = m.Get("c")
-	if !ok || v != 3 {
-		t.Errorf("expected Get(c) = (3, true), got (%d, %v)", v, ok)
-	}
-
-	// Test updating existing key preserves position
-	m.Set("a", 10)
-	v, ok = m.Get("a")
-	if !ok || v != 10 {
-		t.Errorf("expected Get(a) = (10, true), got (%d, %v)", v, ok)
-	}
-	if m.Len() != 3 {
-		t.Errorf("expected Len() = 3 after update, got %d", m.Len())
-	}
-}
-
-func TestMap_InsertionOrderPreserved(t *testing.T) {
-	m := New[string, int]()
-
-	// Insert in non-alphabetical order
-	m.Set("z", 1)
-	m.Set("a", 2)
-	m.Set("m", 3)
-	m.Set("b", 4)
-
-	// Verify iteration order matches insertion order
-	var keys []string
-	var values []int
-	for k, v := range m.All() {
-		keys = append(keys, k)
-		values = append(values, v)
-	}
-
-	expectedKeys := []string{"z", "a", "m", "b"}
-	expectedValues := []int{1, 2, 3, 4}
-
-	if !slices.Equal(keys, expectedKeys) {
-		t.Errorf("expected keys %v, got %v", expectedKeys, keys)
-	}
-	if !slices.Equal(values, expectedValues) {
-		t.Errorf("expected values %v, got %v", expectedValues, values)
-	}
-}
-
-func TestMap_UpdatePreservesPosition(t *testing.T) {
-	m := New[string, int]()
-
-	m.Set("first", 1)
-	m.Set("second", 2)
-	m.Set("third", 3)
-
-	// Update middle element
-	m.Set("second", 20)
-
-	var keys []string
-	for k := range m.All() {
-		keys = append(keys, k)
-	}
-
-	// Order should still be first, second, third
-	expected := []string{"first", "second", "third"}
-	if !slices.Equal(keys, expected) {
-		t.Errorf("expected keys %v, got %v", expected, keys)
-	}
-}
-
-func TestMap_MarshalJSON_PreservesOrder(t *testing.T) {
-	m := New[string, int]()
-
-	// Insert in non-alphabetical order
-	m.Set("z", 1)
-	m.Set("a", 2)
-	m.Set("m", 3)
-
-	data, err := json.Marshal(m)
-	if err != nil {
-		t.Fatalf("Marshal failed: %v", err)
-	}
-
-	// JSON should preserve insertion order, not alphabetical
-	expected := `{"z":1,"a":2,"m":3}`
-	if string(data) != expected {
-		t.Errorf("expected %s, got %s", expected, string(data))
-	}
-}
-
-func TestMap_UnmarshalJSON_PreservesOrder(t *testing.T) {
-	// JSON with non-alphabetical key order
-	jsonData := `{"z":1,"a":2,"m":3}`
-
-	m := New[string, int]()
-	if err := json.Unmarshal([]byte(jsonData), m); err != nil {
-		t.Fatalf("Unmarshal failed: %v", err)
-	}
-
-	// Verify iteration order matches JSON order
-	var keys []string
-	for k := range m.All() {
-		keys = append(keys, k)
-	}
-
-	expected := []string{"z", "a", "m"}
-	if !slices.Equal(keys, expected) {
-		t.Errorf("expected keys %v, got %v", expected, keys)
-	}
-}
-
-func TestMap_JSONRoundTrip(t *testing.T) {
-	// Test that unmarshal -> marshal produces identical JSON
-	original := `{"zebra":"z","apple":"a","mango":"m","banana":"b"}`
-
-	m := New[string, string]()
-	if err := json.Unmarshal([]byte(original), m); err != nil {
-		t.Fatalf("Unmarshal failed: %v", err)
-	}
-
-	data, err := json.Marshal(m)
-	if err != nil {
-		t.Fatalf("Marshal failed: %v", err)
-	}
-
-	if string(data) != original {
-		t.Errorf("round trip failed: expected %s, got %s", original, string(data))
-	}
-}
-
-func TestMap_ToMap(t *testing.T) {
-	m := New[string, int]()
-	m.Set("a", 1)
-	m.Set("b", 2)
-
-	regular := m.ToMap()
-
-	if len(regular) != 2 {
-		t.Errorf("expected len 2, got %d", len(regular))
-	}
-	if regular["a"] != 1 {
-		t.Errorf("expected regular[a] = 1, got %d", regular["a"])
-	}
-	if regular["b"] != 2 {
-		t.Errorf("expected regular[b] = 2, got %d", regular["b"])
-	}
-}
-
-func TestMap_NilSafety(t *testing.T) {
-	var m *Map[string, int]
-
-	// All operations should be safe on nil
-	if m.Len() != 0 {
-		t.Errorf("expected Len() = 0 on nil map, got %d", m.Len())
-	}
-
-	v, ok := m.Get("a")
-	if ok {
-		t.Error("expected Get on nil map to return false")
-	}
-	if v != 0 {
-		t.Errorf("expected zero value from nil map, got %d", v)
-	}
-
-	// Set on nil is a no-op
-	m.Set("a", 1)
-	if m.Len() != 0 {
-		t.Errorf("expected Len() = 0 after Set on nil, got %d", m.Len())
-	}
-
-	// All returns empty iterator
-	var keys []string
-	for k := range m.All() {
-		keys = append(keys, k)
-	}
-	if len(keys) != 0 {
-		t.Errorf("expected empty iteration on nil map, got %v", keys)
-	}
-
-	// ToMap returns nil
-	if m.ToMap() != nil {
-		t.Error("expected ToMap to return nil on nil map")
-	}
-
-	// MarshalJSON returns null
-	data, err := json.Marshal(m)
-	if err != nil {
-		t.Fatalf("Marshal failed: %v", err)
-	}
-	if string(data) != "null" {
-		t.Errorf("expected null, got %s", string(data))
-	}
-}
-
-func TestMap_EmptyMapMarshal(t *testing.T) {
-	m := New[string, int]()
-
-	data, err := json.Marshal(m)
-	if err != nil {
-		t.Fatalf("Marshal failed: %v", err)
-	}
-	if string(data) != "{}" {
-		t.Errorf("expected {}, got %s", string(data))
-	}
-}
-
-func TestMap_NestedValues(t *testing.T) {
-	m := New[string, any]()
-	m.Set("string", "hello")
-	m.Set("number", 42)
-	m.Set("bool", true)
-	m.Set("nested", map[string]int{"x": 1})
-
-	data, err := json.Marshal(m)
-	if err != nil {
-		t.Fatalf("Marshal failed: %v", err)
-	}
-
-	expected := `{"string":"hello","number":42,"bool":true,"nested":{"x":1}}`
-	if string(data) != expected {
-		t.Errorf("expected %s, got %s", expected, string(data))
-	}
-}
-
-func TestMap_AllIteratorEarlyExit(t *testing.T) {
-	m := New[string, int]()
-	m.Set("a", 1)
-	m.Set("b", 2)
-	m.Set("c", 3)
-	m.Set("d", 4)
-
-	// Collect only first 2
-	var keys []string
-	for k := range m.All() {
-		keys = append(keys, k)
-		if len(keys) == 2 {
-			break
-		}
-	}
-
-	expected := []string{"a", "b"}
-	if !slices.Equal(keys, expected) {
-		t.Errorf("expected %v, got %v", expected, keys)
-	}
-}
-
-func TestMap_IntegerKeys(t *testing.T) {
-	m := New[int, string]()
-	m.Set(3, "three")
-	m.Set(1, "one")
-	m.Set(2, "two")
-
-	var keys []int
-	for k := range m.All() {
-		keys = append(keys, k)
-	}
-
-	// Should preserve insertion order, not numerical order
-	expected := []int{3, 1, 2}
-	if !slices.Equal(keys, expected) {
-		t.Errorf("expected %v, got %v", expected, keys)
-	}
-}
-
-func TestMap_UnmarshalIntoExisting(t *testing.T) {
-	m := New[string, int]()
-	m.Set("existing", 999)
-
-	// Unmarshal should replace contents
-	if err := json.Unmarshal([]byte(`{"new":1}`), m); err != nil {
-		t.Fatalf("Unmarshal failed: %v", err)
-	}
-
-	_, ok := m.Get("existing")
-	if ok {
-		t.Error("existing key should be gone after unmarshal")
-	}
-
-	v, ok := m.Get("new")
-	if !ok || v != 1 {
-		t.Errorf("expected Get(new) = (1, true), got (%d, %v)", v, ok)
-	}
-}
-
-func TestMap_LargeOrderPreservation(t *testing.T) {
-	m := New[string, int]()
-
-	// Create many keys in specific order
-	keys := make([]string, 100)
-	for i := range 100 {
-		keys[i] = string(rune('a' + (99 - i))) // reverse order: 'd', 'c', 'b', 'a' (extended)
-		if i >= 26 {
-			keys[i] = string(rune('A'+i-26)) + string(rune('a'+i%26))
-		}
-	}
-
-	for i, k := range keys {
-		m.Set(k, i)
-	}
-
-	// Verify order preserved
-	var resultKeys []string
-	for k := range m.All() {
-		resultKeys = append(resultKeys, k)
-	}
-
-	if !slices.Equal(keys, resultKeys) {
-		t.Error("large map should preserve insertion order")
-	}
-}

kvcache/causal.go

@@ -3,6 +3,7 @@ package kvcache
 import (
 	"errors"
 	"fmt"
+	"log/slog"
 	"math"
 	"slices"
 
@@ -39,18 +40,18 @@ type Causal struct {
 
 	// ** current forward pass **
 
+	// the active layer for Get and Put
+	curLayer int
+
+	// starting location for data storage for this batch
+	curLoc int
+
 	// size of the current batch
 	curBatchSize int
 
-	// locations for data storage for this batch
-	curLoc ml.Tensor
-
 	// mask of the cache as used by this batch
 	curMask ml.Tensor
 
-	// the active layer for Get and Put
-	curLayer int
-
 	// locations in the cache that are needed for this batch
 	curCellRange cellRange
 
@@ -140,6 +141,10 @@ func (c *Causal) Init(backend ml.Backend, dtype ml.DType, maxSequences, capacity
 		c.config.CachePadding = 1
 	}
 
+	if c.config.MaskBatchPadding == 0 {
+		c.config.MaskBatchPadding = 1
+	}
+
 	if c.config.MaskDType == ml.DTypeOther {
 		c.config.MaskDType = ml.DTypeF32
 	}
@@ -201,47 +206,45 @@ func (c *Causal) StartForward(ctx ml.Context, batch input.Batch, reserve bool) e
 	c.curPositions = batch.Positions
 	c.opts.Except = nil
 
-	var locs []int32
 	if !reserve {
 		c.updateSlidingWindow()
 
 		var err error
-		locs, err = c.findLocs()
+		c.curLoc, err = c.findStartLoc()
+		if errors.Is(err, ErrKvCacheFull) {
+			c.defrag()
+			c.curLoc, err = c.findStartLoc()
+		}
 		if err != nil {
 			return err
 		}
 
 		for i, pos := range batch.Positions {
 			seq := batch.Sequences[i]
-			loc := int(locs[i])
 
-			c.cells[loc] = cacheCell{pos: pos, sequences: []int{seq}}
+			c.cells[c.curLoc+i] = cacheCell{pos: pos, sequences: []int{seq}}
 
 			seqRange, ok := c.cellRanges[seq]
 			if !ok {
 				seqRange = newRange()
 			}
 
-			seqRange.min = min(seqRange.min, loc)
+			seqRange.min = min(seqRange.min, c.curLoc+i)
-			c.curCellRange.min = min(c.curCellRange.min, loc)
+			c.curCellRange.min = min(c.curCellRange.min, c.curLoc+i)
 
-			seqRange.max = max(seqRange.max, loc)
+			seqRange.max = max(seqRange.max, c.curLoc+i)
-			c.curCellRange.max = max(c.curCellRange.max, loc)
+			c.curCellRange.max = max(c.curCellRange.max, c.curLoc+i)
 
 			c.cellRanges[seq] = seqRange
 		}
 	} else {
 		// If we are reserving memory, don't update any of the cache metadata but set the size
 		// to the worst case.
-		locs = make([]int32, c.curBatchSize)
+		c.curLoc = 0
-		for i := range locs {
-			locs[i] = int32(i)
-		}
 		c.curCellRange.min = 0
 		c.curCellRange.max = len(c.cells) - 1
 	}
 
-	c.curLoc = ctx.Input().FromInts(locs, len(locs))
 	c.curMask = c.buildMask(ctx)
 
 	return nil
@@ -254,20 +257,22 @@ func newRange() cellRange {
 	}
 }
 
-// Returns a slice of locations where each token in the batch should be stored
+// Find the first contiguous block of at least curBatchSize
-func (c *Causal) findLocs() ([]int32, error) {
+func (c *Causal) findStartLoc() (int, error) {
-	loc := make([]int32, 0, c.curBatchSize)
+	var start, count int
-
 	for i := range c.cells {
 		if len(c.cells[i].sequences) == 0 {
-			loc = append(loc, int32(i))
+			count++
-			if len(loc) >= c.curBatchSize {
+			if count >= c.curBatchSize {
-				return loc, nil
+				return start, nil
 			}
+		} else {
+			start = i + 1
+			count = 0
 		}
 	}
 
-	return nil, fmt.Errorf("%w (cache: %v batch: %v)", ErrKvCacheFull, len(c.cells), c.curBatchSize)
+	return 0, fmt.Errorf("%w (cache: %v batch: %v)", ErrKvCacheFull, len(c.cells), c.curBatchSize)
 }
 
 func (c *Causal) updateSlidingWindow() {
@@ -360,12 +365,15 @@ func roundUp(length, pad int) int {
 // token in the history should apply. This is based on both the sequence and causality (the
 // position of the history is not ahead of the token in the batch).
 func (c *Causal) buildMask(ctx ml.Context) ml.Tensor {
+	// Align and pad the two dimensions as required by the backend
+	batchSize := roundUp(c.curBatchSize, c.config.MaskBatchPadding)
+
 	c.curCellRange.min = roundDown(c.curCellRange.min, c.config.CachePadding)
 	c.curCellRange.max = roundUp(c.curCellRange.max+1, c.config.CachePadding) - 1
 
 	length := c.curCellRange.max - c.curCellRange.min + 1
 
-	mask := make([]float32, c.curBatchSize*length)
+	mask := make([]float32, batchSize*length)
 
 	for i := range c.curBatchSize {
 		enabled := !slices.Contains(c.opts.Except, i)
@@ -379,7 +387,13 @@ func (c *Causal) buildMask(ctx ml.Context) ml.Tensor {
 		}
 	}
 
-	maskTensor := ctx.Input().FromFloats(mask, length, c.curBatchSize)
+	// Mask out any padding tokens we added. For padding that we added to the cache history, this
+	// has already been masked out because the sequence doesn't match.
+	for i := c.curBatchSize * length; i < len(mask); i++ {
+		mask[i] = float32(math.Inf(-1))
+	}
+
+	maskTensor := ctx.Input().FromFloats(mask, length, batchSize)
 
 	if c.config.MaskDType != ml.DTypeF32 {
 		maskTensor = maskTensor.Cast(ctx, c.config.MaskDType)
@@ -388,6 +402,145 @@ func (c *Causal) buildMask(ctx ml.Context) ml.Tensor {
 	return maskTensor
 }
 
+func (c *Causal) moveCells(ctx ml.Context, src, dst, length int) {
+	for i, key := range c.keys {
+		if key == nil {
+			continue
+		}
+
+		kHeadDim := key.Dim(0)
+		numKVHeads := key.Dim(1)
+		rowSize := key.Stride(2)
+
+		kSrcView := key.View(ctx, rowSize*src, kHeadDim*numKVHeads*length)
+		kDstView := key.View(ctx, rowSize*dst, kHeadDim*numKVHeads*length)
+
+		value := c.values[i]
+		var vSrcView, vDstView ml.Tensor
+		if c.config.PermutedV {
+			vHeadDim := value.Dim(1)
+			elemSize := value.Stride(0)
+
+			vSrcView = value.View(ctx, elemSize*src, length, len(c.cells)*elemSize, vHeadDim*numKVHeads)
+			vDstView = value.View(ctx, elemSize*dst, length, len(c.cells)*elemSize, vHeadDim*numKVHeads)
+		} else {
+			vHeadDim := value.Dim(0)
+			rowSize := value.Stride(2)
+
+			vSrcView = value.View(ctx, rowSize*src, vHeadDim*numKVHeads*length)
+			vDstView = value.View(ctx, rowSize*dst, vHeadDim*numKVHeads*length)
+		}
+
+		ctx.Forward(
+			kSrcView.Copy(ctx, kDstView),
+			vSrcView.Copy(ctx, vDstView),
+		)
+	}
+}
+
+func (c *Causal) defrag() {
+	slog.Debug("defragmenting kv cache")
+
+	// Defrag strategy:
+	// - Search for empty holes at the beginning of the cache,
+	//   filling them with active data starting at the end
+	// - If there are contiguous elements that need to be moved,
+	//   combine them into a single operation by holding new moves
+	//   until we see that the next one is non-contiguous
+	// - Fill up the context with the maximum number of operations it
+	//   can hold then compute that and continue with a new context
+	//
+	// We could try to optimize placement by grouping blocks from
+	// the same sequences together but most likely the next forward
+	// pass will disrupt this anyways, so the real world benefit
+	// seems limited as this time.
+
+	ctx := c.backend.NewContext()
+
+	// For every move, 6 tensors are required per layer (2 views and a
+	// copy for each of k and v). We also need to refer to the original
+	// k and v cache tensors - once per layer, not per move.
+	layers := 0
+	for _, key := range c.keys {
+		if key == nil {
+			continue
+		}
+		layers++
+	}
+
+	maxMoves := (ctx.MaxGraphNodes() - 2*layers) / (6 * layers)
+	moves := 0
+
+	var pendingSrc, pendingDst, pendingLen int
+	src := len(c.cells) - 1
+
+	for dst := 0; dst < src; dst++ {
+		if len(c.cells[dst].sequences) == 0 {
+			for ; src > dst; src-- {
+				if len(c.cells[src].sequences) != 0 {
+					c.cells[dst] = c.cells[src]
+					c.cells[src] = cacheCell{}
+
+					if pendingLen > 0 {
+						if src == pendingSrc-pendingLen && dst == pendingDst+pendingLen {
+							pendingSrc = src
+							pendingLen++
+							break
+						} else {
+							c.moveCells(ctx, pendingSrc, pendingDst, pendingLen)
+							moves++
+						}
+					}
+
+					pendingSrc = src
+					pendingDst = dst
+					pendingLen = 1
+
+					break
+				}
+			}
+		}
+
+		if moves >= maxMoves {
+			ctx.Compute()
+			ctx.Close()
+			ctx = c.backend.NewContext()
+
+			moves = 0
+		}
+	}
+
+	if pendingLen > 0 {
+		c.moveCells(ctx, pendingSrc, pendingDst, pendingLen)
+		moves++
+	}
+
+	if moves > 0 {
+		ctx.Compute()
+	}
+	ctx.Close()
+
+	// Reset range metadata
+	for seq := range c.cellRanges {
+		seqRange := newRange()
+
+		for i, cell := range c.cells {
+			if slices.Contains(cell.sequences, seq) {
+				if i < seqRange.min {
+					seqRange.min = i
+				}
+				if i > seqRange.max {
+					seqRange.max = i
+				}
+			}
+		}
+
+		c.cellRanges[seq] = seqRange
+	}
+
+	c.updateSlidingWindow()
+}
+
 func (c *Causal) SetLayer(layer int) {
 	c.curLayer = layer
 }
@@ -472,25 +625,18 @@ func (c *Causal) Put(ctx ml.Context, key, value ml.Tensor) {
 		}
 	}
 
-	key = key.Reshape(ctx, kHeadDim*numKVHeads, batchSize)
+	rowSize := c.keys[c.curLayer].Stride(2)
-	keyCache := c.keys[c.curLayer]
+	ctx.Forward(key.Copy(ctx, c.keys[c.curLayer].View(ctx, rowSize*c.curLoc, kHeadDim*numKVHeads*batchSize)))
-	keyCache = keyCache.Reshape(ctx, kHeadDim*numKVHeads, len(c.cells))
-	ctx.Forward(keyCache.SetRows(ctx, key, c.curLoc))
 
 	if c.config.PermutedV {
-		value = value.Reshape(ctx, vHeadDim*numKVHeads, 1, batchSize)
+		elemSize := c.values[c.curLayer].Stride(0)
-		value = value.Permute(ctx, 2, 0, 1, 3)
 
-		valueCache := c.values[c.curLayer]
+		value = value.Permute(ctx, 1, 2, 0, 3)
-		valueCache = valueCache.Reshape(ctx, 1, len(c.cells), vHeadDim*numKVHeads)
+		ctx.Forward(value.Copy(ctx, c.values[c.curLayer].View(ctx, elemSize*c.curLoc, batchSize, len(c.cells)*elemSize, vHeadDim*numKVHeads)))
-
-		ctx.Forward(valueCache.SetRows(ctx, value, c.curLoc))
 	} else {
-		value = value.Reshape(ctx, vHeadDim*numKVHeads, batchSize)
+		rowSize := c.values[c.curLayer].Stride(2)
-		valueCache := c.values[c.curLayer]
-		valueCache = valueCache.Reshape(ctx, vHeadDim*numKVHeads, len(c.cells))
 
-		ctx.Forward(valueCache.SetRows(ctx, value, c.curLoc))
+		ctx.Forward(value.Copy(ctx, c.values[c.curLayer].View(ctx, rowSize*c.curLoc, vHeadDim*numKVHeads*batchSize)))
 	}
 }
 

llama/build-info.cpp

@@ -1,4 +1,4 @@
 int LLAMA_BUILD_NUMBER = 0;
-char const *LLAMA_COMMIT = "ec98e2002";
+char const *LLAMA_COMMIT = "3cfa9c3f125763305b4226bc032f1954f08990dc";
 char const *LLAMA_COMPILER = "";
 char const *LLAMA_BUILD_TARGET = "";

llama/llama.cpp/.rsync-filter

@@ -17,17 +17,11 @@ include /tools/mtmd/clip.cpp
 include /tools/mtmd/mtmd.cpp
 include /tools/mtmd/mtmd-audio.cpp
 include /tools/mtmd/mtmd-helper.cpp
-include /tools/mtmd/models/
-include /tools/mtmd/models/*.h
-include /tools/mtmd/models/*.cpp
 include /src/
 include /src/llama.*
 include /src/llama-*.*
 include /src/unicode-data.*
 include /src/unicode.*
-include /src/models/
-include /src/models/*.h
-include /src/models/*.cpp
 include /vendor/
 include /vendor/miniaudio/
 include /vendor/miniaudio/*.h

llama/llama.cpp/common/common.cpp

@@ -8,7 +8,6 @@
 #include "common.h"
 #include "log.h"
 #include "llama.h"
-#include "sampling.h"
 
 #include <algorithm>
 #include <cinttypes>
@@ -27,6 +26,7 @@
 #include <sstream>
 #include <string>
 #include <thread>
+#include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
@@ -60,14 +60,6 @@
 #pragma warning(disable: 4244 4267) // possible loss of data
 #endif
 
-common_time_meas::common_time_meas(int64_t & t_acc, bool disable) : t_start_us(disable ? -1 : ggml_time_us()), t_acc(t_acc) {}
-
-common_time_meas::~common_time_meas() {
-    if (t_start_us >= 0) {
-        t_acc += ggml_time_us() - t_start_us;
-    }
-}
-
 //
 // CPU utils
 //
@@ -363,7 +355,11 @@ bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREAD
 }
 
 void common_init() {
-    llama_log_set(common_log_default_callback, NULL);
+    llama_log_set([](ggml_log_level level, const char * text, void * /*user_data*/) {
+        if (LOG_DEFAULT_LLAMA <= common_log_verbosity_thold) {
+            common_log_add(common_log_main(), level, "%s", text);
+        }
+    }, NULL);
 
 #ifdef NDEBUG
     const char * build_type = "";
@@ -694,7 +690,7 @@ bool string_parse_kv_override(const char * data, std::vector<llama_model_kv_over
 
 // Validate if a filename is safe to use
 // To validate a full path, split the path by the OS-specific path separator, and validate each part with this function
-bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
+bool fs_validate_filename(const std::string & filename) {
     if (!filename.length()) {
         // Empty filename invalid
         return false;
@@ -754,14 +750,10 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
             || (c >= 0xD800 && c <= 0xDFFF) // UTF-16 surrogate pairs
             || c == 0xFFFD // Replacement Character (UTF-8)
             || c == 0xFEFF // Byte Order Mark (BOM)
-            || c == ':' || c == '*' // Illegal characters
+            || c == '/' || c == '\\' || c == ':' || c == '*' // Illegal characters
             || c == '?' || c == '"' || c == '<' || c == '>' || c == '|') {
             return false;
         }
-        if (!allow_subdirs && (c == '/' || c == '\\')) {
-            // Subdirectories not allowed, reject path separators
-            return false;
-        }
     }
 
     // Reject any leading or trailing ' ', or any trailing '.', these are stripped on Windows and will cause a different filename
@@ -786,29 +778,11 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
 #include <iostream>
 
 
-#ifdef _WIN32
-static std::wstring utf8_to_wstring(const std::string & str) {
-    if (str.empty()) {
-        return std::wstring();
-    }
-
-    int size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0);
-
-    if (size <= 0) {
-        return std::wstring();
-    }
-
-    std::wstring wstr(size, 0);
-    MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), &wstr[0], size);
-
-    return wstr;
-}
-#endif
-
 // returns true if successful, false otherwise
 bool fs_create_directory_with_parents(const std::string & path) {
 #ifdef _WIN32
-    std::wstring wpath = utf8_to_wstring(path);
+    std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
+    std::wstring wpath = converter.from_bytes(path);
 
     // if the path already exists, check whether it's a directory
     const DWORD attributes = GetFileAttributesW(wpath.c_str());
@@ -881,11 +855,6 @@ bool fs_create_directory_with_parents(const std::string & path) {
 #endif // _WIN32
 }
 
-bool fs_is_directory(const std::string & path) {
-    std::filesystem::path dir(path);
-    return std::filesystem::exists(dir) && std::filesystem::is_directory(dir);
-}
-
 std::string fs_get_cache_directory() {
     std::string cache_directory = "";
     auto ensure_trailing_slash = [](std::string p) {
@@ -920,8 +889,6 @@ std::string fs_get_cache_directory() {
         cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
 #elif defined(_WIN32)
         cache_directory = std::getenv("LOCALAPPDATA");
-#elif defined(__EMSCRIPTEN__)
-        GGML_ABORT("not implemented on this platform");
 #else
 #  error Unknown architecture
 #endif
@@ -941,258 +908,34 @@ std::string fs_get_cache_file(const std::string & filename) {
     return cache_directory + filename;
 }
 
-std::vector<common_file_info> fs_list(const std::string & path, bool include_directories) {
-    std::vector<common_file_info> files;
-    if (path.empty()) return files;
-
-    std::filesystem::path dir(path);
-    if (!std::filesystem::exists(dir) || !std::filesystem::is_directory(dir)) {
-        return files;
-    }
-
-    for (const auto & entry : std::filesystem::directory_iterator(dir)) {
-        try {
-            // Only include regular files (skip directories)
-            const auto & p = entry.path();
-            if (std::filesystem::is_regular_file(p)) {
-                common_file_info info;
-                info.path   = p.string();
-                info.name   = p.filename().string();
-                info.is_dir = false;
-                try {
-                    info.size = static_cast<size_t>(std::filesystem::file_size(p));
-                } catch (const std::filesystem::filesystem_error &) {
-                    info.size = 0;
-                }
-                files.push_back(std::move(info));
-            } else if (include_directories && std::filesystem::is_directory(p)) {
-                common_file_info info;
-                info.path   = p.string();
-                info.name   = p.filename().string();
-                info.size   = 0; // Directories have no size
-                info.is_dir = true;
-                files.push_back(std::move(info));
-            }
-        } catch (const std::filesystem::filesystem_error &) {
-            // skip entries we cannot inspect
-            continue;
-        }
-    }
-
-    return files;
-}
-
-//
-// TTY utils
-//
-
-bool tty_can_use_colors() {
-    // Check NO_COLOR environment variable (https://no-color.org/)
-    if (const char * no_color = std::getenv("NO_COLOR")) {
-        if (no_color[0] != '\0') {
-            return false;
-        }
-    }
-
-    // Check TERM environment variable
-    if (const char * term = std::getenv("TERM")) {
-        if (std::strcmp(term, "dumb") == 0) {
-            return false;
-        }
-    }
-
-    // Check if stdout and stderr are connected to a terminal
-    // We check both because log messages can go to either
-    bool stdout_is_tty = isatty(fileno(stdout));
-    bool stderr_is_tty = isatty(fileno(stderr));
-
-    return stdout_is_tty || stderr_is_tty;
-}
 
 //
 // Model utils
 //
 
-// TODO: move to common/sampling
+struct common_init_result common_init_from_params(common_params & params) {
-static void common_init_sampler_from_model(
+    common_init_result iparams;
-    const llama_model * model,
-    common_params_sampling & sparams) {
-
-    const uint64_t config = sparams.user_sampling_config;
-
-    auto get_int32 = [&](const char * key, int32_t & dst, uint64_t user_config) {
-        if (config & user_config) {
-            return;
-        }
-
-        char buf[64] = {0};
-        if (llama_model_meta_val_str(model, key, buf, sizeof(buf)) > 0) {
-            char * end = nullptr;
-            int32_t v = strtol(buf, &end, 10);
-            if (end && end != buf) {
-                dst = v;
-            }
-        }
-    };
-
-    auto get_float = [&](const char * key, float & dst, uint64_t user_config) {
-        if (config & user_config) {
-            return;
-        }
-
-        char buf[128] = {0};
-        if (llama_model_meta_val_str(model, key, buf, sizeof(buf)) > 0) {
-            char * end = nullptr;
-            float v = strtof(buf, &end);
-            if (end && end != buf) {
-                dst = v;
-            }
-        }
-    };
-
-    // Sampling sequence
-    if (!(config & common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_SAMPLERS)) {
-        char buf[512] = {0};
-        if (llama_model_meta_val_str(model, llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE), buf, sizeof(buf)) > 0) {
-            const std::vector<std::string> sampler_names = string_split<std::string>(std::string(buf), ';');
-            if (!sampler_names.empty()) {
-                sparams.samplers = common_sampler_types_from_names(sampler_names, true);
-            }
-        }
-    }
-
-    get_int32(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_TOP_K),           sparams.top_k,           common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TOP_K);
-    get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_TOP_P),           sparams.top_p,           common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TOP_P);
-    get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_MIN_P),           sparams.min_p,           common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIN_P);
-    get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY), sparams.xtc_probability, common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_XTC_PROBABILITY);
-    get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD),   sparams.xtc_threshold,   common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_XTC_THRESHOLD);
-    get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_TEMP),            sparams.temp,            common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TEMP);
-    get_int32(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N),  sparams.penalty_last_n,  common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_LAST_N);
-    get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT),  sparams.penalty_repeat,  common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT);
-    get_int32(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT),        sparams.mirostat,        common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT);
-    get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU),    sparams.mirostat_tau,    common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_TAU);
-    get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA),    sparams.mirostat_eta,    common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_ETA);
-}
-
-struct common_init_result::impl {
-    impl() = default;
-    ~impl() = default;
-
-    llama_model_ptr   model;
-    llama_context_ptr context;
-
-    std::vector<llama_adapter_lora_ptr> lora;
-
-    std::vector<common_sampler_ptr> samplers;
-};
-
-common_init_result::common_init_result(common_params & params) :
-    pimpl(new impl{}) {
     auto mparams = common_model_params_to_llama(params);
-    auto cparams = common_context_params_to_llama(params);
-
-    if (params.fit_params) {
-        LOG_INF("%s: fitting params to device memory, to report bugs during this step use -fit off (or --verbose if you can't)\n", __func__);
-        llama_params_fit(params.model.path.c_str(), &mparams, &cparams,
-            params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target, params.fit_params_min_ctx,
-            params.verbosity >= 4 ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
-    }
 
     llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
     if (model == NULL) {
-        return;
+        LOG_ERR("%s: failed to load model '%s', try reducing --n-gpu-layers if you're running out of VRAM\n",
+            __func__, params.model.path.c_str());
+        return iparams;
     }
 
-    pimpl->model.reset(model);
-
     const llama_vocab * vocab = llama_model_get_vocab(model);
 
-    // updates params.sampling
+    auto cparams = common_context_params_to_llama(params);
-    // TODO: fix naming
-    common_init_sampler_from_model(model, params.sampling);
-
-    if (params.sampling.ignore_eos && llama_vocab_eos(vocab) == LLAMA_TOKEN_NULL) {
-        LOG_WRN("%s: warning: vocab does not have an EOS token, ignoring --ignore-eos\n", __func__);
-        params.sampling.ignore_eos = false;
-    }
-
-    // initialize once
-    for (llama_token i = 0; i < llama_vocab_n_tokens(vocab); i++) {
-        if (llama_vocab_is_eog(vocab, i)) {
-            LOG_INF("%s: added %s logit bias = %f\n", __func__, common_token_to_piece(vocab, i).c_str(), -INFINITY);
-            params.sampling.logit_bias_eog.push_back({i, -INFINITY});
-        }
-    }
-
-    if (params.sampling.ignore_eos) {
-        // add EOG biases to the active set of logit biases
-        params.sampling.logit_bias.insert(
-                params.sampling.logit_bias.end(),
-                params.sampling.logit_bias_eog.begin(), params.sampling.logit_bias_eog.end());
-    }
-
-    //if (params.sampling.penalty_last_n == -1) {
-    //    LOG_INF("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
-    //    params.sampling.penalty_last_n = llama_n_ctx(lctx);
-    //}
-
-    //if (params.sampling.dry_penalty_last_n == -1) {
-    //    LOG_INF("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
-    //    params.sampling.dry_penalty_last_n = llama_n_ctx(lctx);
-    //}
-
-    pimpl->samplers.resize(cparams.n_seq_max);
-
-    for (int i = 0; i < (int) cparams.n_seq_max; ++i) {
-        pimpl->samplers[i].reset(common_sampler_init(model, params.sampling));
-    }
 
     llama_context * lctx = llama_init_from_model(model, cparams);
     if (lctx == NULL) {
-        LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str());
+        LOG_ERR("%s: failed to create context with model '%s', try reducing --n-gpu-layers if you're running out of VRAM\n",
-        return;
+            __func__, params.model.path.c_str());
+        llama_model_free(model);
+        return iparams;
     }
 
-    pimpl->context.reset(lctx);
-}
-
-llama_model * common_init_result::model() {
-    return pimpl->model.get();
-}
-
-llama_context * common_init_result::context() {
-    return pimpl->context.get();
-}
-
-common_sampler * common_init_result::sampler(llama_seq_id seq_id) {
-    return pimpl->samplers[seq_id].get();
-}
-
-std::vector<llama_adapter_lora_ptr> & common_init_result::lora() {
-    return pimpl->lora;
-}
-
-void common_init_result::free_context() {
-    pimpl->context.reset();
-}
-
-common_init_result_ptr common_init_from_params(common_params & params) {
-    common_init_result_ptr res(new common_init_result(params));
-
-    llama_model * model = res->model();
-    if (model == NULL) {
-        LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());
-        return res;
-    }
-
-    llama_context * lctx = res->context();
-    if (lctx == NULL) {
-        LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str());
-        return res;
-    }
-
-    const llama_vocab * vocab = llama_model_get_vocab(model);
-
     if (params.ctx_shift && !llama_memory_can_shift(llama_get_memory(lctx))) {
         LOG_WRN("%s: KV cache shifting is not supported for this context, disabling KV cache shifting\n", __func__);
         params.ctx_shift = false;
@@ -1204,7 +947,10 @@ common_init_result_ptr common_init_from_params(common_params & params) {
 
         const auto cvec = common_control_vector_load(params.control_vectors);
         if (cvec.n_embd == -1) {
-            return res;
+            llama_free(lctx);
+            llama_model_free(model);
+
+            return iparams;
         }
 
         int err = llama_apply_adapter_cvec(
@@ -1215,7 +961,10 @@ common_init_result_ptr common_init_from_params(common_params & params) {
                 params.control_vector_layer_start,
                 params.control_vector_layer_end);
         if (err) {
-            return res;
+            llama_free(lctx);
+            llama_model_free(model);
+
+            return iparams;
         }
     }
 
@@ -1239,7 +988,10 @@ common_init_result_ptr common_init_from_params(common_params & params) {
         }
 
         if (!ok) {
-            return res;
+            llama_free(lctx);
+            llama_model_free(model);
+
+            return iparams;
         }
     }
 
@@ -1249,7 +1001,9 @@ common_init_result_ptr common_init_from_params(common_params & params) {
         lora.reset(llama_adapter_lora_init(model, la.path.c_str()));
         if (lora == nullptr) {
             LOG_ERR("%s: failed to apply lora adapter '%s'\n", __func__, la.path.c_str());
-            return res;
+            llama_free(lctx);
+            llama_model_free(model);
+            return iparams;
         }
 
         char buf[1024];
@@ -1258,13 +1012,43 @@ common_init_result_ptr common_init_from_params(common_params & params) {
         la.task_name = buf;
         llama_adapter_meta_val_str(la.ptr, "adapter.lora.prompt_prefix", buf, sizeof(buf));
         la.prompt_prefix = buf;
-        res->lora().emplace_back(std::move(lora)); // copy to list of loaded adapters
+        iparams.lora.emplace_back(std::move(lora)); // copy to list of loaded adapters
     }
 
     if (!params.lora_init_without_apply) {
         common_set_adapter_lora(lctx, params.lora_adapters);
     }
 
+    if (params.sampling.ignore_eos && llama_vocab_eos(vocab) == LLAMA_TOKEN_NULL) {
+        LOG_WRN("%s: warning: vocab does not have an EOS token, ignoring --ignore-eos\n", __func__);
+        params.sampling.ignore_eos = false;
+    }
+
+    // initialize once
+    for (llama_token i = 0; i < llama_vocab_n_tokens(vocab); i++) {
+        if (llama_vocab_is_eog(vocab, i)) {
+            LOG_INF("%s: added %s logit bias = %f\n", __func__, common_token_to_piece(lctx, i).c_str(), -INFINITY);
+            params.sampling.logit_bias_eog.push_back({i, -INFINITY});
+        }
+    }
+
+    if (params.sampling.ignore_eos) {
+        // add EOG biases to the active set of logit biases
+        params.sampling.logit_bias.insert(
+                params.sampling.logit_bias.end(),
+                params.sampling.logit_bias_eog.begin(), params.sampling.logit_bias_eog.end());
+    }
+
+    if (params.sampling.penalty_last_n == -1) {
+        LOG_INF("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
+        params.sampling.penalty_last_n = llama_n_ctx(lctx);
+    }
+
+    if (params.sampling.dry_penalty_last_n == -1) {
+        LOG_INF("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
+        params.sampling.dry_penalty_last_n = llama_n_ctx(lctx);
+    }
+
     if (params.warmup) {
         LOG_WRN("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__);
 
@@ -1303,10 +1087,11 @@ common_init_result_ptr common_init_from_params(common_params & params) {
         llama_set_warmup(lctx, false);
     }
 
-    return res;
+    iparams.model.reset(model);
-}
+    iparams.context.reset(lctx);
 
-common_init_result::~common_init_result() = default;
+    return iparams;
+}
 
 std::string get_model_endpoint() {
     const char * model_endpoint_env = getenv("MODEL_ENDPOINT");
@@ -1316,9 +1101,7 @@ std::string get_model_endpoint() {
     std::string model_endpoint = "https://huggingface.co/";
     if (endpoint_env) {
         model_endpoint = endpoint_env;
-        if (model_endpoint.back() != '/') {
+        if (model_endpoint.back() != '/') model_endpoint += '/';
-            model_endpoint += '/';
-        }
     }
     return model_endpoint;
 }

llama/llama.cpp/common/common.h

@@ -2,19 +2,17 @@
 
 #pragma once
 
-#include "ggml-opt.h"
-#include "llama-cpp.h"
-
 #include <set>
 #include <sstream>
 #include <string>
 #include <string_view>
 #include <vector>
 #include <map>
+#include <sstream>
+#include <cmath>
 
-#if defined(_WIN32) && !defined(_WIN32_WINNT)
+#include "ggml-opt.h"
-#define _WIN32_WINNT 0x0A00
+#include "llama-cpp.h"
-#endif
 
 #ifdef _WIN32
 #define DIRECTORY_SEPARATOR '\\'
@@ -30,14 +28,7 @@
     fprintf(stderr, "%s: built with %s for %s\n", __func__, LLAMA_COMPILER, LLAMA_BUILD_TARGET);    \
 } while(0)
 
-struct common_time_meas {
+#define DEFAULT_MODEL_PATH "models/7B/ggml-model-f16.gguf"
-    common_time_meas(int64_t & t_acc, bool disable = false);
-    ~common_time_meas();
-
-    const int64_t t_start_us;
-
-    int64_t & t_acc;
-};
 
 struct common_adapter_lora_info {
     std::string path;
@@ -82,8 +73,7 @@ int32_t cpu_get_num_math();
 enum llama_example {
     LLAMA_EXAMPLE_COMMON,
     LLAMA_EXAMPLE_SPECULATIVE,
-    LLAMA_EXAMPLE_COMPLETION,
+    LLAMA_EXAMPLE_MAIN,
-    LLAMA_EXAMPLE_CLI,
     LLAMA_EXAMPLE_EMBEDDING,
     LLAMA_EXAMPLE_PERPLEXITY,
     LLAMA_EXAMPLE_RETRIEVAL,
@@ -99,7 +89,6 @@ enum llama_example {
     LLAMA_EXAMPLE_TTS,
     LLAMA_EXAMPLE_DIFFUSION,
     LLAMA_EXAMPLE_FINETUNE,
-    LLAMA_EXAMPLE_FIT_PARAMS,
 
     LLAMA_EXAMPLE_COUNT,
 };
@@ -144,22 +133,6 @@ struct common_grammar_trigger {
     llama_token token = LLAMA_TOKEN_NULL;
 };
 
-enum common_params_sampling_config : uint64_t {
-    COMMON_PARAMS_SAMPLING_CONFIG_SAMPLERS        = 1 << 0,
-    COMMON_PARAMS_SAMPLING_CONFIG_TOP_K           = 1 << 1,
-    COMMON_PARAMS_SAMPLING_CONFIG_TOP_P           = 1 << 2,
-    COMMON_PARAMS_SAMPLING_CONFIG_MIN_P           = 1 << 3,
-    COMMON_PARAMS_SAMPLING_CONFIG_XTC_PROBABILITY = 1 << 4,
-    COMMON_PARAMS_SAMPLING_CONFIG_XTC_THRESHOLD   = 1 << 5,
-    COMMON_PARAMS_SAMPLING_CONFIG_TEMP            = 1 << 6,
-    COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_LAST_N  = 1 << 7,
-    COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT  = 1 << 8,
-    COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT        = 1 << 9,
-    COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_TAU    = 1 << 10,
-    COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_ETA    = 1 << 11,
-};
-
-
 // sampling parameters
 struct common_params_sampling {
     uint32_t seed = LLAMA_DEFAULT_SEED; // the seed used to initialize llama_sampler
@@ -192,10 +165,9 @@ struct common_params_sampling {
     bool    no_perf            = false; // disable performance metrics
     bool    timing_per_token   = false;
 
-    uint64_t user_sampling_config = 0; // bitfield to track user-specified samplers
-
     std::vector<std::string> dry_sequence_breakers = {"\n", ":", "\"", "*"};     // default sequence breakers for DRY
 
+
     std::vector<enum common_sampler_type> samplers = {
         COMMON_SAMPLER_TYPE_PENALTIES,
         COMMON_SAMPLER_TYPE_DRY,
@@ -216,10 +188,6 @@ struct common_params_sampling {
     std::vector<llama_logit_bias> logit_bias;     // logit biases to apply
     std::vector<llama_logit_bias> logit_bias_eog; // pre-calculated logit biases for EOG tokens
 
-    bool has_logit_bias() const {
-        return !logit_bias.empty();
-    }
-
     // print the parameters into a string
     std::string print() const;
 };
@@ -230,7 +198,6 @@ struct common_params_model {
     std::string hf_repo     = ""; // HF repo                                                // NOLINT
     std::string hf_file     = ""; // HF file                                                // NOLINT
     std::string docker_repo = ""; // Docker repo                                            // NOLINT
-    std::string name        = ""; // in format <user>/<model>[:<tag>] (tag is optional)     // NOLINT
 };
 
 struct common_params_speculative {
@@ -307,8 +274,8 @@ struct lr_opt {
 struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);
 
 struct common_params {
-    int32_t n_predict             =    -1; // max. number of new tokens to predict, -1 == no limit
+    int32_t n_predict             =    -1; // new tokens to predict
-    int32_t n_ctx                 =     0; // context size, 0 == context the model was trained with
+    int32_t n_ctx                 =  4096; // context size
     int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
     int32_t n_ubatch              =   512; // physical batch size for prompt processing (must be >=32 to use BLAS)
     int32_t n_keep                =     0; // number of tokens to keep from initial prompt
@@ -329,12 +296,9 @@ struct common_params {
     // offload params
     std::vector<ggml_backend_dev_t> devices; // devices to use for offloading
 
-    int32_t n_gpu_layers       = -1;               // number of layers to store in VRAM (-1 - use default)
+    int32_t n_gpu_layers      = -1;  // number of layers to store in VRAM (-1 - use default)
-    int32_t main_gpu           = 0;                // the GPU that is used for scratch and small tensors
+    int32_t main_gpu          = 0;   // the GPU that is used for scratch and small tensors
-    float   tensor_split[128]  = {0};              // how split tensors should be distributed across GPUs
+    float   tensor_split[128] = {0}; // how split tensors should be distributed across GPUs
-    bool    fit_params         = true;             // whether to fit unset model/context parameters to free device memory
-    size_t  fit_params_target  = 1024 * 1024*1024; // margin per device in bytes for fitting parameters to free memory
-    int32_t fit_params_min_ctx = 4096;             // minimum context size to set when trying to reduce memory use
 
     enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
 
@@ -380,7 +344,7 @@ struct common_params {
 
     std::vector<common_control_vector_load_info> control_vectors; // control vector with user defined scale
 
-    int32_t verbosity                  = 3;  // LOG_LEVEL_INFO
+    int32_t verbosity                  = 0;
     int32_t control_vector_layer_start = -1; // layer range for control vector
     int32_t control_vector_layer_end   = -1; // layer range for control vector
     bool    offline                    = false;
@@ -414,7 +378,6 @@ struct common_params {
     bool simple_io         = false; // improves compatibility with subprocesses and limited consoles
     bool cont_batching     = true;  // insert new sequences for decoding on-the-fly
     bool no_perf           = false; // disable performance metrics
-    bool show_timings      = true;  // show timing information on CLI
     bool ctx_shift         = false; // context shift on infinite text generation
     bool swa_full          = false; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
     bool kv_unified        = false; // enable unified KV cache
@@ -443,8 +406,6 @@ struct common_params {
     bool mmproj_use_gpu = true;     // use GPU for multimodal model
     bool no_mmproj = false;         // explicitly disable multimodal model
     std::vector<std::string> image; // path to image file(s)
-    int image_min_tokens = -1;
-    int image_max_tokens = -1;
 
     // finetune
     struct lr_opt lr;
@@ -471,7 +432,7 @@ struct common_params {
     std::string public_path   = "";                                                                         // NOLINT
     std::string api_prefix    = "";                                                                         // NOLINT
     std::string chat_template = "";                                                                         // NOLINT
-    bool use_jinja = true;                                                                                  // NOLINT
+    bool use_jinja = false;                                                                                 // NOLINT
     bool enable_chat_template = true;
     common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
     int reasoning_budget = -1;
@@ -490,22 +451,14 @@ struct common_params {
     bool endpoint_props   = false; // only control POST requests, not GET
     bool endpoint_metrics = false;
 
-    // router server configs
-    std::string models_dir    = ""; // directory containing models for the router server
-    std::string models_preset = ""; // directory containing model presets for the router server
-    int models_max = 4;             // maximum number of models to load simultaneously
-    bool models_autoload = true;    // automatically load models when requested via the router server
-
     bool log_json = false;
 
     std::string slot_save_path;
-    std::string media_path; // path to directory for loading media files
 
     float slot_prompt_similarity = 0.1f;
 
     // batched-bench params
-    bool is_pp_shared   = false;
+    bool is_pp_shared = false;
-    bool is_tg_separate = false;
 
     std::vector<int32_t> n_pp;
     std::vector<int32_t> n_tg;
@@ -552,10 +505,6 @@ struct common_params {
     // return false from callback to abort model loading or true to continue
     llama_progress_callback load_progress_callback = NULL;
     void *                  load_progress_callback_user_data = NULL;
-
-    bool has_speculative() const {
-        return !speculative.model.path.empty() || !speculative.model.hf_repo.empty();
-    }
 };
 
 // call once at the start of a program if it uses libcommon
@@ -650,55 +599,25 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat
 // Filesystem utils
 //
 
-bool fs_validate_filename(const std::string & filename, bool allow_subdirs = false);
+bool fs_validate_filename(const std::string & filename);
 bool fs_create_directory_with_parents(const std::string & path);
-bool fs_is_directory(const std::string & path);
 
 std::string fs_get_cache_directory();
 std::string fs_get_cache_file(const std::string & filename);
 
-struct common_file_info {
-    std::string path;
-    std::string name;
-    size_t      size = 0; // in bytes
-    bool        is_dir = false;
-};
-std::vector<common_file_info> fs_list(const std::string & path, bool include_directories);
-
-//
-// TTY utils
-//
-
-// Auto-detect if colors can be enabled based on terminal and environment
-bool tty_can_use_colors();
-
 //
 // Model utils
 //
 
-struct common_sampler;
+// note: defines object's lifetime
-
-// note: defines the model, context, samplers, ets. lifetimes
 struct common_init_result {
-    common_init_result(common_params & params);
+    llama_model_ptr   model;
-    ~common_init_result();
+    llama_context_ptr context;
 
-    llama_model * model();
+    std::vector<llama_adapter_lora_ptr> lora;
-    llama_context * context();
-    common_sampler * sampler(llama_seq_id seq_id);
-
-    std::vector<llama_adapter_lora_ptr> & lora();
-
-    void free_context();
-
-private:
-    struct impl;
-    std::unique_ptr<impl> pimpl;
 };
 
-using common_init_result_ptr = std::unique_ptr<common_init_result>;
+struct common_init_result     common_init_from_params(common_params & params);
-
-common_init_result_ptr common_init_from_params(common_params & params);
 
 struct llama_model_params     common_model_params_to_llama  (      common_params & params);
 struct llama_context_params   common_context_params_to_llama(const common_params & params);

llama/llama.cpp/common/json-schema-to-grammar.cpp

@@ -268,10 +268,10 @@ static bool is_reserved_name(const std::string & name) {
 }
 
 std::regex INVALID_RULE_CHARS_RE("[^a-zA-Z0-9-]+");
-std::regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"\\\\]");
+std::regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"]");
 std::regex GRAMMAR_RANGE_LITERAL_ESCAPE_RE("[\r\n\"\\]\\-\\\\]");
 std::unordered_map<char, std::string> GRAMMAR_LITERAL_ESCAPES = {
-    {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}, {'\\', "\\\\"}
+    {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}
 };
 
 std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'};
@@ -303,11 +303,8 @@ static std::string format_literal(const std::string & literal) {
     return "\"" + escaped + "\"";
 }
 
-std::string gbnf_format_literal(const std::string & literal) { return format_literal(literal); }
+class SchemaConverter {
-
-class common_schema_converter {
 private:
-    friend class common_schema_info;
     friend std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options);
     std::function<json(const std::string &)> _fetch_json;
     bool _dotall;
@@ -604,10 +601,7 @@ private:
     }
 
     std::string _resolve_ref(const std::string & ref) {
-        auto it = ref.find('#');
+        std::string ref_name = ref.substr(ref.find_last_of('/') + 1);
-        std::string ref_fragment = it != std::string::npos ? ref.substr(it + 1) : ref;
-        static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)");
-        std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-");
         if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) {
             _refs_being_resolved.insert(ref);
             json resolved = _refs[ref];
@@ -730,7 +724,7 @@ private:
     }
 
 public:
-    common_schema_converter(
+    SchemaConverter(
         const std::function<json(const std::string &)> & fetch_json,
         bool dotall)
           : _fetch_json(fetch_json), _dotall(dotall)
@@ -780,24 +774,11 @@ public:
                         std::vector<std::string> tokens = string_split(pointer, "/");
                         for (size_t i = 1; i < tokens.size(); ++i) {
                             std::string sel = tokens[i];
-                            if (target.is_object() && target.contains(sel)) {
+                            if (target.is_null() || !target.contains(sel)) {
-                                target = target[sel];
-                            } else if (target.is_array()) {
-                                size_t sel_index;
-                                try {
-                                    sel_index = std::stoul(sel);
-                                } catch (const std::invalid_argument & e) {
-                                    sel_index = target.size();
-                                }
-                                if (sel_index >= target.size()) {
-                                    _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
-                                    return;
-                                }
-                                target = target[sel_index];
-                            } else {
                                 _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
                                 return;
                             }
+                            target = target[sel];
                         }
                         _refs[ref] = target;
                     }
@@ -975,7 +956,7 @@ public:
 
     void check_errors() {
         if (!_errors.empty()) {
-            throw std::invalid_argument("JSON schema conversion failed:\n" + string_join(_errors, "\n"));
+            throw std::runtime_error("JSON schema conversion failed:\n" + string_join(_errors, "\n"));
         }
         if (!_warnings.empty()) {
             fprintf(stderr, "WARNING: JSON schema conversion was incomplete: %s\n", string_join(_warnings, "; ").c_str());
@@ -991,134 +972,6 @@ public:
     }
 };
 
-// common_schema_info implementation (pimpl)
-
-common_schema_info::common_schema_info()
-    : impl_(std::make_unique<common_schema_converter>(
-        [](const std::string &) { return json(); },
-        false)) {}
-
-common_schema_info::~common_schema_info() = default;
-
-common_schema_info::common_schema_info(common_schema_info &&) noexcept = default;
-common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default;
-
-void common_schema_info::resolve_refs(nlohmann::ordered_json & schema) {
-    impl_->resolve_refs(schema, "");
-}
-
-// Determines if a JSON schema can resolve to a string type through any path.
-// Some models emit raw string values rather than JSON-encoded strings for string parameters.
-// If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns
-// true, allowing callers to handle the value as a raw string for simplicity.
-bool common_schema_info::resolves_to_string(const nlohmann::ordered_json & schema) {
-    std::unordered_set<std::string> visited_refs;
-
-    std::function<bool(const json &)> check = [&](const json & s) -> bool {
-        if (!s.is_object()) {
-            return false;
-        }
-
-        // Handle $ref
-        if (s.contains("$ref")) {
-            const std::string & ref = s["$ref"];
-            if (visited_refs.find(ref) != visited_refs.end()) {
-                // Circular reference, assume not a string to be safe
-                return false;
-            }
-            visited_refs.insert(ref);
-            auto it = impl_->_refs.find(ref);
-            if (it != impl_->_refs.end()) {
-                return check(it->second);
-            }
-            return false;
-        }
-
-        // Check type field
-        if (s.contains("type")) {
-            const json & schema_type = s["type"];
-            if (schema_type.is_string()) {
-                if (schema_type == "string") {
-                    return true;
-                }
-            } else if (schema_type.is_array()) {
-                // Type can be an array like ["string", "null"]
-                for (const auto & t : schema_type) {
-                    if (t == "string") {
-                        return true;
-                    }
-                }
-            }
-        }
-
-        // Check oneOf/anyOf - if any alternative can be a string
-        if (s.contains("oneOf")) {
-            for (const auto & alt : s["oneOf"]) {
-                if (check(alt)) {
-                    return true;
-                }
-            }
-        }
-        if (s.contains("anyOf")) {
-            for (const auto & alt : s["anyOf"]) {
-                if (check(alt)) {
-                    return true;
-                }
-            }
-        }
-
-        // Check allOf - all components must be compatible with string type
-        if (s.contains("allOf")) {
-            bool all_string = true;
-            for (const auto & component : s["allOf"]) {
-                if (!check(component)) {
-                    all_string = false;
-                    break;
-                }
-            }
-            if (all_string) {
-                return true;
-            }
-        }
-
-        // Check const - if the constant value is a string
-        if (s.contains("const")) {
-            if (s["const"].is_string()) {
-                return true;
-            }
-        }
-
-        // Check enum - if any enum value is a string
-        if (s.contains("enum")) {
-            for (const auto & val : s["enum"]) {
-                if (val.is_string()) {
-                    return true;
-                }
-            }
-        }
-
-        // String-specific keywords imply string type
-        if (s.contains("pattern") || s.contains("minLength") || s.contains("maxLength")) {
-            return true;
-        }
-
-        // Check format - many formats imply string
-        if (s.contains("format")) {
-            const std::string & fmt = s["format"];
-            if (fmt == "date" || fmt == "time" || fmt == "date-time" ||
-                fmt == "uri" || fmt == "email" || fmt == "hostname" ||
-                fmt == "ipv4" || fmt == "ipv6" || fmt == "uuid" ||
-                fmt.find("uuid") == 0) {
-                return true;
-            }
-        }
-
-        return false;
-    };
-
-    return check(schema);
-}
-
 std::string json_schema_to_grammar(const json & schema, bool force_gbnf) {
 #ifdef LLAMA_USE_LLGUIDANCE
     if (!force_gbnf) {
@@ -1135,7 +988,7 @@ std::string json_schema_to_grammar(const json & schema, bool force_gbnf) {
 }
 
 std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options) {
-    common_schema_converter converter([&](const std::string &) { return json(); }, options.dotall);
+    SchemaConverter converter([&](const std::string &) { return json(); }, options.dotall);
     common_grammar_builder builder {
         /* .add_rule = */ [&](const std::string & name, const std::string & rule) {
             return converter._add_rule(name, rule);

llama/llama.cpp/common/json-schema-to-grammar.h

@@ -3,31 +3,11 @@
 #include <nlohmann/json_fwd.hpp>
 
 #include <functional>
-#include <memory>
 #include <string>
 
 std::string json_schema_to_grammar(const nlohmann::ordered_json & schema,
                                    bool force_gbnf = false);
 
-class common_schema_converter;
-
-// Probes a JSON schema to extract information about its structure and type constraints.
-class common_schema_info {
-    std::unique_ptr<common_schema_converter> impl_;
-
-  public:
-    common_schema_info();
-    ~common_schema_info();
-
-    common_schema_info(const common_schema_info &) = delete;
-    common_schema_info & operator=(const common_schema_info &) = delete;
-    common_schema_info(common_schema_info &&) noexcept;
-    common_schema_info & operator=(common_schema_info &&) noexcept;
-
-    void resolve_refs(nlohmann::ordered_json & schema);
-    bool resolves_to_string(const nlohmann::ordered_json & schema);
-};
-
 struct common_grammar_builder {
     std::function<std::string(const std::string &, const std::string &)> add_rule;
     std::function<std::string(const std::string &, const nlohmann::ordered_json &)> add_schema;
@@ -38,6 +18,4 @@ struct common_grammar_options {
     bool dotall = false;
 };
 
-std::string gbnf_format_literal(const std::string & literal);
-
 std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options = {});

llama/llama.cpp/common/log.cpp

@@ -1,4 +1,3 @@
-#include "common.h"
 #include "log.h"
 
 #include <chrono>
@@ -27,6 +26,30 @@ void common_log_set_verbosity_thold(int verbosity) {
     common_log_verbosity_thold = verbosity;
 }
 
+// Auto-detect if colors should be enabled based on terminal and environment
+static bool common_log_should_use_colors_auto() {
+    // Check NO_COLOR environment variable (https://no-color.org/)
+    if (const char * no_color = std::getenv("NO_COLOR")) {
+        if (no_color[0] != '\0') {
+            return false;
+        }
+    }
+
+    // Check TERM environment variable
+    if (const char * term = std::getenv("TERM")) {
+        if (std::strcmp(term, "dumb") == 0) {
+            return false;
+        }
+    }
+
+    // Check if stdout and stderr are connected to a terminal
+    // We check both because log messages can go to either
+    bool stdout_is_tty = isatty(fileno(stdout));
+    bool stderr_is_tty = isatty(fileno(stderr));
+
+    return stdout_is_tty || stderr_is_tty;
+}
+
 static int64_t t_us() {
     return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 }
@@ -368,7 +391,7 @@ struct common_log * common_log_main() {
     static std::once_flag    init_flag;
     std::call_once(init_flag, [&]() {
         // Set default to auto-detect colors
-        log.set_colors(tty_can_use_colors());
+        log.set_colors(common_log_should_use_colors_auto());
     });
 
     return &log;
@@ -399,7 +422,7 @@ void common_log_set_file(struct common_log * log, const char * file) {
 
 void common_log_set_colors(struct common_log * log, log_colors colors) {
     if (colors == LOG_COLORS_AUTO) {
-        log->set_colors(tty_can_use_colors());
+        log->set_colors(common_log_should_use_colors_auto());
         return;
     }
 
@@ -419,28 +442,3 @@ void common_log_set_prefix(struct common_log * log, bool prefix) {
 void common_log_set_timestamps(struct common_log * log, bool timestamps) {
     log->set_timestamps(timestamps);
 }
-
-void common_log_flush(struct common_log * log) {
-    log->pause();
-    log->resume();
-}
-
-static int common_get_verbosity(enum ggml_log_level level) {
-    switch (level) {
-        case GGML_LOG_LEVEL_DEBUG: return LOG_LEVEL_DEBUG;
-        case GGML_LOG_LEVEL_INFO:  return LOG_LEVEL_INFO;
-        case GGML_LOG_LEVEL_WARN:  return LOG_LEVEL_WARN;
-        case GGML_LOG_LEVEL_ERROR: return LOG_LEVEL_ERROR;
-        case GGML_LOG_LEVEL_CONT:  return LOG_LEVEL_INFO; // same as INFO
-        case GGML_LOG_LEVEL_NONE:
-        default:
-            return LOG_LEVEL_OUTPUT;
-    }
-}
-
-void common_log_default_callback(enum ggml_log_level level, const char * text, void * /*user_data*/) {
-    auto verbosity = common_get_verbosity(level);
-    if (verbosity <= common_log_verbosity_thold) {
-        common_log_add(common_log_main(), level, "%s", text);
-    }
-}

llama/llama.cpp/common/log.h

@@ -21,14 +21,8 @@
 #    define LOG_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
 #endif
 
-#define LOG_LEVEL_DEBUG  4
+#define LOG_DEFAULT_DEBUG 1
-#define LOG_LEVEL_INFO   3
+#define LOG_DEFAULT_LLAMA 0
-#define LOG_LEVEL_WARN   2
-#define LOG_LEVEL_ERROR  1
-#define LOG_LEVEL_OUTPUT 0 // output data from tools
-
-#define LOG_DEFAULT_DEBUG LOG_LEVEL_DEBUG
-#define LOG_DEFAULT_LLAMA LOG_LEVEL_INFO
 
 enum log_colors {
     LOG_COLORS_AUTO     = -1,
@@ -42,8 +36,6 @@ extern int common_log_verbosity_thold;
 
 void common_log_set_verbosity_thold(int verbosity); // not thread-safe
 
-void common_log_default_callback(enum ggml_log_level level, const char * text, void * user_data);
-
 // the common_log uses an internal worker thread to print/write log messages
 // when the worker thread is paused, incoming log messages are discarded
 struct common_log;
@@ -73,18 +65,16 @@ void common_log_add(struct common_log * log, enum ggml_log_level level, const ch
 //   0.00.090.578 I llm_load_tensors: offloading 32 repeating layers to GPU
 //   0.00.090.579 I llm_load_tensors: offloading non-repeating layers to GPU
 //
+// I - info    (stdout, V = 0)
+// W - warning (stderr, V = 0)
+// E - error   (stderr, V = 0)
 // D - debug   (stderr, V = LOG_DEFAULT_DEBUG)
-// I - info    (stdout, V = LOG_DEFAULT_INFO)
-// W - warning (stderr, V = LOG_DEFAULT_WARN)
-// E - error   (stderr, V = LOG_DEFAULT_ERROR)
-// O - output  (stdout, V = LOG_DEFAULT_OUTPUT)
 //
 
 void common_log_set_file      (struct common_log * log, const char * file); // not thread-safe
 void common_log_set_colors    (struct common_log * log, log_colors colors); // not thread-safe
 void common_log_set_prefix    (struct common_log * log, bool prefix);       // whether to output prefix to each log
 void common_log_set_timestamps(struct common_log * log, bool timestamps);   // whether to output timestamps in the prefix
-void common_log_flush         (struct common_log * log);                    // flush all pending log messages
 
 // helper macros for logging
 // use these to avoid computing log arguments if the verbosity of the log is higher than the threshold
@@ -103,14 +93,14 @@ void common_log_flush         (struct common_log * log);                    // f
         } \
     } while (0)
 
-#define LOG(...)             LOG_TMPL(GGML_LOG_LEVEL_NONE, LOG_LEVEL_OUTPUT, __VA_ARGS__)
+#define LOG(...)             LOG_TMPL(GGML_LOG_LEVEL_NONE, 0,         __VA_ARGS__)
-#define LOGV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_NONE, verbosity,        __VA_ARGS__)
+#define LOGV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_NONE, verbosity, __VA_ARGS__)
 
-#define LOG_DBG(...) LOG_TMPL(GGML_LOG_LEVEL_DEBUG, LOG_LEVEL_DEBUG,  __VA_ARGS__)
+#define LOG_INF(...) LOG_TMPL(GGML_LOG_LEVEL_INFO,  0,                 __VA_ARGS__)
-#define LOG_INF(...) LOG_TMPL(GGML_LOG_LEVEL_INFO,  LOG_LEVEL_INFO,   __VA_ARGS__)
+#define LOG_WRN(...) LOG_TMPL(GGML_LOG_LEVEL_WARN,  0,                 __VA_ARGS__)
-#define LOG_WRN(...) LOG_TMPL(GGML_LOG_LEVEL_WARN,  LOG_LEVEL_WARN,   __VA_ARGS__)
+#define LOG_ERR(...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, 0,                 __VA_ARGS__)
-#define LOG_ERR(...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, LOG_LEVEL_ERROR,  __VA_ARGS__)
+#define LOG_DBG(...) LOG_TMPL(GGML_LOG_LEVEL_DEBUG, LOG_DEFAULT_DEBUG, __VA_ARGS__)
-#define LOG_CNT(...) LOG_TMPL(GGML_LOG_LEVEL_CONT,  LOG_LEVEL_INFO,   __VA_ARGS__) // same as INFO
+#define LOG_CNT(...) LOG_TMPL(GGML_LOG_LEVEL_CONT,  0,                 __VA_ARGS__)
 
 #define LOG_INFV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_INFO,  verbosity, __VA_ARGS__)
 #define LOG_WRNV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_WARN,  verbosity, __VA_ARGS__)

llama/llama.cpp/common/sampling.cpp

@@ -3,10 +3,9 @@
 #include "common.h"
 #include "log.h"
 
-#include <algorithm>
 #include <cmath>
-#include <cstring>
 #include <unordered_map>
+#include <algorithm>
 
 // the ring buffer works similarly to std::deque, but with a fixed capacity
 // TODO: deduplicate with llama-impl.h
@@ -104,22 +103,15 @@ struct ring_buffer {
 struct common_sampler {
     common_params_sampling params;
 
+    struct llama_sampler * grmr;
     struct llama_sampler * chain;
 
-    bool grammar;
-
     ring_buffer<llama_token> prev;
 
     std::vector<llama_token_data> cur;
 
     llama_token_data_array cur_p;
 
-    void reset() {
-        prev.clear();
-
-        llama_sampler_reset(chain);
-    }
-
     void set_logits(struct llama_context * ctx, int idx) {
         const auto * logits = llama_get_logits_ith(ctx, idx);
 
@@ -136,12 +128,6 @@ struct common_sampler {
 
         cur_p = { cur.data(), cur.size(), -1, false };
     }
-
-    common_time_meas tm() {
-        return common_time_meas(t_total_us, params.no_perf);
-    }
-
-    mutable int64_t t_total_us = 0;
 };
 
 std::string common_params_sampling::print() const {
@@ -167,15 +153,10 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, co
 
     lparams.no_perf = params.no_perf;
 
-    llama_sampler * chain = llama_sampler_chain_init(lparams);
+    struct llama_sampler * grmr;
-
-    bool grammar = false;
-    std::vector<llama_sampler *> samplers;
-
     if (params.grammar.compare(0, 11, "%llguidance") == 0) {
 #ifdef LLAMA_USE_LLGUIDANCE
-        samplers.push_back(llama_sampler_init_llg(vocab, "lark", params.grammar.c_str()));
+        grmr = llama_sampler_init_llg(vocab, "lark", params.grammar.c_str());
-        grammar = true;
 #else
         GGML_ABORT("llguidance (cmake -DLLAMA_LLGUIDANCE=ON) is not enabled");
 #endif // LLAMA_USE_LLGUIDANCE
@@ -222,23 +203,30 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, co
             trigger_patterns_c.push_back(regex.c_str());
         }
 
-        if (!params.grammar.empty()) {
+        grmr = params.grammar_lazy
-             if (params.grammar_lazy) {
+             ? llama_sampler_init_grammar_lazy_patterns(vocab, params.grammar.c_str(), "root",
-                 samplers.push_back(
+                                                        trigger_patterns_c.data(), trigger_patterns_c.size(),
-                         llama_sampler_init_grammar_lazy_patterns(vocab, params.grammar.c_str(), "root",
+                                                        trigger_tokens.data(), trigger_tokens.size())
-                             trigger_patterns_c.data(), trigger_patterns_c.size(),
+             :      llama_sampler_init_grammar(vocab, params.grammar.c_str(), "root");
-                             trigger_tokens.data(),     trigger_tokens.size()));
+        if (!grmr) {
-             } else {
+            return nullptr;
-                 samplers.push_back(llama_sampler_init_grammar(vocab, params.grammar.c_str(), "root"));
-             }
-
-             grammar = true;
         }
     }
 
-    if (params.has_logit_bias()) {
+    auto * result = new common_sampler {
-        samplers.push_back(llama_sampler_init_logit_bias(llama_vocab_n_tokens(vocab), params.logit_bias.size(), params.logit_bias.data()));
+        /* .params = */ params,
-    }
+        /* .grmr   = */ grmr,
+        /* .chain  = */ llama_sampler_chain_init(lparams),
+        /* .prev   = */ ring_buffer<llama_token>(std::max(32, params.n_prev)),
+        /* .cur    = */ {},
+        /* .cur_p  = */ {},
+    };
+
+    llama_sampler_chain_add(result->chain,
+            llama_sampler_init_logit_bias(
+                llama_vocab_n_tokens(vocab),
+                params.logit_bias.size(),
+                params.logit_bias.data()));
 
     if (params.mirostat == 0) {
         for (const auto & cnstr : params.samplers) {
@@ -251,70 +239,58 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, co
                             c_breakers.push_back(str.c_str());
                         }
 
-                        samplers.push_back(llama_sampler_init_dry    (vocab, llama_model_n_ctx_train(model), params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size()));
+                        llama_sampler_chain_add(result->chain, llama_sampler_init_dry      (vocab, llama_model_n_ctx_train(model), params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size()));
                     }
                     break;
                 case COMMON_SAMPLER_TYPE_TOP_K:
-                    samplers.push_back(llama_sampler_init_top_k      (params.top_k));
+                    llama_sampler_chain_add(result->chain, llama_sampler_init_top_k       (params.top_k));
                     break;
                 case COMMON_SAMPLER_TYPE_TOP_P:
-                    samplers.push_back(llama_sampler_init_top_p      (params.top_p, params.min_keep));
+                    llama_sampler_chain_add(result->chain, llama_sampler_init_top_p       (params.top_p, params.min_keep));
                     break;
                 case COMMON_SAMPLER_TYPE_TOP_N_SIGMA:
-                    samplers.push_back(llama_sampler_init_top_n_sigma(params.top_n_sigma));
+                    llama_sampler_chain_add(result->chain, llama_sampler_init_top_n_sigma (params.top_n_sigma));
                     break;
                 case COMMON_SAMPLER_TYPE_MIN_P:
-                    samplers.push_back(llama_sampler_init_min_p      (params.min_p, params.min_keep));
+                    llama_sampler_chain_add(result->chain, llama_sampler_init_min_p       (params.min_p, params.min_keep));
                     break;
                 case COMMON_SAMPLER_TYPE_XTC:
-                    samplers.push_back(llama_sampler_init_xtc        (params.xtc_probability, params.xtc_threshold, params.min_keep, params.seed));
+                    llama_sampler_chain_add(result->chain, llama_sampler_init_xtc         (params.xtc_probability, params.xtc_threshold, params.min_keep, params.seed));
                     break;
                 case COMMON_SAMPLER_TYPE_TYPICAL_P:
-                    samplers.push_back(llama_sampler_init_typical    (params.typ_p, params.min_keep));
+                    llama_sampler_chain_add(result->chain, llama_sampler_init_typical     (params.typ_p, params.min_keep));
                     break;
                 case COMMON_SAMPLER_TYPE_TEMPERATURE:
-                    samplers.push_back(llama_sampler_init_temp_ext   (params.temp, params.dynatemp_range, params.dynatemp_exponent));
+                    llama_sampler_chain_add(result->chain, llama_sampler_init_temp_ext    (params.temp, params.dynatemp_range, params.dynatemp_exponent));
                     break;
                 case COMMON_SAMPLER_TYPE_INFILL:
-                    samplers.push_back(llama_sampler_init_infill     (vocab));
+                    llama_sampler_chain_add(result->chain, llama_sampler_init_infill      (vocab));
                     break;
                 case COMMON_SAMPLER_TYPE_PENALTIES:
-                    samplers.push_back(llama_sampler_init_penalties  (params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present));
+                    llama_sampler_chain_add(result->chain, llama_sampler_init_penalties   (params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present));
                     break;
                 default:
                     GGML_ASSERT(false && "unknown sampler type");
             }
         }
-
+        llama_sampler_chain_add(result->chain, llama_sampler_init_dist(params.seed));
-        samplers.push_back(llama_sampler_init_dist(params.seed));
     } else if (params.mirostat == 1) {
-        samplers.push_back(llama_sampler_init_temp(params.temp));
+        llama_sampler_chain_add(result->chain, llama_sampler_init_temp(params.temp));
-        samplers.push_back(llama_sampler_init_mirostat(llama_vocab_n_tokens(vocab), params.seed, params.mirostat_tau, params.mirostat_eta, 100));
+        llama_sampler_chain_add(result->chain, llama_sampler_init_mirostat(llama_vocab_n_tokens(vocab), params.seed, params.mirostat_tau, params.mirostat_eta, 100));
     } else if (params.mirostat == 2) {
-        samplers.push_back(llama_sampler_init_temp(params.temp));
+        llama_sampler_chain_add(result->chain, llama_sampler_init_temp(params.temp));
-        samplers.push_back(llama_sampler_init_mirostat_v2(params.seed, params.mirostat_tau, params.mirostat_eta));
+        llama_sampler_chain_add(result->chain, llama_sampler_init_mirostat_v2(params.seed, params.mirostat_tau, params.mirostat_eta));
     } else {
         GGML_ASSERT(false && "unknown mirostat version");
     }
 
-    for (auto * smpl : samplers) {
-        llama_sampler_chain_add(chain, smpl);
-    }
-
-    auto * result = new common_sampler {
-        /* .params  = */ params,
-        /* .chain   = */ chain,
-        /* .grammar = */ grammar,
-        /* .prev    = */ ring_buffer<llama_token>(std::max(32, params.n_prev)),
-        /* .cur     = */ {},
-        /* .cur_p   = */ {},
-    };
-
     return result;
 }
 
 void common_sampler_free(struct common_sampler * gsmpl) {
     if (gsmpl) {
+        llama_sampler_free(gsmpl->grmr);
+
         llama_sampler_free(gsmpl->chain);
 
         delete gsmpl;
@@ -322,117 +298,91 @@ void common_sampler_free(struct common_sampler * gsmpl) {
 }
 
 void common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool accept_grammar) {
-    const auto tm = gsmpl->tm();
+    if (accept_grammar) {
-
+        llama_sampler_accept(gsmpl->grmr, token);
-    if (gsmpl->grammar) {
-        const int n_smpl = llama_sampler_chain_n(gsmpl->chain);
-
-        for (int i = 0; i < n_smpl; i++) {
-            auto * smpl = llama_sampler_chain_get(gsmpl->chain, i);
-
-            // the grammar sampler is always the first one
-            if (i == 0) {
-                if (accept_grammar) {
-                    llama_sampler_accept(smpl, token);
-                }
-            } else {
-                llama_sampler_accept(smpl, token);
-            }
-        }
-    } else {
-        llama_sampler_accept(gsmpl->chain, token);
     }
 
+    llama_sampler_accept(gsmpl->chain, token);
+
     gsmpl->prev.push_back(token);
 }
 
 void common_sampler_reset(struct common_sampler * gsmpl) {
-    gsmpl->reset();
+    llama_sampler_reset(gsmpl->grmr);
+
+    llama_sampler_reset(gsmpl->chain);
 }
 
 struct common_sampler * common_sampler_clone(common_sampler * gsmpl) {
     return new common_sampler {
-        /* .params  = */ gsmpl->params,
+        /* .params = */ gsmpl->params,
-        /* .chain   = */ llama_sampler_clone(gsmpl->chain),
+        /* .grmr   = */ llama_sampler_clone(gsmpl->grmr),
-        /* .grammar = */ gsmpl->grammar,
+        /* .chain  = */ llama_sampler_clone(gsmpl->chain),
-        /* .prev    = */ gsmpl->prev,
+        /* .prev   = */ gsmpl->prev,
-        /* .cur     = */ gsmpl->cur,
+        /* .cur    = */ gsmpl->cur,
-        /* .cur_p   = */ gsmpl->cur_p,
+        /* .cur_p  = */ gsmpl->cur_p,
     };
 }
 
 void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl) {
     // TODO: measure grammar performance
 
-    const double t_sampling_ms = gsmpl ? 1e-3*gsmpl->t_total_us : 0;
-
-    llama_perf_sampler_data data_smpl;
-    llama_perf_context_data data_ctx;
-
-    memset(&data_smpl, 0, sizeof(data_smpl));
-    memset(&data_ctx,  0, sizeof(data_ctx));
-
     if (gsmpl) {
-        auto & data = data_smpl;
+        llama_perf_sampler_print(gsmpl->chain);
-
-        data = llama_perf_sampler(gsmpl->chain);
-
-        // note: the sampling time includes the samplers time + extra time spent in common/sampling
-        LOG_INF("%s:    sampling time = %10.2f ms\n", __func__, t_sampling_ms);
-        LOG_INF("%s:    samplers time = %10.2f ms / %5d tokens\n", __func__, data.t_sample_ms, data.n_sample);
     }
-
     if (ctx) {
-        auto & data = data_ctx;
+        llama_perf_context_print(ctx);
-
-        data = llama_perf_context(ctx);
-
-        const double t_end_ms = 1e-3 * ggml_time_us();
-
-        const double t_total_ms = t_end_ms - data.t_start_ms;
-        const double t_unacc_ms = t_total_ms - (t_sampling_ms + data.t_p_eval_ms + data.t_eval_ms);
-        const double t_unacc_pc = 100.0 * t_unacc_ms /  t_total_ms;
-
-        LOG_INF("%s:        load time = %10.2f ms\n", __func__, data.t_load_ms);
-        LOG_INF("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",
-                __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval);
-        LOG_INF("%s:        eval time = %10.2f ms / %5d runs   (%8.2f ms per token, %8.2f tokens per second)\n",
-                __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval);
-        LOG_INF("%s:       total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));
-        LOG_INF("%s: unaccounted time = %10.2f ms / %5.1f %%      (total - sampling - prompt eval - eval) / (total)\n", __func__, t_unacc_ms, t_unacc_pc);
-        LOG_INF("%s:    graphs reused = %10d\n", __func__, data.n_reused);
-
         llama_memory_breakdown_print(ctx);
     }
 }
 
-struct llama_sampler * common_sampler_get(const struct common_sampler * gsmpl) {
+llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_context * ctx, int idx, bool grammar_first) {
-    return gsmpl->chain;
+    gsmpl->set_logits(ctx, idx);
-}
-
-llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_context * ctx, int idx) {
-    llama_synchronize(ctx);
-
-    // start measuring sampling time after the llama_context synchronization in order to not measure any ongoing async operations
-    const auto tm = gsmpl->tm();
-
-    llama_token id = LLAMA_TOKEN_NULL;
 
+    auto & grmr  = gsmpl->grmr;
     auto & chain = gsmpl->chain;
     auto & cur_p = gsmpl->cur_p; // initialized by set_logits
 
-    gsmpl->set_logits(ctx, idx);
+    if (grammar_first) {
+        llama_sampler_apply(grmr, &cur_p);
+    }
 
     llama_sampler_apply(chain, &cur_p);
 
     GGML_ASSERT(cur_p.selected != -1 && "no selected token during sampling - check your sampling configuration");
 
-    id = cur_p.data[cur_p.selected].id;
+    const llama_token id = cur_p.data[cur_p.selected].id;
 
-    return id;
+    if (grammar_first) {
+        return id;
+    }
+
+    // check if it the sampled token fits the grammar
+    {
+        llama_token_data       single_token_data       = { id, 1.0f, 0.0f };
+        llama_token_data_array single_token_data_array = { &single_token_data, 1, -1, false };
+
+        llama_sampler_apply(grmr, &single_token_data_array);
+
+        const bool is_valid = single_token_data_array.data[0].logit != -INFINITY;
+        if (is_valid) {
+            return id;
+        }
+    }
+
+    // resampling:
+    // if the token is not valid, sample again, but first apply the grammar sampler and then the sampling chain
+    gsmpl->set_logits(ctx, idx);
+
+    llama_sampler_apply(grmr,  &cur_p);
+    llama_sampler_apply(chain, &cur_p);
+
+    GGML_ASSERT(cur_p.selected != -1 && "no selected token during re-sampling - check your sampling configuration");
+
+    return cur_p.data[cur_p.selected].id;
 }
 
-std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const std::vector<int> & idxs, const llama_tokens & draft) {
+std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const std::vector<int> & idxs, const llama_tokens & draft, bool grammar_first) {
     GGML_ASSERT(idxs.size() == draft.size() + 1 && "idxs.size() must be draft.size() + 1");
 
     std::vector<llama_token> result;
@@ -440,7 +390,7 @@ std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sample
 
     size_t i = 0;
     for (; i < draft.size(); i++) {
-        const llama_token id = common_sampler_sample(gsmpl, ctx, idxs[i]);
+        const llama_token id = common_sampler_sample(gsmpl, ctx, idxs[i], grammar_first);
 
         common_sampler_accept(gsmpl, id, true);
 
@@ -452,7 +402,7 @@ std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sample
     }
 
     if (i == draft.size()) {
-        const llama_token id = common_sampler_sample(gsmpl, ctx, idxs[i]);
+        const llama_token id = common_sampler_sample(gsmpl, ctx, idxs[i], grammar_first);
 
         common_sampler_accept(gsmpl, id, true);
 
@@ -462,13 +412,13 @@ std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sample
     return result;
 }
 
-std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const llama_tokens & draft) {
+std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const llama_tokens & draft, bool grammar_first) {
     std::vector<int> idxs(draft.size() + 1);
     for (size_t i = 0; i < idxs.size(); ++i) {
         idxs[i] = i;
     }
 
-    return common_sampler_sample_and_accept_n(gsmpl, ctx, idxs, draft);
+    return common_sampler_sample_and_accept_n(gsmpl, ctx, idxs, draft, grammar_first);
 }
 
 uint32_t common_sampler_get_seed(const struct common_sampler * gsmpl) {
@@ -478,8 +428,6 @@ uint32_t common_sampler_get_seed(const struct common_sampler * gsmpl) {
 // helpers
 
 llama_token_data_array * common_sampler_get_candidates(struct common_sampler * gsmpl, bool do_sort) {
-    const auto tm = gsmpl->tm();
-
     auto * res = &gsmpl->cur_p;
 
     if (do_sort && !res->sorted) {
@@ -513,8 +461,7 @@ std::string common_sampler_print(const struct common_sampler * gsmpl) {
 
     for (int i = 0; i < llama_sampler_chain_n(gsmpl->chain); i++) {
         const auto * smpl = llama_sampler_chain_get(gsmpl->chain, i);
-        result += std::string("-> ");
+        result += std::string("-> ") + llama_sampler_name(smpl) + " ";
-        result += std::string(llama_sampler_name(smpl)) + " ";
     }
 
     return result;

llama/llama.cpp/common/sampling.h

@@ -48,8 +48,6 @@ struct common_sampler * common_sampler_clone (struct common_sampler * gsmpl);
 // arguments can be nullptr to skip printing
 void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl);
 
-struct llama_sampler * common_sampler_get(const struct common_sampler * gsmpl);
-
 // extended sampling implementation:
 //
 // - set logits
@@ -57,7 +55,10 @@ struct llama_sampler * common_sampler_get(const struct common_sampler * gsmpl);
 // - check if the token fits the grammar (if any)
 // - if not: resample by first applying the grammar constraints and then sampling again (slower path)
 //
-llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_context * ctx, int idx);
+// if grammar_first is true, the grammar is applied before the samplers (slower)
+// useful in cases where all the resulting candidates (not just the sampled one) must fit the grammar
+//
+llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_context * ctx, int idx, bool grammar_first = false);
 
 // generalized version of common_sampler_sample
 //
@@ -75,10 +76,10 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co
 //
 // returns at least 1 token, up to idxs.size()
 //
-std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const std::vector<int> & idxs, const llama_tokens & draft);
+std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const std::vector<int> & idxs, const llama_tokens & draft, bool grammar_first = false);
 
 // assume idxs == [ 0, 1, 2, ..., draft.size() ]
-std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const llama_tokens & draft);
+std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const llama_tokens & draft, bool grammar_first = false);
 
 uint32_t common_sampler_get_seed(const struct common_sampler * gsmpl);
 
@@ -106,9 +107,3 @@ std::vector<enum common_sampler_type> common_sampler_types_from_chars(const std:
 
 llama_sampler * llama_sampler_init_llg(const llama_vocab * vocab,
                 const char * grammar_kind, const char * grammar_data);
-
-struct common_sampler_deleter {
-    void operator()(common_sampler * s) { common_sampler_free(s); }
-};
-
-typedef std::unique_ptr<common_sampler, common_sampler_deleter> common_sampler_ptr;

llama/llama.cpp/include/llama.h

@@ -83,7 +83,6 @@ extern "C" {
         LLAMA_ROPE_TYPE_NORM   = 0,
         LLAMA_ROPE_TYPE_NEOX   = GGML_ROPE_TYPE_NEOX,
         LLAMA_ROPE_TYPE_MROPE  = GGML_ROPE_TYPE_MROPE,
-        LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE,
         LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,
     };
 
@@ -246,21 +245,6 @@ extern "C" {
         LLAMA_KV_OVERRIDE_TYPE_STR,
     };
 
-    enum llama_model_meta_key {
-        LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE,
-        LLAMA_MODEL_META_KEY_SAMPLING_TOP_K,
-        LLAMA_MODEL_META_KEY_SAMPLING_TOP_P,
-        LLAMA_MODEL_META_KEY_SAMPLING_MIN_P,
-        LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY,
-        LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD,
-        LLAMA_MODEL_META_KEY_SAMPLING_TEMP,
-        LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N,
-        LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT,
-        LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT,
-        LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU,
-        LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA,
-    };
-
     struct llama_model_kv_override {
         enum llama_model_kv_override_type tag;
 
@@ -313,7 +297,6 @@ extern "C" {
         bool check_tensors;   // validate model tensor data
         bool use_extra_bufts; // use extra buffer types (used for weight repacking)
         bool no_host;         // bypass host buffer allowing extra buffers to be used
-        bool no_alloc;        // only load metadata and simulate memory allocations
     };
 
     // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations
@@ -467,35 +450,17 @@ extern "C" {
     // Frees all allocated memory
     LLAMA_API void llama_free(struct llama_context * ctx);
 
-    // fits mparams and cparams to free device memory (assumes system memory is unlimited)
-    // returns true if the parameters could be successfully modified to fit device memory
-    // this function is NOT thread safe because it modifies the global llama logger state
-    LLAMA_API bool llama_params_fit(
-                                   const char   * path_model,
-                    struct llama_model_params   * mparams,
-                    struct llama_context_params * cparams,
-                                          float * tensor_split,          // writable buffer for tensor split, needs at least llama_max_devices elements
-        struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
-                                         size_t   margin,                // margin of memory to leave per device in bytes
-                                       uint32_t   n_ctx_min,             // minimum context size to set when trying to reduce memory use
-                            enum ggml_log_level   log_level);            // minimum log level to print during fitting, lower levels go to debug log
-
     LLAMA_API int64_t llama_time_us(void);
 
     LLAMA_API size_t llama_max_devices(void);
     LLAMA_API size_t llama_max_parallel_sequences(void);
-    LLAMA_API size_t llama_max_tensor_buft_overrides(void);
 
     LLAMA_API bool llama_supports_mmap       (void);
     LLAMA_API bool llama_supports_mlock      (void);
     LLAMA_API bool llama_supports_gpu_offload(void);
     LLAMA_API bool llama_supports_rpc        (void);
 
-    // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions
-    //       In some cases the requested values via llama_context_params may differ from the actual values used by the context
-    //       ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732
     LLAMA_API uint32_t llama_n_ctx      (const struct llama_context * ctx);
-    LLAMA_API uint32_t llama_n_ctx_seq  (const struct llama_context * ctx);
     LLAMA_API uint32_t llama_n_batch    (const struct llama_context * ctx);
     LLAMA_API uint32_t llama_n_ubatch   (const struct llama_context * ctx);
     LLAMA_API uint32_t llama_n_seq_max  (const struct llama_context * ctx);
@@ -516,7 +481,6 @@ extern "C" {
 
     LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model);
     LLAMA_API int32_t llama_model_n_embd     (const struct llama_model * model);
-    LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model);
     LLAMA_API int32_t llama_model_n_layer    (const struct llama_model * model);
     LLAMA_API int32_t llama_model_n_head     (const struct llama_model * model);
     LLAMA_API int32_t llama_model_n_head_kv  (const struct llama_model * model);
@@ -548,9 +512,6 @@ extern "C" {
     // Get the number of metadata key/value pairs
     LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);
 
-    // Get sampling metadata key name. Returns nullptr if the key is invalid
-    LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key);
-
     // Get metadata key name by index
     LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
 
@@ -623,7 +584,7 @@ extern "C" {
     LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);
 
     // Manually free a LoRA adapter
-    // NOTE: loaded adapters will be free when the associated model is deleted
+    // Note: loaded adapters will be free when the associated model is deleted
     LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter);
 
     // Get the invocation tokens if the current lora is an alora
@@ -1149,6 +1110,8 @@ extern "C" {
     //        // sample from the logits of the last token in the batch
     //        const llama_token id = llama_sampler_sample(smpl, ctx, -1);
     //
+    //        // accepting the token updates the internal state of certain samplers (e.g. grammar, repetition, etc.)
+    //        llama_sampler_accept(smpl, id);
     //        ...
     //    }
     //
@@ -1369,9 +1332,7 @@ extern "C" {
 
     // Set callback for all future logging events.
     // If this is not called, or NULL is supplied, everything is output on stderr.
-    // The logger state is global so these functions are NOT thread safe.
+    LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data);
-    LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data);
-    LLAMA_API void llama_log_set(ggml_log_callback   log_callback, void *  user_data);
 
     //
     // Performance utils

llama/llama.cpp/src/llama-arch.h

@@ -3,7 +3,6 @@
 #include "ggml.h" // ggml_op
 
 #include <string>
-#include <set>
 
 //
 // gguf constants (sync with gguf.py)
@@ -37,9 +36,6 @@ enum llm_arch {
     LLM_ARCH_QWEN2VL,
     LLM_ARCH_QWEN3,
     LLM_ARCH_QWEN3MOE,
-    LLM_ARCH_QWEN3NEXT,
-    LLM_ARCH_QWEN3VL,
-    LLM_ARCH_QWEN3VLMOE,
     LLM_ARCH_PHI2,
     LLM_ARCH_PHI3,
     LLM_ARCH_PHIMOE,
@@ -80,7 +76,6 @@ enum llm_arch {
     LLM_ARCH_JAIS,
     LLM_ARCH_NEMOTRON,
     LLM_ARCH_NEMOTRON_H,
-    LLM_ARCH_NEMOTRON_H_MOE,
     LLM_ARCH_EXAONE,
     LLM_ARCH_EXAONE4,
     LLM_ARCH_RWKV6,
@@ -98,7 +93,6 @@ enum llm_arch {
     LLM_ARCH_BAILINGMOE2,
     LLM_ARCH_DOTS1,
     LLM_ARCH_ARCEE,
-    LLM_ARCH_AFMOE,
     LLM_ARCH_ERNIE4_5,
     LLM_ARCH_ERNIE4_5_MOE,
     LLM_ARCH_HUNYUAN_MOE,
@@ -114,11 +108,6 @@ enum llm_arch {
     LLM_ARCH_SEED_OSS,
     LLM_ARCH_GROVEMOE,
     LLM_ARCH_APERTUS,
-    LLM_ARCH_MINIMAX_M2,
-    LLM_ARCH_COGVLM,
-    LLM_ARCH_RND1,
-    LLM_ARCH_PANGU_EMBED,
-    LLM_ARCH_MISTRAL3,
     LLM_ARCH_UNKNOWN,
 };
 
@@ -128,18 +117,6 @@ enum llm_kv {
     LLM_KV_GENERAL_QUANTIZATION_VERSION,
     LLM_KV_GENERAL_ALIGNMENT,
     LLM_KV_GENERAL_FILE_TYPE,
-    LLM_KV_GENERAL_SAMPLING_SEQUENCE,
-    LLM_KV_GENERAL_SAMPLING_TOP_K,
-    LLM_KV_GENERAL_SAMPLING_TOP_P,
-    LLM_KV_GENERAL_SAMPLING_MIN_P,
-    LLM_KV_GENERAL_SAMPLING_XTC_PROBABILITY,
-    LLM_KV_GENERAL_SAMPLING_XTC_THRESHOLD,
-    LLM_KV_GENERAL_SAMPLING_TEMP,
-    LLM_KV_GENERAL_SAMPLING_PENALTY_LAST_N,
-    LLM_KV_GENERAL_SAMPLING_PENALTY_REPEAT,
-    LLM_KV_GENERAL_SAMPLING_MIROSTAT,
-    LLM_KV_GENERAL_SAMPLING_MIROSTAT_TAU,
-    LLM_KV_GENERAL_SAMPLING_MIROSTAT_ETA,
     LLM_KV_GENERAL_NAME,
     LLM_KV_GENERAL_AUTHOR,
     LLM_KV_GENERAL_VERSION,
@@ -173,7 +150,6 @@ enum llm_kv {
     LLM_KV_EXPERTS_PER_GROUP,
     LLM_KV_MOE_EVERY_N_LAYERS,
     LLM_KV_NEXTN_PREDICT_LAYERS,
-    LLM_KV_NUM_DEEPSTACK_LAYERS,
     LLM_KV_POOLING_TYPE,
     LLM_KV_LOGIT_SCALE,
     LLM_KV_DECODER_START_TOKEN_ID,
@@ -212,7 +188,6 @@ enum llm_kv {
     LLM_KV_ATTENTION_SCALE,
     LLM_KV_ATTENTION_OUTPUT_SCALE,
     LLM_KV_ATTENTION_TEMPERATURE_LENGTH,
-    LLM_KV_ATTENTION_TEMPERATURE_SCALE,
     LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION,
     LLM_KV_ATTENTION_KEY_LENGTH_MLA,
     LLM_KV_ATTENTION_VALUE_LENGTH_MLA,
@@ -319,7 +294,6 @@ enum llm_tensor {
     LLM_TENSOR_DENSE_3_OUT,
     LLM_TENSOR_OUTPUT,
     LLM_TENSOR_OUTPUT_NORM,
-    LLM_TENSOR_OUTPUT_NORM_LFM2, // fix for wrong tensor name
     LLM_TENSOR_ROPE_FREQS,
     LLM_TENSOR_ROPE_FACTORS_LONG,
     LLM_TENSOR_ROPE_FACTORS_SHORT,
@@ -334,7 +308,6 @@ enum llm_tensor {
     LLM_TENSOR_ATTN_POST_NORM,
     LLM_TENSOR_ATTN_ROT_EMBD,
     LLM_TENSOR_ATTN_SINKS,
-    LLM_TENSOR_ATTN_GATE,
     LLM_TENSOR_FFN_GATE_INP,
     LLM_TENSOR_FFN_GATE_INP_SHEXP,
     LLM_TENSOR_FFN_NORM,
@@ -384,13 +357,11 @@ enum llm_tensor {
     LLM_TENSOR_SSM_DT,
     LLM_TENSOR_SSM_DT_NORM,
     LLM_TENSOR_SSM_A,
-    LLM_TENSOR_SSM_A_NOSCAN,        // qwen3next special case with MUL instead of SSM_SCAN
     LLM_TENSOR_SSM_B_NORM,
     LLM_TENSOR_SSM_C_NORM,
     LLM_TENSOR_SSM_D,
     LLM_TENSOR_SSM_NORM,
     LLM_TENSOR_SSM_OUT,
-    LLM_TENSOR_SSM_BETA_ALPHA,      // qwen3next
     LLM_TENSOR_TIME_MIX_W0,
     LLM_TENSOR_TIME_MIX_W1,
     LLM_TENSOR_TIME_MIX_W2,
@@ -487,11 +458,6 @@ enum llm_tensor {
     LLM_TENSOR_SHORTCONV_CONV,
     LLM_TENSOR_SHORTCONV_INPROJ,
     LLM_TENSOR_SHORTCONV_OUTPROJ,
-    LLM_TENSOR_VISEXP_ATTN_QKV,
-    LLM_TENSOR_VISEXP_ATTN_OUT,
-    LLM_TENSOR_VISEXP_FFN_GATE,
-    LLM_TENSOR_VISEXP_FFN_DOWN,
-    LLM_TENSOR_VISEXP_FFN_UP,
     LLM_TENSOR_NEXTN_EH_PROJ,
     LLM_TENSOR_NEXTN_EMBED_TOKENS,
     LLM_TENSOR_NEXTN_ENORM,
@@ -531,10 +497,6 @@ struct LLM_TN_IMPL {
     const int bid;
     const int xid;
 
-    const std::set<llm_tensor> model_tensors;
-
-    LLM_TN_IMPL(llm_arch arch, llm_tensor tensor, const char * suffix, int bid, int xid);
-
     std::string str() const;
 
     operator std::string() const {
@@ -556,11 +518,11 @@ struct LLM_TN {
     llm_arch arch;
 
     LLM_TN_IMPL operator()(llm_tensor tensor, const char * suffix, int bid = -1, int xid = -1) const {
-        return LLM_TN_IMPL(arch, tensor, suffix, bid, xid);
+        return { arch, tensor, suffix, bid, xid };
     }
 
     LLM_TN_IMPL operator()(llm_tensor tensor, int bid = -1, int xid = -1) const {
-        return LLM_TN_IMPL(arch, tensor, nullptr, bid, xid);
+        return { arch, tensor, nullptr, bid, xid };
     }
 };
 

llama/llama.cpp/src/llama-batch.cpp

@@ -215,7 +215,6 @@ bool llama_batch_allocr::init(
             /*.n_seq_tokens =*/ (uint32_t) 1,
             /*.n_seqs       =*/ (uint32_t) batch.n_tokens,
             /*.n_seqs_unq   =*/ (uint32_t) this->seq_id_unq.size(),
-            /*.n_pos        =*/ n_pos_per_embd,
             /*.token        =*/ batch.token,
             /*.embd         =*/ batch.embd,
             /*.pos          =*/ batch.pos,
@@ -252,72 +251,46 @@ bool llama_batch_allocr::init(
     // consistency checks
     //
 
-    if (n_pos_per_embd > 1) {
+    for (uint32_t s = 0; s < n_seq_max; ++s) {
-        // M-RoPE case: allow position to "jump" forward only (non-continuous positions are allowed)
+        if (seq_pos[s].empty()) {
-        for (uint32_t s = 0; s < n_seq_max; ++s) {
+            continue;
-            if (seq_pos[s].empty()) {
+        }
-                continue;
-            }
 
-            const llama_pos p0 = memory ? memory->seq_pos_max(s) : -1;
+        const llama_pos p0 = memory ? memory->seq_pos_max(s) : -1;
+
+        if (p0 >= 0) {
+            bool ok = true;
 
             if (batch.token) {
-                if (p0 >= 0 && p0 >= seq_pos_min(s)) {
-                    LLAMA_LOG_ERROR(
-                            "%s: the tokens of sequence %d in the input batch have inconsistent sequence positions:\n"
-                            " - the last position stored in the memory module of the context (i.e. the KV cache) for sequence %d is X = %d\n"
-                            " - the tokens for sequence %d in the input batch have a starting position of Y = %d\n"
-                            " for M-RoPE, it is required that the position satisfies: X < Y\n",
-                            __func__, s, s, p0, s, seq_pos_min(s));
-
-                    return false;
-                }
-            } else {
-                // embedding inputs can have overlapping positions
-                if (p0 >= 0 && p0 > seq_pos_min(s)) {
-                    LLAMA_LOG_ERROR(
-                            "%s: the tokens of sequence %d in the input batch have inconsistent sequence positions:\n"
-                            " - the last position stored in the memory module of the context (i.e. the KV cache) for sequence %d is X = %d\n"
-                            " - the tokens for sequence %d in the input batch have a starting position of Y = %d\n"
-                            " for M-RoPE, it is required that the position satisfies: X <= Y\n",
-                            __func__, s, s, p0, s, seq_pos_min(s));
-
-                    return false;
-                }
-            }
-        }
-    } else {
-        for (uint32_t s = 0; s < n_seq_max; ++s) {
-            if (seq_pos[s].empty()) {
-                continue;
-            }
-
-            const llama_pos p0 = memory ? memory->seq_pos_max(s) : -1;
-
-            if (p0 >= 0) {
-                bool ok = true;
-
                 if (seq_pos_min(s) != p0 + 1) {
                     ok = false;
                 }
+            } else {
+                assert(batch.embd);
 
-                if (!ok) {
+                // for embeddings (typically used as vision input), we allow them to have repeating positions
-                    LLAMA_LOG_ERROR(
+                // ref: https://github.com/ggml-org/llama.cpp/issues/13694#issuecomment-2983871762
-                            "%s: the tokens of sequence %d in the input batch have inconsistent sequence positions:\n"
+                if (seq_pos_min(s) != p0 && seq_pos_min(s) != p0 + 1) {
-                            " - the last position stored in the memory module of the context (i.e. the KV cache) for sequence %d is X = %d\n"
+                    ok = false;
-                            " - the tokens for sequence %d in the input batch have a starting position of Y = %d\n"
-                            " it is required that the sequence positions remain consecutive: Y = X + 1\n",
-                            __func__, s, s, p0, s, seq_pos_min(s));
-
-                    return false;
                 }
             }
 
-            if (seq_pos_max(s) - seq_pos_min(s) + 1 > (int) seq_pos[s].size()) {
+            if (!ok) {
-                LLAMA_LOG_ERROR("%s: sequence %d positions are not continuous\n", __func__, s);
+                LLAMA_LOG_ERROR(
+                        "%s: the tokens of sequence %d in the input batch have inconsistent sequence positions:\n"
+                        " - the last position stored in the memory module of the context (i.e. the KV cache) for sequence %d is X = %d\n"
+                        " - the tokens for sequence %d in the input batch have a starting position of Y = %d\n"
+                        " it is required that the sequence positions remain consecutive: Y = X + 1\n",
+                        __func__, s, s, p0, s, seq_pos_min(s));
+
                 return false;
             }
         }
+
+        if (seq_pos_max(s) - seq_pos_min(s) + 1 > (int) seq_pos[s].size()) {
+            LLAMA_LOG_ERROR("%s: sequence %d positions are not continuous\n", __func__, s);
+            return false;
+        }
     }
 
     if (memory) {
@@ -416,7 +389,6 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
         /*.n_seq_tokens =*/ n_seq_tokens,
         /*.n_seqs       =*/ n_seqs,
         /*.n_seqs_unq   =*/ n_seqs,
-        /*.n_pos        =*/ n_pos_per_embd,
 
         /*.token        =*/ udata->token.data(),
         /*.embd         =*/ nullptr,
@@ -683,8 +655,10 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
 
     auto udata = std::make_shared<llama_ubatch::data_t>();
 
+    const int32_t n_pos_cur = batch.embd ? n_pos_per_embd : 1;
+
     const int64_t n_embd_all = batch.embd ? (int64_t) n_tokens*n_embd : 0;
-    const int64_t n_pos_all  =              (int64_t) n_tokens*n_pos_per_embd;
+    const int64_t n_pos_all  =              (int64_t) n_tokens*n_pos_cur;
 
     udata->token     .resize(n_tokens);
     udata->embd      .resize(n_embd_all);
@@ -695,8 +669,6 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
     udata->seq_idx   .resize(LLAMA_MAX_SEQ, -1);
     udata->output    .resize(n_tokens);
 
-    udata->seq_id_data.reserve(n_tokens);
-
     seq_set_t seq_set_unq;
 
     for (size_t i = 0; i < idxs.size(); ++i) {
@@ -708,23 +680,16 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
             memcpy(udata->embd.data() + i*n_embd, batch.embd + (int64_t) idxs[i]*n_embd, n_embd*sizeof(float));
         }
 
-        for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) {
+        for (int j = 0; j < n_pos_cur; ++j) {
-            // if we are using M-RoPE
+            udata->pos[j*n_tokens + i] = batch.pos[j*batch.n_tokens + idxs[i]];
-            //     if the current batch is text, we need to broadcast the same position across all RoPE sections
-            //     otherwise, the input batch is image embeddings, we copy the positions as-is
-            // if we are not using M-RoPE, there is only one position per token (this loop runs only once)
-            size_t src_off = batch.token ? 0 : j*batch.n_tokens;
-            udata->pos[j*n_tokens + i] = batch.pos[src_off + idxs[i]];
         }
 
         udata->n_seq_id[i] = batch.n_seq_id[idxs[i]];
+        udata->seq_id[i]   = batch.seq_id[idxs[i]];
         udata->output[i]   = batch.logits[idxs[i]];
 
         for (int s = 0; s < udata->n_seq_id[i]; ++s) {
-            const llama_seq_id seq_id = batch.seq_id[idxs[i]][s];
+            seq_set_unq.set(udata->seq_id[i][s]);
-
-            udata->seq_id_data.push_back(seq_id);
-            seq_set_unq.set(seq_id);
         }
 
         if (udata->output[i]) {
@@ -732,12 +697,6 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
         }
     }
 
-    llama_seq_id * seq_id_ptr = udata->seq_id_data.data();
-    for (size_t i = 0; i < idxs.size(); ++i) {
-        udata->seq_id[i] = seq_id_ptr;
-        seq_id_ptr += udata->n_seq_id[i];
-    }
-
     for (uint32_t s = 0; s < n_seq_max; ++s) {
         if (seq_set_unq.test(s)) {
             udata->seq_idx[s] = udata->seq_id_unq.size();
@@ -751,7 +710,6 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
         /*.n_seq_tokens =*/ n_tokens/n_seqs,
         /*.n_seqs       =*/ n_seqs,
         /*.n_seqs_unq   =*/ (uint32_t) udata->seq_id_unq.size(),
-        /*.n_pos        =*/ n_pos_per_embd,
 
         /*.token        =*/ batch.token ? udata->token.data() : nullptr,
         /*.embd         =*/ batch.embd ? udata->embd.data() : nullptr,