tmp

CONTRIBUTING.md

@@ -51,7 +51,7 @@ see if the change were accepted.
 
 The title should look like:
 
-    <package>: <short description>
+   <package>: <short description>
 
 The package is the most affected Go package. If the change does not affect Go
 code, then use the directory name instead. Changes to a single well-known

Dockerfile

@@ -104,8 +104,8 @@ COPY --from=cuda-12 dist/lib/ollama/cuda_v12 /lib/ollama/cuda_v12
 FROM --platform=linux/arm64 scratch AS arm64
 COPY --from=cuda-11 dist/lib/ollama/cuda_v11 /lib/ollama/cuda_v11
 COPY --from=cuda-12 dist/lib/ollama/cuda_v12 /lib/ollama/cuda_v12
-COPY --from=jetpack-5 dist/lib/ollama/cuda_v11 /lib/ollama/cuda_jetpack5
-COPY --from=jetpack-6 dist/lib/ollama/cuda_v12 /lib/ollama/cuda_jetpack6
+COPY --from=jetpack-5 dist/lib/ollama/cuda_v11 lib/ollama/cuda_jetpack5
+COPY --from=jetpack-6 dist/lib/ollama/cuda_v12 lib/ollama/cuda_jetpack6
 
 FROM scratch AS rocm
 COPY --from=rocm-6 dist/lib/ollama/rocm /lib/ollama/rocm

README.md

@@ -291,7 +291,7 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [Typescript UI](https://github.com/ollama-interface/Ollama-Gui?tab=readme-ov-file)
 - [Minimalistic React UI for Ollama Models](https://github.com/richawo/minimal-llm-ui)
 - [Ollamac](https://github.com/kevinhermawan/Ollamac)
-- [big-AGI](https://github.com/enricoros/big-AGI) 
+- [big-AGI](https://github.com/enricoros/big-AGI/blob/main/docs/config-local-ollama.md)
 - [Cheshire Cat assistant framework](https://github.com/cheshire-cat-ai/core)
 - [Amica](https://github.com/semperai/amica)
 - [chatd](https://github.com/BruceMacD/chatd)
@@ -348,7 +348,7 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [PartCAD](https://github.com/openvmp/partcad/) (CAD model generation with OpenSCAD and CadQuery)
 - [Ollama4j Web UI](https://github.com/ollama4j/ollama4j-web-ui) - Java-based Web UI for Ollama built with Vaadin, Spring Boot and Ollama4j
 - [PyOllaMx](https://github.com/kspviswa/pyOllaMx) - macOS application capable of chatting with both Ollama and Apple MLX models.
-- [Cline](https://github.com/cline/cline) - Formerly known as Claude Dev is a VSCode extension for multi-file/whole-repo coding
+- [Claude Dev](https://github.com/saoudrizwan/claude-dev) - VSCode extension for multi-file/whole-repo coding
 - [Cherry Studio](https://github.com/kangfenmao/cherry-studio) (Desktop client with Ollama support)
 - [ConfiChat](https://github.com/1runeberg/confichat) (Lightweight, standalone, multi-platform, and privacy focused LLM chat interface with optional encryption)
 - [Archyve](https://github.com/nickthecook/archyve) (RAG-enabling document library)
@@ -440,7 +440,6 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [DeepShell](https://github.com/Abyss-c0re/deepshell) Your self-hosted AI assistant. Interactive Shell, Files and Folders analysis.
 - [orbiton](https://github.com/xyproto/orbiton) Configuration-free text editor and IDE with support for tab completion with Ollama.
 - [orca-cli](https://github.com/molbal/orca-cli) Ollama Registry CLI Application - Browse, pull and download models from Ollama Registry in your terminal.
-- [GGUF-to-Ollama](https://github.com/jonathanhecl/gguf-to-ollama) - Importing GGUF to Ollama made easy (multiplatform)
 
 ### Apple Vision Pro
 

api/types.go

@@ -163,65 +163,19 @@ func (t *ToolCallFunctionArguments) String() string {
 
 type Tool struct {
 	Type     string       `json:"type"`
-	Items    any          `json:"items,omitempty"`
 	Function ToolFunction `json:"function"`
 }
 
-// PropertyType can be either a string or an array of strings
-type PropertyType []string
-
-// UnmarshalJSON implements the json.Unmarshaler interface
-func (pt *PropertyType) UnmarshalJSON(data []byte) error {
-	// Try to unmarshal as a string first
-	var s string
-	if err := json.Unmarshal(data, &s); err == nil {
-		*pt = []string{s}
-		return nil
-	}
-
-	// If that fails, try to unmarshal as an array of strings
-	var a []string
-	if err := json.Unmarshal(data, &a); err != nil {
-		return err
-	}
-	*pt = a
-	return nil
-}
-
-// MarshalJSON implements the json.Marshaler interface
-func (pt PropertyType) MarshalJSON() ([]byte, error) {
-	if len(pt) == 1 {
-		// If there's only one type, marshal as a string
-		return json.Marshal(pt[0])
-	}
-	// Otherwise marshal as an array
-	return json.Marshal([]string(pt))
-}
-
-// String returns a string representation of the PropertyType
-func (pt PropertyType) String() string {
-	if len(pt) == 0 {
-		return ""
-	}
-	if len(pt) == 1 {
-		return pt[0]
-	}
-	return fmt.Sprintf("%v", []string(pt))
-}
-
 type ToolFunction struct {
 	Name        string `json:"name"`
 	Description string `json:"description"`
 	Parameters  struct {
 		Type       string   `json:"type"`
-		Defs       any      `json:"$defs,omitempty"`
-		Items      any      `json:"items,omitempty"`
 		Required   []string `json:"required"`
 		Properties map[string]struct {
-			Type        PropertyType `json:"type"`
-			Items       any          `json:"items,omitempty"`
-			Description string       `json:"description"`
-			Enum        []any        `json:"enum,omitempty"`
+			Type        string   `json:"type"`
+			Description string   `json:"description"`
+			Enum        []string `json:"enum,omitempty"`
 		} `json:"properties"`
 	} `json:"parameters"`
 }

api/types_test.go

@@ -231,144 +231,3 @@ func TestMessage_UnmarshalJSON(t *testing.T) {
 		}
 	}
 }
-
-func TestToolFunction_UnmarshalJSON(t *testing.T) {
-	tests := []struct {
-		name    string
-		input   string
-		wantErr string
-	}{
-		{
-			name: "valid enum with same types",
-			input: `{
-				"name": "test",
-				"description": "test function",
-				"parameters": {
-					"type": "object",
-					"required": ["test"],
-					"properties": {
-						"test": {
-							"type": "string",
-							"description": "test prop",
-							"enum": ["a", "b", "c"]
-						}
-					}
-				}
-			}`,
-			wantErr: "",
-		},
-		{
-			name: "empty enum array",
-			input: `{
-				"name": "test",
-				"description": "test function",
-				"parameters": {
-					"type": "object",
-					"required": ["test"],
-					"properties": {
-						"test": {
-							"type": "string",
-							"description": "test prop",
-							"enum": []
-						}
-					}
-				}
-			}`,
-			wantErr: "",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			var tf ToolFunction
-			err := json.Unmarshal([]byte(tt.input), &tf)
-
-			if tt.wantErr != "" {
-				require.Error(t, err)
-				assert.Contains(t, err.Error(), tt.wantErr)
-			} else {
-				require.NoError(t, err)
-			}
-		})
-	}
-}
-
-func TestPropertyType_UnmarshalJSON(t *testing.T) {
-	tests := []struct {
-		name     string
-		input    string
-		expected PropertyType
-	}{
-		{
-			name:     "string type",
-			input:    `"string"`,
-			expected: PropertyType{"string"},
-		},
-		{
-			name:     "array of types",
-			input:    `["string", "number"]`,
-			expected: PropertyType{"string", "number"},
-		},
-		{
-			name:     "array with single type",
-			input:    `["string"]`,
-			expected: PropertyType{"string"},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			var pt PropertyType
-			if err := json.Unmarshal([]byte(test.input), &pt); err != nil {
-				t.Errorf("Unexpected error: %v", err)
-			}
-
-			if len(pt) != len(test.expected) {
-				t.Errorf("Length mismatch: got %v, expected %v", len(pt), len(test.expected))
-			}
-
-			for i, v := range pt {
-				if v != test.expected[i] {
-					t.Errorf("Value mismatch at index %d: got %v, expected %v", i, v, test.expected[i])
-				}
-			}
-		})
-	}
-}
-
-func TestPropertyType_MarshalJSON(t *testing.T) {
-	tests := []struct {
-		name     string
-		input    PropertyType
-		expected string
-	}{
-		{
-			name:     "single type",
-			input:    PropertyType{"string"},
-			expected: `"string"`,
-		},
-		{
-			name:     "multiple types",
-			input:    PropertyType{"string", "number"},
-			expected: `["string","number"]`,
-		},
-		{
-			name:     "empty type",
-			input:    PropertyType{},
-			expected: `[]`,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			data, err := json.Marshal(test.input)
-			if err != nil {
-				t.Errorf("Unexpected error: %v", err)
-			}
-
-			if string(data) != test.expected {
-				t.Errorf("Marshaled data mismatch: got %v, expected %v", string(data), test.expected)
-			}
-		})
-	}
-}

cmd/cmd.go

@@ -1381,6 +1381,7 @@ func NewCLI() *cobra.Command {
 				envVars["OLLAMA_NOPRUNE"],
 				envVars["OLLAMA_ORIGINS"],
 				envVars["OLLAMA_SCHED_SPREAD"],
+				envVars["OLLAMA_TMPDIR"],
 				envVars["OLLAMA_FLASH_ATTENTION"],
 				envVars["OLLAMA_KV_CACHE_TYPE"],
 				envVars["OLLAMA_LLM_LIBRARY"],

convert/convert.go

@@ -182,10 +182,8 @@ func ConvertModel(fsys fs.FS, ws io.WriteSeeker) error {
 
 	var conv ModelConverter
 	switch p.Architectures[0] {
-	case "LlamaForCausalLM":
+	case "LlamaForCausalLM", "MistralForCausalLM":
 		conv = &llamaModel{}
-	case "Mistral3ForConditionalGeneration":
-		conv = &mistral3Model{}
 	case "MixtralForCausalLM":
 		conv = &mixtralModel{}
 	case "GemmaForCausalLM":

convert/convert_mistral.go

@@ -1,190 +0,0 @@
-package convert
-
-import (
-	"cmp"
-	"fmt"
-	"strings"
-
-	"github.com/pdevine/tensor"
-	"github.com/pdevine/tensor/native"
-
-	"github.com/ollama/ollama/fs/ggml"
-)
-
-type mistral3Model struct {
-	ModelParameters
-	ImageTokenIndex    uint32 `json:"image_token_index"`
-	SpatialMergeSize   uint32 `json:"spatial_merge_size"`
-	VisionFeatureLayer int32  `json:"vision_feature_layer"`
-	TextModel          struct {
-		NumHiddenLayers       uint32  `json:"num_hidden_layers"`
-		MaxPositionEmbeddings uint32  `json:"max_position_embeddings"`
-		HiddenSize            uint32  `json:"hidden_size"`
-		IntermediateSize      uint32  `json:"intermediate_size"`
-		NumAttentionHeads     uint32  `json:"num_attention_heads"`
-		NumKeyValueHeads      uint32  `json:"num_key_value_heads"`
-		RopeTheta             float32 `json:"rope_theta"`
-		RMSNormEPS            float32 `json:"rms_norm_eps"`
-		HeadDim               uint32  `json:"head_dim"`
-		SlidingWindow         *uint32 `json:"sliding_window"`
-		HiddenAct             string  `json:"hidden_act"`
-		VocabSize             uint32  `json:"vocab_size"`
-	} `json:"text_config"`
-	VisionModel struct {
-		NumAttentionHeads uint32  `json:"num_attention_heads"`
-		NumHiddenLayers   uint32  `json:"num_hidden_layers"`
-		HiddenSize        uint32  `json:"hidden_size"`
-		IntermediateSize  uint32  `json:"intermediate_size"`
-		ImageSize         uint32  `json:"image_size"`
-		NumChannels       uint32  `json:"num_channels"`
-		PatchSize         uint32  `json:"patch_size"`
-		HeadDim           uint32  `json:"head_dim"`
-		HiddenAct         string  `json:"hidden_act"`
-		RopeTheta         float32 `json:"rope_theta"`
-	} `json:"vision_config"`
-	MultiModalProjectorBias bool   `json:"multimodal_projector_bias"`
-	ProjectorHiddenAct      string `json:"projector_hidden_act"`
-}
-
-func (p *mistral3Model) KV(t *Tokenizer) ggml.KV {
-	kv := p.ModelParameters.KV(t)
-	kv["general.architecture"] = "mistral3"
-	kv["mistral3.vocab_size"] = p.TextModel.VocabSize
-
-	// Text configuration
-	kv["mistral3.block_count"] = p.TextModel.NumHiddenLayers
-	kv["mistral3.context_length"] = p.TextModel.MaxPositionEmbeddings
-	kv["mistral3.embedding_length"] = p.TextModel.HiddenSize
-	kv["mistral3.feed_forward_length"] = p.TextModel.IntermediateSize
-	kv["mistral3.attention.head_count"] = p.TextModel.NumAttentionHeads
-	kv["mistral3.attention.head_count_kv"] = p.TextModel.NumKeyValueHeads
-	kv["mistral3.attention.layer_norm_rms_epsilon"] = p.TextModel.RMSNormEPS
-	kv["mistral3.attention.key_length"] = p.TextModel.HeadDim
-	kv["mistral3.attention.value_length"] = p.TextModel.HeadDim
-	kv["mistral3.rope.dimension_count"] = p.TextModel.HiddenSize / p.TextModel.NumHiddenLayers
-	kv["mistral3.rope.freq_base"] = p.TextModel.RopeTheta
-
-	// Vision configuration
-	kv["mistral3.vision.block_count"] = p.VisionModel.NumHiddenLayers
-	kv["mistral3.vision.embedding_length"] = p.VisionModel.HiddenSize
-	kv["mistral3.vision.feed_forward_length"] = p.VisionModel.IntermediateSize
-	kv["mistral3.vision.attention.head_count"] = p.VisionModel.NumAttentionHeads
-	kv["mistral3.vision.attention.key_length"] = p.VisionModel.HeadDim
-	kv["mistral3.vision.image_size"] = p.VisionModel.ImageSize
-	kv["mistral3.vision.patch_size"] = p.VisionModel.PatchSize
-	kv["mistral3.vision.num_channels"] = p.VisionModel.NumChannels
-	// kv["mistral3.vision.attention.layer_norm_epsilon"] = 1e-05 // Default value
-	kv["mistral3.vision.rope.freq_base"] = p.VisionModel.RopeTheta
-
-	// Multimodal configuration
-	kv["mistral3.image_token_index"] = p.ImageTokenIndex
-	kv["mistral3.spatial_merge_size"] = p.SpatialMergeSize
-
-	kv["mistral3.mm.projector_bias"] = p.MultiModalProjectorBias
-
-	if p.ProjectorHiddenAct != "" {
-		kv["mistral3.mm.projector_hidden_act"] = p.ProjectorHiddenAct
-	}
-
-	return kv
-}
-
-func (p *mistral3Model) Tensors(ts []Tensor) []ggml.Tensor {
-	var out []ggml.Tensor
-
-	for _, t := range ts {
-		if !strings.HasPrefix(t.Name(), "v.") {
-			if strings.HasSuffix(t.Name(), ".attn_q.weight") ||
-				strings.HasSuffix(t.Name(), ".attn_k.weight") {
-				t.SetRepacker(p.repack)
-			}
-		}
-
-		out = append(out, ggml.Tensor{
-			Name:     t.Name(),
-			Kind:     t.Kind(),
-			Shape:    t.Shape(),
-			WriterTo: t,
-		})
-	}
-
-	return out
-}
-
-func (p *mistral3Model) Replacements() []string {
-	return []string{
-		"language_model.model.norm", "output_norm",
-		"language_model.model.", "",
-		"language_model.", "",
-		"layers", "blk",
-		"transformer.layers", "blk",
-		"vision_tower", "v",
-		"ln_pre", "encoder_norm",
-		"input_layernorm", "attn_norm",
-		"post_attention_layernorm", "ffn_norm",
-		"embed_tokens", "token_embd",
-		"self_attn.q_proj", "attn_q",
-		"self_attn.k_proj", "attn_k",
-		"self_attn.v_proj", "attn_v",
-		"self_attn.o_proj", "attn_output",
-		"mlp.down_proj", "ffn_down",
-		"mlp.gate_proj", "ffn_gate",
-		"mlp.up_proj", "ffn_up",
-		"attention.q_proj", "attn_q",
-		"attention.k_proj", "attn_k",
-		"attention.v_proj", "attn_v",
-		"attention.o_proj", "attn_output",
-		"attention_norm", "attn_norm",
-		"feed_forward.gate_proj", "ffn_gate",
-		"feed_forward.down_proj", "ffn_down",
-		"feed_forward.up_proj", "ffn_up",
-		"multi_modal_projector", "mm",
-		"ffn_norm", "ffn_norm",
-		"lm_head", "output",
-	}
-}
-
-func (p *mistral3Model) repack(name string, data []float32, shape []uint64) ([]float32, error) {
-	var dims []int
-	for _, dim := range shape {
-		dims = append(dims, int(dim))
-	}
-
-	var heads uint32
-	if strings.HasSuffix(name, ".attn_q.weight") {
-		heads = p.TextModel.NumAttentionHeads
-	} else if strings.HasSuffix(name, ".attn_k.weight") {
-		heads = cmp.Or(p.TextModel.NumKeyValueHeads, p.TextModel.NumAttentionHeads)
-	} else {
-		return nil, fmt.Errorf("unknown tensor for repack: %s", name)
-	}
-
-	n := tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data))
-	if err := n.Reshape(append([]int{int(heads), 2, dims[0] / int(heads) / 2}, dims[1:]...)...); err != nil {
-		return nil, err
-	}
-
-	if err := n.T(0, 2, 1, 3); err != nil {
-		return nil, err
-	}
-
-	if err := n.Reshape(dims...); err != nil {
-		return nil, err
-	}
-
-	if err := n.Transpose(); err != nil {
-		return nil, err
-	}
-
-	ts, err := native.SelectF32(n, 1)
-	if err != nil {
-		return nil, err
-	}
-
-	var f32s []float32
-	for _, t := range ts {
-		f32s = append(f32s, t...)
-	}
-
-	return f32s, nil
-}

convert/reader.go

@@ -62,7 +62,10 @@ func parseTensors(fsys fs.FS, replacer *strings.Replacer) ([]Tensor, error) {
 		Pattern string
 		Func    func(fs.FS, *strings.Replacer, ...string) ([]Tensor, error)
 	}{
-		{"*.safetensors", parseSafetensors},
+		{"model-*-of-*.safetensors", parseSafetensors},
+		{"model.safetensors", parseSafetensors},
+		{"adapters.safetensors", parseSafetensors},
+		{"adapter_model.safetensors", parseSafetensors},
 		{"pytorch_model-*-of-*.bin", parseTorch},
 		{"pytorch_model.bin", parseTorch},
 		{"consolidated.*.pth", parseTorch},

docs/troubleshooting.md

@@ -26,6 +26,7 @@ When you run Ollama on **Windows**, there are a few different locations. You can
 - `explorer %LOCALAPPDATA%\Ollama` to view logs.  The most recent server logs will be in `server.log` and older logs will be in `server-#.log` 
 - `explorer %LOCALAPPDATA%\Programs\Ollama` to browse the binaries (The installer adds this to your user PATH)
 - `explorer %HOMEPATH%\.ollama` to browse where models and configuration is stored
+- `explorer %TEMP%` where temporary executable files are stored in one or more `ollama*` directories
 
 To enable additional debug logging to help troubleshoot problems, first **Quit the running app from the tray menu** then in a powershell terminal
 
@@ -68,6 +69,10 @@ If you run into problems on Linux and want to install an older version, or you'd
 curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.5.7 sh
 ```
 
+## Linux tmp noexec 
+
+If your system is configured with the "noexec" flag where Ollama stores its temporary executable files, you can specify an alternate location by setting OLLAMA_TMPDIR to a location writable by the user ollama runs as. For example OLLAMA_TMPDIR=/usr/share/ollama/
+
 ## Linux docker
 
 If Ollama initially works on the GPU in a docker container, but then switches to running on CPU after some period of time with errors in the server log reporting GPU discovery failures, this can be resolved by disabling systemd cgroup management in Docker.  Edit `/etc/docker/daemon.json` on the host and add `"exec-opts": ["native.cgroupdriver=cgroupfs"]` to the docker configuration.

docs/windows.md

@@ -62,6 +62,7 @@ the explorer window by hitting `<Ctrl>+R` and type in:
     - *upgrade.log* contains log output for upgrades
 - `explorer %LOCALAPPDATA%\Programs\Ollama` contains the binaries (The installer adds this to your user PATH)
 - `explorer %HOMEPATH%\.ollama` contains models and configuration
+- `explorer %TEMP%` contains temporary executable files in one or more `ollama*` directories
 
 ## Uninstall
 

fs/fs.go

@@ -0,0 +1,88 @@
+package fs
+
+import (
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+
+	"github.com/ollama/ollama/fs/ggml"
+)
+
+type DType int
+
+type Model struct {
+	KV      Config
+	Tensors map[string]TensorReader
+}
+
+func (m Model) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.String("architecture", m.KV.Architecture()),
+	)
+}
+
+type Tensor interface {
+	Name() string
+	Shape() []int
+	DType() DType
+	Size() int
+}
+
+type TensorReader interface {
+	Tensor
+	io.Reader
+}
+
+type shimTensorReader struct {
+	internal *ggml.Tensor
+	*io.SectionReader
+}
+
+func (t *shimTensorReader) Name() string {
+	return t.internal.Name
+}
+
+func (t *shimTensorReader) Shape() []int {
+	shape := make([]int, len(t.internal.Shape))
+	for i, s := range t.internal.Shape {
+		shape[i] = int(s)
+	}
+
+	return shape
+}
+
+func (t *shimTensorReader) Size() int {
+	return int(t.internal.Size())
+}
+
+func (t *shimTensorReader) DType() DType {
+	return DType(t.internal.Kind)
+}
+
+func ReadFrom(f *os.File) (*Model, error) {
+	bts, err := io.ReadAll(io.NewSectionReader(f, 0, 4))
+	if err != nil {
+		return nil, err
+	}
+
+	switch ggml.DetectContentType(bts[:4]) {
+	case "gguf":
+		c, _, err := ggml.Decode(f, -1)
+		if err != nil {
+			return nil, err
+		}
+
+		tensors := make(map[string]TensorReader, len(c.Tensors().Items()))
+		for _, t := range c.Tensors().Items() {
+			tensors[t.Name] = &shimTensorReader{
+				internal:      t,
+				SectionReader: io.NewSectionReader(f, int64(c.Tensors().Offset+t.Offset), int64(t.Size())),
+			}
+		}
+
+		return &Model{KV: c.KV(), Tensors: tensors}, nil
+	default:
+		return nil, fmt.Errorf("unsupported file type")
+	}
+}

fs/ggml/ggml.go

@@ -134,10 +134,7 @@ func (kv KV) Floats(key string, defaultValue ...[]float32) []float32 {
 }
 
 func (kv KV) OllamaEngineRequired() bool {
-	return slices.Contains([]string{
-		"gemma3",
-		"mistral3",
-	}, kv.Architecture())
+	return kv.Architecture() == "gemma3"
 }
 
 func keyValue[T string | uint32 | uint64 | float32 | *array | bool](kv KV, key string, defaultValue ...T) T {
@@ -641,7 +638,7 @@ func (llm GGML) VisionGraphSize() (weights, graphSize uint64) {
 			embeddingLength*numPatches*maxNumTiles +
 			9*embeddingLength*numPaddedPatches*maxNumTiles +
 			numPaddedPatches*maxNumTiles*numPaddedPatches*maxNumTiles*headCount)
-	case "gemma3", "mistral3":
+	case "gemma3":
 		graphSize = 4 * (imageSize*imageSize*numChannels +
 			embeddingLength*patchSize +
 			numPatches*numPatches*headCount)

integration/max_queue_test.go

@@ -52,8 +52,8 @@ func TestMaxQueue(t *testing.T) {
 	embedCtx := ctx
 
 	var genwg sync.WaitGroup
-	genwg.Add(1)
 	go func() {
+		genwg.Add(1)
 		defer genwg.Done()
 		slog.Info("Starting generate request")
 		DoGenerate(ctx, t, client, req, resp, 45*time.Second, 5*time.Second)
@@ -71,8 +71,8 @@ func TestMaxQueue(t *testing.T) {
 	counterMu := sync.Mutex{}
 	var embedwg sync.WaitGroup
 	for i := 0; i < threadCount; i++ {
-		embedwg.Add(1)
 		go func(i int) {
+			embedwg.Add(1)
 			defer embedwg.Done()
 			slog.Info("embed started", "id", i)
 			embedReq := api.EmbeddingRequest{

kvcache/cache.go

@@ -56,9 +56,8 @@ type Cache interface {
 
 	// StartForward is called before the start of the model's forward pass.
 	// For each token in the coming batch, there must be a corresponding
-	// entry in positions and seqs. reserve is to preallocate memory
-	// without actually storing data in the cache.
-	StartForward(ctx ml.Context, batch input.Batch, reserve bool) error
+	// entry in positions and seqs.
+	StartForward(ctx ml.Context, batch input.Batch) error
 
 	// CopyPrefix copies tokens in the range [0, len) from srcSeq to dstSeq
 	CopyPrefix(srcSeq, dstSeq int, len int32)

kvcache/causal.go

@@ -146,60 +146,51 @@ func (c *Causal) Close() {
 	}
 }
 
-func (c *Causal) StartForward(ctx ml.Context, batch input.Batch, reserve bool) error {
+func (c *Causal) StartForward(ctx ml.Context, batch input.Batch) error {
 	c.curBatchSize = len(batch.Positions)
 	c.curSequences = batch.Sequences
 	c.curPositions = batch.Positions
 	c.opts.Except = nil
 
-	if !reserve {
-		c.updateSlidingWindow()
-
-		var err error
-		c.curLoc, err = c.findStartLoc()
-		if errors.Is(err, ErrKvCacheFull) {
-			c.defrag()
-			c.curLoc, err = c.findStartLoc()
-		}
-		if err != nil {
-			return err
-		}
-
-		c.curCellRange = newRange()
-		for i, pos := range batch.Positions {
-			seq := batch.Sequences[i]
-
-			c.cells[c.curLoc+i] = cacheCell{pos: pos, sequences: []int{seq}}
-
-			seqRange, ok := c.cellRanges[seq]
-			if !ok {
-				seqRange = newRange()
-			}
-
-			if c.curLoc+i > seqRange.max {
-				seqRange.max = c.curLoc + i
-			}
-			if seqRange.max > c.curCellRange.max {
-				c.curCellRange.max = seqRange.max
-			}
-
-			if c.curLoc+i < seqRange.min {
-				seqRange.min = c.curLoc + i
-			}
-			if seqRange.min < c.curCellRange.min {
-				c.curCellRange.min = seqRange.min
-			}
-			c.cellRanges[seq] = seqRange
-		}
-	} else {
-		// If we are reserving memory, don't update any of the cache metadata but set the size
-		// to the worst case.
-		c.curLoc = 0
-		c.curCellRange.min = 0
-		c.curCellRange.max = len(c.cells) - 1
-	}
+	c.updateSlidingWindow()
 
 	var err error
+	c.curLoc, err = c.findStartLoc()
+	if errors.Is(err, ErrKvCacheFull) {
+		c.defrag()
+		c.curLoc, err = c.findStartLoc()
+	}
+	if err != nil {
+		return err
+	}
+
+	c.curCellRange = newRange()
+	for i, pos := range batch.Positions {
+		seq := batch.Sequences[i]
+
+		c.cells[c.curLoc+i] = cacheCell{pos: pos, sequences: []int{seq}}
+
+		seqRange, ok := c.cellRanges[seq]
+		if !ok {
+			seqRange = newRange()
+		}
+
+		if c.curLoc+i > seqRange.max {
+			seqRange.max = c.curLoc + i
+		}
+		if seqRange.max > c.curCellRange.max {
+			c.curCellRange.max = seqRange.max
+		}
+
+		if c.curLoc+i < seqRange.min {
+			seqRange.min = c.curLoc + i
+		}
+		if seqRange.min < c.curCellRange.min {
+			c.curCellRange.min = seqRange.min
+		}
+		c.cellRanges[seq] = seqRange
+	}
+
 	c.curMask, err = c.buildMask(ctx)
 
 	return err

kvcache/causal_test.go

@@ -5,6 +5,7 @@ import (
 	"slices"
 	"testing"
 
+	"github.com/ollama/ollama/fs"
 	"github.com/ollama/ollama/ml"
 	"github.com/ollama/ollama/model/input"
 )
@@ -280,7 +281,7 @@ func testCache(t *testing.T, backend ml.Backend, cache Cache, tests []testCase)
 			context := backend.NewContext()
 			defer context.Close()
 
-			err := cache.StartForward(context, input.Batch{Positions: test.pos, Sequences: test.seqs}, false)
+			err := cache.StartForward(context, input.Batch{Positions: test.pos, Sequences: test.seqs})
 			if err != nil {
 				panic(err)
 			}
@@ -314,7 +315,7 @@ func TestCanResume(t *testing.T) {
 	err := cache.StartForward(context, input.Batch{
 		Positions: []int32{0, 1, 2, 3},
 		Sequences: []int{0, 0, 0, 0},
-	}, false)
+	})
 	if err != nil {
 		t.Fatalf("StartForward failed: %v", err)
 	}
@@ -341,7 +342,7 @@ func TestCanResume(t *testing.T) {
 	err = cache.StartForward(context, input.Batch{
 		Positions: []int32{4, 5},
 		Sequences: []int{0, 0},
-	}, false)
+	})
 	if err != nil {
 		t.Fatalf("StartForward failed: %v", err)
 	}
@@ -371,8 +372,14 @@ func TestCanResume(t *testing.T) {
 	}
 }
 
-type testBackend struct {
-	ml.Backend
+type testBackend struct{}
+
+func (b *testBackend) Config() fs.Config {
+	panic("not implemented")
+}
+
+func (b *testBackend) Get(name string) ml.Tensor {
+	panic("not implemented")
 }
 
 func (b *testBackend) NewContext() ml.Context {
@@ -383,10 +390,12 @@ func (b *testBackend) NewContextSize(int) ml.Context {
 	return &testContext{}
 }
 
-type testContext struct {
-	ml.Context
+func (b *testBackend) SystemInfo() string {
+	return "not implemented"
 }
 
+type testContext struct{}
+
 func (c *testContext) Empty(dtype ml.DType, shape ...int) ml.Tensor {
 	total := 0
 
@@ -431,8 +440,6 @@ func (c *testContext) Forward(...ml.Tensor) ml.Context { return c }
 
 func (c *testContext) Compute(...ml.Tensor) {}
 
-func (c *testContext) Reserve() error { return nil }
-
 func (c *testContext) MaxGraphNodes() int {
 	return 10
 }
@@ -440,8 +447,6 @@ func (c *testContext) MaxGraphNodes() int {
 func (c *testContext) Close() {}
 
 type testTensor struct {
-	ml.Tensor
-
 	dtype       ml.DType
 	elementSize int
 	data        []float32
@@ -469,20 +474,16 @@ func (t *testTensor) DType() ml.DType {
 	return t.dtype
 }
 
+func (t *testTensor) Bytes() []byte {
+	panic("not implemented")
+}
+
 func (t *testTensor) Floats() []float32 {
 	out := make([]float32, len(t.data))
 	copy(out, t.data)
 	return out
 }
 
-func (t *testTensor) Neg(ctx ml.Context) ml.Tensor {
-	out := ctx.Empty(t.DType(), t.Shape()...).(*testTensor)
-	for i := range out.data {
-		out.data[i] = -t.data[i]
-	}
-	return out
-}
-
 func (t *testTensor) Add(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
 	out := ctx.Empty(t.DType(), t.Shape()...).(*testTensor)
 
@@ -493,6 +494,66 @@ func (t *testTensor) Add(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
 	return out
 }
 
+func (t *testTensor) Mul(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Mulmat(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) MulmatFullPrec(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Softmax(ctx ml.Context) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) LayerNorm(ctx ml.Context, weight, bias ml.Tensor, eps float32) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) RMSNorm(ctx ml.Context, weight ml.Tensor, eps float32) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Scale(ctx ml.Context, s float64) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) AvgPool1D(ctx ml.Context, k, s, p int) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) AvgPool2D(ctx ml.Context, k, s int, p float32) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Conv2D(ctx ml.Context, weight ml.Tensor, s0, s1, p0, p1, d0, d1 int) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) RoPE(ctx ml.Context, positionIDs, ropeFactors ml.Tensor, dim, ropeType uint32, base, scale float32) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Tanh(ctx ml.Context) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) GELU(ctx ml.Context) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) SILU(ctx ml.Context) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Reshape(ctx ml.Context, shape ...int) ml.Tensor {
+	panic("not implemented")
+}
+
 func (t *testTensor) View(ctx ml.Context, offset int, shape ...int) ml.Tensor {
 	offset /= t.elementSize
 
@@ -515,6 +576,38 @@ func (t *testTensor) View(ctx ml.Context, offset int, shape ...int) ml.Tensor {
 	return view
 }
 
+func (t *testTensor) Permute(ctx ml.Context, shape ...int) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Contiguous(ctx ml.Context) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Set(ctx ml.Context, t2 ml.Tensor, offset int, strides ...int) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Pad(ctx ml.Context, shape ...int) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Unpad(ctx ml.Context, shape ...int) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Stack(ctx ml.Context, dim int, s ...ml.Tensor) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Concat(ctx ml.Context, t2 ml.Tensor, dim int) ml.Tensor {
+	panic("not implemented")
+}
+
+func (t *testTensor) Rows(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
+	panic("not implemented")
+}
+
 func (t *testTensor) Copy(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
 	copy(t2.(*testTensor).data, t.data)
 	return nil

kvcache/encoder.go

@@ -27,11 +27,6 @@ type EncoderCache struct {
 	// anything will be stored)
 	curPos int32
 
-	// curReserve indicates that this forward pass is only for
-	// memory reservation and we should not update our metadata
-	// based on it.
-	curReserve bool
-
 	// ** cache metadata **
 
 	// was something stored in the cache?
@@ -88,14 +83,12 @@ func (c *EncoderCache) Close() {
 	}
 }
 
-func (c *EncoderCache) StartForward(ctx ml.Context, batch input.Batch, reserve bool) error {
+func (c *EncoderCache) StartForward(ctx ml.Context, batch input.Batch) error {
 	// We work with the most recent image
 	if len(batch.Multimodal) > 0 {
 		c.curPos = batch.Positions[batch.Multimodal[len(batch.Multimodal)-1].Index]
 	}
 
-	c.curReserve = reserve
-
 	return nil
 }
 
@@ -112,10 +105,8 @@ func (c *EncoderCache) Get(ctx ml.Context) (ml.Tensor, ml.Tensor, ml.Tensor) {
 }
 
 func (c *EncoderCache) Put(ctx ml.Context, key, value ml.Tensor) {
-	if !c.curReserve {
-		c.encoderPos = c.curPos
-		c.encoderCached = true
-	}
+	c.encoderPos = c.curPos
+	c.encoderCached = true
 
 	if c.config.PermutedV {
 		value = value.Permute(ctx, 1, 2, 0, 3)

kvcache/wrapper.go

@@ -41,9 +41,9 @@ func (c *WrapperCache) Close() {
 	}
 }
 
-func (c *WrapperCache) StartForward(ctx ml.Context, batch input.Batch, reserve bool) error {
+func (c *WrapperCache) StartForward(ctx ml.Context, batch input.Batch) error {
 	for i, cache := range c.caches {
-		err := cache.StartForward(ctx, batch, reserve)
+		err := cache.StartForward(ctx, batch)
 		if err != nil {
 			// unwind on error - Remove with endIndex set to math.MaxInt32 does not fail
 			for j := i - 1; j >= 0; j-- {

llama/llama.cpp/src/llama-arch.cpp

@@ -65,7 +65,6 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_CHAMELEON,        "chameleon"        },
     { LLM_ARCH_SOLAR,            "solar"            },
     { LLM_ARCH_WAVTOKENIZER_DEC, "wavtokenizer-dec" },
-    { LLM_ARCH_MISTRAL3,         "mistral3"         },
     { LLM_ARCH_UNKNOWN,          "(unknown)"        },
 };
 
@@ -1372,22 +1371,6 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
             { LLM_TENSOR_POS_NET_ATTN_OUT,  "posnet.%d.attn_output" },
         },
     },
-    {
-        LLM_ARCH_MISTRAL3,
-        {
-            { LLM_TENSOR_TOKEN_EMBD,  "token_embd" },
-            { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
-            { LLM_TENSOR_ATTN_NORM,   "blk.%d.attn_norm" },
-            { LLM_TENSOR_ATTN_Q,      "blk.%d.attn_q" },
-            { LLM_TENSOR_ATTN_K,      "blk.%d.attn_k" },
-            { LLM_TENSOR_ATTN_V,      "blk.%d.attn_v" },
-            { LLM_TENSOR_ATTN_OUT,    "blk.%d.attn_output" },
-            { LLM_TENSOR_FFN_NORM,    "blk.%d.ffn_norm" },
-            { LLM_TENSOR_FFN_GATE,    "blk.%d.ffn_gate" },
-            { LLM_TENSOR_FFN_UP,      "blk.%d.ffn_up" },
-            { LLM_TENSOR_FFN_DOWN,    "blk.%d.ffn_down" },
-        }
-    },
     {
         LLM_ARCH_UNKNOWN,
         {

llama/llama.cpp/src/llama-arch.h

@@ -69,7 +69,6 @@ enum llm_arch {
     LLM_ARCH_CHAMELEON,
     LLM_ARCH_SOLAR,
     LLM_ARCH_WAVTOKENIZER_DEC,
-    LLM_ARCH_MISTRAL3,
     LLM_ARCH_UNKNOWN,
 };
 

llama/llama.cpp/src/llama-model.cpp

@@ -1277,7 +1277,6 @@ void llama_model::load_hparams(llama_model_loader & ml) {
                 ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);
                 ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn);
             } break;
-        case LLM_ARCH_MISTRAL3: break;
         default: throw std::runtime_error("unsupported model architecture");
     }
 
@@ -3538,7 +3537,6 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
                     output   = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {hparams.convnext.n_embd, n_embd}, 0);
                     output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"),   {n_embd}, 0);
                 } break;
-            case LLM_ARCH_MISTRAL3: break;
             default:
                 throw std::runtime_error("unknown architecture");
         }
@@ -4017,7 +4015,6 @@ enum llama_rope_type llama_model_rope_type(const struct llama_model * model) {
         case LLM_ARCH_GRANITE_MOE:
         case LLM_ARCH_CHAMELEON:
         case LLM_ARCH_SOLAR:
-        case LLM_ARCH_MISTRAL3:
             return LLAMA_ROPE_TYPE_NORM;
 
         // the pairs of head values are offset by n_rot/2

llama/llama.cpp/src/llama-quant.cpp

@@ -738,8 +738,13 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
         bool quantize = name.rfind("weight") == name.size() - 6; // ends with 'weight'?
 
         // don't quantize vision stuff
-        quantize &= name.find("v.") == std::string::npos;
-        quantize &= name.find("mm.") == std::string::npos;
+        quantize &= name.find("v.blk.") == std::string::npos;
+
+        quantize &= name.find("mm.mm_input_projection.weight") == std::string::npos;
+        quantize &= name.find("mm.mm_soft_emb_norm.weight") == std::string::npos;
+        quantize &= name.find("v.patch_embedding.weight") == std::string::npos;
+        quantize &= name.find("v.position_embedding.weight") == std::string::npos;
+        quantize &= name.find("v.post_layernorm.weight") == std::string::npos;
 
         // quantize only 2D and 3D tensors (experts)
         quantize &= (ggml_n_dims(tensor) >= 2);

llama/patches/0021-gemma3-quantization.patch

@@ -1,19 +1,17 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Patrick Devine <patrick@infrahq.com>
 Date: Fri, 14 Mar 2025 16:33:23 -0700
-Subject: [PATCH] add model quantizations
+Subject: [PATCH] gemma3 quantization
 
-- gemma3
-- mistral3
 ---
- src/llama-arch.cpp  | 36 ++++++++++++++++++++++++++++++++++++
- src/llama-arch.h    |  2 ++
- src/llama-model.cpp | 10 ++++++++++
- src/llama-quant.cpp |  4 ++++
- 4 files changed, 52 insertions(+)
+ src/llama-arch.cpp  | 19 +++++++++++++++++++
+ src/llama-arch.h    |  1 +
+ src/llama-model.cpp |  7 +++++++
+ src/llama-quant.cpp |  9 +++++++++
+ 4 files changed, 36 insertions(+)
 
 diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
-index b6f20286..13a0a988 100644
+index b6f20286..b443fcd3 100644
 --- a/src/llama-arch.cpp
 +++ b/src/llama-arch.cpp
 @@ -37,6 +37,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
@@ -24,15 +22,7 @@ index b6f20286..13a0a988 100644
      { LLM_ARCH_STARCODER2,       "starcoder2"       },
      { LLM_ARCH_MAMBA,            "mamba"            },
      { LLM_ARCH_XVERSE,           "xverse"           },
-@@ -64,6 +65,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
-     { LLM_ARCH_CHAMELEON,        "chameleon"        },
-     { LLM_ARCH_SOLAR,            "solar"            },
-     { LLM_ARCH_WAVTOKENIZER_DEC, "wavtokenizer-dec" },
-+    { LLM_ARCH_MISTRAL3,         "mistral3"         },
-     { LLM_ARCH_UNKNOWN,          "(unknown)"        },
- };
- 
-@@ -804,6 +806,24 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
+@@ -804,6 +805,24 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
              { LLM_TENSOR_FFN_POST_NORM,   "blk.%d.post_ffw_norm" },
          },
      },
@@ -57,31 +47,8 @@ index b6f20286..13a0a988 100644
      {
          LLM_ARCH_STARCODER2,
          {
-@@ -1352,6 +1372,22 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
-             { LLM_TENSOR_POS_NET_ATTN_OUT,  "posnet.%d.attn_output" },
-         },
-     },
-+    {
-+        LLM_ARCH_MISTRAL3,
-+        {
-+            { LLM_TENSOR_TOKEN_EMBD,  "token_embd" },
-+            { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
-+            { LLM_TENSOR_ATTN_NORM,   "blk.%d.attn_norm" },
-+            { LLM_TENSOR_ATTN_Q,      "blk.%d.attn_q" },
-+            { LLM_TENSOR_ATTN_K,      "blk.%d.attn_k" },
-+            { LLM_TENSOR_ATTN_V,      "blk.%d.attn_v" },
-+            { LLM_TENSOR_ATTN_OUT,    "blk.%d.attn_output" },
-+            { LLM_TENSOR_FFN_NORM,    "blk.%d.ffn_norm" },
-+            { LLM_TENSOR_FFN_GATE,    "blk.%d.ffn_gate" },
-+            { LLM_TENSOR_FFN_UP,      "blk.%d.ffn_up" },
-+            { LLM_TENSOR_FFN_DOWN,    "blk.%d.ffn_down" },
-+        }
-+    },
-     {
-         LLM_ARCH_UNKNOWN,
-         {
 diff --git a/src/llama-arch.h b/src/llama-arch.h
-index ec742224..8476ae0a 100644
+index ec742224..aad92a5d 100644
 --- a/src/llama-arch.h
 +++ b/src/llama-arch.h
 @@ -41,6 +41,7 @@ enum llm_arch {
@@ -92,16 +59,8 @@ index ec742224..8476ae0a 100644
      LLM_ARCH_STARCODER2,
      LLM_ARCH_MAMBA,
      LLM_ARCH_XVERSE,
-@@ -68,6 +69,7 @@ enum llm_arch {
-     LLM_ARCH_CHAMELEON,
-     LLM_ARCH_SOLAR,
-     LLM_ARCH_WAVTOKENIZER_DEC,
-+    LLM_ARCH_MISTRAL3,
-     LLM_ARCH_UNKNOWN,
- };
- 
 diff --git a/src/llama-model.cpp b/src/llama-model.cpp
-index ab1a07d1..db4f2685 100644
+index ab1a07d1..70183041 100644
 --- a/src/llama-model.cpp
 +++ b/src/llama-model.cpp
 @@ -878,6 +878,9 @@ void llama_model::load_hparams(llama_model_loader & ml) {
@@ -114,15 +73,7 @@ index ab1a07d1..db4f2685 100644
          case LLM_ARCH_STARCODER2:
              {
                  ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
-@@ -1274,6 +1277,7 @@ void llama_model::load_hparams(llama_model_loader & ml) {
-                 ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);
-                 ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn);
-             } break;
-+        case LLM_ARCH_MISTRAL3: break;
-         default: throw std::runtime_error("unsupported model architecture");
-     }
- 
-@@ -2537,6 +2541,9 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
+@@ -2537,6 +2540,9 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
                          layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);
                      }
                  } break;
@@ -132,23 +83,7 @@ index ab1a07d1..db4f2685 100644
              case LLM_ARCH_STARCODER2:
                  {
                      tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
-@@ -3531,6 +3538,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
-                     output   = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {hparams.convnext.n_embd, n_embd}, 0);
-                     output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"),   {n_embd}, 0);
-                 } break;
-+            case LLM_ARCH_MISTRAL3: break;
-             default:
-                 throw std::runtime_error("unknown architecture");
-         }
-@@ -4009,6 +4017,7 @@ enum llama_rope_type llama_model_rope_type(const struct llama_model * model) {
-         case LLM_ARCH_GRANITE_MOE:
-         case LLM_ARCH_CHAMELEON:
-         case LLM_ARCH_SOLAR:
-+        case LLM_ARCH_MISTRAL3:
-             return LLAMA_ROPE_TYPE_NORM;
- 
-         // the pairs of head values are offset by n_rot/2
-@@ -4029,6 +4038,7 @@ enum llama_rope_type llama_model_rope_type(const struct llama_model * model) {
+@@ -4029,6 +4035,7 @@ enum llama_rope_type llama_model_rope_type(const struct llama_model * model) {
          case LLM_ARCH_PHIMOE:
          case LLM_ARCH_GEMMA:
          case LLM_ARCH_GEMMA2:
@@ -157,16 +92,21 @@ index ab1a07d1..db4f2685 100644
          case LLM_ARCH_OPENELM:
          case LLM_ARCH_GPTNEOX:
 diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
-index 6eb1da08..ebcbafa1 100644
+index 6eb1da08..d2f3a510 100644
 --- a/src/llama-quant.cpp
 +++ b/src/llama-quant.cpp
-@@ -737,6 +737,10 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
+@@ -737,6 +737,15 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
          // This used to be a regex, but <regex> has an extreme cost to compile times.
          bool quantize = name.rfind("weight") == name.size() - 6; // ends with 'weight'?
  
 +        // don't quantize vision stuff
-+        quantize &= name.find("v.") == std::string::npos;
-+        quantize &= name.find("mm.") == std::string::npos;
++        quantize &= name.find("v.blk.") == std::string::npos;
++
++        quantize &= name.find("mm.mm_input_projection.weight") == std::string::npos;
++        quantize &= name.find("mm.mm_soft_emb_norm.weight") == std::string::npos;
++        quantize &= name.find("v.patch_embedding.weight") == std::string::npos;
++        quantize &= name.find("v.position_embedding.weight") == std::string::npos;
++        quantize &= name.find("v.post_layernorm.weight") == std::string::npos;
 +
          // quantize only 2D and 3D tensors (experts)
          quantize &= (ggml_n_dims(tensor) >= 2);

llama/patches/0022-metal-add-op_neg.patch

@@ -1,75 +0,0 @@
-From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
-From: Michael Yang <git@mxy.ng>
-Date: Wed, 2 Apr 2025 15:26:15 -0700
-Subject: [PATCH] metal: add op_neg
-
----
- ggml/src/ggml-metal/ggml-metal.m     | 15 +++++++++++++++
- ggml/src/ggml-metal/ggml-metal.metal |  7 +++++++
- 2 files changed, 22 insertions(+)
-
-diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m
-index e4c093f9..d8422f1b 100644
---- a/ggml/src/ggml-metal/ggml-metal.m
-+++ b/ggml/src/ggml-metal/ggml-metal.m
-@@ -423,6 +423,7 @@ enum ggml_metal_kernel_type {
-     GGML_METAL_KERNEL_TYPE_SQRT,
-     GGML_METAL_KERNEL_TYPE_SIN,
-     GGML_METAL_KERNEL_TYPE_COS,
-+    GGML_METAL_KERNEL_TYPE_NEG,
-     GGML_METAL_KERNEL_TYPE_SUM_ROWS,
-     GGML_METAL_KERNEL_TYPE_POOL_2D_AVG_F32,
-     GGML_METAL_KERNEL_TYPE_POOL_2D_MAX_F32,
-@@ -1039,6 +1040,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de
-         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQRT,                          sqrt,                           true);
-         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SIN,                           sin,                            true);
-         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_COS,                           cos,                            true);
-+        GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NEG,                           neg,                            true);
-         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS,                      sum_rows,                       true);
-         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGMAX,                        argmax,                         true);
-         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_POOL_2D_AVG_F32,               pool_2d_avg_f32,                true);
-@@ -1202,6 +1204,7 @@ static bool ggml_metal_supports_op(const struct ggml_backend_metal_device_contex
-                 case GGML_UNARY_OP_GELU_QUICK:
-                 case GGML_UNARY_OP_SILU:
-                 case GGML_UNARY_OP_ELU:
-+                case GGML_UNARY_OP_NEG:
-                     return ggml_is_contiguous(op->src[0]);
-                 default:
-                     return false;
-@@ -1873,6 +1876,18 @@ static void ggml_metal_encode_node(
- 
-                     [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
-                 } break;
-+                case GGML_UNARY_OP_NEG:
-+                {
-+                    id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_NEG].pipeline;
-+
-+                    [encoder setComputePipelineState:pipeline];
-+                    [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
-+                    [encoder setBuffer:id_dst  offset:offs_dst  atIndex:1];
-+
-+                    const int64_t n = ggml_nelements(dst);
-+
-+                    [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
-+                } break;
-                 default:
-                 {
-                     GGML_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, idx, ggml_op_name(dst->op));
-diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal
-index f38909d0..bb0ff668 100644
---- a/ggml/src/ggml-metal/ggml-metal.metal
-+++ b/ggml/src/ggml-metal/ggml-metal.metal
-@@ -945,6 +945,13 @@ kernel void kernel_cos(
-     dst[tpig] = cos(src0[tpig]);
- }
- 
-+kernel void kernel_neg(
-+        device const float * src0,
-+        device       float * dst,
-+        uint tpig[[thread_position_in_grid]]) {
-+    dst[tpig] = -src0[tpig];
-+}
-+
- kernel void kernel_sum_rows(
-         device const float * src0,
-         device       float * dst,

llm/server.go

@@ -675,32 +675,9 @@ type CompletionRequest struct {
 	Grammar string // set before sending the request to the subprocess
 }
 
-// DoneReason represents the reason why a completion response is done
-type DoneReason int
-
-const (
-	// DoneReasonStop indicates the completion stopped naturally
-	DoneReasonStop DoneReason = iota
-	// DoneReasonLength indicates the completion stopped due to length limits
-	DoneReasonLength
-	// DoneReasonConnectionClosed indicates the completion stopped due to the connection being closed
-	DoneReasonConnectionClosed
-)
-
-func (d DoneReason) String() string {
-	switch d {
-	case DoneReasonLength:
-		return "length"
-	case DoneReasonStop:
-		return "stop"
-	default:
-		return "" // closed
-	}
-}
-
 type CompletionResponse struct {
 	Content            string        `json:"content"`
-	DoneReason         DoneReason    `json:"done_reason"`
+	DoneReason         string        `json:"done_reason"`
 	Done               bool          `json:"done"`
 	PromptEvalCount    int           `json:"prompt_eval_count"`
 	PromptEvalDuration time.Duration `json:"prompt_eval_duration"`
@@ -809,6 +786,7 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu
 				continue
 			}
 
+			// slog.Debug("got line", "line", string(line))
 			evt, ok := bytes.CutPrefix(line, []byte("data: "))
 			if !ok {
 				evt = line

ml/backend.go

@@ -97,13 +97,6 @@ type Context interface {
 
 	Forward(...Tensor) Context
 	Compute(...Tensor)
-
-	// Reserve is analogous to Compute but rather than executing a
-	// graph, simply preallocates memory. Typically called with a
-	// worst case graph to ensure all resources are available for
-	// for future inference.
-	Reserve() error
-
 	MaxGraphNodes() int
 	Close()
 
@@ -125,7 +118,6 @@ type Tensor interface {
 	Bytes() []byte
 	Floats() []float32
 
-	Neg(ctx Context) Tensor
 	Add(ctx Context, t2 Tensor) Tensor
 	Mul(ctx Context, t2 Tensor) Tensor
 	Mulmat(ctx Context, t2 Tensor) Tensor
@@ -140,10 +132,7 @@ type Tensor interface {
 	Conv2D(ctx Context, weight Tensor, s0, s1, p0, p1, d0, d1 int) Tensor
 
 	RoPE(ctx Context, positionIDs, ropeFactors Tensor, dim, ropeType uint32, base, scale float32) Tensor
-	IM2Col(ctx Context, weight Tensor, s0, s1, p0, p1, d0, d1 int) Tensor
 
-	Sin(ctx Context) Tensor
-	Cos(ctx Context) Tensor
 	Tanh(ctx Context) Tensor
 	GELU(ctx Context) Tensor
 	SILU(ctx Context) Tensor
@@ -158,13 +147,9 @@ type Tensor interface {
 	Unpad(ctx Context, shape ...int) Tensor
 
 	Stack(ctx Context, dim int, s ...Tensor) Tensor
-
-	// Repeat repeats the tensor n times along dimension dim
-	Repeat(ctx Context, dim, n int) Tensor
 	Concat(ctx Context, t2 Tensor, dim int) Tensor
 	Rows(ctx Context, t2 Tensor) Tensor
 	Copy(ctx Context, t2 Tensor) Tensor
-	Duplicate(ctx Context) Tensor
 }
 
 // ScaledDotProductAttention implements a fused attention
@@ -229,7 +214,7 @@ func Dump(ctx Context, t Tensor, opts ...DumpOptions) string {
 			return strconv.FormatFloat(float64(f), 'f', opts[0].Precision, 32)
 		})
 	case DTypeF16, DTypeQ80, DTypeQ40:
-		f32 := ctx.Input().Empty(DTypeF32, t.Shape()...)
+		f32 := ctx.Empty(DTypeF32, t.Shape()...)
 		f32 = t.Copy(ctx, f32)
 		return dump[[]float32](ctx, f32, opts[0].Items, func(f float32) string {
 			return strconv.FormatFloat(float64(f), 'f', opts[0].Precision, 32)

ml/backend/ggml/ggml.go

@@ -9,6 +9,7 @@ package ggml
 import "C"
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -20,6 +21,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"unicode"
 	"unsafe"
@@ -43,12 +45,8 @@ func devices() []*C.struct_ggml_backend_device {
 }
 
 type Backend struct {
-	meta *fsggml.GGML
-
-	sched         *C.struct_ggml_backend_sched
-	schedBackends []*C.struct_ggml_backend
-	schedBufts    []*C.struct_ggml_backend_buffer_type
-
+	meta    *fsggml.GGML
+	sched   *C.struct_ggml_backend_sched
 	tensors map[string]*C.struct_ggml_tensor
 
 	// input is the backend used for inputs
@@ -286,10 +284,6 @@ func New(ctx context.Context, r *os.File, params ml.BackendParams) (ml.Backend,
 		}
 
 		b := C.ggml_backend_alloc_ctx_tensors_from_buft(c, bt)
-		if b == nil {
-			return nil, fmt.Errorf("unable to allocate memory from device %v for model weights", C.GoString(C.ggml_backend_buft_name(bt)))
-		}
-
 		C.ggml_backend_buffer_set_usage(b, C.GGML_BACKEND_BUFFER_USAGE_WEIGHTS)
 		bbs[c] = b
 	}
@@ -308,6 +302,11 @@ func New(ctx context.Context, r *os.File, params ml.BackendParams) (ml.Backend,
 
 	var doneBytes atomic.Uint64
 	totalBytes := uint64(n) - meta.Tensors().Offset
+	pool := sync.Pool{
+		New: func() any {
+			return new(bytes.Buffer)
+		},
+	}
 
 	g, ctx := errgroup.WithContext(ctx)
 	g.SetLimit(runtime.GOMAXPROCS(0))
@@ -328,27 +327,33 @@ func New(ctx context.Context, r *os.File, params ml.BackendParams) (ml.Backend,
 				tts[i] = tt
 			}
 
-			// Create a new FD for each goroutine so that each FD is read sequentially, rather than
-			// seeking around within an FD shared between all goroutines.
-			file, err := os.Open(r.Name())
-			if err != nil {
-				return err
-			}
-			defer file.Close()
-			sr := io.NewSectionReader(file, int64(meta.Tensors().Offset+t.Offset), int64(t.Size()))
-			bts := make([]byte, 128*format.KibiByte)
+			sr := io.NewSectionReader(r, int64(meta.Tensors().Offset+t.Offset), int64(t.Size()))
+			// bts := make([]byte, 128*format.KibiByte)
 
 			var s uint64
 			for s < t.Size() {
-				n, err := io.ReadFull(sr, bts[:min(len(bts), int(t.Size()-s))])
-				if err != nil {
+				b := pool.Get().(*bytes.Buffer)
+				b.Reset()
+
+				// n, err := io.ReadFull(sr, bts[:min(len(bts), int(t.Size()-s))])
+				// if err != nil {
+				// 	return err
+				// }
+				n, err := io.CopyN(b, sr, 256*format.KibiByte)
+				if n > 0 {
+				} else if errors.Is(err, io.EOF) {
+					break
+				} else if err != nil {
 					return err
 				}
 
+				bts := b.Bytes()
 				for _, tt := range tts {
 					C.ggml_backend_tensor_set(tt, unsafe.Pointer(&bts[0]), C.size_t(s), C.size_t(n))
 				}
 
+				pool.Put(b)
+
 				s += uint64(n)
 
 				if params.Progress != nil {
@@ -394,6 +399,8 @@ func New(ctx context.Context, r *os.File, params ml.BackendParams) (ml.Backend,
 		schedBackends = append(schedBackends, b)
 		schedBufts = append(schedBufts, bt)
 
+		slog.Info("compute graph", "backend", C.GoString(C.ggml_backend_name(b)), "buffer_type", C.GoString(C.ggml_backend_buft_name(bt)))
+
 		if C.ggml_backend_is_cpu(b) {
 			// set number of threads for cpu backend
 			C.ggml_backend_cpu_set_n_threads(b, C.int(Threads(params.NumThreads)))
@@ -412,9 +419,7 @@ func New(ctx context.Context, r *os.File, params ml.BackendParams) (ml.Backend,
 			C.size_t(maxGraphNodes),
 			C._Bool(len(gpus) > 1 && slices.Contains(gpus, output.d)),
 		),
-		schedBackends: schedBackends,
-		schedBufts:    schedBufts,
-		input:         deviceBufferTypes[input.d],
+		input: deviceBufferTypes[input.d],
 		layers: func() map[int]*C.struct_ggml_backend_buffer_type {
 			m := make(map[int]*C.struct_ggml_backend_buffer_type)
 			for i, layer := range layers {
@@ -539,24 +544,6 @@ func (c Context) Compute(tensors ...ml.Tensor) {
 	}
 }
 
-func (c Context) Reserve() error {
-	if !C.ggml_backend_sched_reserve(c.b.sched, c.graph) {
-		C.ggml_backend_sched_reset(c.b.sched)
-		return errors.New("failed to reserve graph")
-	}
-
-	slog.Debug("compute graph", "nodes", C.ggml_graph_n_nodes(c.graph), "splits", C.ggml_backend_sched_get_n_splits(c.b.sched))
-	for i := range c.b.schedBackends {
-		size := C.ggml_backend_sched_get_buffer_size(c.b.sched, c.b.schedBackends[i])
-		slog.Info("compute graph", "backend", C.GoString(C.ggml_backend_name(c.b.schedBackends[i])), "buffer_type", C.GoString(C.ggml_backend_buft_name(c.b.schedBufts[i])),
-			"size", format.HumanBytes2(uint64(size)))
-	}
-
-	C.ggml_backend_sched_reset(c.b.sched)
-
-	return nil
-}
-
 func (c Context) MaxGraphNodes() int {
 	return c.maxGraphNodes
 }
@@ -574,9 +561,9 @@ func pad(length, pad C.size_t) C.size_t {
 	return ((length + pad - 1) / pad) * pad
 }
 
-func (c Context) newTensor(dtype ml.DType, shape []int) (ml.Tensor, error) {
+func (c Context) newTensor(dtype ml.DType, shape []int) ml.Tensor {
 	if c.buft == nil {
-		panic("set Input or Layer before creating tensors")
+		panic("set Input, Output, or Layer before creating tensors")
 	}
 
 	var cdtype uint32
@@ -597,7 +584,7 @@ func (c Context) newTensor(dtype ml.DType, shape []int) (ml.Tensor, error) {
 
 	if len(shape) < 1 || shape[0] == 0 {
 		var shape C.int64_t = 0
-		return &Tensor{b: c.b, t: C.ggml_new_tensor(c.ctx, cdtype, 1, &shape)}, nil
+		return &Tensor{b: c.b, t: C.ggml_new_tensor(c.ctx, cdtype, 1, &shape)}
 	} else if len(shape) > 4 {
 		panic("unsupported number of dimensions")
 	}
@@ -611,29 +598,16 @@ func (c Context) newTensor(dtype ml.DType, shape []int) (ml.Tensor, error) {
 	t := C.ggml_new_tensor(c.ctx, cdtype, C.int(len(shape)), shapeToGGML(shape))
 	size := pad(C.ggml_backend_buft_get_alloc_size(c.buft, t), C.ggml_backend_buft_get_alignment(c.buft))
 	b := C.ggml_backend_buft_alloc_buffer(c.buft, size)
-	if b == nil {
-		return nil, fmt.Errorf("unable to allocate %v from device %v for new tensor", format.HumanBytes2(uint64(size)), C.GoString(C.ggml_backend_buft_name(c.buft)))
-	}
-
 	C.ggml_backend_tensor_alloc(b, t, C.ggml_backend_buffer_get_base(b))
-	return &Tensor{b: c.b, t: t}, nil
+	return &Tensor{b: c.b, t: t}
 }
 
 func (c Context) Empty(dtype ml.DType, shape ...int) ml.Tensor {
-	t, err := c.newTensor(dtype, shape)
-	if err != nil {
-		panic(err)
-	}
-
-	return t
+	return c.newTensor(dtype, shape)
 }
 
 func (c Context) Zeros(dtype ml.DType, shape ...int) ml.Tensor {
-	t, err := c.newTensor(dtype, shape)
-	if err != nil {
-		panic(err)
-	}
-
+	t := c.newTensor(dtype, shape)
 	C.ggml_set_zero(t.(*Tensor).t)
 	return t
 }
@@ -661,11 +635,7 @@ func (c Context) FromFloatSlice(s []float32, shape ...int) (ml.Tensor, error) {
 		return nil, err
 	}
 
-	t, err := c.newTensor(ml.DTypeF32, shape)
-	if err != nil {
-		return nil, err
-	}
-
+	t := c.newTensor(ml.DTypeF32, shape)
 	if len(s) > 0 {
 		C.ggml_backend_tensor_set(t.(*Tensor).t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t.(*Tensor).t))
 	}
@@ -678,11 +648,7 @@ func (c Context) FromIntSlice(s []int32, shape ...int) (ml.Tensor, error) {
 		return nil, err
 	}
 
-	t, err := c.newTensor(ml.DTypeI32, shape)
-	if err != nil {
-		return nil, err
-	}
-
+	t := c.newTensor(ml.DTypeI32, shape)
 	if len(s) > 0 {
 		C.ggml_backend_tensor_set(t.(*Tensor).t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t.(*Tensor).t))
 	}
@@ -766,13 +732,6 @@ func (t *Tensor) DType() ml.DType {
 	}
 }
 
-func (t *Tensor) Neg(ctx ml.Context) ml.Tensor {
-	return &Tensor{
-		b: t.b,
-		t: C.ggml_neg(ctx.(*Context).ctx, t.t),
-	}
-}
-
 func (t *Tensor) Add(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
 	return &Tensor{
 		b: t.b,
@@ -780,27 +739,6 @@ func (t *Tensor) Add(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
 	}
 }
 
-func (t *Tensor) Repeat(ctx ml.Context, dim, n int) ml.Tensor {
-	if dim < 0 || dim >= C.GGML_MAX_DIMS {
-		panic("invalid dimension")
-	}
-
-	shape := make([]C.int64_t, C.GGML_MAX_DIMS)
-	for i := range C.GGML_MAX_DIMS {
-		if i == dim {
-			shape[i] = C.int64_t(t.Dim(i) * n)
-		} else {
-			shape[i] = C.int64_t(t.Dim(i))
-		}
-	}
-
-	tmpl := C.ggml_new_tensor(ctx.(*Context).ctx, t.t._type, C.int(len(shape)), unsafe.SliceData(shape))
-	return &Tensor{
-		b: t.b,
-		t: C.ggml_repeat(ctx.(*Context).ctx, t.t, tmpl),
-	}
-}
-
 func (t *Tensor) Stack(ctx ml.Context, dim int, s ...ml.Tensor) ml.Tensor {
 	if len(s) > 0 {
 		return t.Concat(ctx, s[0].Stack(ctx, dim, s[1:]...), dim)
@@ -937,20 +875,6 @@ func (t *Tensor) Softmax(ctx ml.Context) ml.Tensor {
 	}
 }
 
-func (t *Tensor) Sin(ctx ml.Context) ml.Tensor {
-	return &Tensor{
-		b: t.b,
-		t: C.ggml_sin(ctx.(*Context).ctx, t.t),
-	}
-}
-
-func (t *Tensor) Cos(ctx ml.Context) ml.Tensor {
-	return &Tensor{
-		b: t.b,
-		t: C.ggml_cos(ctx.(*Context).ctx, t.t),
-	}
-}
-
 func (t *Tensor) Tanh(ctx ml.Context) ml.Tensor {
 	return &Tensor{
 		b: t.b,
@@ -1039,13 +963,6 @@ func (t *Tensor) RoPE(ctx ml.Context, positionIDs, ropeFactors ml.Tensor, ropeDi
 	}
 }
 
-func (t *Tensor) IM2Col(ctx ml.Context, t2 ml.Tensor, s0, s1, p0, p1, d0, d1 int) ml.Tensor {
-	return &Tensor{
-		b: t.b,
-		t: C.ggml_im2col(ctx.(*Context).ctx, t.t, t2.(*Tensor).t, C.int(s0), C.int(s1), C.int(p0), C.int(p1), C.int(d0), C.int(d1), true, C.GGML_TYPE_F32),
-	}
-}
-
 func (t *Tensor) GELU(ctx ml.Context) ml.Tensor {
 	return &Tensor{
 		b: t.b,
@@ -1114,10 +1031,3 @@ func (t *Tensor) ScaledDotProductAttention(ctx ml.Context, key, value, mask ml.T
 		return kqv.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
 	}
 }
-
-func (t *Tensor) Duplicate(ctx ml.Context) ml.Tensor {
-	return &Tensor{
-		b: t.b,
-		t: C.ggml_dup(ctx.(*Context).ctx, t.t),
-	}
-}

ml/backend/ggml/ggml/src/ggml-metal/ggml-metal-embed.metal

@@ -3083,13 +3083,6 @@ kernel void kernel_cos(
     dst[tpig] = cos(src0[tpig]);
 }
 
-kernel void kernel_neg(
-        device const float * src0,
-        device       float * dst,
-        uint tpig[[thread_position_in_grid]]) {
-    dst[tpig] = -src0[tpig];
-}
-
 kernel void kernel_sum_rows(
         device const float * src0,
         device       float * dst,

ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.m

@@ -423,7 +423,6 @@ enum ggml_metal_kernel_type {
     GGML_METAL_KERNEL_TYPE_SQRT,
     GGML_METAL_KERNEL_TYPE_SIN,
     GGML_METAL_KERNEL_TYPE_COS,
-    GGML_METAL_KERNEL_TYPE_NEG,
     GGML_METAL_KERNEL_TYPE_SUM_ROWS,
     GGML_METAL_KERNEL_TYPE_POOL_2D_AVG_F32,
     GGML_METAL_KERNEL_TYPE_POOL_2D_MAX_F32,
@@ -1040,7 +1039,6 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de
         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQRT,                          sqrt,                           true);
         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SIN,                           sin,                            true);
         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_COS,                           cos,                            true);
-        GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NEG,                           neg,                            true);
         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS,                      sum_rows,                       true);
         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGMAX,                        argmax,                         true);
         GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_POOL_2D_AVG_F32,               pool_2d_avg_f32,                true);
@@ -1204,7 +1202,6 @@ static bool ggml_metal_supports_op(const struct ggml_backend_metal_device_contex
                 case GGML_UNARY_OP_GELU_QUICK:
                 case GGML_UNARY_OP_SILU:
                 case GGML_UNARY_OP_ELU:
-                case GGML_UNARY_OP_NEG:
                     return ggml_is_contiguous(op->src[0]);
                 default:
                     return false;
@@ -1876,18 +1873,6 @@ static void ggml_metal_encode_node(
 
                     [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
                 } break;
-                case GGML_UNARY_OP_NEG:
-                {
-                    id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_NEG].pipeline;
-
-                    [encoder setComputePipelineState:pipeline];
-                    [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
-                    [encoder setBuffer:id_dst  offset:offs_dst  atIndex:1];
-
-                    const int64_t n = ggml_nelements(dst);
-
-                    [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
-                } break;
                 default:
                 {
                     GGML_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, idx, ggml_op_name(dst->op));

ml/backend/ggml/ggml/src/ggml-metal/ggml-metal.metal

@@ -945,13 +945,6 @@ kernel void kernel_cos(
     dst[tpig] = cos(src0[tpig]);
 }
 
-kernel void kernel_neg(
-        device const float * src0,
-        device       float * dst,
-        uint tpig[[thread_position_in_grid]]) {
-    dst[tpig] = -src0[tpig];
-}
-
 kernel void kernel_sum_rows(
         device const float * src0,
         device       float * dst,

ml/backend/ggml/ggml2.go

@@ -0,0 +1,273 @@
+package ggml
+
+// #cgo CPPFLAGS: -I${SRCDIR}/ggml/include
+// #include <stdlib.h>
+// #include <stdint.h>
+// #include "ggml.h"
+// #include "ggml-cpu.h"
+// #include "ggml-backend.h"
+import "C"
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"runtime"
+	"sync"
+	"unsafe"
+
+	"github.com/ollama/ollama/format"
+	"github.com/ollama/ollama/fs"
+	"github.com/ollama/ollama/ml"
+	ggml "github.com/ollama/ollama/ml/backend/ggml/ggml/src"
+	"golang.org/x/sync/errgroup"
+)
+
+type backend struct {
+	gpus, cpus []*C.struct_ggml_backend_device
+	bufts      map[*C.struct_ggml_backend_device][]*C.struct_ggml_backend_buffer_type
+	ctxs       map[*C.struct_ggml_backend_buffer_type]*C.struct_ggml_context
+	bbs        map[*C.struct_ggml_backend_buffer_type]*C.struct_ggml_backend_buffer
+	readers    map[*C.struct_ggml_tensor]io.Reader
+	reserved   map[*C.struct_ggml_context]uint64
+
+	onceScheduler sync.Once
+	scheduler     *scheduler
+}
+
+var _ ml.Backend2 = (*backend)(nil)
+
+func New2() (ml.Backend2, error) {
+	ggml.OnceLoad()
+
+	var cpus, accels, gpus []*C.struct_ggml_backend_device
+	for i := range C.ggml_backend_dev_count() {
+		d := C.ggml_backend_dev_get(C.size_t(i))
+		switch C.ggml_backend_dev_type(d) {
+		case C.GGML_BACKEND_DEVICE_TYPE_CPU:
+			// only the first cpu device should be used
+			if len(cpus) > 0 {
+				continue
+			}
+
+			cpus = append(cpus, d)
+		case C.GGML_BACKEND_DEVICE_TYPE_ACCEL:
+			accels = append(accels, d)
+		case C.GGML_BACKEND_DEVICE_TYPE_GPU:
+			gpus = append(gpus, d)
+		}
+	}
+
+	bufts := make(map[*C.struct_ggml_backend_device][]*C.struct_ggml_backend_buffer_type)
+
+	cpu := C.ggml_backend_dev_by_type(C.GGML_BACKEND_DEVICE_TYPE_CPU)
+	for _, d := range append(accels, cpus...) {
+		bufts[cpu] = append(bufts[cpu], C.ggml_backend_dev_buffer_type(d))
+	}
+
+	for _, d := range gpus {
+		bufts[d] = append(bufts[d], append([]*C.struct_ggml_backend_buffer_type{C.ggml_backend_dev_buffer_type(d)}, bufts[cpu]...)...)
+	}
+
+	return &backend{
+		// merge accels and cpus
+		gpus:     gpus,
+		cpus:     append(accels, cpus...),
+		bufts:    bufts,
+		ctxs:     make(map[*C.struct_ggml_backend_buffer_type]*C.struct_ggml_context, len(bufts)),
+		bbs:      make(map[*C.struct_ggml_backend_buffer_type]*C.struct_ggml_backend_buffer, len(bufts)),
+		readers:  make(map[*C.struct_ggml_tensor]io.Reader),
+		reserved: make(map[*C.struct_ggml_context]uint64),
+	}, nil
+}
+
+func (b *backend) Close() {
+}
+
+func (b *backend) NewContext() ml.Context {
+	return &Context{
+		b: &Backend{
+			input:  b.bufts[b.cpus[0]][0],
+			output: b.bufts[b.cpus[0]][0],
+			layers: func() map[int]*C.struct_ggml_backend_buffer_type {
+				m := make(map[int]*C.struct_ggml_backend_buffer_type)
+				for i := range 100 {
+					m[i] = b.bufts[b.gpus[0]][0]
+				}
+				return m
+			}(),
+			sched: func() *C.struct_ggml_backend_sched {
+				return b.Scheduler().(*scheduler).s
+			}(),
+			maxGraphNodes: 8192,
+		},
+		ctx: C.ggml_init(C.struct_ggml_init_params{
+			mem_size: C.ggml_tensor_overhead() * C.size_t(4000),
+			no_alloc: true,
+		}),
+		buft:          b.bufts[b.cpus[0]][0],
+		maxGraphNodes: 8192,
+	}
+}
+
+func (b *backend) Get(tensorReader fs.TensorReader, preferredDevice ml.Device) ml.Tensor {
+	var ctx *C.struct_ggml_context
+
+	var devices []*C.struct_ggml_backend_device
+	if preferredDevice == ml.GPU {
+		devices = b.gpus
+	}
+
+	for _, d := range append(devices, b.cpus...) {
+		var free, total C.size_t
+		C.ggml_backend_dev_memory(d, &free, &total)
+
+		for _, buft := range b.bufts[d] {
+			if _, ok := b.ctxs[buft]; !ok {
+				b.ctxs[buft] = C.ggml_init(C.struct_ggml_init_params{
+					mem_size: C.ggml_tensor_overhead() * C.size_t(1000),
+					no_alloc: true,
+				})
+			}
+
+			ctx = b.ctxs[buft]
+			if free > 0 && b.reserved[ctx]+uint64(tensorReader.Size()) >= uint64(free) {
+				slog.Info("no space available", "device", C.GoString(C.ggml_backend_dev_name(d)), "free", format.HumanBytes2(uint64(free)), "total", format.HumanBytes2(uint64(total)), "reserve", format.HumanBytes2(b.reserved[ctx]), "size", format.HumanBytes2(uint64(tensorReader.Size())))
+				continue
+			}
+
+			cname := C.CString(tensorReader.Name())
+			defer C.free(unsafe.Pointer(cname))
+
+			if t := C.ggml_get_tensor(ctx, cname); t != nil {
+				slog.Info("using existing tensor in buffer type", "name", tensorReader.Name(), "buffer_type", C.GoString(C.ggml_backend_buft_name(buft)))
+				return &Tensor{t: t}
+			}
+
+			shape := make([]C.int64_t, len(tensorReader.Shape()))
+			for i, s := range tensorReader.Shape() {
+				shape[i] = C.int64_t(s)
+			}
+
+			t := C.ggml_new_tensor(ctx, uint32(tensorReader.DType()), C.int(len(tensorReader.Shape())), unsafe.SliceData(shape))
+			C.ggml_set_name(t, cname)
+
+			b.readers[t] = tensorReader
+			b.reserved[ctx] += uint64(tensorReader.Size())
+
+			slog.Info("creating new tensor in buffer type", "name", tensorReader.Name(), "buffer_type", C.GoString(C.ggml_backend_buft_name(buft)), "reserve", format.HumanBytes2(b.reserved[ctx]))
+			return &Tensor{t: t}
+		}
+	}
+
+	panic("no device available")
+}
+
+func (b *backend) LoadAll(ctx context.Context) error {
+	// allocate buffers for each context
+	for buft, ctx := range b.ctxs {
+		if C.ggml_get_first_tensor(ctx) == nil {
+			continue
+		}
+
+		bb := C.ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft)
+		C.ggml_backend_buffer_set_usage(bb, C.GGML_BACKEND_BUFFER_USAGE_WEIGHTS)
+		b.bbs[buft] = bb
+	}
+
+	for _, bb := range b.bbs {
+		slog.Info("", "buffer.size", C.ggml_backend_buffer_get_size(bb), "buffer.usage", C.ggml_backend_buffer_get_usage(bb))
+	}
+
+	pool := sync.Pool{
+		New: func() any {
+			return new(bytes.Buffer)
+		},
+	}
+
+	g, ctx := errgroup.WithContext(context.Background())
+	g.SetLimit(runtime.GOMAXPROCS(0))
+	for t, r := range b.readers {
+		g.Go(func() error {
+			var s uint64
+
+			for {
+				b := pool.Get().(*bytes.Buffer)
+				b.Reset()
+
+				n, err := io.CopyN(b, r, 32*format.KibiByte)
+				if n > 0 {
+				} else if errors.Is(err, io.EOF) {
+					break
+				} else if err != nil {
+					return err
+				}
+
+				C.ggml_backend_tensor_set(t, unsafe.Pointer(&b.Bytes()[0]), C.size_t(s), C.size_t(n))
+				pool.Put(b)
+			}
+
+			return nil
+		})
+	}
+
+	go func() {
+		<-ctx.Done()
+		g.Go(func() error {
+			return ctx.Err()
+		})
+	}()
+
+	return g.Wait()
+}
+
+type scheduler struct {
+	s *C.struct_ggml_backend_sched
+}
+
+var (
+	_ ml.Scheduler = (*scheduler)(nil)
+	_ ml.Reserver  = (*scheduler)(nil)
+)
+
+func (b *backend) Scheduler() ml.Scheduler {
+	b.onceScheduler.Do(func() {
+		devices := append(b.gpus, b.cpus...)
+		backends := make([]C.ggml_backend_t, len(devices))
+		bufts := make([]C.ggml_backend_buffer_type_t, len(devices))
+		for i, device := range devices {
+			backend := C.ggml_backend_dev_init(device, nil)
+			buft := C.ggml_backend_get_default_buffer_type(backend)
+			if d := C.ggml_backend_get_device(backend); C.ggml_backend_dev_type(d) == C.GGML_BACKEND_DEVICE_TYPE_CPU && len(b.gpus) > 0 {
+				if hbt := C.ggml_backend_dev_host_buffer_type(b.gpus[0]); hbt != nil {
+					buft = hbt
+				}
+			}
+
+			slog.Info("scheduler", "backend", C.GoString(C.ggml_backend_name(backend)), "buffer_type", C.GoString(C.ggml_backend_buft_name(buft)))
+			backends[i] = backend
+			bufts[i] = buft
+		}
+
+		maxGraphNodes := max(8192, 1)
+		b.scheduler = &scheduler{
+			s: C.ggml_backend_sched_new(
+				unsafe.SliceData(backends),
+				unsafe.SliceData(bufts),
+				C.int(len(backends)),
+				C.size_t(maxGraphNodes),
+				C._Bool(len(b.gpus) > 1),
+			),
+		}
+	})
+
+	return b.scheduler
+}
+
+func (s scheduler) Schedule() {
+}
+
+func (s scheduler) Reserve() {
+}

ml/backend2.go

@@ -0,0 +1,25 @@
+package ml
+
+import (
+	"context"
+
+	"github.com/ollama/ollama/fs"
+)
+
+type Device int
+
+const (
+	CPU Device = iota
+	GPU
+)
+
+type Backend2 interface {
+	Close()
+
+	NewContext() Context
+
+	Scheduler() Scheduler
+
+	Get(fs.TensorReader, Device) Tensor
+	LoadAll(context.Context) error
+}

ml/scheduler.go

@@ -0,0 +1,11 @@
+package ml
+
+// Scheduler is an interface that can be implemented by a Backend to schedule resources.
+type Scheduler interface {
+	Schedule()
+}
+
+// Reserver is an optional interface that can be implemented by a Scheduler to reserve resources for the compute graph.
+type Reserver interface {
+	Reserve()
+}

model/model.go

@@ -256,16 +256,23 @@ func setPointer(base Base, v reflect.Value, tags []Tag) {
 type Tag struct {
 	Name      string
 	Alternate []string
+	Root      bool
+	Device    ml.Device
 }
 
 func ParseTags(s string) (tag Tag) {
 	parts := strings.Split(s, ",")
 	if len(parts) > 0 {
 		tag.Name = parts[0]
+		tag.Device = ml.GPU
 
 		for _, part := range parts[1:] {
 			if value, ok := strings.CutPrefix(part, "alt:"); ok {
 				tag.Alternate = append(tag.Alternate, value)
+			} else if value, ok := strings.CutPrefix(part, "root:"); ok {
+				tag.Root, _ = strconv.ParseBool(value)
+			} else if part == "cpu" {
+				tag.Device = ml.CPU
 			}
 		}
 	}
@@ -299,7 +306,7 @@ func Forward(ctx ml.Context, m Model, inputs []int32, batch input.Batch) (ml.Ten
 
 	cache := m.Config().Cache
 	if cache != nil {
-		err := cache.StartForward(ctx, batch, false)
+		err := cache.StartForward(ctx, batch)
 		if err != nil {
 			return nil, err
 		}

model/model2.go

@@ -0,0 +1,139 @@
+package model
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/ollama/ollama/fs"
+	"github.com/ollama/ollama/ml"
+)
+
+type Model2 struct {
+	ml.Backend2
+	Model
+}
+
+func New2(cfg *fs.Model, b ml.Backend2) (*Model2, error) {
+	fn, ok := models[cfg.KV.Architecture()]
+	if !ok {
+		return nil, fmt.Errorf("unsupported model architecture %q", cfg.KV.Architecture())
+	}
+
+	m, err := fn(cfg.KV)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: load tensors from the model into the backend
+	v := reflect.ValueOf(m)
+	v.Elem().Set(temp(b, cfg.Tensors, v.Elem()))
+
+	if r, ok := b.Scheduler().(ml.Reserver); ok {
+		// TODO: build a graph of the model and reserve the necessary resources
+		r.Reserve()
+	}
+
+	return &Model2{b, m}, nil
+}
+
+func temp(b ml.Backend2, tensors map[string]fs.TensorReader, v reflect.Value, tags ...Tag) reflect.Value {
+	t := v.Type()
+	if t.Kind() != reflect.Struct {
+		return v
+	}
+
+	allNil := true
+	for i := range t.NumField() {
+		tt := t.Field(i).Type
+		vv := v.Field(i)
+		if !vv.CanSet() {
+			continue
+		}
+
+		tagsCopy := tags
+		if s := t.Field(i).Tag.Get("gguf"); s != "" {
+			tag := ParseTags(s)
+			if tag.Root {
+				tagsCopy = []Tag{tag}
+			} else {
+				tagsCopy = append(tagsCopy, ParseTags(s))
+			}
+		}
+
+		switch {
+		case tt == reflect.TypeOf((*ml.Tensor)(nil)).Elem():
+			var permute func([]Tag) [][]string
+			permute = func(tags []Tag) (values [][]string) {
+				if len(tags) < 1 {
+					return nil
+				}
+
+				values = [][]string{{tags[0].Name}}
+				for _, alt := range tags[0].Alternate {
+					values = append(values, []string{alt})
+				}
+
+				for i, value := range values {
+					for _, rest := range permute(tags[1:]) {
+						value = append(value, rest...)
+					}
+
+					values[i] = value
+				}
+
+				return values
+			}
+
+			names := permute(tagsCopy)
+			for _, name := range names {
+				if tensor, ok := tensors[strings.Join(name, ".")]; ok {
+					vv.Set(reflect.ValueOf(b.Get(tensor, tags[0].Device)))
+					break
+				}
+			}
+		case tt.Kind() == reflect.Pointer || tt.Kind() == reflect.Interface:
+			setPointer2(b, tensors, vv, tagsCopy)
+		case tt.Kind() == reflect.Slice || tt.Kind() == reflect.Array:
+			for i := vv.Len() - 1; i >= 0; i-- {
+				vvv := vv.Index(i)
+				if vvv.Kind() == reflect.Pointer || vvv.Kind() == reflect.Interface {
+					setPointer2(b, tensors, vvv, append(tagsCopy, Tag{Name: strconv.Itoa(i)}))
+				} else {
+					vvv.Set(temp(b, tensors, vvv, append(tagsCopy, Tag{Name: strconv.Itoa(i)})...))
+				}
+			}
+		}
+
+		if !canNil(tt) || !vv.IsNil() {
+			allNil = false
+		}
+	}
+
+	if allNil {
+		return reflect.Zero(t)
+	}
+
+	return v
+}
+
+func setPointer2(b ml.Backend2, tensors map[string]fs.TensorReader, v reflect.Value, tags []Tag) {
+	vv := v
+	if v.Kind() == reflect.Interface {
+		if v.IsNil() {
+			return
+		}
+
+		vv = vv.Elem()
+	}
+
+	vv = vv.Elem()
+	if v.IsNil() {
+		vv = reflect.New(v.Type().Elem()).Elem()
+	}
+
+	if f := temp(b, tensors, vv, tags...); f.CanAddr() {
+		v.Set(f.Addr())
+	}
+}

model/models/gemma3/model_text.go

@@ -11,7 +11,7 @@ import (
 	"github.com/ollama/ollama/model/input"
 )
 
-type TextConfig struct {
+type TextOptions struct {
 	hiddenSize, numHeads, numKVHeads int
 	attnKeyLen, attnValLen           int
 	eps, ropeScale                   float32
@@ -28,7 +28,7 @@ type TextModel struct {
 	OutputNorm     *nn.RMSNorm   `gguf:"output_norm"`
 	Output         *nn.Linear    `gguf:"output,alt:token_embd"`
 
-	*TextConfig
+	*TextOptions
 }
 
 const (
@@ -55,7 +55,7 @@ func newTextModel(c fs.Config) *TextModel {
 			},
 		),
 		Layers: make([]TextLayer, numBlocks),
-		TextConfig: &TextConfig{
+		TextOptions: &TextOptions{
 			hiddenSize:     int(c.Uint("embedding_length")),
 			numHeads:       int(c.Uint("attention.head_count")),
 			numKVHeads:     int(c.Uint("attention.head_count_kv")),
@@ -84,7 +84,7 @@ type TextSelfAttention struct {
 	Output    *nn.Linear  `gguf:"attn_output"`
 }
 
-func (sa *TextSelfAttention) Forward(ctx ml.Context, layer int, hiddenState, positionIDs ml.Tensor, cache kvcache.Cache, opts *TextConfig) ml.Tensor {
+func (sa *TextSelfAttention) Forward(ctx ml.Context, layer int, hiddenState, positionIDs ml.Tensor, cache kvcache.Cache, opts *TextOptions) ml.Tensor {
 	batchSize := hiddenState.Dim(1)
 	ropeType := uint32(2)
 
@@ -120,12 +120,12 @@ func (sa *TextSelfAttention) Forward(ctx ml.Context, layer int, hiddenState, pos
 }
 
 func (m *TextModel) Shift(ctx ml.Context, layer int, key, shift ml.Tensor) (ml.Tensor, error) {
-	ropeBase := m.TextConfig.ropeLocalBase
+	ropeBase := m.TextOptions.ropeLocalBase
 	if (layer+1)%gemmaGlobalCacheCount == 0 {
-		ropeBase = m.TextConfig.ropeGlobalBase
+		ropeBase = m.TextOptions.ropeGlobalBase
 	}
 
-	return key.RoPE(ctx, shift, nil, uint32(m.TextConfig.attnKeyLen), uint32(2), ropeBase, m.TextConfig.ropeScale), nil
+	return key.RoPE(ctx, shift, nil, uint32(m.TextOptions.attnKeyLen), uint32(2), ropeBase, m.TextOptions.ropeScale), nil
 }
 
 type TextMLP struct {
@@ -134,7 +134,7 @@ type TextMLP struct {
 	Gate *nn.Linear `gguf:"ffn_gate"`
 }
 
-func (mlp *TextMLP) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *TextConfig) ml.Tensor {
+func (mlp *TextMLP) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *TextOptions) ml.Tensor {
 	hiddenState = mlp.Gate.Forward(ctx, hiddenState).GELU(ctx).Mul(ctx, mlp.Up.Forward(ctx, hiddenState))
 	return mlp.Down.Forward(ctx, hiddenState)
 }
@@ -148,7 +148,7 @@ type TextLayer struct {
 	PostMLPNorm       *nn.RMSNorm `gguf:"post_ffw_norm"`
 }
 
-func (l *TextLayer) Forward(ctx ml.Context, layer int, hiddenState, positionIDs, outputs ml.Tensor, cache kvcache.Cache, opts *TextConfig) ml.Tensor {
+func (l *TextLayer) Forward(ctx ml.Context, layer int, hiddenState, positionIDs, outputs ml.Tensor, cache kvcache.Cache, opts *TextOptions) ml.Tensor {
 	residual := hiddenState
 
 	hiddenState = l.AttentionNorm.Forward(ctx, hiddenState, opts.eps)
@@ -173,7 +173,7 @@ func (l *TextLayer) Forward(ctx ml.Context, layer int, hiddenState, positionIDs,
 
 func (m *TextModel) Forward(ctx ml.Context, inputs, positions, outputs ml.Tensor, batch input.Batch, cache kvcache.Cache) ml.Tensor {
 	hiddenState := m.TokenEmbedding.Forward(ctx, inputs)
-	hiddenState = hiddenState.Scale(ctx, math.Sqrt(float64(m.TextConfig.hiddenSize)))
+	hiddenState = hiddenState.Scale(ctx, math.Sqrt(float64(m.TextOptions.hiddenSize)))
 
 	// set image embeddings
 	var except []int
@@ -206,7 +206,7 @@ func (m *TextModel) Forward(ctx ml.Context, inputs, positions, outputs ml.Tensor
 			lastLayerOutputs = outputs
 		}
 
-		hiddenState = layer.Forward(ctx, i, hiddenState, positions, lastLayerOutputs, cache, m.TextConfig)
+		hiddenState = layer.Forward(ctx, i, hiddenState, positions, lastLayerOutputs, cache, m.TextOptions)
 	}
 
 	hiddenState = m.OutputNorm.Forward(ctx, hiddenState, m.eps)

model/models/llama/model.go

@@ -23,7 +23,7 @@ type Model struct {
 	model.Base
 	model.BytePairEncoding
 
-	TokenEmbedding *nn.Embedding `gguf:"token_embd"`
+	TokenEmbedding *nn.Embedding `gguf:"token_embd,cpu"`
 	Layers         []Layer       `gguf:"blk"`
 	OutputNorm     *nn.RMSNorm   `gguf:"output_norm"`
 	Output         *nn.Linear    `gguf:"output,alt:token_embd"`
@@ -61,7 +61,7 @@ func New(c fs.Config) (model.Model, error) {
 		},
 	}
 
-	m.Cache = kvcache.NewCausalCache(m.Shift)
+	// m.Cache = kvcache.NewCausalCache(m.Shift)
 
 	return &m, nil
 }
@@ -71,7 +71,7 @@ type SelfAttention struct {
 	Key         *nn.Linear `gguf:"attn_k"`
 	Value       *nn.Linear `gguf:"attn_v"`
 	Output      *nn.Linear `gguf:"attn_output"`
-	RopeFactors ml.Tensor  `gguf:"rope_freqs.weight"`
+	RopeFactors ml.Tensor  `gguf:"rope_freqs.weight,root:true"`
 }
 
 func (sa *SelfAttention) Forward(ctx ml.Context, hiddenState, positionIDs ml.Tensor, cache kvcache.Cache, opts *Options) ml.Tensor {
@@ -91,7 +91,7 @@ func (sa *SelfAttention) Forward(ctx ml.Context, hiddenState, positionIDs ml.Ten
 	v = v.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
 
 	scaleFactor := 1.0 / math.Sqrt(float64(headDim))
-	kqv := nn.Attention(ctx, q, k, v, scaleFactor, cache)
+	kqv := nn.Attention(ctx, q, k, v, scaleFactor, nil)
 	kqv = kqv.Reshape(ctx, opts.hiddenSize, batchSize)
 
 	return sa.Output.Forward(ctx, kqv)
@@ -154,7 +154,7 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {
 	hiddenState := m.TokenEmbedding.Forward(ctx, batch.Inputs)
 
 	for i, layer := range m.Layers {
-		m.Cache.SetLayer(i)
+		// m.Cache.SetLayer(i)
 
 		var lastLayerOutputs ml.Tensor
 		if i == len(m.Layers)-1 {

model/models/mistral3/imageproc.go

@@ -1,56 +0,0 @@
-package mistral3
-
-import (
-	"image"
-	_ "image/jpeg"
-	_ "image/png"
-	"math"
-
-	"github.com/ollama/ollama/fs"
-	"github.com/ollama/ollama/model/imageproc"
-)
-
-type ImageProcessor struct {
-	imageSize   int
-	patchSize   int
-	numChannels int
-	longestEdge int
-}
-
-func newImageProcessor(c fs.Config) ImageProcessor {
-	return ImageProcessor{
-		imageSize:   int(c.Uint("vision.image_size", 1540)),
-		patchSize:   int(c.Uint("vision.patch_size", 14)),
-		numChannels: int(c.Uint("vision.num_channels", 3)),
-		longestEdge: int(c.Uint("vision.longest_edge", 1540)),
-	}
-}
-
-// ProcessImage prepares an image for the vision model by:
-// 1. Compositing transparent images
-// 2. Resizing to fit model constraints while preserving aspect ratio
-// 3. Normalizing pixel values
-// Returns normalized image data and the final size in pixels
-func (p *ImageProcessor) ProcessImage(img image.Image) ([]float32, image.Point, error) {
-	img = imageproc.Composite(img)
-
-	size := img.Bounds().Size()
-	ratio := max(float64(size.Y)/float64(p.longestEdge), float64(size.X)/float64(p.longestEdge))
-	if ratio > 1.0 {
-		size = image.Point{
-			int(math.Floor(float64(size.X) / ratio)),
-			int(math.Floor(float64(size.Y) / ratio)),
-		}
-	}
-
-	patchesX := (size.X-1)/p.patchSize + 1
-	patchesY := (size.Y-1)/p.patchSize + 1
-	size = image.Point{
-		patchesX * p.patchSize,
-		patchesY * p.patchSize,
-	}
-
-	img = imageproc.Resize(img, size, imageproc.ResizeBilinear)
-	data := imageproc.Normalize(img, imageproc.ClipDefaultMean, imageproc.ClipDefaultSTD, true, true)
-	return data, size, nil
-}

model/models/mistral3/model.go

@@ -1,189 +0,0 @@
-package mistral3
-
-import (
-	"bytes"
-	"image"
-	"slices"
-	"sync"
-
-	"github.com/ollama/ollama/fs"
-	"github.com/ollama/ollama/kvcache"
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-	"github.com/ollama/ollama/model"
-	"github.com/ollama/ollama/model/input"
-)
-
-type Model struct {
-	model.Base
-	*TextModel
-	*VisionModel         `gguf:"v,vision"`
-	*MultiModalProjector `gguf:"mm"`
-
-	ImageProcessor
-}
-
-// Implement MultimodalProcessor interface
-var _ model.MultimodalProcessor = (*Model)(nil)
-
-func New(c fs.Config) (model.Model, error) {
-	textModel, err := NewTextModel(c)
-	if err != nil {
-		return nil, err
-	}
-
-	m := &Model{
-		TextModel:           textModel,
-		VisionModel:         newVisionModel(c),
-		ImageProcessor:      newImageProcessor(c),
-		MultiModalProjector: newMultiModalProjector(c),
-	}
-
-	m.Cache = kvcache.NewCausalCache(m.TextModel.Shift)
-
-	return m, nil
-}
-
-type PatchMerger struct {
-	MergingLayer *nn.Linear `gguf:"merging_layer"`
-}
-
-func (pm *PatchMerger) Forward(ctx ml.Context, visionOutputs ml.Tensor, size image.Point, spatialMergeSize int) ml.Tensor {
-	d := visionOutputs.Dim(0)
-	imageGrid := visionOutputs.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx).Reshape(ctx, size.X, size.Y, d)
-	kernel := ctx.Input().Empty(ml.DTypeF32, spatialMergeSize, spatialMergeSize, d)
-	patches := kernel.IM2Col(ctx, imageGrid, spatialMergeSize, spatialMergeSize, 0, 0, 1, 1)
-	reshaped := patches.Reshape(ctx, d*spatialMergeSize*spatialMergeSize, patches.Dim(1)*patches.Dim(2))
-	return pm.MergingLayer.Forward(ctx, reshaped)
-}
-
-type MultiModalProjector struct {
-	Norm        *nn.RMSNorm  `gguf:"norm"`
-	Linear1     *nn.Linear   `gguf:"linear_1"`
-	Linear2     *nn.Linear   `gguf:"linear_2"`
-	PatchMerger *PatchMerger `gguf:"patch_merger"`
-
-	spatialMergeSize int
-	eps              float32
-	patchSize        int
-}
-
-func (p *MultiModalProjector) Forward(ctx ml.Context, visionOutputs ml.Tensor, size image.Point) (ml.Tensor, image.Point) {
-	visionOutputs = p.Norm.Forward(ctx, visionOutputs, p.eps)
-	patchSizes := image.Point{size.X / p.patchSize, size.Y / p.patchSize}
-	visionOutputs = p.PatchMerger.Forward(ctx, visionOutputs, patchSizes, p.spatialMergeSize)
-	visionOutputs = p.Linear1.Forward(ctx, visionOutputs)
-	visionOutputs = visionOutputs.GELU(ctx)
-	return p.Linear2.Forward(ctx, visionOutputs), image.Point{patchSizes.X / p.spatialMergeSize, patchSizes.Y / p.spatialMergeSize}
-}
-
-func newMultiModalProjector(c fs.Config) *MultiModalProjector {
-	return &MultiModalProjector{
-		spatialMergeSize: int(c.Uint("spatial_merge_size", 2)),
-		eps:              c.Float("text_config.rms_norm_eps", 1e-5),
-		patchSize:        int(c.Uint("vision.patch_size", 14)),
-	}
-}
-
-func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) (any, error) {
-	if len(m.VisionModel.Layers) == 0 {
-		return nil, model.ErrNoVisionModel
-	}
-
-	image, _, err := image.Decode(bytes.NewReader(multimodalData))
-	if err != nil {
-		return nil, err
-	}
-
-	f32s, size, err := m.ImageProcessor.ProcessImage(image)
-	if err != nil {
-		return nil, err
-	}
-
-	pixelValues, err := ctx.Input().FromFloatSlice(f32s, size.X, size.Y, m.ImageProcessor.numChannels)
-	if err != nil {
-		return nil, err
-	}
-
-	visionOutputs := m.VisionModel.Forward(ctx, pixelValues)
-	features, size := m.MultiModalProjector.Forward(ctx, visionOutputs, size)
-
-	// split into patches to be sent to the text transformer
-	parent := imageFeatures{tensor: features}
-	rows := make([]*imageRow, size.Y)
-	for i := range rows {
-		rows[i] = &imageRow{parent: &parent, s: i, shape: []int{features.Dim(0), size.X}}
-	}
-
-	return rows, nil
-}
-
-type imageFeatures struct {
-	tensor ml.Tensor
-
-	dataOnce sync.Once
-	data     []float32
-}
-
-type imageRow struct {
-	parent *imageFeatures
-	s      int
-	shape  []int
-}
-
-func (r *imageRow) data() []float32 {
-	n := 1
-	for _, s := range r.shape {
-		n *= s
-	}
-
-	return r.parent.data[r.s*n : (r.s+1)*n]
-}
-
-// PostTokenize arranges Mistral 3's inputs for the forward pass
-// In Mistral 3 and Pixtral, the input patches are arranged as follows:
-// [IMG]...[IMG][IMG_BREAK][IMG]...[IMG][IMG_BREAK][IMG]...[IMG][IMG_END]
-// Each sequence of [IMG]...[IMG] is a set of patches of vision embeddings
-// that can be processed together.
-func (m *Model) PostTokenize(inputs []input.Input) ([]input.Input, error) {
-	var result []input.Input
-	for _, inp := range inputs {
-		if inp.Multimodal == nil {
-			result = append(result, inp)
-		} else {
-			inputMultimodal := inp.Multimodal.([]*imageRow)
-			for i, row := range inputMultimodal {
-				// [IMG]
-				result = append(result, input.Input{Token: 10, Multimodal: row, MultimodalHash: inp.MultimodalHash, SameBatch: row.shape[1]})
-				result = append(result, slices.Repeat([]input.Input{{Token: 10}}, row.shape[1]-1)...)
-				if i == len(inputMultimodal)-1 {
-					// [IMG_END]
-					result = append(result, input.Input{Token: 13})
-				} else {
-					// [IMG_BREAK]
-					result = append(result, input.Input{Token: 12})
-				}
-			}
-		}
-	}
-
-	return result, nil
-}
-
-func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {
-	positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions))
-	if err != nil {
-		return nil, err
-	}
-
-	outputs, err := ctx.Input().FromIntSlice(batch.Outputs, len(batch.Outputs))
-	if err != nil {
-		return nil, err
-	}
-
-	return m.TextModel.Forward(ctx, batch.Inputs, positions, outputs, batch, m.Cache), nil
-}
-
-func init() {
-	model.Register("mistral3", New)
-}

model/models/mistral3/model_text.go

@@ -1,177 +0,0 @@
-package mistral3
-
-import (
-	"fmt"
-	"math"
-	"strings"
-
-	"github.com/ollama/ollama/fs"
-	"github.com/ollama/ollama/kvcache"
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-	"github.com/ollama/ollama/model"
-	"github.com/ollama/ollama/model/input"
-)
-
-type TextOptions struct {
-	hiddenSize, numHeads, numKVHeads, headDim int
-	eps, ropeBase, ropeScale                  float32
-	ropeDim                                   uint32
-}
-
-type TextModel struct {
-	model.Base
-	model.BytePairEncoding
-
-	TokenEmbedding *nn.Embedding `gguf:"token_embd"`
-	Layers         []Layer       `gguf:"blk"`
-	OutputNorm     *nn.RMSNorm   `gguf:"output_norm"`
-	Output         *nn.Linear    `gguf:"output,alt:token_embd"`
-
-	*TextOptions
-}
-
-type SelfAttention struct {
-	Query  *nn.Linear `gguf:"attn_q"`
-	Key    *nn.Linear `gguf:"attn_k"`
-	Value  *nn.Linear `gguf:"attn_v"`
-	Output *nn.Linear `gguf:"attn_output"`
-}
-
-func (sa *SelfAttention) Forward(ctx ml.Context, hiddenState, positionIDs ml.Tensor, cache kvcache.Cache, opts *TextOptions) ml.Tensor {
-	batchSize := hiddenState.Dim(1)
-	ropeType := uint32(0)
-	headDim := opts.headDim
-	if headDim == 0 {
-		headDim = opts.hiddenSize / opts.numHeads
-	}
-
-	q := sa.Query.Forward(ctx, hiddenState)
-	q = q.Reshape(ctx, headDim, opts.numHeads, batchSize)
-	q = q.RoPE(ctx, positionIDs, nil, opts.ropeDim, ropeType, opts.ropeBase, opts.ropeScale)
-
-	k := sa.Key.Forward(ctx, hiddenState)
-	k = k.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
-	k = k.RoPE(ctx, positionIDs, nil, opts.ropeDim, ropeType, opts.ropeBase, opts.ropeScale)
-
-	v := sa.Value.Forward(ctx, hiddenState)
-	v = v.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
-
-	kqv := nn.Attention(ctx, q, k, v, 1.0/math.Sqrt(float64(headDim)), cache)
-	kqv = kqv.Reshape(ctx, headDim*opts.numHeads, batchSize)
-	return sa.Output.Forward(ctx, kqv)
-}
-
-func (m *TextModel) Shift(ctx ml.Context, layer int, key, shift ml.Tensor) (ml.Tensor, error) {
-	return key.RoPE(ctx, shift, nil, uint32(0), m.ropeDim, m.ropeBase, m.ropeScale), nil
-}
-
-type MLP struct {
-	Up   *nn.Linear `gguf:"ffn_up"`
-	Down *nn.Linear `gguf:"ffn_down"`
-	Gate *nn.Linear `gguf:"ffn_gate"`
-}
-
-func (mlp *MLP) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *TextOptions) ml.Tensor {
-	hiddenState = mlp.Gate.Forward(ctx, hiddenState).SILU(ctx).Mul(ctx, mlp.Up.Forward(ctx, hiddenState))
-	return mlp.Down.Forward(ctx, hiddenState)
-}
-
-type Layer struct {
-	AttentionNorm *nn.RMSNorm `gguf:"attn_norm"`
-	SelfAttention *SelfAttention
-	MLPNorm       *nn.RMSNorm `gguf:"ffn_norm"`
-	MLP           *MLP
-}
-
-func (l *Layer) Forward(ctx ml.Context, hiddenState, positionIDs, outputs ml.Tensor, cache kvcache.Cache, opts *TextOptions) ml.Tensor {
-	residual := hiddenState
-
-	hiddenState = l.AttentionNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = l.SelfAttention.Forward(ctx, hiddenState, positionIDs, cache, opts)
-
-	// In the final layer (outputs != nil), optimize by pruning to just the token positions
-	// we need logits for.
-	if outputs != nil {
-		hiddenState = hiddenState.Rows(ctx, outputs)
-		residual = residual.Rows(ctx, outputs)
-	}
-
-	hiddenState = hiddenState.Add(ctx, residual)
-	residual = hiddenState
-
-	hiddenState = l.MLPNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = l.MLP.Forward(ctx, hiddenState, opts)
-	return hiddenState.Add(ctx, residual)
-}
-
-func (m *TextModel) Forward(ctx ml.Context, inputs, positions, outputs ml.Tensor, batch input.Batch, cache kvcache.Cache) ml.Tensor {
-	hiddenState := m.TokenEmbedding.Forward(ctx, inputs).Duplicate(ctx)
-
-	// image embeddings
-	for _, image := range batch.Multimodal {
-		row := image.Multimodal.(*imageRow)
-		row.parent.dataOnce.Do(func() {
-			// use a new, throwaway context so the image tensor is not added to the graph
-			temp := m.Backend().NewContext()
-			temp.Forward(row.parent.tensor).Compute(row.parent.tensor)
-			row.parent.data = row.parent.tensor.Floats()
-			temp.Close()
-		})
-
-		imageFeature, err := ctx.Input().FromFloatSlice(row.data(), row.shape...)
-		if err != nil {
-			panic(err)
-		}
-
-		ctx.Forward(imageFeature.Copy(ctx, hiddenState.View(ctx, image.Index*hiddenState.Stride(1), imageFeature.Dim(0)*imageFeature.Dim(1))))
-	}
-
-	for i, layer := range m.Layers {
-		cache.SetLayer(i)
-
-		var lastLayerOutputs ml.Tensor
-		if i == len(m.Layers)-1 {
-			lastLayerOutputs = outputs
-		}
-
-		hiddenState = layer.Forward(ctx, hiddenState, positions, lastLayerOutputs, cache, m.TextOptions)
-	}
-
-	hiddenState = m.OutputNorm.Forward(ctx, hiddenState, m.eps)
-	return m.Output.Forward(ctx, hiddenState)
-}
-
-func NewTextModel(c fs.Config) (*TextModel, error) {
-	if !strings.EqualFold(c.String("tokenizer.ggml.model"), "gpt2") {
-		return nil, fmt.Errorf("tokenizer %s not yet supported", c.String("tokenizer.ggml.model"))
-	}
-
-	textModel := &TextModel{
-		BytePairEncoding: model.NewBytePairEncoding(
-			c.String("tokenizer.ggml.pretokenizer", `[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`),
-			&model.Vocabulary{
-				Values: c.Strings("tokenizer.ggml.tokens"),
-				Types:  c.Uints("tokenizer.ggml.token_type"),
-				Merges: c.Strings("tokenizer.ggml.merges"),
-				BOS:    int32(c.Uint("tokenizer.ggml.bos_token_id", 1)),
-				AddBOS: c.Bool("tokenizer.ggml.add_bos_token", true),
-				EOS:    int32(c.Uint("tokenizer.ggml.eos_token_id", 2)),
-				AddEOS: c.Bool("tokenizer.ggml.add_eos_token", false),
-			},
-		),
-		Layers: make([]Layer, c.Uint("block_count")),
-		TextOptions: &TextOptions{
-			hiddenSize: int(c.Uint("embedding_length")),
-			numHeads:   int(c.Uint("attention.head_count")),
-			numKVHeads: int(c.Uint("attention.head_count_kv")),
-			headDim:    int(c.Uint("attention.key_length")),
-			eps:        c.Float("attention.layer_norm_rms_epsilon"),
-			ropeBase:   c.Float("rope.freq_base"),
-			ropeScale:  c.Float("rope.freq_scale", 1),
-			ropeDim:    c.Uint("rope.dimension_count"),
-		},
-	}
-
-	return textModel, nil
-}

model/models/mistral3/model_vision.go

@@ -1,186 +0,0 @@
-package mistral3
-
-import (
-	"math"
-
-	"github.com/ollama/ollama/fs"
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-)
-
-var batchSize int = 1
-
-func rotateHalf(ctx ml.Context, t ml.Tensor) ml.Tensor {
-	x1 := t.View(ctx, 0, t.Dim(0)/2, t.Stride(1), t.Dim(1), t.Stride(2), t.Dim(2), t.Stride(3), t.Dim(3))
-	x2 := t.View(ctx, t.Stride(0)*t.Dim(0)/2, t.Dim(0)/2, t.Stride(1), t.Dim(1), t.Stride(2), t.Dim(2), t.Stride(3), t.Dim(3)).Contiguous(ctx)
-	return x2.Neg(ctx).Concat(ctx, x1, 0)
-}
-
-func applyRotaryPositionalEmbedding(ctx ml.Context, t, cos, sin ml.Tensor) ml.Tensor {
-	return t.Mul(ctx, cos).Add(ctx, rotateHalf(ctx, t).Mul(ctx, sin))
-}
-
-type VisionSelfAttention struct {
-	Query  *nn.Linear `gguf:"attn_q"`
-	Key    *nn.Linear `gguf:"attn_k"`
-	Value  *nn.Linear `gguf:"attn_v"`
-	Output *nn.Linear `gguf:"attn_output"`
-}
-
-func (sa *VisionSelfAttention) Forward(ctx ml.Context, hiddenStates, cos, sin ml.Tensor, opts *VisionModelOptions) ml.Tensor {
-	query := sa.Query.Forward(ctx, hiddenStates)
-	key := sa.Key.Forward(ctx, hiddenStates)
-	value := sa.Value.Forward(ctx, hiddenStates)
-
-	query = query.Reshape(ctx, opts.headDim, opts.numHeads, query.Dim(1), batchSize)
-	key = key.Reshape(ctx, opts.headDim, opts.numHeads, key.Dim(1), batchSize)
-	value = value.Reshape(ctx, opts.headDim, opts.numHeads, value.Dim(1), batchSize)
-
-	query = applyRotaryPositionalEmbedding(ctx, query, cos, sin)
-	key = applyRotaryPositionalEmbedding(ctx, key, cos, sin)
-
-	attention := nn.Attention(ctx, query, key, value, 1./math.Sqrt(float64(opts.headDim)), nil)
-	attention = attention.Reshape(ctx, opts.hiddenSize, attention.Dim(2), batchSize)
-	return sa.Output.Forward(ctx, attention)
-}
-
-type VisionMLP struct {
-	Gate *nn.Linear `gguf:"ffn_gate"`
-	Up   *nn.Linear `gguf:"ffn_up"`
-	Down *nn.Linear `gguf:"ffn_down"`
-}
-
-func (mlp *VisionMLP) Forward(ctx ml.Context, hiddenStates ml.Tensor, opts *VisionModelOptions) ml.Tensor {
-	hiddenStates = mlp.Gate.Forward(ctx, hiddenStates).SILU(ctx).Mul(ctx, mlp.Up.Forward(ctx, hiddenStates))
-	return mlp.Down.Forward(ctx, hiddenStates)
-}
-
-type VisionEncoderLayer struct {
-	AttentionNorm *nn.RMSNorm `gguf:"attn_norm"`
-	SelfAttention *VisionSelfAttention
-	FFNNorm       *nn.RMSNorm `gguf:"ffn_norm"`
-	MLP           *VisionMLP
-}
-
-func (e *VisionEncoderLayer) Forward(ctx ml.Context, hiddenStates, cos, sin ml.Tensor, opts *VisionModelOptions) ml.Tensor {
-	residual := hiddenStates
-	hiddenStates = e.AttentionNorm.Forward(ctx, hiddenStates, opts.eps)
-	hiddenStates = e.SelfAttention.Forward(ctx, hiddenStates, cos, sin, opts)
-	hiddenStates = hiddenStates.Add(ctx, residual)
-
-	residual = hiddenStates
-	hiddenStates = e.FFNNorm.Forward(ctx, hiddenStates, opts.eps)
-	hiddenStates = e.MLP.Forward(ctx, hiddenStates, opts)
-	return hiddenStates.Add(ctx, residual)
-}
-
-type VisionModelOptions struct {
-	hiddenSize       int
-	numHeads         int
-	headDim          int
-	intermediateSize int
-	imageSize        int
-	patchSize        int
-	numChannels      int
-	eps              float32
-	ropeBase         float32
-}
-
-type VisionModel struct {
-	PatchEmbedding *nn.Conv2D           `gguf:"patch_conv"`
-	EncoderNorm    *nn.RMSNorm          `gguf:"encoder_norm"`
-	Layers         []VisionEncoderLayer `gguf:"blk"`
-
-	*VisionModelOptions
-}
-
-func (m *VisionModel) positionalEmbedding(ctx ml.Context, positionIDs ml.Tensor) ml.Tensor {
-	maxPatchesPerSide := m.imageSize / m.patchSize
-	frequencies := m.headDim / 2
-	frequenciesHeight := make([]float32, frequencies/2*maxPatchesPerSide)
-	frequenciesWidth := make([]float32, frequencies/2*maxPatchesPerSide)
-	for i := range frequencies {
-		for j := range maxPatchesPerSide {
-			frequency := float32(j) / float32(math.Pow(float64(m.ropeBase), float64(i)*2/float64(m.headDim)))
-			if i%2 == 0 {
-				frequenciesHeight[i/2*maxPatchesPerSide+j] = frequency
-			} else {
-				frequenciesWidth[i/2*maxPatchesPerSide+j] = frequency
-			}
-		}
-	}
-
-	h, err := ctx.Input().FromFloatSlice(frequenciesHeight, maxPatchesPerSide, frequencies/2)
-	if err != nil {
-		panic(err)
-	}
-
-	w, err := ctx.Input().FromFloatSlice(frequenciesWidth, maxPatchesPerSide, frequencies/2)
-	if err != nil {
-		panic(err)
-	}
-
-	h = h.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)
-	w = w.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)
-
-	h = h.Repeat(ctx, 1, maxPatchesPerSide)
-	h = h.Reshape(ctx, frequencies/2, maxPatchesPerSide, maxPatchesPerSide).Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	w = w.Repeat(ctx, 2, maxPatchesPerSide)
-
-	inverseFrequencies := h.Concat(ctx, w, 0).Reshape(ctx, frequencies, maxPatchesPerSide*maxPatchesPerSide)
-	inverseFrequencies = inverseFrequencies.Concat(ctx, inverseFrequencies, 0)
-	return inverseFrequencies.Rows(ctx, positionIDs)
-}
-
-func (m *VisionModel) Forward(ctx ml.Context, pixelValues ml.Tensor) ml.Tensor {
-	numPatchesW := pixelValues.Dim(0) / m.patchSize
-	numPatchesH := pixelValues.Dim(1) / m.patchSize
-	numPatches := numPatchesW * numPatchesH
-
-	hiddenStates := m.PatchEmbedding.Forward(ctx, pixelValues, m.patchSize, m.patchSize, 0, 0, 1, 1)
-	hiddenStates = hiddenStates.Reshape(ctx, numPatches, m.hiddenSize)
-	hiddenStates = hiddenStates.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)
-	hiddenStates = m.EncoderNorm.Forward(ctx, hiddenStates, m.VisionModelOptions.eps)
-
-	// Prepare position IDs for 2D rope
-	positions := make([]int32, numPatches)
-	for h := range numPatchesH {
-		for w := range numPatchesW {
-			idx := h*numPatchesW + w
-			positions[idx] = int32(h*m.imageSize/m.patchSize + w)
-		}
-	}
-
-	positionIDs, err := ctx.Input().FromIntSlice(positions, len(positions))
-	if err != nil {
-		panic(err)
-	}
-
-	positionEmbedding := m.positionalEmbedding(ctx, positionIDs)
-	cos, sin := positionEmbedding.Cos(ctx), positionEmbedding.Sin(ctx)
-	cos = cos.Reshape(ctx, cos.Dim(0), 1, cos.Dim(1))
-	sin = sin.Reshape(ctx, sin.Dim(0), 1, sin.Dim(1))
-
-	for _, layer := range m.Layers {
-		hiddenStates = layer.Forward(ctx, hiddenStates, cos, sin, m.VisionModelOptions)
-	}
-
-	return hiddenStates
-}
-
-func newVisionModel(c fs.Config) *VisionModel {
-	return &VisionModel{
-		Layers: make([]VisionEncoderLayer, c.Uint("vision.block_count", 24)),
-		VisionModelOptions: &VisionModelOptions{
-			hiddenSize:       int(c.Uint("vision.embedding_length", 1024)),
-			numHeads:         int(c.Uint("vision.attention.head_count", 16)),
-			headDim:          int(c.Uint("vision.attention.key_length", 64)),
-			intermediateSize: int(c.Uint("vision.feed_forward_length", 4096)),
-			imageSize:        int(c.Uint("vision.image_size", 1540)),
-			patchSize:        int(c.Uint("vision.patch_size", 14)),
-			numChannels:      int(c.Uint("vision.num_channels", 3)),
-			eps:              c.Float("vision.attention.layer_norm_epsilon", 1e-5),
-			ropeBase:         c.Float("vision.rope.freq_base", 10000.0),
-		},
-	}
-}

model/models/mllama/model_vision.go

@@ -186,7 +186,7 @@ func (m *VisionModel) Forward(ctx ml.Context, pixelValues, positionIDs, aspectRa
 	hiddenState = hiddenState.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)
 
 	hiddenState = m.PreTilePositionEmbedding.Forward(ctx, hiddenState, aspectRatioIDs, m.VisionModelOptions)
-	hiddenState = m.ClassEmbedding.Repeat(ctx, 2, m.numTiles).Concat(ctx, hiddenState, 1)
+	hiddenState = m.ClassEmbedding.Stack(ctx, 2, slices.Repeat([]ml.Tensor{m.ClassEmbedding}, m.numTiles-1)...).Concat(ctx, hiddenState, 1)
 
 	hiddenState = m.PositionEmbedding.Forward(ctx, hiddenState, positionIDs, aspectRatioIDs, numPositions, m.VisionModelOptions)
 	hiddenState = m.PreLayerNorm.Forward(ctx, hiddenState, m.eps)

model/models/models.go

@@ -4,6 +4,5 @@ import (
 	_ "github.com/ollama/ollama/model/models/gemma2"
 	_ "github.com/ollama/ollama/model/models/gemma3"
 	_ "github.com/ollama/ollama/model/models/llama"
-	_ "github.com/ollama/ollama/model/models/mistral3"
 	_ "github.com/ollama/ollama/model/models/mllama"
 )

model/models/pixtral/imageproc.go

@@ -0,0 +1,68 @@
+package pixtral
+
+import (
+	"fmt"
+	"image"
+	_ "image/jpeg"
+	_ "image/png"
+	"io"
+	"math"
+
+	"github.com/ollama/ollama/model/imageproc"
+)
+
+func getNumImageTokens(imageSize, patchSize image.Point) image.Point {
+	return image.Point{
+		(imageSize.X-1)/patchSize.X + 1,
+		(imageSize.Y-1)/patchSize.Y + 1,
+	}
+}
+
+func getResizeOutputImageSize(img image.Image, longestEdge int, patchSize image.Point) image.Point {
+	b := img.Bounds()
+	le := float64(longestEdge)
+	ratio := math.Max(float64(b.Max.Y)/le, float64(b.Max.X)/le)
+
+	newSize := img.Bounds().Max
+
+	if ratio > 1.0 {
+		newSize = image.Point{
+			int(math.Ceil(float64(b.Max.X) / ratio)),
+			int(math.Ceil(float64(b.Max.Y) / ratio)),
+		}
+	}
+
+	tokens := getNumImageTokens(newSize, patchSize)
+	return image.Point{
+		tokens.X * patchSize.X,
+		tokens.Y * patchSize.Y,
+	}
+}
+
+func resizeImage(img image.Image, format string, longestEdge int, patchSize image.Point) image.Image {
+	if format == "png" {
+		img = imageproc.Composite(img)
+	}
+
+	newSize := getResizeOutputImageSize(img, longestEdge, patchSize)
+
+	// todo should be ResizeBicubic, but it doesn't exist
+	return imageproc.Resize(img, newSize, imageproc.ResizeBilinear)
+}
+
+func Preprocess(imageData io.Reader) ([]float32, map[string]any, error) {
+	img, format, err := image.Decode(imageData)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to decode image: %w", err)
+	}
+
+	longestEdge := 1024
+	patchSize := image.Point{16, 16}
+
+	img = resizeImage(img, format, longestEdge, patchSize)
+
+	data := imageproc.Normalize(img, imageproc.ClipDefaultMean, imageproc.ClipDefaultSTD, true, true)
+
+	opts := map[string]any{}
+	return data, opts, nil
+}

model/models/pixtral/imageproc_test.go

@@ -0,0 +1,219 @@
+package pixtral
+
+import (
+	"bytes"
+	"encoding/binary"
+	"image"
+	"image/png"
+	"math"
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestGetNumImageTokens(t *testing.T) {
+	type numImageTokensCase struct {
+		ImageSize image.Point
+		PatchSize image.Point
+		Expected  image.Point
+	}
+
+	cases := []numImageTokensCase{
+		{
+			ImageSize: image.Point{1024, 764},
+			PatchSize: image.Point{16, 16},
+			Expected:  image.Point{64, 48},
+		},
+		{
+			ImageSize: image.Point{800, 600},
+			PatchSize: image.Point{16, 16},
+			Expected:  image.Point{50, 38},
+		},
+		{
+			ImageSize: image.Point{640, 480},
+			PatchSize: image.Point{16, 16},
+			Expected:  image.Point{40, 30},
+		},
+		{
+			ImageSize: image.Point{320, 200},
+			PatchSize: image.Point{16, 16},
+			Expected:  image.Point{20, 13},
+		},
+		{
+			ImageSize: image.Point{1320, 200},
+			PatchSize: image.Point{16, 16},
+			Expected:  image.Point{83, 13},
+		},
+		{
+			ImageSize: image.Point{2000, 200},
+			PatchSize: image.Point{16, 16},
+			Expected:  image.Point{125, 13},
+		},
+		{
+			ImageSize: image.Point{10000, 200},
+			PatchSize: image.Point{16, 16},
+			Expected:  image.Point{625, 13},
+		},
+		{
+			ImageSize: image.Point{1131, 577},
+			PatchSize: image.Point{16, 16},
+			Expected:  image.Point{71, 37},
+		},
+		{
+			ImageSize: image.Point{16, 16},
+			PatchSize: image.Point{16, 16},
+			Expected:  image.Point{1, 1},
+		},
+	}
+
+	for _, c := range cases {
+		actual := getNumImageTokens(c.ImageSize, c.PatchSize)
+
+		if diff := cmp.Diff(actual, c.Expected); diff != "" {
+			t.Errorf("mismatch (-got +want):\n%s", diff)
+		}
+	}
+}
+
+func TestGetResizeOutputImageSize(t *testing.T) {
+	type resizeCase struct {
+		Image       image.Image
+		LongestEdge int
+		PatchSize   image.Point
+		Expected    image.Point
+	}
+
+	cases := []resizeCase{
+		{
+			Image:       image.NewRGBA(image.Rect(0, 0, 1024, 768)),
+			LongestEdge: 1024,
+			PatchSize:   image.Point{16, 16},
+			Expected:    image.Point{1024, 768},
+		},
+		{
+			Image:       image.NewRGBA(image.Rect(0, 0, 1162, 690)),
+			LongestEdge: 1024,
+			PatchSize:   image.Point{16, 16},
+			Expected:    image.Point{1024, 624},
+		},
+		{
+			Image:       image.NewRGBA(image.Rect(0, 0, 300, 200)),
+			LongestEdge: 1024,
+			PatchSize:   image.Point{16, 16},
+			Expected:    image.Point{304, 208},
+		},
+		{
+			Image:       image.NewRGBA(image.Rect(0, 0, 1862, 522)),
+			LongestEdge: 1024,
+			PatchSize:   image.Point{16, 16},
+			Expected:    image.Point{1024, 288},
+		},
+	}
+
+	for _, c := range cases {
+		actual := getResizeOutputImageSize(c.Image, c.LongestEdge, c.PatchSize)
+
+		if diff := cmp.Diff(actual, c.Expected); diff != "" {
+			t.Errorf("mismatch (-got +want):\n%s", diff)
+		}
+	}
+}
+
+func TestResize(t *testing.T) {
+	type resizeCase struct {
+		Image       image.Image
+		LongestEdge int
+		PatchSize   image.Point
+		Expected    image.Image
+	}
+
+	cases := []resizeCase{
+		{
+			Image:       image.NewRGBA(image.Rect(0, 0, 1862, 522)),
+			LongestEdge: 1024,
+			PatchSize:   image.Point{16, 16},
+			Expected:    image.NewRGBA(image.Rect(0, 0, 1024, 288)),
+		},
+		{
+			Image:       image.NewRGBA(image.Rect(0, 0, 10, 10)),
+			LongestEdge: 1024,
+			PatchSize:   image.Point{16, 16},
+			Expected:    image.NewRGBA(image.Rect(0, 0, 16, 16)),
+		},
+	}
+
+	for _, c := range cases {
+		actual := resizeImage(c.Image, "png", c.LongestEdge, c.PatchSize)
+
+		if actual.Bounds() != c.Expected.Bounds() {
+			t.Errorf("image size incorrect: '%#v': expected: '%#v'", actual.Bounds(), c.Expected.Bounds())
+		}
+	}
+}
+
+func TestPreprocess(t *testing.T) {
+	type preprocessCase struct {
+		TestImage   image.Image
+		ExpectedLen int
+	}
+
+	cases := []preprocessCase{
+		{
+			TestImage:   image.NewRGBA(image.Rect(0, 0, 10, 10)),
+			ExpectedLen: 16 * 16 * 3 * 1,
+		},
+		{
+			TestImage:   image.NewRGBA(image.Rect(0, 0, 2000, 2000)),
+			ExpectedLen: 1024 * 1024 * 3 * 1,
+		},
+	}
+
+	for _, c := range cases {
+		var buf bytes.Buffer
+		err := png.Encode(&buf, c.TestImage)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		imgData, _, err := Preprocess(&buf)
+		if err != nil {
+			t.Fatalf("error processing: %q", err)
+		}
+
+		switch len(imgData) {
+		case 0:
+			t.Errorf("no image data returned")
+		case c.ExpectedLen:
+			// ok
+		default:
+			t.Errorf("unexpected image data length: %d, expected: %d", len(imgData), c.ExpectedLen)
+		}
+	}
+}
+
+func TestPreprocessImages(t *testing.T) {
+	for _, testFile := range []string{"flight.png", "sportsball.png"} {
+		f, err := os.Open(testFile)
+		if err != nil {
+			t.Skipf("skipping test, no test image found at %s", testFile)
+		}
+		defer f.Close()
+
+		imgData, _, err := Preprocess(f)
+		if err != nil {
+			t.Fatalf("error processing: %q", err)
+		}
+
+		byteData := make([]byte, len(imgData)*4) // float32 is 4 bytes
+		for i, f := range imgData {
+			binary.LittleEndian.PutUint32(byteData[i*4:], math.Float32bits(f))
+		}
+
+		outputPath := "processed_" + testFile + ".bin"
+		err = os.WriteFile(outputPath, byteData, 0o644)
+		if err != nil {
+			t.Fatalf("error writing processed image: %q", err)
+		}
+	}
+}

model/process_text.go

@@ -263,10 +263,6 @@ func (bpe BytePairEncoding) Encode(s string, addSpecial bool) ([]int32, error) {
 					continue
 				}
 
-				if id := bpe.vocab.Encode(pair.value); id < 0 {
-					continue
-				}
-
 				merges[pair.a].runes = append(left.runes, right.runes...)
 				merges[pair.b].runes = nil
 

openai/openai_test.go

@@ -281,31 +281,27 @@ func TestChatMiddleware(t *testing.T) {
 							Description: "Get the current weather",
 							Parameters: struct {
 								Type       string   `json:"type"`
-								Defs       any      `json:"$defs,omitempty"`
-								Items      any      `json:"items,omitempty"`
 								Required   []string `json:"required"`
 								Properties map[string]struct {
-									Type        api.PropertyType `json:"type"`
-									Items       any              `json:"items,omitempty"`
-									Description string           `json:"description"`
-									Enum        []any            `json:"enum,omitempty"`
+									Type        string   `json:"type"`
+									Description string   `json:"description"`
+									Enum        []string `json:"enum,omitempty"`
 								} `json:"properties"`
 							}{
 								Type:     "object",
 								Required: []string{"location"},
 								Properties: map[string]struct {
-									Type        api.PropertyType `json:"type"`
-									Items       any              `json:"items,omitempty"`
-									Description string           `json:"description"`
-									Enum        []any            `json:"enum,omitempty"`
+									Type        string   `json:"type"`
+									Description string   `json:"description"`
+									Enum        []string `json:"enum,omitempty"`
 								}{
 									"location": {
-										Type:        api.PropertyType{"string"},
+										Type:        "string",
 										Description: "The city and state",
 									},
 									"unit": {
-										Type: api.PropertyType{"string"},
-										Enum: []any{"celsius", "fahrenheit"},
+										Type: "string",
+										Enum: []string{"celsius", "fahrenheit"},
 									},
 								},
 							},

parser/parser.go

@@ -11,13 +11,10 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
-	"runtime"
 	"slices"
 	"strconv"
 	"strings"
-	"sync"
 
-	"golang.org/x/sync/errgroup"
 	"golang.org/x/text/encoding/unicode"
 	"golang.org/x/text/transform"
 
@@ -147,25 +144,12 @@ func fileDigestMap(path string) (map[string]string, error) {
 		files = []string{path}
 	}
 
-	var mu sync.Mutex
-	var g errgroup.Group
-	g.SetLimit(max(runtime.GOMAXPROCS(0)-1, 1))
 	for _, f := range files {
-		g.Go(func() error {
-			digest, err := digestForFile(f)
-			if err != nil {
-				return err
-			}
-
-			mu.Lock()
-			defer mu.Unlock()
-			fl[f] = digest
-			return nil
-		})
-	}
-
-	if err := g.Wait(); err != nil {
-		return nil, err
+		digest, err := digestForFile(f)
+		if err != nil {
+			return nil, err
+		}
+		fl[f] = digest
 	}
 
 	return fl, nil
@@ -227,10 +211,16 @@ func filesForModel(path string) ([]string, error) {
 	}
 
 	var files []string
-	if st, _ := glob(filepath.Join(path, "*.safetensors"), "application/octet-stream"); len(st) > 0 {
+	if st, _ := glob(filepath.Join(path, "model*.safetensors"), "application/octet-stream"); len(st) > 0 {
 		// safetensors files might be unresolved git lfs references; skip if they are
 		// covers model-x-of-y.safetensors, model.fp32-x-of-y.safetensors, model.safetensors
 		files = append(files, st...)
+	} else if st, _ := glob(filepath.Join(path, "adapters.safetensors"), "application/octet-stream"); len(st) > 0 {
+		// covers adapters.safetensors
+		files = append(files, st...)
+	} else if st, _ := glob(filepath.Join(path, "adapter_model.safetensors"), "application/octet-stream"); len(st) > 0 {
+		// covers adapter_model.safetensors
+		files = append(files, st...)
 	} else if pt, _ := glob(filepath.Join(path, "pytorch_model*.bin"), "application/zip"); len(pt) > 0 {
 		// pytorch files might also be unresolved git lfs references; skip if they are
 		// covers pytorch_model-x-of-y.bin, pytorch_model.fp32-x-of-y.bin, pytorch_model.bin

runner/llamarunner/runner.go

@@ -83,7 +83,7 @@ type Sequence struct {
 	// true if an embedding are to be returned instead of text generation
 	embeddingOnly bool
 
-	doneReason llm.DoneReason
+	doneReason string
 
 	// Metrics
 	startProcessingTime time.Time
@@ -301,7 +301,7 @@ func flushPending(seq *Sequence) bool {
 	}
 }
 
-func (s *Server) removeSequence(seqIndex int, reason llm.DoneReason) {
+func (s *Server) removeSequence(seqIndex int, reason string) {
 	seq := s.seqs[seqIndex]
 
 	flushPending(seq)
@@ -380,7 +380,7 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
 
 		// if past the num predict limit
 		if seq.numPredict > 0 && seq.numPredicted >= seq.numPredict {
-			s.removeSequence(seqIdx, llm.DoneReasonLength)
+			s.removeSequence(seqIdx, "limit")
 			continue
 		}
 
@@ -482,7 +482,7 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
 			}
 
 			seq.embedding <- embed
-			s.removeSequence(i, llm.DoneReasonStop)
+			s.removeSequence(i, "")
 			continue
 		}
 
@@ -499,7 +499,7 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
 			// as it's important for the /api/generate context
 			// seq.responses <- piece
 
-			s.removeSequence(i, llm.DoneReasonStop)
+			s.removeSequence(i, "stop")
 			continue
 		}
 
@@ -530,7 +530,7 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
 			}
 			seq.cache.Inputs = seq.cache.Inputs[:tokenLen]
 
-			s.removeSequence(i, llm.DoneReasonStop)
+			s.removeSequence(i, "stop")
 			continue
 		}
 
@@ -543,7 +543,7 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
 		}
 
 		if !flushPending(seq) {
-			s.removeSequence(i, llm.DoneReasonConnectionClosed)
+			s.removeSequence(i, "connection")
 		}
 	}
 
@@ -657,9 +657,14 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) {
 
 				flusher.Flush()
 			} else {
+				// Send the final response
+				doneReason := "stop"
+				if seq.doneReason == "limit" {
+					doneReason = "length"
+				}
 				if err := json.NewEncoder(w).Encode(&llm.CompletionResponse{
 					Done:               true,
-					DoneReason:         seq.doneReason,
+					DoneReason:         doneReason,
 					PromptEvalCount:    seq.numPromptInputs,
 					PromptEvalDuration: seq.startGenerationTime.Sub(seq.startProcessingTime),
 					EvalCount:          seq.numDecoded,

runner/ollamarunner/cache_test.go

@@ -448,7 +448,7 @@ func (m *mockCache) Get(ctx ml.Context) (ml.Tensor, ml.Tensor, ml.Tensor)
 func (m *mockCache) Put(ctx ml.Context, key, value ml.Tensor)                                      {}
 func (m *mockCache) Init(backend ml.Backend, dtype ml.DType, maxSequences, capacity, maxBatch int) {}
 func (m *mockCache) Close()                                                                        {}
-func (m *mockCache) StartForward(ctx ml.Context, batch input.Batch, reserve bool) error            { return nil }
+func (m *mockCache) StartForward(ctx ml.Context, batch input.Batch) error                          { return nil }
 func (m *mockCache) CopyPrefix(srcSeq, dstSeq int, len int32)                                      {}
 func (m *mockCache) SetConfig(ml.CacheConfig)                                                      {}
 func (m *mockCache) CanResume(seq int, pos int32) bool                                             { return true }

runner/ollamarunner/runner.go

@@ -82,7 +82,7 @@ type Sequence struct {
 	// true if an embedding are to be returned instead of text generation
 	embeddingOnly bool
 
-	doneReason llm.DoneReason
+	doneReason string
 
 	// Metrics
 	startProcessingTime time.Time
@@ -341,7 +341,7 @@ func flushPending(seq *Sequence) bool {
 	}
 }
 
-func (s *Server) removeSequence(seqIndex int, reason llm.DoneReason) {
+func (s *Server) removeSequence(seqIndex int, reason string) {
 	seq := s.seqs[seqIndex]
 
 	flushPending(seq)
@@ -391,7 +391,7 @@ func (s *Server) processBatch() error {
 
 		// if past the num predict limit
 		if seq.numPredict > 0 && seq.numPredicted >= seq.numPredict {
-			s.removeSequence(seqIdx, llm.DoneReasonLength)
+			s.removeSequence(seqIdx, "limit")
 			continue
 		}
 
@@ -510,7 +510,7 @@ func (s *Server) processBatch() error {
 		if seq.embeddingOnly {
 			// TODO(jessegross): Embedding support
 			slog.Warn("generation of embedding outputs not yet supported")
-			s.removeSequence(i, llm.DoneReasonStop)
+			s.removeSequence(i, "")
 			continue
 		}
 
@@ -528,7 +528,7 @@ func (s *Server) processBatch() error {
 			// as it's important for the /api/generate context
 			// seq.responses <- piece
 
-			s.removeSequence(i, llm.DoneReasonStop)
+			s.removeSequence(i, "stop")
 			continue
 		}
 
@@ -564,7 +564,7 @@ func (s *Server) processBatch() error {
 			}
 			seq.cache.Inputs = seq.cache.Inputs[:tokenLen]
 
-			s.removeSequence(i, llm.DoneReasonStop)
+			s.removeSequence(i, "stop")
 			continue
 		}
 
@@ -577,7 +577,7 @@ func (s *Server) processBatch() error {
 		}
 
 		if !flushPending(seq) {
-			s.removeSequence(i, llm.DoneReasonConnectionClosed)
+			s.removeSequence(i, "connection")
 		}
 	}
 
@@ -690,9 +690,14 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) {
 
 				flusher.Flush()
 			} else {
+				// Send the final response
+				doneReason := "stop"
+				if seq.doneReason == "limit" {
+					doneReason = "length"
+				}
 				if err := json.NewEncoder(w).Encode(&llm.CompletionResponse{
 					Done:               true,
-					DoneReason:         seq.doneReason,
+					DoneReason:         doneReason,
 					PromptEvalCount:    seq.numPromptInputs,
 					PromptEvalDuration: seq.startGenerationTime.Sub(seq.startProcessingTime),
 					EvalCount:          seq.numPredicted,
@@ -728,51 +733,6 @@ func (m *multiLPath) String() string {
 	return strings.Join(*m, ", ")
 }
 
-func (s *Server) reserveWorstCaseGraph() error {
-	ctx := s.model.Backend().NewContext()
-	defer ctx.Close()
-
-	var batch input.Batch
-
-	inputs := make([]int32, s.batchSize)
-	batch.Positions = make([]int32, len(inputs))
-	batch.Sequences = make([]int, len(inputs))
-	for i := range inputs {
-		batch.Positions[i] = int32(i)
-	}
-
-	batch.Outputs = make([]int32, s.parallel)
-	for i := range batch.Outputs {
-		batch.Outputs[i] = int32(i)
-	}
-
-	var err error
-	batch.Inputs, err = ctx.Input().FromIntSlice(inputs, len(inputs))
-	if err != nil {
-		return err
-	}
-
-	cache := s.model.Config().Cache
-	if cache != nil {
-		err := cache.StartForward(ctx, batch, true)
-		if err != nil {
-			return err
-		}
-	}
-
-	t, err := s.model.Forward(ctx, batch)
-	if err != nil {
-		return err
-	}
-
-	err = ctx.Forward(t).Reserve()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (s *Server) loadModel(
 	ctx context.Context,
 	mpath string,
@@ -810,11 +770,6 @@ func (s *Server) loadModel(
 	s.seqs = make([]*Sequence, s.parallel)
 	s.seqsSem = semaphore.NewWeighted(int64(s.parallel))
 
-	err = s.reserveWorstCaseGraph()
-	if err != nil {
-		panic(err)
-	}
-
 	s.status = llm.ServerStatusReady
 	s.ready.Done()
 }

server/model.go

@@ -10,7 +10,6 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
-	"regexp"
 	"slices"
 	"strings"
 	"text/template/parse"
@@ -154,342 +153,99 @@ func parseObjects(s string) []map[string]any {
 	return objs
 }
 
-// Get tool call token from model template
-func (m *Model) TemplateToolToken() (string, string, bool) {
-	// Try to detect the tool call format from the model's template
+// parseToolCalls attempts to parse a JSON string into a slice of ToolCalls.
+// mxyng: this only really works if the input contains tool calls in some JSON format
+func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) {
+	// create a subtree from the node that ranges over .ToolCalls
 	tmpl := m.Template.Subtree(func(n parse.Node) bool {
 		if t, ok := n.(*parse.RangeNode); ok {
 			return slices.Contains(template.Identifiers(t.Pipe), "ToolCalls")
 		}
+
 		return false
 	})
 
-	// fmt.Println("tool call template", tmpl)
-	if tmpl != nil {
-		// Execute template with test data to see the format
-		var b bytes.Buffer
-		if err := tmpl.Execute(&b, map[string][]api.ToolCall{
-			"ToolCalls": {
-				{
-					Function: api.ToolCallFunction{
-						Name: "function_name",
-						Arguments: api.ToolCallFunctionArguments{
-							"argument1": "value1",
-							// "argument2": "value2",
-						},
-					},
-				},
-			},
-		}); err == nil {
-			// Look for special tokens in the template output
-			output := strings.TrimSpace(b.String())
-			slog.Debug("tool call template output", "output", output)
-			if strings.Contains(output, "<") {
-				// Extract the special token between < and >
-				start := strings.Index(output, "<")
-				end := strings.Index(output, ">")
-				if start >= 0 && end > start {
-					token := output[start : end+1]
-					return output, token, true
-				}
-			} else if strings.Contains(output, "[") {
-				// Check if it's a tool call token rather than JSON array
-				start := strings.Index(output, "[")
-				end := strings.Index(output, "]")
-				if start >= 0 && end > start {
-					token := output[start : end+1]
-					// Only consider it a token if it's not valid JSON
-					var jsonTest any
-					if err := json.Unmarshal([]byte(token), &jsonTest); err != nil {
-						return output, token, true
-					}
-				}
-			}
-		}
-	}
-	return "", "", false
-}
-
-func parsePythonFunctionCall(s string) ([]api.ToolCall, bool) {
-	re := regexp.MustCompile(`(\w+)\((.*?)\)`)
-	matches := re.FindAllStringSubmatchIndex(s, -1)
-	if len(matches) == 0 {
+	if tmpl == nil {
 		return nil, false
 	}
 
-	var toolCalls []api.ToolCall
-	for _, match := range matches {
-		name := s[match[2]:match[3]]
-		args := s[match[4]:match[5]]
+	var b bytes.Buffer
+	if err := tmpl.Execute(&b, map[string][]api.ToolCall{
+		"ToolCalls": {
+			{
+				Function: api.ToolCallFunction{
+					Name: "@@name@@",
+					Arguments: api.ToolCallFunctionArguments{
+						"@@argument@@": 1,
+					},
+				},
+			},
+		},
+	}); err != nil {
+		return nil, false
+	}
 
-		arguments := make(api.ToolCallFunctionArguments)
-		if strings.Contains(args, "=") { // Keyword args
-			pairs := strings.SplitSeq(args, ",")
-			for pair := range pairs {
-				pair = strings.TrimSpace(pair)
-				kv := strings.Split(pair, "=")
-				if len(kv) == 2 {
-					key := strings.TrimSpace(kv[0])
-					value := strings.TrimSpace(kv[1])
-					arguments[key] = value
-				}
+	templateObjects := parseObjects(b.String())
+	if len(templateObjects) == 0 {
+		return nil, false
+	}
+
+	// find the keys that correspond to the name and arguments fields
+	var name, arguments string
+	for k, v := range templateObjects[0] {
+		switch v.(type) {
+		case string:
+			name = k
+		case map[string]any:
+			arguments = k
+		}
+	}
+
+	if name == "" || arguments == "" {
+		return nil, false
+	}
+
+	responseObjects := parseObjects(s)
+	if len(responseObjects) == 0 {
+		return nil, false
+	}
+
+	// collect all nested objects
+	var collect func(any) []map[string]any
+	collect = func(obj any) (all []map[string]any) {
+		switch o := obj.(type) {
+		case map[string]any:
+			all = append(all, o)
+			for _, v := range o {
+				all = append(all, collect(v)...)
 			}
+		case []any:
+			for _, v := range o {
+				all = append(all, collect(v)...)
+			}
+		}
+
+		return all
+	}
+
+	var objs []map[string]any
+	for _, p := range responseObjects {
+		objs = append(objs, collect(p)...)
+	}
+
+	var toolCalls []api.ToolCall
+	for _, kv := range objs {
+		n, nok := kv[name].(string)
+		a, aok := kv[arguments].(map[string]any)
+		if nok && aok {
 			toolCalls = append(toolCalls, api.ToolCall{
 				Function: api.ToolCallFunction{
-					Name:      name,
-					Arguments: arguments,
+					Name:      n,
+					Arguments: a,
 				},
 			})
 		}
 	}
 
-	if len(toolCalls) > 0 {
-		return toolCalls, true
-	}
-	return nil, false
-}
-
-// ToolCallFormat represents different possible formats for tool calls
-type toolCallFormat struct {
-	// Direct format
-	Name      string         `json:"name,omitempty"`
-	Arguments map[string]any `json:"arguments,omitempty"`
-
-	// Command-r-plus format
-	ToolName   string         `json:"tool_name,omitempty"`
-	Parameters map[string]any `json:"parameters,omitempty"`
-
-	// Function format
-	Function *struct {
-		Name       string         `json:"name"`
-		Arguments  map[string]any `json:"arguments,omitempty"`
-		Parameters map[string]any `json:"parameters,omitempty"`
-	} `json:"function,omitempty"`
-
-	// Xlam format
-	ToolCalls []toolCallFormat `json:"tool_calls,omitempty"`
-}
-
-func parseJSONToolCalls(obj map[string]any) ([]api.ToolCall, bool) {
-	// Helper to convert any to []any safely
-	toArray := func(v any) []any {
-		if arr, ok := v.([]any); ok {
-			return arr
-		}
-		return nil
-	}
-
-	// Convert a single format to a tool call
-	makeToolCall := func(f toolCallFormat) (api.ToolCall, bool) {
-		switch {
-		case f.Name != "" && f.Arguments != nil:
-			return api.ToolCall{
-				Function: api.ToolCallFunction{
-					Name:      f.Name,
-					Arguments: f.Arguments,
-				},
-			}, true
-		case f.Name != "" && f.Parameters != nil: // Handle parameters field
-			return api.ToolCall{
-				Function: api.ToolCallFunction{
-					Name:      f.Name,
-					Arguments: f.Parameters,
-				},
-			}, true
-		case f.ToolName != "" && f.Parameters != nil:
-			return api.ToolCall{
-				Function: api.ToolCallFunction{
-					Name:      f.ToolName,
-					Arguments: f.Parameters,
-				},
-			}, true
-		case f.Function != nil && f.Function.Name != "":
-			args := f.Function.Arguments
-			if args == nil {
-				args = f.Function.Parameters
-			}
-			if args != nil {
-				return api.ToolCall{
-					Function: api.ToolCallFunction{
-						Name:      f.Function.Name,
-						Arguments: args,
-					},
-				}, true
-			}
-		}
-		return api.ToolCall{}, false
-	}
-
-	// Try parsing as array first
-	if arr := toArray(obj); arr != nil {
-		var calls []api.ToolCall
-		for _, item := range arr {
-			if itemMap, ok := item.(map[string]any); ok {
-				var format toolCallFormat
-				data, _ := json.Marshal(itemMap)
-				if err := json.Unmarshal(data, &format); err == nil {
-					if call, ok := makeToolCall(format); ok {
-						calls = append(calls, call)
-					}
-				}
-			}
-		}
-		if len(calls) > 0 {
-			return calls, true
-		}
-	}
-
-	// Try parsing as single object
-	var format toolCallFormat
-	data, _ := json.Marshal(obj)
-	if err := json.Unmarshal(data, &format); err != nil {
-		return nil, false
-	}
-
-	// Handle xlam format (tool_calls array)
-	if len(format.ToolCalls) > 0 {
-		var calls []api.ToolCall
-		for _, f := range format.ToolCalls {
-			if call, ok := makeToolCall(f); ok {
-				calls = append(calls, call)
-			}
-		}
-		if len(calls) > 0 {
-			return calls, true
-		}
-	}
-
-	// Try as single tool call
-	if call, ok := makeToolCall(format); ok {
-		return []api.ToolCall{call}, true
-	}
-
-	return nil, false
-}
-
-// token, partial, success
-func deriveToolToken(s string, prefix string) (string, bool, bool) {
-	// There shouldn't be spaces in a tool token
-	if len(strings.Fields(s)) > 1 {
-		return "", false, false
-	}
-
-	if prefix == "[" && len(s) > 1 && s[len(s)-1] == ']' {
-		return s, false, true
-	} else if prefix == "<" && len(s) > 1 && s[len(s)-1] == '>' {
-		return s, false, true
-	}
-	return "", true, true
-}
-
-func parseJSON(s string) ([]api.ToolCall, bool) {
-	objs := parseObjects(s)
-	tcs := []api.ToolCall{}
-	for _, obj := range objs {
-		toolCalls, ok := parseJSONToolCalls(obj)
-		if ok {
-			tcs = append(tcs, toolCalls...)
-		}
-	}
-	if len(tcs) > 0 {
-		return tcs, true
-	}
-	return nil, false
-}
-
-// returns tool calls, partial, success
-func (m *Model) ParseToolCalls(s string, toolToken *string) ([]api.ToolCall, bool, bool) {
-	// [ case can either be JSON, Python or a Tool Token
-	s = strings.TrimSpace(s)
-	fmt.Printf("ParseToolCallsNew input: %q\n", s)
-	if len(s) == 0 {
-		return nil, false, false
-	}
-
-	if strings.HasPrefix(s, "[") {
-		fmt.Println("Found [ prefix")
-		// JSON case
-		// we do not consider array JSONs as tool calls
-		if strings.HasPrefix(s, "[{") {
-			fmt.Println("Found [{ prefix - attempting JSON parse")
-			// TODO: mark as JSON partial
-			if calls, ok := parseJSON(s); ok {
-				fmt.Printf("Successfully parsed JSON, found %d calls\n", len(calls))
-				return calls, false, true
-			}
-			return nil, true, true
-		}
-		// Python Case
-		// We just do a full python check here
-		fmt.Println("Attempting Python function parse")
-		tc, ok := parsePythonFunctionCall(s)
-		if ok {
-			fmt.Printf("Successfully parsed Python function: %+v\n", tc)
-			return tc, false, true
-		}
-		// Tool Token Case - this is okay if it's a real tool token and we couldn't get from template
-		fmt.Println("Attempting to derive tool token")
-		if toolToken == nil || *toolToken == "" {
-			toolTok, partial, ok := deriveToolToken(s, "[")
-			if !ok {
-				return nil, false, false
-			}
-			if partial {
-				return nil, true, true
-			}
-			*toolToken = toolTok
-		}
-		fmt.Printf("Found tool token: %q\n", *toolToken)
-		s = strings.TrimSpace(s[len(*toolToken):])
-		fmt.Printf("Recursing with remaining string: %q\n", s)
-		if toolCalls, partial, ok := m.ParseToolCalls(s, toolToken); ok {
-			return toolCalls, partial, true
-		}
-		return nil, true, true
-	} else if strings.HasPrefix(s, "{") || strings.HasPrefix(s, "```") {
-		// // TODO: temp fix
-		// if strings.HasPrefix(s, "```") && len(s) == 3 {
-		// 	return nil, false, false
-		// }
-		fmt.Println("Found { prefix - attempting JSON parse with ", s)
-		if calls, ok := parseJSON(s); ok {
-			fmt.Printf("Successfully parsed JSON object, found %d calls\n", len(calls))
-			return calls, false, true
-		}
-		fmt.Println("Failed to parse JSON in JSON case")
-		// TODO: possible case where it never finishes parsing - then what?
-		return nil, true, true
-	} else if strings.HasPrefix(s, "<") {
-		fmt.Println("Found < prefix - attempting to derive tool token")
-		if toolToken == nil || *toolToken == "" {
-			toolTok, partial, ok := deriveToolToken(s, "<")
-			if !ok {
-				return nil, false, false
-			}
-			if partial {
-				return nil, true, true
-			}
-			*toolToken = toolTok
-			fmt.Printf("Found tool token: %q\n", *toolToken)
-		}
-		fmt.Printf("Found tool token: %q\n", *toolToken)
-		s = strings.TrimSpace(s[len(*toolToken):])
-		fmt.Printf("Recursing with remaining string: %q\n", s)
-		if toolCalls, partial, ok := m.ParseToolCalls(s, toolToken); ok {
-			return toolCalls, partial, true
-		}
-		return nil, true, true
-	} else if strings.Contains(s, "(") || len(strings.Fields(s)) == 1 {
-		fmt.Println("Attempting Python function parse")
-		tc, ok := parsePythonFunctionCall(s)
-		if ok {
-			fmt.Printf("Successfully parsed Python function: %+v\n", tc)
-			return tc, false, true
-		}
-		fmt.Printf("Failed to parse Python function: %q, returning partial", s)
-		return nil, true, true
-	}
-	fmt.Println("No successful parse paths found")
-	fmt.Printf("failed string: %q\n", s)
-	return nil, false, false
+	return toolCalls, len(toolCalls) > 0
 }

server/routes.go

@@ -308,10 +308,11 @@ func (s *Server) GenerateHandler(c *gin.Context) {
 			Options: opts,
 		}, func(cr llm.CompletionResponse) {
 			res := api.GenerateResponse{
-				Model:     req.Model,
-				CreatedAt: time.Now().UTC(),
-				Response:  cr.Content,
-				Done:      cr.Done,
+				Model:      req.Model,
+				CreatedAt:  time.Now().UTC(),
+				Response:   cr.Content,
+				Done:       cr.Done,
+				DoneReason: cr.DoneReason,
 				Metrics: api.Metrics{
 					PromptEvalCount:    cr.PromptEvalCount,
 					PromptEvalDuration: cr.PromptEvalDuration,
@@ -325,7 +326,6 @@ func (s *Server) GenerateHandler(c *gin.Context) {
 			}
 
 			if cr.Done {
-				res.DoneReason = cr.DoneReason.String()
 				res.TotalDuration = time.Since(checkpointStart)
 				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
 
@@ -1526,17 +1526,6 @@ func (s *Server) ChatHandler(c *gin.Context) {
 		defer close(ch)
 		var sb strings.Builder
 		var toolCallIndex int = 0
-		var sentWithTools int = 0
-		// var prefix string
-		// var templateToolToken string
-		_, templateToolToken, _ := m.TemplateToolToken()
-		// fmt.Println("special token", templateToolToken)
-
-		var minDuration time.Duration = math.MaxInt64
-		var maxDuration time.Duration
-		var totalDuration time.Duration
-		var checkCount int
-		const maxToolTokens = 1
 		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
 			Prompt:  prompt,
 			Images:  images,
@@ -1544,10 +1533,11 @@ func (s *Server) ChatHandler(c *gin.Context) {
 			Options: opts,
 		}, func(r llm.CompletionResponse) {
 			res := api.ChatResponse{
-				Model:     req.Model,
-				CreatedAt: time.Now().UTC(),
-				Message:   api.Message{Role: "assistant", Content: r.Content},
-				Done:      r.Done,
+				Model:      req.Model,
+				CreatedAt:  time.Now().UTC(),
+				Message:    api.Message{Role: "assistant", Content: r.Content},
+				Done:       r.Done,
+				DoneReason: r.DoneReason,
 				Metrics: api.Metrics{
 					PromptEvalCount:    r.PromptEvalCount,
 					PromptEvalDuration: r.PromptEvalDuration,
@@ -1557,15 +1547,6 @@ func (s *Server) ChatHandler(c *gin.Context) {
 			}
 
 			if r.Done {
-				slog.Debug("min duration", "duration", minDuration)
-				slog.Debug("max duration", "duration", maxDuration)
-				slog.Debug("total duration", "duration", totalDuration)
-				slog.Debug("check count", "count", checkCount)
-				// slog.Debug("average duration", "duration", totalDuration/time.Duration(checkCount))
-				// if sb.Len() > 0 {
-				// 	res.Message.Content = sb.String()
-				// }
-				res.DoneReason = r.DoneReason.String()
 				res.TotalDuration = time.Since(checkpointStart)
 				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
 			}
@@ -1582,48 +1563,25 @@ func (s *Server) ChatHandler(c *gin.Context) {
 			// If tools are recognized, use a flag to track the sending of a tool downstream
 			// This ensures that content is cleared from the message on the last chunk sent
 			sb.WriteString(r.Content)
-			startTime := time.Now()
-			// TODO: work max tool tok logic
-			if len(req.Tools) > 0 && sentWithTools < maxToolTokens {
-				toolCalls, partial, ok := m.ParseToolCalls(sb.String(), &templateToolToken)
-				duration := time.Since(startTime)
-				checkCount++
-				minDuration = min(minDuration, duration)
-				maxDuration = max(maxDuration, duration)
-				totalDuration += duration
-				slog.Debug("tool call duration", "duration", duration)
-				if ok {
-					// fmt.Println("toolCalls", toolCalls, partial, ok, duration)
-					if partial {
-						// If the tool call is partial, we need to wait for the next chunk
-						return
-					}
-					slog.Debug("toolCalls", "toolCalls", toolCalls, "partial", partial, "ok", ok)
-					res.Message.ToolCalls = toolCalls
-					for i := range toolCalls {
-						toolCalls[i].Function.Index = toolCallIndex
-						toolCallIndex++
-					}
-					sentWithTools = 0
-					// prefix = ""
-					templateToolToken = ""
-					res.Message.Content = ""
-					sb.Reset()
-					ch <- res
-					// TODO: revisit this
-					sentWithTools++
-					slog.Debug("fired on tool call", "toolCalls", toolCalls, "toolCallIndex", toolCallIndex)
-					return
+			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
+				res.Message.ToolCalls = toolCalls
+				for i := range toolCalls {
+					toolCalls[i].Function.Index = toolCallIndex
+					toolCallIndex++
 				}
+				res.Message.Content = ""
+				sb.Reset()
+				ch <- res
+				return
 			}
 
-			// Send any remaining content if no tool calls were detected
-			// if toolCallIndex == 0 {
-			// fmt.Println("toolCallIndex", toolCallIndex)
-			sentWithTools++
-			res.Message.Content = sb.String()
-			sb.Reset()
-			ch <- res
+			if r.Done {
+				// Send any remaining content if no tool calls were detected
+				if toolCallIndex == 0 {
+					res.Message.Content = sb.String()
+				}
+				ch <- res
+			}
 		}); err != nil {
 			ch <- gin.H{"error": err.Error()}
 		}
@@ -1632,33 +1590,11 @@ func (s *Server) ChatHandler(c *gin.Context) {
 	if req.Stream != nil && !*req.Stream {
 		var resp api.ChatResponse
 		var sb strings.Builder
-		var toolCalls []api.ToolCall
-		const MAX_TOOL_TOKENS = 1
-		sentWithTools := 0
-		var tb strings.Builder
-		_, templateToolToken, _ := m.TemplateToolToken()
 		for rr := range ch {
 			switch t := rr.(type) {
 			case api.ChatResponse:
 				sb.WriteString(t.Message.Content)
 				resp = t
-				// TODO: work max tool tok logic
-				if len(req.Tools) > 0 && sentWithTools < MAX_TOOL_TOKENS {
-					tb.WriteString(t.Message.Content)
-					if tcs, partial, ok := m.ParseToolCalls(tb.String(), &templateToolToken); ok {
-						if !partial {
-							// resp.Message.ToolCalls = toolCalls
-							toolCalls = append(toolCalls, tcs...)
-							resp.Message.Content = ""
-							tb.Reset()
-						}
-					} else {
-						// equivalent to no partial - send the content downstream
-						tb.Reset()
-						sentWithTools++
-
-					}
-				}
 			case gin.H:
 				msg, ok := t["error"].(string)
 				if !ok {
@@ -1674,17 +1610,13 @@ func (s *Server) ChatHandler(c *gin.Context) {
 		}
 
 		resp.Message.Content = sb.String()
-		if len(toolCalls) > 0 {
-			resp.Message.ToolCalls = toolCalls
-			// resp.Message.Content = ""
-		}
 
-		// if len(req.Tools) > 0 {
-		// 	if toolCalls, ok := m.ParseToolCalls(sb.String()); ok {
-		// 		resp.Message.ToolCalls = toolCalls
-		// 		resp.Message.Content = ""
-		// 	}
-		// }
+		if len(req.Tools) > 0 {
+			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
+				resp.Message.ToolCalls = toolCalls
+				resp.Message.Content = ""
+			}
+		}
 
 		c.JSON(http.StatusOK, resp)
 		return

server/routes_generate_test.go

@@ -58,7 +58,7 @@ func TestGenerateChat(t *testing.T) {
 	mock := mockRunner{
 		CompletionResponse: llm.CompletionResponse{
 			Done:               true,
-			DoneReason:         llm.DoneReasonStop,
+			DoneReason:         "stop",
 			PromptEvalCount:    1,
 			PromptEvalDuration: 1,
 			EvalCount:          1,
@@ -370,31 +370,27 @@ func TestGenerateChat(t *testing.T) {
 					Description: "Get the current weather",
 					Parameters: struct {
 						Type       string   `json:"type"`
-						Defs       any      `json:"$defs,omitempty"`
-						Items      any      `json:"items,omitempty"`
 						Required   []string `json:"required"`
 						Properties map[string]struct {
-							Type        api.PropertyType `json:"type"`
-							Items       any              `json:"items,omitempty"`
-							Description string           `json:"description"`
-							Enum        []any            `json:"enum,omitempty"`
+							Type        string   `json:"type"`
+							Description string   `json:"description"`
+							Enum        []string `json:"enum,omitempty"`
 						} `json:"properties"`
 					}{
 						Type:     "object",
 						Required: []string{"location"},
 						Properties: map[string]struct {
-							Type        api.PropertyType `json:"type"`
-							Items       any              `json:"items,omitempty"`
-							Description string           `json:"description"`
-							Enum        []any            `json:"enum,omitempty"`
+							Type        string   `json:"type"`
+							Description string   `json:"description"`
+							Enum        []string `json:"enum,omitempty"`
 						}{
 							"location": {
-								Type:        api.PropertyType{"string"},
+								Type:        "string",
 								Description: "The city and state",
 							},
 							"unit": {
-								Type: api.PropertyType{"string"},
-								Enum: []any{"celsius", "fahrenheit"},
+								Type: "string",
+								Enum: []string{"celsius", "fahrenheit"},
 							},
 						},
 					},
@@ -405,7 +401,7 @@ func TestGenerateChat(t *testing.T) {
 		mock.CompletionResponse = llm.CompletionResponse{
 			Content:            `{"name":"get_weather","arguments":{"location":"Seattle, WA","unit":"celsius"}}`,
 			Done:               true,
-			DoneReason:         llm.DoneReasonStop,
+			DoneReason:         "done",
 			PromptEvalCount:    1,
 			PromptEvalDuration: 1,
 			EvalCount:          1,
@@ -471,31 +467,27 @@ func TestGenerateChat(t *testing.T) {
 					Description: "Get the current weather",
 					Parameters: struct {
 						Type       string   `json:"type"`
-						Defs       any      `json:"$defs,omitempty"`
-						Items      any      `json:"items,omitempty"`
 						Required   []string `json:"required"`
 						Properties map[string]struct {
-							Type        api.PropertyType `json:"type"`
-							Items       any              `json:"items,omitempty"`
-							Description string           `json:"description"`
-							Enum        []any            `json:"enum,omitempty"`
+							Type        string   `json:"type"`
+							Description string   `json:"description"`
+							Enum        []string `json:"enum,omitempty"`
 						} `json:"properties"`
 					}{
 						Type:     "object",
 						Required: []string{"location"},
 						Properties: map[string]struct {
-							Type        api.PropertyType `json:"type"`
-							Items       any              `json:"items,omitempty"`
-							Description string           `json:"description"`
-							Enum        []any            `json:"enum,omitempty"`
+							Type        string   `json:"type"`
+							Description string   `json:"description"`
+							Enum        []string `json:"enum,omitempty"`
 						}{
 							"location": {
-								Type:        api.PropertyType{"string"},
+								Type:        "string",
 								Description: "The city and state",
 							},
 							"unit": {
-								Type: api.PropertyType{"string"},
-								Enum: []any{"celsius", "fahrenheit"},
+								Type: "string",
+								Enum: []string{"celsius", "fahrenheit"},
 							},
 						},
 					},
@@ -527,7 +519,7 @@ func TestGenerateChat(t *testing.T) {
 				{
 					Content:            `, WA","unit":"celsius"}}`,
 					Done:               true,
-					DoneReason:         llm.DoneReasonStop,
+					DoneReason:         "tool_call",
 					PromptEvalCount:    3,
 					PromptEvalDuration: 1,
 				},
@@ -602,7 +594,7 @@ func TestGenerate(t *testing.T) {
 	mock := mockRunner{
 		CompletionResponse: llm.CompletionResponse{
 			Done:               true,
-			DoneReason:         llm.DoneReasonStop,
+			DoneReason:         "stop",
 			PromptEvalCount:    1,
 			PromptEvalDuration: 1,
 			EvalCount:          1,

server/sched.go

@@ -667,19 +667,13 @@ func (runner *runnerRef) waitForVRAMRecovery() chan any {
 	return finished
 }
 
-type ByDurationAndName []*runnerRef
+type ByDuration []*runnerRef
 
-func (a ByDurationAndName) Len() int      { return len(a) }
-func (a ByDurationAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a ByDurationAndName) Less(i, j int) bool {
-	// Primary sort by session duration (uint64 to handle negatives)
-	d1 := uint64(a[i].sessionDuration)
-	d2 := uint64(a[j].sessionDuration)
-	if d1 != d2 {
-		return d1 < d2
-	}
-	// Secondary sort by model path lex order
-	return a[i].modelPath < a[j].modelPath
+func (a ByDuration) Len() int      { return len(a) }
+func (a ByDuration) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByDuration) Less(i, j int) bool {
+	// uint64 to turn negative time (never unload) to largest
+	return uint64(a[i].sessionDuration) < uint64(a[j].sessionDuration)
 }
 
 // TODO - future consideration to pick runners based on size
@@ -781,7 +775,7 @@ func (s *Scheduler) findRunnerToUnload() *runnerRef {
 
 	// In the future we can enhance the algorithm to be smarter about picking the optimal runner to unload
 	// e.g., if we have multiple options, will one make room for the request?
-	sort.Sort(ByDurationAndName(runnerList))
+	sort.Sort(ByDuration(runnerList))
 
 	// First try to find a runner that's already idle
 	for _, runner := range runnerList {