Merge pull request #3684 from ollama/mxyng/scale-graph

scale graph based on gpu count

gpu/gpu_darwin.go

@@ -55,6 +55,6 @@ func getCPUMem() (memInfo, error) {
 	return memInfo{
 		TotalMemory: uint64(C.getPhysicalMemory()),
 		FreeMemory:  0,
-		DeviceCount: 0,
+		DeviceCount: 1,
 	}, nil
 }

llm/ext_server/server.cpp

@@ -39,6 +39,10 @@
 #include "httplib.h"
 #include "json.hpp"
 
+#if defined(_WIN32)
+#include <windows.h>
+#endif
+
 #include <cstddef>
 #include <thread>
 #include <chrono>
@@ -2770,8 +2774,28 @@ inline void signal_handler(int signal) {
     shutdown_handler(signal);
 }
 
-int main(int argc, char **argv)
+#if defined(_WIN32)
-{
+char* wchar_to_char(const wchar_t* wstr) {
+    if (wstr == nullptr) return nullptr;
+
+    // Determine the number of bytes needed for the UTF-8 string
+    int bytes = WideCharToMultiByte(CP_UTF8, 0, wstr, -1, nullptr, 0, nullptr, nullptr);
+    char* str = new char[bytes];
+
+    // Convert the wide-character string to a UTF-8 string
+    WideCharToMultiByte(CP_UTF8, 0, wstr, -1, str, bytes, nullptr, nullptr);
+    return str;
+}
+
+int wmain(int argc, wchar_t **wargv) {
+    char** argv = new char*[argc];
+    for (int i = 0; i < argc; ++i) {
+        argv[i] = wchar_to_char(wargv[i]);
+    }
+#else
+int main(int argc, char **argv) {
+#endif
+
 #if SERVER_VERBOSE != 1
     log_disable();
 #endif
@@ -3282,6 +3306,11 @@ int main(int argc, char **argv)
         return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
     };
     SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
+
+    for (int i = 0; i < argc; ++i) {
+        delete[] argv[i];
+    }
+    delete[] argv;
 #endif
     llama.queue_tasks.start_loop();
     svr.stop();

llm/server.go

@@ -79,6 +79,9 @@ func NewLlamaServer(model string, adapters, projectors []string, opts api.Option
 		graphFullOffload = graphPartialOffload
 	}
 
+	graphFullOffload *= uint64(info.DeviceCount)
+	graphPartialOffload *= uint64(info.DeviceCount)
+
 	// memoryRequiredTotal represents the memory required for full GPU offloading (all layers)
 	memoryRequiredTotal := memoryMinimum + graphFullOffload
 

types/model/digest.go

@@ -1,7 +1,6 @@
 package model
 
 import (
-	"fmt"
 	"log/slog"
 	"strings"
 	"unicode"
@@ -48,11 +47,8 @@ var (
 // Digest.
 func ParseDigest(s string) Digest {
 	typ, digest, ok := strings.Cut(s, "-")
-	if !ok {
-		typ, digest, ok = strings.Cut(s, ":")
-	}
 	if ok && isValidDigestType(typ) && isValidHex(digest) {
-		return Digest{s: fmt.Sprintf("%s-%s", typ, digest)}
+		return Digest{s: s}
 	}
 	return Digest{}
 }

types/model/name.go

@@ -185,8 +185,8 @@ func parseMask(s string) Name {
 	return r
 }
 
-func MustParseName(s, fill string) Name {
+func MustParseName(s, defaults string) Name {
-	r := ParseName(s, fill)
+	r := ParseName(s, "")
 	if !r.IsValid() {
 		panic("invalid Name: " + s)
 	}
@@ -643,15 +643,6 @@ func (r Name) Filepath() string {
 	return filepath.Join(r.parts[:]...)
 }
 
-// FilepathNoBuild returns a complete, canonicalized, relative file path using
-// the parts of a complete Name, but without the build part.
-func (r Name) FilepathNoBuild() string {
-	for i := range PartBuild {
-		r.parts[i] = strings.ToLower(r.parts[i])
-	}
-	return filepath.Join(r.parts[:PartBuild]...)
-}
-
 // isValidPart reports if s contains all valid characters for the given
 // part kind.
 func isValidPart(kind PartKind, s string) bool {

types/model/name_test.go

@@ -318,13 +318,6 @@ func TestNameGoString(t *testing.T) {
 	}
 }
 
-func TestDisplayLongest(t *testing.T) {
-	g := ParseName("example.com/library/mistral:latest+Q4_0", FillNothing).DisplayLongest()
-	if g != "example.com/library/mistral:latest" {
-		t.Errorf("got = %q; want %q", g, "example.com/library/mistral:latest")
-	}
-}
-
 func TestDisplayShortest(t *testing.T) {
 	cases := []struct {
 		in        string
@@ -485,36 +478,30 @@ func TestNamePath(t *testing.T) {
 	}
 }
 
-func TestNameFilepath(t *testing.T) {
+func TestNameFromFilepath(t *testing.T) {
 	cases := []struct {
-		in          string
+		in   string
-		want        string
+		want string
-		wantNoBuild string
 	}{
 		{
-			in:          "example.com/library/mistral:latest+Q4_0",
+			in:   "example.com/library/mistral:latest+Q4_0",
-			want:        "example.com/library/mistral/latest/Q4_0",
+			want: "example.com/library/mistral/latest/Q4_0",
-			wantNoBuild: "example.com/library/mistral/latest",
 		},
 		{
-			in:          "Example.Com/Library/Mistral:Latest+Q4_0",
+			in:   "Example.Com/Library/Mistral:Latest+Q4_0",
-			want:        "example.com/library/mistral/latest/Q4_0",
+			want: "example.com/library/mistral/latest/Q4_0",
-			wantNoBuild: "example.com/library/mistral/latest",
 		},
 		{
-			in:          "Example.Com/Library/Mistral:Latest+Q4_0",
+			in:   "Example.Com/Library/Mistral:Latest+Q4_0",
-			want:        "example.com/library/mistral/latest/Q4_0",
+			want: "example.com/library/mistral/latest/Q4_0",
-			wantNoBuild: "example.com/library/mistral/latest",
 		},
 		{
-			in:          "example.com/library/mistral:latest",
+			in:   "example.com/library/mistral:latest",
-			want:        "example.com/library/mistral/latest",
+			want: "example.com/library/mistral/latest",
-			wantNoBuild: "example.com/library/mistral/latest",
 		},
 		{
-			in:          "",
+			in:   "",
-			want:        "",
+			want: "",
-			wantNoBuild: "",
 		},
 	}
 	for _, tt := range cases {
@@ -526,11 +513,6 @@ func TestNameFilepath(t *testing.T) {
 			if g != tt.want {
 				t.Errorf("got = %q; want %q", g, tt.want)
 			}
-			g = p.FilepathNoBuild()
-			g = filepath.ToSlash(g)
-			if g != tt.wantNoBuild {
-				t.Errorf("got = %q; want %q", g, tt.wantNoBuild)
-			}
 		})
 	}
 }