pali112
/
ollama
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "server: model info caching system for improved performance" 

This reverts commit 8ef643d4978168a8563ae24434a424358ce390e3.
This commit is contained in:
Bruce MacDonald 2025-06-11 11:04:33 -07:00
parent 883f655dd6
commit 2348fef568
4 changed files with 5 additions and 367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
server/cache.go
View File

@ -1,95 +0,0 @@
package server
import (
"log/slog"
"os"
"sync"
"time"
)
type ModelCache struct {
mu sync.RWMutex
cache map[string]*CachedModel
}
type CachedModel struct {
model *Model
modTime time.Time
fileSize int64
}
var modelCache = &ModelCache{
cache: make(map[string]*CachedModel),
}
func init() {
modelCache.fill()
}
func (c *ModelCache) fill() {
manifests, err := Manifests(true) // continues on error
if err != nil {
slog.Warn("Failed to get manifests during cache fill", "error", err)
return
}
for modelName := range manifests {
nameStr := modelName.String()
// Load the model (this will populate the cache via GetModel -> set)
_, err := GetModel(nameStr)
if err != nil {
slog.Debug("Failed to load model during cache fill", "name", nameStr, "error", err)
continue
}
}
slog.Debug("Model cache filled")
}
func (c *ModelCache) get(name string) (*Model, bool) {
mp := ParseModelPath(name)
manifestPath, err := mp.GetManifestPath()
if err != nil {
return nil, false
}
// Check manifest file modification time
info, err := os.Stat(manifestPath)
if err != nil {
return nil, false
}
cached, exists := c.cache[name]
if exists && cached.modTime.Equal(info.ModTime()) && cached.fileSize == info.Size() {
// Cache hit - return cached model
return cached.model, true
}
// Cache miss or stale
return nil, false
}
func (c *ModelCache) set(name string, model *Model) {
mp := ParseModelPath(name)
manifestPath, err := mp.GetManifestPath()
if err != nil {
slog.Debug("Failed to get manifest path for model", "name", name, "error", err)
return
}
info, err := os.Stat(manifestPath)
if err != nil {
slog.Debug("Failed to stat manifest file", "path", manifestPath, "error", err)
return
}
c.mu.Lock()
defer c.mu.Unlock()
c.cache[name] = &CachedModel{
model: model,
modTime: info.ModTime(),
fileSize: info.Size(),
}
}

257
server/cache_test.go
View File

@ -1,257 +0,0 @@
package server
import (
"os"
"path/filepath"
"testing"
"time"
)
func TestModelCacheGet(t *testing.T) {
testModel := &Model{
Name: "test-model:latest",
ShortName: "test-model",
}
tests := []struct {
name string
modelName string
setupFunc func(t *testing.T, modelsDir string, cache *ModelCache) string // returns manifest path
expectedModel *Model
expectedExists bool
}{
{
name: "cache hit - valid cached model",
modelName: "test-model:latest",
setupFunc: func(t *testing.T, modelsDir string, cache *ModelCache) string {
createTestModel(t, modelsDir, "test-model", []Layer{
{MediaType: "application/vnd.ollama.image.model", Digest: "sha256-abc123", Size: 1000},
})
manifestPath := filepath.Join(modelsDir, "manifests", "registry.ollama.ai", "library", "test-model", "latest")
info, err := os.Stat(manifestPath)
if err != nil {
t.Fatal(err)
}
cache.cache["test-model:latest"] = &CachedModel{
model: testModel,
modTime: info.ModTime(),
fileSize: info.Size(),
}
return manifestPath
},
expectedModel: testModel,
expectedExists: true,
},
{
name: "cache miss - no cached entry",
modelName: "missing-model:latest",
setupFunc: func(t *testing.T, modelsDir string, cache *ModelCache) string {
createTestModel(t, modelsDir, "missing-model", []Layer{
{MediaType: "application/vnd.ollama.image.model", Digest: "sha256-def456", Size: 2000},
})
return filepath.Join(modelsDir, "manifests", "registry.ollama.ai", "library", "missing-model", "latest")
},
expectedModel: nil,
expectedExists: false,
},
{
name: "cache stale - modification time changed",
modelName: "stale-model:latest",
setupFunc: func(t *testing.T, modelsDir string, cache *ModelCache) string {
createTestModel(t, modelsDir, "stale-model", []Layer{
{MediaType: "application/vnd.ollama.image.model", Digest: "sha256-ghi789", Size: 3000},
})
manifestPath := filepath.Join(modelsDir, "manifests", "registry.ollama.ai", "library", "stale-model", "latest")
info, err := os.Stat(manifestPath)
if err != nil {
t.Fatal(err)
}
cache.cache["stale-model:latest"] = &CachedModel{
model: testModel,
modTime: info.ModTime().Add(-time.Hour), // Stale time
fileSize: info.Size(),
}
return manifestPath
},
expectedModel: nil,
expectedExists: false,
},
{
name: "cache stale - file size changed",
modelName: "stale-size-model:latest",
setupFunc: func(t *testing.T, modelsDir string, cache *ModelCache) string {
createTestModel(t, modelsDir, "stale-size-model", []Layer{
{MediaType: "application/vnd.ollama.image.model", Digest: "sha256-jkl012", Size: 4000},
})
manifestPath := filepath.Join(modelsDir, "manifests", "registry.ollama.ai", "library", "stale-size-model", "latest")
info, err := os.Stat(manifestPath)
if err != nil {
t.Fatal(err)
}
cache.cache["stale-size-model:latest"] = &CachedModel{
model: testModel,
modTime: info.ModTime(),
fileSize: info.Size() + 100, // Different size
}
return manifestPath
},
expectedModel: nil,
expectedExists: false,
},
{
name: "manifest file does not exist",
modelName: "nonexistent-model:latest",
setupFunc: func(t *testing.T, modelsDir string, cache *ModelCache) string {
// Add to cache but don't create manifest file
cache.cache["nonexistent-model:latest"] = &CachedModel{
model: testModel,
modTime: time.Now(),
fileSize: 100,
}
return "" // No manifest created
},
expectedModel: nil,
expectedExists: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
modelsDir := t.TempDir()
t.Setenv("OLLAMA_MODELS", modelsDir)
// Create fresh cache instance for each test
cache := &ModelCache{
cache: make(map[string]*CachedModel),
}
if tt.setupFunc != nil {
tt.setupFunc(t, modelsDir, cache)
}
model, exists := cache.get(tt.modelName)
if exists != tt.expectedExists {
t.Errorf("get() exists = %v, want %v", exists, tt.expectedExists)
}
if tt.expectedModel != nil && model != tt.expectedModel {
t.Errorf("get() model = %v, want %v", model, tt.expectedModel)
}
if tt.expectedModel == nil && model != nil {
t.Errorf("get() model = %v, want nil", model)
}
})
}
}
func TestModelCacheSet(t *testing.T) {
testModel := &Model{
Name: "test-model:latest",
ShortName: "test-model",
}
tests := []struct {
name string
modelName string
model *Model
setupFunc func(t *testing.T, modelsDir string) string // returns manifest path
expectCached bool
}{
{
name: "successful cache set",
modelName: "test-model:latest",
model: testModel,
setupFunc: func(t *testing.T, modelsDir string) string {
createTestModel(t, modelsDir, "test-model", []Layer{
{MediaType: "application/vnd.ollama.image.model", Digest: "sha256-abc123", Size: 1000},
})
return filepath.Join(modelsDir, "manifests", "registry.ollama.ai", "library", "test-model", "latest")
},
expectCached: true,
},
{
name: "manifest file does not exist",
modelName: "nonexistent-model:latest",
model: testModel,
setupFunc: func(t *testing.T, modelsDir string) string {
// Don't create manifest file
return ""
},
expectCached: false,
},
{
name: "overwrite existing cache entry",
modelName: "existing-model:latest",
model: testModel,
setupFunc: func(t *testing.T, modelsDir string) string {
createTestModel(t, modelsDir, "existing-model", []Layer{
{MediaType: "application/vnd.ollama.image.model", Digest: "sha256-def456", Size: 2000},
})
return filepath.Join(modelsDir, "manifests", "registry.ollama.ai", "library", "existing-model", "latest")
},
expectCached: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
modelsDir := t.TempDir()
t.Setenv("OLLAMA_MODELS", modelsDir)
// Create fresh cache instance for each test
cache := &ModelCache{
cache: make(map[string]*CachedModel),
}
if tt.name == "overwrite existing cache entry" {
cache.cache[tt.modelName] = &CachedModel{
model: &Model{Name: "old-model"},
modTime: time.Now().Add(-time.Hour),
fileSize: 50,
}
}
if tt.setupFunc != nil {
tt.setupFunc(t, modelsDir)
}
cache.set(tt.modelName, tt.model)
cached, exists := cache.cache[tt.modelName]
if tt.expectCached {
if !exists {
t.Errorf("set() expected model to be cached, but it wasn't")
return
}
if cached.model != tt.model {
t.Errorf("set() cached model = %v, want %v", cached.model, tt.model)
}
// Verify file info is captured correctly if manifest exists
expectedManifestPath := filepath.Join(modelsDir, "manifests", "registry.ollama.ai", "library", tt.modelName[:len(tt.modelName)-7], "latest")
if info, err := os.Stat(expectedManifestPath); err == nil {
if !cached.modTime.Equal(info.ModTime()) {
t.Errorf("set() cached modTime = %v, want %v", cached.modTime, info.ModTime())
}
if cached.fileSize != info.Size() {
t.Errorf("set() cached fileSize = %v, want %v", cached.fileSize, info.Size())
}
}
} else {
if exists {
t.Errorf("set() expected model not to be cached, but it was")
}
}
})
}
}

18
server/images.go
View File

@ -65,12 +65,11 @@ type Model struct {
Options map[string]any
Messages []api.Message
Capabilities []model.Capability
Template *template.Template
Template *template.Template
}
// Capabilities returns the capabilities that the model supports
func Capabilities(m *Model) []model.Capability {
func (m *Model) Capabilities() []model.Capability {
capabilities := []model.Capability{}
// Check for completion capability
@ -122,6 +121,7 @@ func Capabilities(m *Model) []model.Capability {
// CheckCapabilities checks if the model has the specified capabilities returning an error describing
// any missing or unknown capabilities
func (m *Model) CheckCapabilities(want ...model.Capability) error {
available := m.Capabilities()
var errs []error
// Map capabilities to their corresponding error
@ -141,7 +141,7 @@ func (m *Model) CheckCapabilities(want ...model.Capability) error {
return fmt.Errorf("unknown capability: %s", cap)
}
if !slices.Contains(m.Capabilities, cap) {
if !slices.Contains(available, cap) {
errs = append(errs, err)
}
}
@ -272,12 +272,6 @@ func GetManifest(mp ModelPath) (*Manifest, string, error) {
}
func GetModel(name string) (*Model, error) {
// Try cache first
if model, hit := modelCache.get(name); hit {
return model, nil
}
// Cache miss, load the model from disk
mp := ParseModelPath(name)
manifest, digest, err := GetManifest(mp)
if err != nil {
@ -374,10 +368,6 @@ func GetModel(name string) (*Model, error) {
}
}
model.Capabilities = Capabilities(model)
modelCache.set(name, model)
return model, nil
}

2
server/routes.go
View File

@ -824,7 +824,7 @@ func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
Template: m.Template.String(),
Details: modelDetails,
Messages: msgs,
Capabilities: m.Capabilities,
Capabilities: m.Capabilities(),
ModifiedAt: manifest.fi.ModTime(),
}