diff --git a/convert/convert_mistrallarge3.go b/convert/convert_mistrallarge3.go index d09c0f707..158667fd6 100644 --- a/convert/convert_mistrallarge3.go +++ b/convert/convert_mistrallarge3.go @@ -147,8 +147,8 @@ func (p *mistralLarge3Model) KV(t *Tokenizer) ggml.KV { kv["deepseek2.spatial_merge_size"] = p.VisionEncoder.SpatialMergeSize } - // Set tokenizer type - use deepseek-v3 since we're using deepseek2 architecture - kv["tokenizer.ggml.pre"] = "deepseek-v3" + // Set tokenizer type - use default for Mistral models + kv["tokenizer.ggml.pre"] = "tekken" // Let it use the default tokenizer preprocessing return kv } diff --git a/model/models/deepseek2/model.go b/model/models/deepseek2/model.go index c6a4ed685..cd8892891 100644 --- a/model/models/deepseek2/model.go +++ b/model/models/deepseek2/model.go @@ -4,6 +4,7 @@ package deepseek2 import ( "cmp" + "fmt" "math" "github.com/ollama/ollama/fs" @@ -215,6 +216,7 @@ type Layer struct { } func (t *Layer) Forward(ctx ml.Context, hiddenStates, positions, attentionScales, outputs ml.Tensor, cache kvcache.Cache, opts *Options) ml.Tensor { + fmt.Println("[LAYER] In the new engine") residual := hiddenStates hiddenStates = t.AttentionNorm.Forward(ctx, hiddenStates, opts.eps) hiddenStates = t.Attention.Forward(ctx, hiddenStates, positions, attentionScales, cache, opts) @@ -247,8 +249,8 @@ type Model struct { } func New(c fs.Config) (model.Model, error) { - // layers := make([]Layer, c.Uint("block_count")) - layers := make([]Layer, 4) + layers := make([]Layer, c.Uint("block_count")) + fmt.Printf("[MODEL DEBUG] Creating model with %d layers\n", c.Uint("block_count")) firstDenseLayerIndex := int(c.Uint("leading_dense_block_count")) for i := range layers { @@ -267,6 +269,7 @@ func New(c fs.Config) (model.Model, error) { valueLength := int(cmp.Or(c.Uint("attention.value_length_mla"), c.Uint("attention.value_length"))) var pre []string + fmt.Println("[TOKENIZER] Using tokenizer", c.String("tokenizer.ggml.pre")) switch c.String("tokenizer.ggml.pre") { case "deepseek-v3": pre = []string{ @@ -275,6 +278,11 @@ func New(c fs.Config) (model.Model, error) { `[一-龥぀-ゟ゠-ヿ]+`, "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+[\r\n]*|\\s*[\r\n]+|\\s+(?!\\S)|\\s+", } + case "tekken": + fmt.Println("[TOKENIZER] Using Tekken tokenizer") + pre = []string{ + "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + } case "deepseek-llm": // TODO: these models haven't been vetted so skip for now // pre = []string{ @@ -290,13 +298,37 @@ func New(c fs.Config) (model.Model, error) { return nil, model.ErrUnsupportedTokenizer } + // DEBUG: Check tokenizer vocabulary loading + tokens := c.Strings("tokenizer.ggml.tokens") + tokenTypes := c.Ints("tokenizer.ggml.token_type") + merges := c.Strings("tokenizer.ggml.merges") + + fmt.Printf("[TOKENIZER DEBUG] Loading vocabulary:\n") + fmt.Printf("[TOKENIZER DEBUG] - Tokens count: %d\n", len(tokens)) + fmt.Printf("[TOKENIZER DEBUG] - Token types count: %d\n", len(tokenTypes)) + fmt.Printf("[TOKENIZER DEBUG] - Merges count: %d\n", len(merges)) + fmt.Printf("[TOKENIZER DEBUG] - BOS token ID: %d\n", c.Uint("tokenizer.ggml.bos_token_id")) + fmt.Printf("[TOKENIZER DEBUG] - EOS token ID: %d\n", c.Uint("tokenizer.ggml.eos_token_id")) + fmt.Printf("[TOKENIZER DEBUG] - Add BOS: %v\n", c.Bool("tokenizer.ggml.add_bos_token", true)) + fmt.Printf("[TOKENIZER DEBUG] - Add EOS: %v\n", c.Bool("tokenizer.ggml.add_eos_token", false)) + + if len(tokens) > 0 { + maxShow := 10 + if len(tokens) < maxShow { + maxShow = len(tokens) + } + fmt.Printf("[TOKENIZER DEBUG] First %d tokens: %v\n", maxShow, tokens[:maxShow]) + } else { + fmt.Printf("[TOKENIZER DEBUG] ERROR: No tokens loaded from GGUF!\n") + } + m := Model{ BytePairEncoding: model.NewBytePairEncoding( &model.Vocabulary{ - Values: c.Strings("tokenizer.ggml.tokens"), - Types: c.Ints("tokenizer.ggml.token_type"), - Merges: c.Strings("tokenizer.ggml.merges"), - AddBOS: c.Bool("tokenizer.ggml.add_bos_token", true), + Values: tokens, + Types: tokenTypes, + Merges: merges, + AddBOS: false, // c.Bool("tokenizer.ggml.add_bos_token", true), BOS: []int32{int32(c.Uint("tokenizer.ggml.bos_token_id"))}, AddEOS: c.Bool("tokenizer.ggml.add_eos_token", false), EOS: append( diff --git a/runner/ollamarunner/runner.go b/runner/ollamarunner/runner.go index a756cba23..8d0c101e2 100644 --- a/runner/ollamarunner/runner.go +++ b/runner/ollamarunner/runner.go @@ -213,6 +213,7 @@ func (s *Server) NewSequence(prompt string, images []llm.ImageData, params NewSe func calculateLogprobs(logits []float32, selectedToken int32, topK int, textProcessor model.TextProcessor) []llm.Logprob { decoder := func(tokenID int) string { text, _ := textProcessor.Decode([]int32{int32(tokenID)}) + fmt.Printf("[TOKENIZER] Decoded token %d to: %q\n", tokenID, text) return text } return common.CalculateLogprobs(logits, int(selectedToken), topK, decoder) @@ -242,10 +243,52 @@ func (s *Server) inputs(prompt string, images []llm.ImageData) ([]*input.Input, for i, part := range parts { // text - tokenize + fmt.Printf("[TOKENIZER] Encoding text: %q\n", part) + + // Debug: Test what token 0 decodes to + token0Text, _ := s.model.(model.TextProcessor).Decode([]int32{0}) + fmt.Printf("[TOKENIZER] Token 0 decodes to: %q\n", token0Text) + + // Debug: Test a few other common tokens + for testToken := int32(1); testToken <= 10; testToken++ { + testText, _ := s.model.(model.TextProcessor).Decode([]int32{testToken}) + fmt.Printf("[TOKENIZER] Token %d decodes to: %q\n", testToken, testText) + } + + // Debug: Test higher token IDs where real vocabulary might be + fmt.Printf("[TOKENIZER] Testing higher token IDs:\n") + testHighTokens := []int32{100, 1000, 10000, 50000, 100000, 131000} + for _, testToken := range testHighTokens { + testText, _ := s.model.(model.TextProcessor).Decode([]int32{testToken}) + fmt.Printf("[TOKENIZER] Token %d decodes to: %q\n", testToken, testText) + } + tokens, err := s.model.(model.TextProcessor).Encode(part, i == 0) if err != nil { return nil, nil, nil, err } + fmt.Printf("[TOKENIZER] Encoded to %d tokens: %v\n", len(tokens), tokens) + + // Debug: Decode the encoded tokens back to text + if len(tokens) > 0 { + decodedText, _ := s.model.(model.TextProcessor).Decode(tokens) + fmt.Printf("[TOKENIZER] Tokens %v decode back to: %q\n", tokens, decodedText) + + // Debug: Show each token individually + fmt.Printf("[TOKENIZER] Individual tokens:\n") + for i, token := range tokens { + singleText, _ := s.model.(model.TextProcessor).Decode([]int32{token}) + fmt.Printf("[TOKENIZER] Token %d: %d → %q (hex: %x)\n", i, token, singleText, []byte(singleText)) + } + + // Debug: Test specific tokens that should be clean + fmt.Printf("[TOKENIZER] Testing specific clean tokens:\n") + testTokens := []int32{8101, 1033, 29706} // hi, !, hello + for _, testToken := range testTokens { + testText, _ := s.model.(model.TextProcessor).Decode([]int32{testToken}) + fmt.Printf("[TOKENIZER] Clean test %d → %q (hex: %x)\n", testToken, testText, []byte(testText)) + } + } for _, t := range tokens { inputs = append(inputs, &input.Input{Token: t}) @@ -780,6 +823,9 @@ func (s *Server) computeBatch(activeBatch batchState) { panic("failed to decode token") } + // DEBUG: Show what token is being generated + fmt.Printf("[GENERATION] Token %d → %q (hex: %x)\n", token, piece, []byte(piece)) + // Calculate logprobs if requested (after EOS check to avoid logprobs for EOS tokens) if seq.logprobs { logprobs := calculateLogprobs(logits, token, seq.topLogprobs, s.model.(model.TextProcessor))