pali112
/
ollama
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

skip quantizing per_layer_token_embd (#11207) 

this tensor isn't compatible with cuda when quantized to q4_K so skip it
This commit is contained in:
Michael Yang 2025-06-26 21:49:35 -07:00 committed by GitHub
parent 11ffc36157
commit d0b32def60
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/quantization.go
View File

@ -231,6 +231,8 @@ func newType(t *fsggml.Tensor, kv fsggml.KV, qs *quantizeState, ftype fsggml.Fil
// do not quantize relative position bias (T5) // do not quantize relative position bias (T5)
quantize = quantize && !strings.Contains(name, "attn_rel_b.weight") quantize = quantize && !strings.Contains(name, "attn_rel_b.weight")
quantize = quantize && !strings.Contains(name, "per_layer_token_embd.weight")
newType := fsggml.TensorType(t.Kind) newType := fsggml.TensorType(t.Kind)
if quantize { if quantize {
// get more optimal quantization type based on the tensor shape, layer, etc. // get more optimal quantization type based on the tensor shape, layer, etc.