matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-12 16:57:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

backend: API to support full precision matmul

Browse Source 
Most tensor backends try to optimize performance by using a lower
precision for matmuls. However, some operations (such as kq) on
some models are sensitive to this and require full precision.
This commit is contained in:
Jesse Gross
2025-02-13 10:01:14 -08:00
committed by Jesse Gross
parent 4d4463b2bd
commit d773b7d671
4 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
model/llama/model.go
View File

@@ -80,7 +80,7 @@ func (sa *SelfAttention) Forward(ctx ml.Context, hiddenState, positionIDs ml.Ten
k = k.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
v = v.Permute(ctx, 1, 2, 0, 3).Contiguous(ctx)
kq := k.Mulmat(ctx, q)
kq := k.MulmatFullPrec(ctx, q)
kq = kq.Scale(ctx, 1.0/math.Sqrt(float64(headDim)))
kq = kq.Softmax(ctx)