matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-11 16:26:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: qwen3 dense and sparse models (#10708)

Browse Source 
* feat: qwen3 dense
* feat: qwen3moe
* fix llama4 moe
This commit is contained in:
Michael Yang
2025-05-21 10:21:07 -07:00
committed by GitHub
parent 139f84cf21
commit e0ed984cde
5 changed files with 258 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
model/models/llama4/model_text.go
View File

@@ -82,7 +82,7 @@ func (e *TextExperts) Forward(ctx ml.Context, hiddenStates, routerLogits ml.Tens
nextStates := downStates.View(ctx, 0, hiddenStates.Dim(0), downStates.Stride(2), hiddenStates.Dim(2))
for i := 1; i < opts.numExpertsUsed; i++ {
nextStates.Add(ctx, downStates.View(ctx, i*downStates.Stride(1), hiddenStates.Dim(0), downStates.Stride(2), hiddenStates.Dim(2)))
nextStates = nextStates.Add(ctx, downStates.View(ctx, i*downStates.Stride(1), hiddenStates.Dim(0), downStates.Stride(2), hiddenStates.Dim(2)))
}
return nextStates