matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-10 07:46:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: regression unsupported metal types

Browse Source 
omitting `--n-gpu-layers` means use metal on macos which isn't correct
since ollama uses `num_gpu=0` to explicitly disable gpu for file types
that are not implemented in metal
This commit is contained in:
Michael Yang
2023-10-16 14:37:17 -07:00
parent 9ef2fce33a
commit cb4a80b693
1 changed files with 1 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
llm/llama.go
View File

@@ -292,13 +292,10 @@ func newLlama(model string, adapters []string, runners []ModelRunner, numLayers
"--rope-freq-base", fmt.Sprintf("%f", opts.RopeFrequencyBase),
"--rope-freq-scale", fmt.Sprintf("%f", opts.RopeFrequencyScale),
"--batch-size", fmt.Sprintf("%d", opts.NumBatch),
"--n-gpu-layers", fmt.Sprintf("%d", numGPU),
"--embedding",
}
if numGPU > 0 {
params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", numGPU))
}
if opts.NumGQA > 0 {
params = append(params, "--gqa", fmt.Sprintf("%d", opts.NumGQA))
}