enable metal gpu acceleration

ggml-metal.metal must be in the same directory as the ollama binary
otherwise llama.cpp will not be able to find it and load it.

1. go generate llama/llama_metal.go
2. go build .
3. ./ollama serve
This commit is contained in:
Michael Yang
2023-07-05 17:45:11 -07:00
committed by Jeffrey Morgan
parent b9fb988112
commit 1b7183c5a1
3 changed files with 15 additions and 15 deletions

View File

@@ -22,7 +22,7 @@ func pull(c *gin.Context) {
func generate(c *gin.Context) {
// TODO: these should be request parameters
gpulayers := 0
gpulayers := 1
tokens := 512
threads := runtime.NumCPU()
// TODO: set prompt from template