matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-13 17:27:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

no gpu for 70B model

Browse Source
This commit is contained in:
Michael Yang
2023-08-01 16:57:14 -07:00
parent 7a1c3e62dc
commit 74a5f7e698
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
llama/llama.go
View File

@@ -128,6 +128,11 @@ func New(model string, opts api.Options) (*LLM, error) {
C.llama_backend_init(C.bool(llm.UseNUMA)) C.llama_backend_init(C.bool(llm.UseNUMA))
// TODO: GQA == 8 suggests 70B model which doesn't support metal
if llm.NumGQA == 8 {
llm.NumGPU = 0
}
params := C.llama_context_default_params() params := C.llama_context_default_params()
params.seed = C.uint(llm.Seed) params.seed = C.uint(llm.Seed)
params.n_ctx = C.int(llm.NumCtx) params.n_ctx = C.int(llm.NumCtx)