matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-12 08:47:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix CPU performance on hyperthreaded systems

Browse Source 
The default thread count logic was broken and resulted in 2x the number
of threads as it should on a hyperthreading CPU
resulting in thrashing and poor performance.
This commit is contained in:
Daniel Hiltgen
2023-12-21 16:23:36 -08:00
parent d9cd3d9667
commit 325d74985b
2 changed files with 9 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
llm/ext_server.go
View File

@@ -37,7 +37,6 @@ import (
"fmt"
"log"
"os"
"runtime"
"strings"
"sync"
"time"
@@ -185,11 +184,7 @@ func newExtServer(server extServer, model string, adapters, projectors []string,
sparams.mmproj = nil
}
if opts.NumThread > 0 {
sparams.n_threads = C.uint(opts.NumThread)
} else {
sparams.n_threads = C.uint(runtime.NumCPU())
}
sparams.n_threads = C.uint(opts.NumThread)
log.Printf("Initializing internal llama server")
resp := newExtServerResp(128)