matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-11 16:26:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

windows CUDA support (#1262)

Browse Source 
* Support cuda build in Windows
* Enable dynamic NumGPU allocation for Windows
This commit is contained in:
Jing Zhang
2023-11-24 14:16:36 -08:00
committed by GitHub
parent 12e8c12d2b
commit 82b9b329ff
3 changed files with 11 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
llm/llama.go
View File

@@ -84,6 +84,7 @@ func chooseRunners(workDir, runnerType string) []ModelRunner {
case "windows":
// TODO: select windows GPU runner here when available
runners = []ModelRunner{
{Path: path.Join(buildPath, "cuda", "bin", "Release", "ollama-runner.exe"), Accelerated: true},
{Path: path.Join(buildPath, "cpu", "bin", "Release", "ollama-runner.exe")},
}
default:
@@ -269,7 +270,7 @@ func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int {
if opts.NumGPU != -1 {
return opts.NumGPU
}
if runtime.GOOS == "linux" {
if runtime.GOOS == "linux" || runtime.GOOS == "windows" {
freeBytes, err := CheckVRAM()
if err != nil {
if !errors.Is(err, errNvidiaSMI) {