matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-11 16:26:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

model and file type as strings

Browse Source
This commit is contained in:
Michael Yang
2023-08-17 11:37:27 -07:00
parent 519f4d98ef
commit a894cc792d
4 changed files with 133 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
llm/llm.go
View File

@@ -35,10 +35,10 @@ func New(model string, adapters []string, opts api.Options) (LLM, error) {
return nil, err
}
switch ggml.FileType {
case FileTypeF32, FileTypeF16, FileTypeQ5_0, FileTypeQ5_1, FileTypeQ8_0:
switch ggml.FileType().String() {
case "F32", "F16", "Q5_0", "Q5_1", "Q8_0":
if opts.NumGPU != 0 {
// Q5_0, Q5_1, and Q8_0 do not support Metal API and will
// F32, F16, Q5_0, Q5_1, and Q8_0 do not support Metal API and will
// cause the runner to segmentation fault so disable GPU
log.Printf("WARNING: GPU disabled for F32, F16, Q5_0, Q5_1, and Q8_0")
opts.NumGPU = 0
@@ -46,7 +46,7 @@ func New(model string, adapters []string, opts api.Options) (LLM, error) {
}
totalResidentMemory := memory.TotalMemory()
switch ggml.ModelType {
switch ggml.ModelType() {
case ModelType3B, ModelType7B:
if totalResidentMemory < 8*1024*1024 {
return nil, fmt.Errorf("model requires at least 8GB of memory")
@@ -65,10 +65,10 @@ func New(model string, adapters []string, opts api.Options) (LLM, error) {
}
}
switch ggml.ModelFamily {
switch ggml.ModelFamily() {
case ModelFamilyLlama:
return newLlama(model, adapters, opts)
default:
return nil, fmt.Errorf("unknown ggml type: %s", ggml.ModelFamily)
return nil, fmt.Errorf("unknown ggml type: %s", ggml.ModelFamily())
}
}