matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-11 00:07:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

quantize any fp16/fp32 model

Browse Source 
- FROM /path/to/{safetensors,pytorch}
- FROM /path/to/fp{16,32}.bin
- FROM model:fp{16,32}
This commit is contained in:
Michael Yang
2024-04-12 13:55:12 -07:00
parent d091fe3c21
commit 9685c34509
12 changed files with 654 additions and 556 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
llm/filetype.go Normal file
View File

@@ -0,0 +1,138 @@
package llm
import "fmt"
type filetype uint32
const (
filetypeF32 filetype = iota
filetypeF16
filetypeQ4_0
filetypeQ4_1
filetypeQ4_1_F16
filetypeQ8_0 filetype = iota + 2
filetypeQ5_0
filetypeQ5_1
filetypeQ2_K
filetypeQ3_K_S
filetypeQ3_K_M
filetypeQ3_K_L
filetypeQ4_K_S
filetypeQ4_K_M
filetypeQ5_K_S
filetypeQ5_K_M
filetypeQ6_K
filetypeIQ2_XXS
filetypeIQ2_XS
filetypeQ2_K_S
filetypeQ3_K_XS
filetypeIQ3_XXS
filetypeUnknown
)
func ParseFileType(s string) (filetype, error) {
switch s {
case "F32":
return filetypeF32, nil
case "F16":
return filetypeF16, nil
case "Q4_0":
return filetypeQ4_0, nil
case "Q4_1":
return filetypeQ4_1, nil
case "Q4_1_F16":
return filetypeQ4_1_F16, nil
case "Q8_0":
return filetypeQ8_0, nil
case "Q5_0":
return filetypeQ5_0, nil
case "Q5_1":
return filetypeQ5_1, nil
case "Q2_K":
return filetypeQ2_K, nil
case "Q3_K_S":
return filetypeQ3_K_S, nil
case "Q3_K_M":
return filetypeQ3_K_M, nil
case "Q3_K_L":
return filetypeQ3_K_L, nil
case "Q4_K_S":
return filetypeQ4_K_S, nil
case "Q4_K_M":
return filetypeQ4_K_M, nil
case "Q5_K_S":
return filetypeQ5_K_S, nil
case "Q5_K_M":
return filetypeQ5_K_M, nil
case "Q6_K":
return filetypeQ6_K, nil
case "IQ2_XXS":
return filetypeIQ2_XXS, nil
case "IQ2_XS":
return filetypeIQ2_XS, nil
case "Q2_K_S":
return filetypeQ2_K_S, nil
case "Q3_K_XS":
return filetypeQ3_K_XS, nil
case "IQ3_XXS":
return filetypeIQ3_XXS, nil
default:
return filetypeUnknown, fmt.Errorf("unknown filetype: %s", s)
}
}
func (t filetype) String() string {
switch t {
case filetypeF32:
return "F32"
case filetypeF16:
return "F16"
case filetypeQ4_0:
return "Q4_0"
case filetypeQ4_1:
return "Q4_1"
case filetypeQ4_1_F16:
return "Q4_1_F16"
case filetypeQ8_0:
return "Q8_0"
case filetypeQ5_0:
return "Q5_0"
case filetypeQ5_1:
return "Q5_1"
case filetypeQ2_K:
return "Q2_K"
case filetypeQ3_K_S:
return "Q3_K_S"
case filetypeQ3_K_M:
return "Q3_K_M"
case filetypeQ3_K_L:
return "Q3_K_L"
case filetypeQ4_K_S:
return "Q4_K_S"
case filetypeQ4_K_M:
return "Q4_K_M"
case filetypeQ5_K_S:
return "Q5_K_S"
case filetypeQ5_K_M:
return "Q5_K_M"
case filetypeQ6_K:
return "Q6_K"
case filetypeIQ2_XXS:
return "IQ2_XXS"
case filetypeIQ2_XS:
return "IQ2_XS"
case filetypeQ2_K_S:
return "Q2_K_S"
case filetypeQ3_K_XS:
return "Q3_K_XS"
case filetypeIQ3_XXS:
return "IQ3_XXS"
default:
return "unknown"
}
}
func (t filetype) Value() uint32 {
return uint32(t)
}

95
llm/ggml.go
View File

@@ -13,82 +13,6 @@ type GGML struct {
model
}
const (
fileTypeF32 uint32 = iota
fileTypeF16
fileTypeQ4_0
fileTypeQ4_1
fileTypeQ4_1_F16
fileTypeQ8_0 uint32 = iota + 2
fileTypeQ5_0
fileTypeQ5_1
fileTypeQ2_K
fileTypeQ3_K_S
fileTypeQ3_K_M
fileTypeQ3_K_L
fileTypeQ4_K_S
fileTypeQ4_K_M
fileTypeQ5_K_S
fileTypeQ5_K_M
fileTypeQ6_K
fileTypeIQ2_XXS
fileTypeIQ2_XS
fileTypeQ2_K_S
fileTypeQ3_K_XS
fileTypeIQ3_XXS
)
func fileType(fileType uint32) string {
switch fileType {
case fileTypeF32:
return "F32"
case fileTypeF16:
return "F16"
case fileTypeQ4_0:
return "Q4_0"
case fileTypeQ4_1:
return "Q4_1"
case fileTypeQ4_1_F16:
return "Q4_1_F16"
case fileTypeQ8_0:
return "Q8_0"
case fileTypeQ5_0:
return "Q5_0"
case fileTypeQ5_1:
return "Q5_1"
case fileTypeQ2_K:
return "Q2_K"
case fileTypeQ3_K_S:
return "Q3_K_S"
case fileTypeQ3_K_M:
return "Q3_K_M"
case fileTypeQ3_K_L:
return "Q3_K_L"
case fileTypeQ4_K_S:
return "Q4_K_S"
case fileTypeQ4_K_M:
return "Q4_K_M"
case fileTypeQ5_K_S:
return "Q5_K_S"
case fileTypeQ5_K_M:
return "Q5_K_M"
case fileTypeQ6_K:
return "Q6_K"
case fileTypeIQ2_XXS:
return "IQ2_XXS"
case fileTypeIQ2_XS:
return "IQ2_XS"
case fileTypeQ2_K_S:
return "Q2_K_S"
case fileTypeQ3_K_XS:
return "Q3_K_XS"
case fileTypeIQ3_XXS:
return "IQ3_XXS"
default:
return "unknown"
}
}
type model interface {
KV() KV
Tensors() Tensors
@@ -123,7 +47,7 @@ func (kv KV) ParameterCount() uint64 {
func (kv KV) FileType() string {
if u64 := kv.u64("general.file_type"); u64 > 0 {
return fileType(uint32(u64))
return filetype(uint32(u64)).String()
}
return "unknown"
@@ -286,6 +210,23 @@ const (
var ErrUnsupportedFormat = errors.New("unsupported model format")
func DetectGGMLType(b []byte) string {
switch binary.LittleEndian.Uint32(b[:4]) {
case FILE_MAGIC_GGML:
return "ggml"
case FILE_MAGIC_GGMF:
return "ggmf"
case FILE_MAGIC_GGJT:
return "ggjt"
case FILE_MAGIC_GGLA:
return "ggla"
case FILE_MAGIC_GGUF_LE, FILE_MAGIC_GGUF_BE:
return "gguf"
default:
return ""
}
}
func DecodeGGML(rs io.ReadSeeker) (*GGML, int64, error) {
var magic uint32
if err := binary.Read(rs, binary.LittleEndian, &magic); err != nil {

56
llm/llm.go
View File

@@ -20,7 +20,7 @@ func SystemInfo() string {
return C.GoString(C.llama_print_system_info())
}
func Quantize(infile, outfile, filetype string) error {
func Quantize(infile, outfile string, ftype filetype) error {
cinfile := C.CString(infile)
defer C.free(unsafe.Pointer(cinfile))
@@ -29,58 +29,10 @@ func Quantize(infile, outfile, filetype string) error {
params := C.llama_model_quantize_default_params()
params.nthread = -1
params.ftype = ftype.Value()
switch filetype {
case "F32":
params.ftype = fileTypeF32
case "F16":
params.ftype = fileTypeF16
case "Q4_0":
params.ftype = fileTypeQ4_0
case "Q4_1":
params.ftype = fileTypeQ4_1
case "Q4_1_F16":
params.ftype = fileTypeQ4_1_F16
case "Q8_0":
params.ftype = fileTypeQ8_0
case "Q5_0":
params.ftype = fileTypeQ5_0
case "Q5_1":
params.ftype = fileTypeQ5_1
case "Q2_K":
params.ftype = fileTypeQ2_K
case "Q3_K_S":
params.ftype = fileTypeQ3_K_S
case "Q3_K_M":
params.ftype = fileTypeQ3_K_M
case "Q3_K_L":
params.ftype = fileTypeQ3_K_L
case "Q4_K_S":
params.ftype = fileTypeQ4_K_S
case "Q4_K_M":
params.ftype = fileTypeQ4_K_M
case "Q5_K_S":
params.ftype = fileTypeQ5_K_S
case "Q5_K_M":
params.ftype = fileTypeQ5_K_M
case "Q6_K":
params.ftype = fileTypeQ6_K
case "IQ2_XXS":
params.ftype = fileTypeIQ2_XXS
case "IQ2_XS":
params.ftype = fileTypeIQ2_XS
case "Q2_K_S":
params.ftype = fileTypeQ2_K_S
case "Q3_K_XS":
params.ftype = fileTypeQ3_K_XS
case "IQ3_XXS":
params.ftype = fileTypeIQ3_XXS
default:
return fmt.Errorf("unknown filetype: %s", filetype)
}
if retval := C.llama_model_quantize(cinfile, coutfile, &params); retval != 0 {
return fmt.Errorf("llama_model_quantize: %d", retval)
if rc := C.llama_model_quantize(cinfile, coutfile, &params); rc != 0 {
return fmt.Errorf("llama_model_quantize: %d", rc)
}
return nil