matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-11 00:07:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add gemma safetensors conversion (#3250)

Browse Source 
Co-authored-by: Michael Yang <mxyng@pm.me>
This commit is contained in:
Patrick Devine
2024-03-28 18:54:01 -07:00
committed by GitHub
parent 97ae517fbf
commit 5a5efee46b
11 changed files with 949 additions and 833 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
server/images.go
View File

@@ -321,7 +321,7 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
pathName := realpath(modelFileDir, c.Args)
ggufName, err := convertSafetensors(name, pathName)
ggufName, err := convertSafetensors(name, pathName, fn)
if err != nil {
var pathErr *fs.PathError
switch {
@@ -336,6 +336,7 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
if ggufName != "" {
pathName = ggufName
slog.Debug(fmt.Sprintf("new image layer path: %s", pathName))
defer os.RemoveAll(ggufName)
}
@@ -422,10 +423,13 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
CREATE:
for {
fn(api.ProgressResponse{Status: "creating model layer"})
if _, err := bin.Seek(offset, io.SeekStart); err != nil {
return err
}
bin.Seek(offset, io.SeekStart)
ggml, err := llm.DecodeGGML(bin)
if err != nil {
slog.Error(fmt.Sprintf("error decoding gguf file: %q", err))
switch {
case errors.Is(err, io.EOF):
break CREATE
@@ -621,8 +625,8 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
return nil
}
func convertSafetensors(name, fn string) (string, error) {
r, err := zip.OpenReader(fn)
func convertSafetensors(name, path string, fn func(resp api.ProgressResponse)) (string, error) {
r, err := zip.OpenReader(path)
if err != nil {
return "", err
}
@@ -634,6 +638,7 @@ func convertSafetensors(name, fn string) (string, error) {
}
defer os.RemoveAll(tempDir)
fn(api.ProgressResponse{Status: "unpacking model metadata"})
for _, f := range r.File {
fpath := filepath.Join(tempDir, f.Name)
outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
@@ -662,6 +667,7 @@ func convertSafetensors(name, fn string) (string, error) {
SupportedArchs := []string{
"MistralForCausalLM",
"GemmaForCausalLM",
}
for _, arch := range params.Architectures {
@@ -670,22 +676,24 @@ func convertSafetensors(name, fn string) (string, error) {
}
}
t, err := convert.GetSafeTensors(tempDir)
fn(api.ProgressResponse{Status: "processing safetensors"})
t, err := convert.GetSafeTensors(tempDir, params)
if err != nil {
return "", err
}
vocab, err := convert.LoadTokens(tempDir)
vocab, err := convert.LoadTokens(tempDir, params)
if err != nil {
return "", err
}
fn, err = convert.WriteGGUF(name, t, params, vocab)
fn(api.ProgressResponse{Status: "converting model"})
path, err = convert.WriteGGUF(name, t, params, vocab)
if err != nil {
return "", err
}
return fn, nil
return path, nil
}
func CopyModel(src, dest string) error {

14
server/routes_test.go
View File

@@ -3,6 +3,7 @@ package server
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
"fmt"
"io"
@@ -31,13 +32,22 @@ func Test_Routes(t *testing.T) {
}
createTestFile := func(t *testing.T, name string) string {
t.Helper()
f, err := os.CreateTemp(t.TempDir(), name)
assert.Nil(t, err)
defer f.Close()
_, err = f.Write([]byte("GGUF"))
err = binary.Write(f, binary.LittleEndian, []byte("GGUF"))
assert.Nil(t, err)
_, err = f.Write([]byte{0x2, 0})
err = binary.Write(f, binary.LittleEndian, uint32(3))
assert.Nil(t, err)
err = binary.Write(f, binary.LittleEndian, uint64(0))
assert.Nil(t, err)
err = binary.Write(f, binary.LittleEndian, uint64(0))
assert.Nil(t, err)
return f.Name()