matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-10 07:46:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

convert: only extract large files

Browse Source
This commit is contained in:
Michael Yang
2024-06-29 16:53:59 -07:00
parent 781fc2d576
commit eafc607abb
10 changed files with 120 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
convert/tokenizer_spm.go
View File

@@ -5,8 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"google.golang.org/protobuf/proto"
@@ -14,8 +14,8 @@ import (
"github.com/ollama/ollama/convert/sentencepiece"
)
func parseSentencePiece(d string) (*Vocabulary, error) {
bts, err := os.ReadFile(filepath.Join(d, "tokenizer.model"))
func parseSentencePiece(fsys fs.FS) (*Vocabulary, error) {
bts, err := fs.ReadFile(fsys, "tokenizer.model")
if err != nil {
return nil, err
}
@@ -41,7 +41,7 @@ func parseSentencePiece(d string) (*Vocabulary, error) {
}
}
f, err := os.Open(filepath.Join(d, "added_tokens.json"))
f, err := fsys.Open("added_tokens.json")
if errors.Is(err, os.ErrNotExist) {
return &v, nil
} else if err != nil {