matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-10 15:57:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

clean up convert tokenizer

Browse Source
This commit is contained in:
Michael Yang
2024-08-27 10:45:39 -07:00
parent 3eb08377f8
commit eae3af6807
2 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
convert/convert_test.go
View File

@@ -89,7 +89,7 @@ func TestMain(m *testing.M) {
os.Exit(m.Run()) os.Exit(m.Run())
} }
func TestConvertFull(t *testing.T) { func TestConvertModel(t *testing.T) {
cases := []string{ cases := []string{
"Meta-Llama-3-8B-Instruct", "Meta-Llama-3-8B-Instruct",
"Meta-Llama-3.1-8B-Instruct", "Meta-Llama-3.1-8B-Instruct",

3
convert/tokenizer.go
View File

@@ -154,7 +154,6 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error)
} }
type tokenizer struct { type tokenizer struct {
Version string `json:"version"`
AddedTokens []token `json:"added_tokens"` AddedTokens []token `json:"added_tokens"`
Model struct { Model struct {
Type string `json:"type"` Type string `json:"type"`
@@ -252,7 +251,7 @@ func parseVocabulary(fsys fs.FS) (*Vocabulary, error) {
return pattern.Func(fsys) return pattern.Func(fsys)
} }
return nil, errors.New("unknown tensor format") return nil, errors.New("unknown tokenizer format")
} }
type SpecialVocabulary struct { type SpecialVocabulary struct {