matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-11 00:07:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

model: treat 'user defined' tokens as special tokens (#11077)

Browse Source
This commit is contained in:
Jeffrey Morgan
2025-06-16 16:03:16 -07:00
committed by GitHub
parent a6fbfc880c
commit 9e125d884c
2 changed files with 17 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
model/vocabulary.go
View File

@@ -87,7 +87,7 @@ func (v *Vocabulary) Decode(id int32) string {
func (v *Vocabulary) SpecialVocabulary() []string {
v.specialOnce.Do(func() {
for i := range v.Values {
if v.Types[i] == TOKEN_TYPE_CONTROL {
if v.Types[i] == TOKEN_TYPE_CONTROL || v.Types[i] == TOKEN_TYPE_USER_DEFINED {
v.special = append(v.special, v.Values[i])
}
}