matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-12 16:57:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

model: validate left and right pairs before merging them

Browse Source
This commit is contained in:
jmorganca
2025-03-11 18:40:55 +01:00
committed by Michael Yang
parent 63a394068c
commit 20e3593863
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
model/process_text_spm.go
View File

@@ -169,6 +169,10 @@ func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error)
continue continue
} }
if id := spm.vocab.Encode(string(left.runes) + string(right.runes)); id < 0 {
continue
}
merges[pair.a].runes = append(left.runes, right.runes...) merges[pair.a].runes = append(left.runes, right.runes...)
merges[pair.b].runes = nil merges[pair.b].runes = nil
merges[pair.a].n = right.n merges[pair.a].n = right.n