fix drift from main

This commit is contained in:
Jesse Gross
2025-03-07 09:30:10 -08:00
committed by Michael Yang
parent 4b037a97dc
commit 4346c2409d
5 changed files with 49 additions and 22 deletions

View File

@@ -73,7 +73,7 @@ func TestSentencePieceEncode(t *testing.T) {
}
for _, want := range cases {
ids, err := tokenizer.Encode(want)
ids, err := tokenizer.Encode(want, true)
if err != nil {
t.Fatal(err)
}
@@ -98,7 +98,7 @@ func TestSentencePieceEncode(t *testing.T) {
}
for _, want := range cases {
ids, err := tokenizer.Encode(want.token)
ids, err := tokenizer.Encode(want.token, true)
if err != nil {
t.Fatal(err)
}