matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-12 08:47:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

runner.go: Add unit tests for context shifting

Browse Source 
This also makes it easier to truncate long inputs the same as
shifting but does not actually implement it. This type of
truncation has a trade off between quality and time to first
token.
This commit is contained in:
Jesse Gross
2024-11-25 14:49:38 -08:00
committed by Jesse Gross
parent 52bbad12f9
commit 2cd11ae365
3 changed files with 82 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
llama/runner/cache.go
View File

@@ -199,6 +199,20 @@ func countCommonPrefix(a []input, b []input) int {
return count
}
func (c *InputCache) ShiftDiscard(inputLen int, numKeep int) int {
targetFree := (c.numCtx - numKeep) / 2
targetFree = max(targetFree, 1)
currentFree := c.numCtx - inputLen
discard := targetFree - currentFree
if discard < 0 {
discard = 0
}
return discard
}
// Frees up space in the KV cache by deleting the oldest half of history and shifting
// the newest half into that space (saving numKeep inputs at the beginning).
//
@@ -208,11 +222,7 @@ func (c *InputCache) ShiftCacheSlot(slot *InputCacheSlot, numKeep int) error {
return fmt.Errorf("unable to shift context - keep exceeds context (keep: %v context: %v)", numKeep, c.numCtx)
}
targetFree := (c.numCtx - numKeep) / 2
targetFree = max(targetFree, 1)
currentFree := c.numCtx - len(slot.Inputs)
discard := targetFree - currentFree
discard := c.ShiftDiscard(len(slot.Inputs), numKeep)
if discard <= 0 {
return nil