matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-12 00:37:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

runner.go: Retry decoding after defragmentation if needed

Browse Source 
Fragmentation of the KV cache can occur due to cache shifting or
different sequences getting processed. Decode uses a heuristic to
decide if it should defrag. However, this heuristic isn't 100%
accurate, so decoding can sometimes fail by surprise.

For these cases, if decode indicates that there is no KV cache space,
we should defrag and then try again.
This commit is contained in:
Jesse Gross
2024-11-19 11:00:41 -08:00
committed by Jesse Gross
parent 5f68fcab12
commit 7121dfa309
3 changed files with 50 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
llama/runner/runner.go
View File

@@ -426,8 +426,15 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
err := s.lc.Decode(batch)
if err != nil {
slog.Error("failed to decode batch", "error", err)
return
if errors.Is(err, llama.ErrKvCacheFull) {
slog.Debug("defragmenting kv cache")
s.cache.lc.KvCacheDefrag()
err = s.lc.Decode(batch)
}
if err != nil {
slog.Error("failed to decode batch", "error", err)
return
}
}
if crossAttention {