mirror of
https://github.com/dogkeeper886/ollama37.git
synced 2025-12-13 01:07:12 +00:00
Allow models to force a new batch
This is useful for a few things: - Work around bugs, such as having 2 images in one batch - Keep the image in a single batch for fully connected attention - Improve performance by not evaluating embeddings multiple times
This commit is contained in:
committed by
Michael Yang
parent
a8e83a7654
commit
06007c0a18
@@ -363,7 +363,7 @@ func (s *Server) processBatch() error {
|
||||
}
|
||||
}
|
||||
|
||||
if j >= s.batchSize {
|
||||
if j >= s.batchSize || (inp.BatchBreak && len(seq.pendingInputs) != 0) {
|
||||
break
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user