increase default batch size to 1024

2025-12-10 15:57:04 +00:00 · 2023-07-27 16:51:01 -04:00
parent cca61181cb
commit 688661ab9b
1 changed files with 1 additions and 1 deletions
--- a/api/types.go
+++ b/api/types.go
@@ -188,7 +188,7 @@ func DefaultOptions() Options {
 		UseNUMA: false,

 		NumCtx:   2048,
-		NumBatch: 32,
+		NumBatch: 1024,
 		NumGPU:   1,
 		LowVRAM:  false,
 		F16KV:    true,