Fix clearing kv cache between requests with the same prompt (#2186)

* Fix clearing kv cache between requests with the same prompt * fix powershell script
2025-12-11 00:07:07 +00:00 · 2024-01-25 13:46:20 -08:00
parent 7c40a67841
commit a64570dcae
4 changed files with 65 additions and 0 deletions
--- a/llm/dyn_ext_server.go
+++ b/llm/dyn_ext_server.go
@@ -190,6 +190,7 @@ func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn fu
 		"seed":              predict.Options.Seed,
 		"stop":              predict.Options.Stop,
 		"image_data":        imageData,
+		"cache_prompt":      true,
 	}

 	if predict.Format == "json" {