Shutdown faster

Make sure that when a shutdown signal comes, we shutdown quickly instead
of waiting for a potentially long exchange to wrap up.
This commit is contained in:
Daniel Hiltgen
2024-02-08 22:22:50 -08:00
parent 42b797ed9c
commit 6680761596
2 changed files with 44 additions and 1 deletions

View File

@@ -258,7 +258,7 @@ func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn fu
})
}
if p.Stop {
if p.Stop || bool(result.stop) {
fn(PredictResult{
Done: true,
PromptEvalCount: p.Timings.PromptN,