support raw generation requests (#952)

- add the optional `raw` generate request parameter to bypass prompt formatting and response context
-add raw request to docs
This commit is contained in:
Bruce MacDonald
2023-11-08 14:05:02 -08:00
committed by GitHub
parent ec84c02d54
commit ec2a31e9b3
3 changed files with 50 additions and 5 deletions

View File

@@ -158,9 +158,14 @@ func GenerateHandler(c *gin.Context) {
return
}
if req.Model == "" {
// validate the request
switch {
case req.Model == "":
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
return
case req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0):
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
return
}
model, err := GetModel(req.Model)
@@ -189,10 +194,13 @@ func GenerateHandler(c *gin.Context) {
checkpointLoaded := time.Now()
prompt, err := model.Prompt(req)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
prompt := req.Prompt
if !req.Raw {
prompt, err = model.Prompt(req)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
}
ch := make(chan any)
@@ -215,6 +223,11 @@ func GenerateHandler(c *gin.Context) {
r.LoadDuration = checkpointLoaded.Sub(checkpointStart)
}
if req.Raw {
// in raw mode the client must manage history on their own
r.Context = nil
}
ch <- r
}