package renderers import ( "encoding/json" "strings" "github.com/ollama/ollama/api" ) func marshalWithSpaces(v any) ([]byte, error) { b, err := json.Marshal(v) if err != nil { return nil, err } out := make([]byte, 0, len(b)+len(b)/8) inStr, esc := false, false for _, c := range b { if inStr { out = append(out, c) if esc { esc = false continue } if c == '\\' { esc = true continue } if c == '"' { inStr = false } continue } switch c { case '"': inStr = true out = append(out, c) case ':': out = append(out, ':', ' ') case ',': out = append(out, ',', ' ') default: out = append(out, c) } } return out, nil } type Qwen3VLRenderer struct { isThinking bool useImgTags bool } func (r *Qwen3VLRenderer) renderContent(content api.Message) string { // This assumes all images are at the front of the message - same assumption as ollama/ollama/runner.go var subSb strings.Builder for range content.Images { // TODO: (jmorganca): how to render this is different for different // model backends, and so we should eventually parameterize this or // only output a placeholder such as [img] if r.useImgTags { subSb.WriteString("[img]") } else { subSb.WriteString("<|vision_start|><|image_pad|><|vision_end|>") } } // TODO: support videos subSb.WriteString(content.Content) return subSb.String() } func (r *Qwen3VLRenderer) Render(messages []api.Message, tools []api.Tool, _ *api.ThinkValue) (string, error) { var sb strings.Builder if len(tools) > 0 { sb.WriteString(imStartTag + "system\n") if len(messages) > 0 && messages[0].Role == "system" { sb.WriteString(messages[0].Content + "\n\n") } sb.WriteString("# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n") for _, tool := range tools { sb.WriteString("\n") if b, err := marshalWithSpaces(tool); err == nil { sb.Write(b) } } sb.WriteString("\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n") } else if len(messages) > 0 && messages[0].Role == "system" { sb.WriteString("<|im_start|>system\n" + messages[0].Content + "<|im_end|>\n") } multiStepTool := true lastQueryIndex := len(messages) - 1 // so this is the last user message for i := len(messages) - 1; i >= 0; i-- { message := messages[i] if multiStepTool && message.Role == "user" { // Check if content starts with and ends with content := r.renderContent(message) if !(strings.HasPrefix(content, "") && strings.HasSuffix(content, "")) { multiStepTool = false lastQueryIndex = i } } } for i, message := range messages { content := r.renderContent(message) lastMessage := i == len(messages)-1 prefill := lastMessage && message.Role == "assistant" if message.Role == "user" || message.Role == "system" && i != 0 { sb.WriteString("<|im_start|>" + message.Role + "\n" + content + "<|im_end|>\n") } else if message.Role == "assistant" { contentReasoning := "" if r.isThinking { if message.Thinking != "" { contentReasoning = message.Thinking } } if r.isThinking && i > lastQueryIndex { if i == len(messages)-1 || contentReasoning != "" { sb.WriteString("<|im_start|>" + message.Role + "\n\n" + strings.Trim(contentReasoning, "\n")) // do we want to add a new line here? if content != "" { sb.WriteString("\n\n\n" + strings.TrimLeft(content, "\n")) } } else { sb.WriteString("<|im_start|>" + message.Role + "\n" + content) } } else { sb.WriteString("<|im_start|>" + message.Role + "\n" + content) } if len(message.ToolCalls) > 0 { for j, toolCall := range message.ToolCalls { if j > 0 || content != "" { sb.WriteString("\n") } sb.WriteString("\n{\"name\": \"" + toolCall.Function.Name + "\", \"arguments\": ") if b, err := marshalWithSpaces(toolCall.Function.Arguments); err == nil { sb.Write(b) } sb.WriteString("}\n") } } if !prefill { sb.WriteString("<|im_end|>\n") } } else if message.Role == "tool" { if i == 0 || messages[i-1].Role != "tool" { sb.WriteString("<|im_start|>user") } sb.WriteString("\n\n" + message.Content + "\n") if i == len(messages)-1 || messages[i+1].Role != "tool" { sb.WriteString("<|im_end|>\n") } } // prefill at the end if lastMessage && !prefill { sb.WriteString("<|im_start|>assistant\n") if r.isThinking { sb.WriteString("\n") } } } return sb.String(), nil }