matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-10 15:57:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Runner for Ollama engine

Browse Source 
This provides integration with the new Ollama engine
(5824541 next ollama runner (#7913)) and the rest of the Ollama
infrastructure such as the runner and Ollama server.

In addition, it also builds out the KV cache infrastructure to
support requirements of how Ollama runs models such as:
 - Parallel processing
 - Memory management for defragmentation and shifting
 - Multi-modal modals

Both old and new engines continue to be supported. By default, only
the old engine is used. To enable the new engine:

Start the server with the OLLAMA_NEW_ENGINE environment variable set:
OLLAMA_NEW_ENGINE=1 ./ollama serve

Start a model that is supported by the Ollama engine. This one is Llama 3.1 8b Q4_K_M:
./ollama run jessegross/llama3.1
This commit is contained in:
Jesse Gross
2024-12-17 19:59:41 -08:00
committed by Jesse Gross
parent 6945617af5
commit ed443a0393
31 changed files with 2952 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
ml/backend.go
View File

@@ -50,6 +50,7 @@ type Context interface {
Forward(Tensor)
Compute(...Tensor)
MaxTensors() int
Close()
}
@@ -118,7 +119,7 @@ type DumpOptions struct {
Precision int
}
func Dump(t Tensor, opts ...DumpOptions) string {
func Dump(ctx Context, t Tensor, opts ...DumpOptions) string {
if len(opts) < 1 {
opts = append(opts, DumpOptions{
Items: 3,
@@ -128,11 +129,17 @@ func Dump(t Tensor, opts ...DumpOptions) string {
switch t.DType() {
case DTypeF32:
return dump[[]float32](t, opts[0].Items, func(f float32) string {
return dump[[]float32](ctx, t, opts[0].Items, func(f float32) string {
return strconv.FormatFloat(float64(f), 'f', opts[0].Precision, 32)
})
case DTypeF16:
f32 := ctx.Zeros(DTypeF32, t.Shape()...)
f32 = t.Copy(ctx, f32)
return dump[[]float32](ctx, f32, opts[0].Items, func(f float32) string {
return strconv.FormatFloat(float64(f), 'f', opts[0].Precision, 32)
})
case DTypeI32:
return dump[[]int32](t, opts[0].Items, func(i int32) string {
return dump[[]int32](ctx, t, opts[0].Items, func(i int32) string {
return strconv.FormatInt(int64(i), 10)
})
default:
@@ -140,10 +147,10 @@ func Dump(t Tensor, opts ...DumpOptions) string {
}
}
func dump[S ~[]E, E number](t Tensor, items int, fn func(E) string) string {
bts := t.Bytes()
if bts == nil {
return "<nil>"
func dump[S ~[]E, E number](ctx Context, t Tensor, items int, fn func(E) string) string {
if t.Bytes() == nil {
ctx.Forward(t)
ctx.Compute(t)
}
s := make(S, mul(t.Shape()...))
@@ -191,7 +198,8 @@ func dump[S ~[]E, E number](t Tensor, items int, fn func(E) string) string {
type DType int
const (
DTypeF32 DType = iota
DTypeOther DType = iota
DTypeF32
DTypeF16
DTypeI32
DTypeOther
)