Add timing instrumentation and user progress messages for model loading

Problem: Model loading takes 2-3 minutes on first load with no user feedback,
causing confusion about whether the system is frozen or working.

Root Cause: GPU initialization (reserveWorstCaseGraph) takes ~164 seconds on
Tesla K80 GPUs due to CUDA kernel compilation (PTX JIT for compute 3.7). This
is by design - it validates GPU compatibility before committing to full load.

Solution:
1. Add comprehensive timing instrumentation to identify bottlenecks
2. Add user-facing progress messages explaining the delay

Changes:
- cmd/cmd.go: Update spinner with informative message for users
- llama/llama.go: Add timing logs for CGO model loading
- runner/llamarunner/runner.go: Add detailed timing for llama runner
- runner/ollamarunner/runner.go: Add timing + stderr messages for new engine
- server/sched.go: Add timing for scheduler load operation

User Experience:
Before: Silent wait with blinking cursor for 2-3 minutes
After: Rotating spinner with message "loading model (may take 1-3 min on first load)"

Performance Metrics Captured:
- GGUF file reading: ~0.4s
- GPU kernel compilation: ~164s (bottleneck identified)
- Model weight loading: ~0.002s
- Total end-to-end: ~165s

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

cmd/cmd.go

@@ -272,7 +272,9 @@ func loadOrUnloadModel(cmd *cobra.Command, opts *runOptions) error {
 	p := progress.NewProgress(os.Stderr)
 	defer p.StopAndClear()
 
-	spinner := progress.NewSpinner("")
+	// Show a message explaining potential delays on first load
+	// For older GPUs (Tesla K80, etc.), GPU initialization can take 1-3 minutes
+	spinner := progress.NewSpinner("loading model (may take 1-3 min on first load)")
 	p.Add("", spinner)
 
 	client, err := api.ClientFromEnvironment()

llama/llama.go

@@ -51,6 +51,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"time"
 	"unsafe"
 
 	_ "github.com/ollama/ollama/llama/llama.cpp/common"
@@ -329,6 +330,9 @@ func llamaProgressCallback(progress C.float, userData unsafe.Pointer) C.bool {
 // - *Model: Loaded model ready for inference
 // - error: If file not found, incompatible format, or out of memory
 func LoadModelFromFile(modelPath string, params ModelParams) (*Model, error) {
+	loadStart := time.Now()
+	slog.Info("LoadModelFromFile: starting", "path", modelPath, "num_gpu_layers", params.NumGpuLayers, "use_mmap", params.UseMmap)
+
 	// Initialize C parameters structure
 	cparams := C.llama_model_default_params()
 	cparams.n_gpu_layers = C.int(params.NumGpuLayers) // Layers to offload to GPU
@@ -367,11 +371,16 @@ func LoadModelFromFile(modelPath string, params ModelParams) (*Model, error) {
 	// 2. Allocates CPU/GPU memory for tensors
 	// 3. Loads/mmaps weights into memory
 	// 4. For Tesla K80: compiles CUDA kernels via PTX JIT (compute 3.7)
+	slog.Info("LoadModelFromFile: calling llama_model_load_from_file (CGO -> C++)")
+	cgoStart := time.Now()
 	m := Model{c: C.llama_model_load_from_file(C.CString(modelPath), cparams)}
+	slog.Info("LoadModelFromFile: llama_model_load_from_file returned", "duration_sec", time.Since(cgoStart).Seconds())
+
 	if m.c == nil {
 		return nil, fmt.Errorf("unable to load model: %s", modelPath)
 	}
 
+	slog.Info("LoadModelFromFile: COMPLETE", "total_duration_sec", time.Since(loadStart).Seconds())
 	return &m, nil
 }
 

runner/llamarunner/runner.go

@@ -876,40 +876,56 @@ func (s *Server) loadModel(
 	threads int,
 	multiUserCache bool,
 ) {
+	loadStartTime := time.Now()
+	slog.Info("loadModel: starting model load", "model_path", mpath, "num_gpu_layers", params.NumGpuLayers)
+
 	var err error
+	modelLoadStart := time.Now()
 	s.model, err = llama.LoadModelFromFile(mpath, params)
 	if err != nil {
 		panic(err)
 	}
+	slog.Info("loadModel: model weights loaded from disk", "duration_sec", time.Since(modelLoadStart).Seconds())
 
+	ctxStart := time.Now()
 	ctxParams := llama.NewContextParams(kvSize, s.batchSize*s.parallel, s.parallel, threads, flashAttention, kvCacheType)
 	s.lc, err = llama.NewContextWithModel(s.model, ctxParams)
 	if err != nil {
 		panic(err)
 	}
+	slog.Info("loadModel: context and KV cache initialized", "kv_size", kvSize, "duration_sec", time.Since(ctxStart).Seconds())
 
-	for _, path := range lpath {
+	if len(lpath) > 0 {
-		err := s.model.ApplyLoraFromFile(s.lc, path, 1.0, threads)
+		loraStart := time.Now()
-		if err != nil {
+		for _, path := range lpath {
-			panic(err)
+			err := s.model.ApplyLoraFromFile(s.lc, path, 1.0, threads)
+			if err != nil {
+				panic(err)
+			}
 		}
+		slog.Info("loadModel: LoRA adapters applied", "count", len(lpath), "duration_sec", time.Since(loraStart).Seconds())
 	}
 
 	if ppath != "" {
+		projectorStart := time.Now()
 		var err error
 		s.image, err = NewImageContext(s.lc, ppath)
 		if err != nil {
 			panic(err)
 		}
+		slog.Info("loadModel: vision projector loaded", "projector_path", ppath, "duration_sec", time.Since(projectorStart).Seconds())
 	}
 
+	cacheStart := time.Now()
 	s.cache, err = NewInputCache(s.lc, kvSize, s.parallel, multiUserCache)
 	if err != nil {
 		panic(err)
 	}
+	slog.Info("loadModel: input cache initialized", "duration_sec", time.Since(cacheStart).Seconds())
 
 	s.status = llm.ServerStatusReady
 	s.ready.Done()
+	slog.Info("loadModel: COMPLETE - model ready for inference", "total_duration_sec", time.Since(loadStartTime).Seconds())
 }
 
 // load is the handler called by the Ollama server to process different

runner/ollamarunner/runner.go

@@ -1011,6 +1011,10 @@ func (s *Server) health(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *Server) reserveWorstCaseGraph(prompt bool) error {
+	// Update progress to show we're initializing GPU
+	// This is displayed to the user during model loading
+	s.progress = 0.5
+
 	ctx := s.model.Backend().NewContext()
 	defer ctx.Close()
 
@@ -1112,6 +1116,9 @@ func (s *Server) reserveWorstCaseGraph(prompt bool) error {
 	ctx.SetBatchSize(batchSize)
 	ctx.Forward(t).Reserve()
 
+	// Update progress to show GPU initialization complete
+	s.progress = 0.9
+
 	return nil
 }
 
@@ -1126,6 +1133,9 @@ func (s *Server) allocModel(
 	kvSize int,
 	multiUserCache bool,
 ) (panicErr error) {
+	allocStart := time.Now()
+	slog.Info("allocModel: starting", "model_path", mpath, "alloc_memory", params.AllocMemory, "gpu_layers", len(params.GPULayers))
+
 	// Convert memory allocation panics to errors
 	defer func() {
 		if r := recover(); r != nil {
@@ -1143,7 +1153,10 @@ func (s *Server) allocModel(
 	}()
 
 	var err error
+	modelNewStart := time.Now()
+	slog.Info("allocModel: calling model.New() to read GGUF file")
 	s.model, err = model.New(mpath, params)
+	slog.Info("allocModel: model.New() completed", "duration_sec", time.Since(modelNewStart).Seconds())
 	if err != nil {
 		return err
 	}
@@ -1153,10 +1166,12 @@ func (s *Server) allocModel(
 		return errors.New("loras are not yet implemented")
 	}
 
+	cacheStart := time.Now()
 	s.cache, err = NewInputCache(s.model, kvCacheType, int32(kvSize), parallel, s.batchSize, multiUserCache)
 	if err != nil {
 		return err
 	}
+	slog.Info("allocModel: input cache created", "duration_sec", time.Since(cacheStart).Seconds())
 
 	if !s.cache.enabled && parallel > 1 {
 		parallel = 1
@@ -1167,12 +1182,24 @@ func (s *Server) allocModel(
 	s.seqs = make([]*Sequence, s.parallel)
 	s.seqsSem = semaphore.NewWeighted(int64(s.parallel))
 
+	graphStart := time.Now()
+
+	// Inform user about GPU initialization process
+	// For older GPUs (e.g., Tesla K80 with compute 3.7), this can take 1-3 minutes
+	// on first model load due to CUDA kernel compilation (PTX JIT).
+	fmt.Fprintf(os.Stderr, "initializing GPU compute kernels (may take 1-3 minutes on first load)...\n")
+	slog.Info("allocModel: reserving compute graphs (this tests GPU compatibility and measures memory)")
+
 	err = s.reserveWorstCaseGraph(true)
 	if err != nil {
 		return nil
 	}
 
-	return s.reserveWorstCaseGraph(false)
+	err = s.reserveWorstCaseGraph(false)
+	fmt.Fprintf(os.Stderr, "GPU initialization complete\n")
+	slog.Info("allocModel: compute graphs reserved", "duration_sec", time.Since(graphStart).Seconds())
+	slog.Info("allocModel: COMPLETE", "total_duration_sec", time.Since(allocStart).Seconds())
+	return err
 }
 
 // closeModel frees all memory associated with a model

server/sched.go

@@ -480,7 +480,10 @@ func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, systemInfo ml.SystemInfo
 	// Returns:
 	// - gpuIDs: List of GPU device IDs where model layers were loaded
 	// - err: Error if model doesn't fit or loading fails
+	slog.Info("scheduler.load: calling llama.Load() to load model into memory")
+	loadStart := time.Now()
 	gpuIDs, err := llama.Load(req.ctx, systemInfo, gpus, requireFull)
+	slog.Info("scheduler.load: llama.Load() completed", "duration_sec", time.Since(loadStart).Seconds(), "error", err)
 	if err != nil {
 		if errors.Is(err, llm.ErrLoadRequiredFull) {
 			if !requireFull {