Merge branch 'main' into archlinux

llm/dyn_ext_server.go

@@ -75,7 +75,7 @@ func newDynExtServer(library, model string, adapters, projectors []string, opts
 	updatePath(filepath.Dir(library))
 	libPath := C.CString(library)
 	defer C.free(unsafe.Pointer(libPath))
-	resp := newExtServerResp(128)
+	resp := newExtServerResp(512)
 	defer freeExtServerResp(resp)
 	var srv C.struct_dynamic_llama_server
 	C.dyn_init(libPath, &srv, &resp)
@@ -181,7 +181,6 @@ func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn fu
 		"seed":              predict.Options.Seed,
 		"stop":              predict.Options.Stop,
 		"image_data":        imageData,
-		"cache_prompt":      true,
 	}
 
 	if predict.Format == "json" {

llm/generate/gen_darwin.sh

@@ -14,9 +14,11 @@ BUILD_DIR="${LLAMACPP_DIR}/build/darwin/metal"
 case "${GOARCH}" in
 "amd64")
     CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DLLAMA_METAL=off -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
+    ARCH="x86_64"
     ;;
 "arm64")
     CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 -DLLAMA_METAL=on ${CMAKE_DEFS}"
+    ARCH="arm64"
     ;;
 *)
     echo "GOARCH must be set"
@@ -30,6 +32,7 @@ apply_patches
 build
 install
 gcc -fPIC -g -shared -o ${BUILD_DIR}/lib/libext_server.so \
+    -arch ${ARCH} \
     -Wl,-force_load ${BUILD_DIR}/lib/libext_server.a \
     ${BUILD_DIR}/lib/libcommon.a \
     ${BUILD_DIR}/lib/libllama.a \

llm/generate/gen_linux.sh

@@ -39,8 +39,13 @@ amdGPUs() {
 }
 
 echo "Starting linux generate script"
-if [ -z "${CUDACXX}" -a -x /usr/local/cuda/bin/nvcc ]; then
-    export CUDACXX=/usr/local/cuda/bin/nvcc
+if [ -z "${CUDACXX}" ]; then
+    if [ -x /usr/local/cuda/bin/nvcc ]; then
+        export CUDACXX=/usr/local/cuda/bin/nvcc
+    else
+        # Try the default location in case it exists
+        export CUDACXX=$(command -v nvcc)
+    fi
 fi
 COMMON_CMAKE_DEFS="-DCMAKE_POSITION_INDEPENDENT_CODE=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off"
 source $(dirname $0)/gen_common.sh
@@ -109,33 +114,41 @@ else
     echo "Skipping CPU generation step as requested"
 fi
 
-for cudalibpath in "/usr/local/cuda/lib64" "/opt/cuda/targets/x86_64-linux/lib"; do
-    if [ -d "$cudalibpath" ]; then
-        echo "CUDA libraries detected - building dynamic CUDA library"
-        init_vars
-        CUDA_MAJOR=$(find "$cudalibpath" -name 'libcudart.so.*' -print | head -1 | cut -f3 -d. || true)
-        if [ -n "${CUDA_MAJOR}" ]; then
-            CUDA_VARIANT="_v${CUDA_MAJOR}"
-        fi
-        CMAKE_DEFS="-DLLAMA_CUBLAS=on ${COMMON_CMAKE_DEFS} ${CMAKE_DEFS}"
-        BUILD_DIR="${LLAMACPP_DIR}/build/linux/cuda${CUDA_VARIANT}"
-        CUDA_LIB_DIR="$cudalibpath"
-        build
-        install
-        gcc -fPIC -g -shared -o "${BUILD_DIR}/lib/libext_server.so" \
-            -Wl,--whole-archive \
-            "${BUILD_DIR}/lib/libext_server.a" \
-            "${BUILD_DIR}/lib/libcommon.a" \
-            "${BUILD_DIR}/lib/libllama.a" \
-            -Wl,--no-whole-archive \
-            "${CUDA_LIB_DIR}/libcudart_static.a" \
-            "${CUDA_LIB_DIR}/libcublas_static.a" \
-            "${CUDA_LIB_DIR}/libcublasLt_static.a" \
-            "${CUDA_LIB_DIR}/libcudadevrt.a" \
-            "${CUDA_LIB_DIR}/libculibos.a" \
-            -lrt -lpthread -ldl -lstdc++ -lm
+# If needed, look for the default CUDA toolkit location
+if [ -z "${CUDA_LIB_DIR}" ] && [ -d /usr/local/cuda/lib64 ]; then
+    CUDA_LIB_DIR=/usr/local/cuda/lib64
+fi
+
+# If needed, look for CUDA on Arch Linux
+if [ -z "${CUDA_LIB_DIR}" ] && [ -d /opt/cuda/targets/x86_64-linux/lib ]; then
+    CUDA_LIB_DIR=/opt/cuda/targets/x86_64-linux/lib
+fi
+
+if [ -d "${CUDA_LIB_DIR}" ]; then
+    echo "CUDA libraries detected - building dynamic CUDA library"
+    init_vars
+    CUDA_MAJOR=$(ls "${CUDA_LIB_DIR}"/libcudart.so.* | head -1 | cut -f3 -d. || true)
+    if [ -n "${CUDA_MAJOR}" ]; then
+        CUDA_VARIANT=_v${CUDA_MAJOR}
     fi
-done
+    CMAKE_DEFS="-DLLAMA_CUBLAS=on ${COMMON_CMAKE_DEFS} ${CMAKE_DEFS}"
+    BUILD_DIR="${LLAMACPP_DIR}/build/linux/cuda${CUDA_VARIANT}"
+    build
+    install
+    gcc -fPIC -g -shared -o ${BUILD_DIR}/lib/libext_server.so \
+        -Wl,--whole-archive \
+        ${BUILD_DIR}/lib/libext_server.a \
+        ${BUILD_DIR}/lib/libcommon.a \
+        ${BUILD_DIR}/lib/libllama.a \
+        -Wl,--no-whole-archive \
+        ${CUDA_LIB_DIR}/libcudart_static.a \
+        ${CUDA_LIB_DIR}/libcublas_static.a \
+        ${CUDA_LIB_DIR}/libcublasLt_static.a \
+        ${CUDA_LIB_DIR}/libcudadevrt.a \
+        ${CUDA_LIB_DIR}/libculibos.a \
+        -lcuda \
+        -lrt -lpthread -ldl -lstdc++ -lm
+fi
 
 if [ -z "${ROCM_PATH}" ]; then
     # Try the default location in case it exists