matt/ollama37
1
0
Fork 0
mirror of https://github.com/dogkeeper886/ollama37.git synced 2025-12-21 05:07:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add LogCollector for precise test log boundaries

Browse Source 
Problem: Tests used `docker compose logs --since=5m` which caused:
- Log overlap between tests
- Logs from previous tests included
- Missing logs if test exceeded 5 minutes

Solution:
- New LogCollector class runs `docker compose logs --follow`
- Marks test start/end boundaries
- Writes test-specific logs to /tmp/test-{testId}-logs.txt
- Test steps access via TEST_ID environment variable

Changes:
- tests/src/log-collector.ts: New LogCollector class
- tests/src/executor.ts: Integrate LogCollector, set TEST_ID env
- tests/src/cli.ts: Start/stop LogCollector for runtime/inference
- All test cases: Use log collector with fallback to docker compose

Also updated docs/CICD.md with:
- Test runner CLI documentation
- Judge modes (simple, llm, dual)
- Log collector integration
- Updated test case list (12b, 27b models)
- Model unload strategy
- Troubleshooting guide

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Shang Chieh Tseng
2025-12-17 17:46:49 +08:00
parent 82ab6cc96e
commit 2c5094db92
12 changed files with 702 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
tests/testcases/inference/TC-INFERENCE-001.yml
View File

@@ -27,8 +27,12 @@ steps:
- name: Verify model loading in logs
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== Model Loading Check ==="
@@ -58,8 +62,12 @@ steps:
- name: Verify llama runner started
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== Llama Runner Check ==="
@@ -74,8 +82,12 @@ steps:
- name: Check for model loading errors
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== Model Loading Error Check ==="
@@ -97,8 +109,12 @@ steps:
- name: Display model memory allocation from logs
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== Model Memory Allocation ==="
echo "$LOGS" | grep -E '(model weights|kv cache|compute graph|total memory).*device=' | tail -8

24
tests/testcases/inference/TC-INFERENCE-002.yml
View File

@@ -17,8 +17,12 @@ steps:
- name: Check for inference errors in logs
command: |
cd docker
LOGS=$(docker compose logs --since=5m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=5m 2>&1)
fi
echo "=== Inference Error Check ==="
@@ -47,8 +51,12 @@ steps:
- name: Verify inference request in logs
command: |
cd docker
LOGS=$(docker compose logs --since=5m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=5m 2>&1)
fi
echo "=== Inference Request Verification ==="
@@ -69,8 +77,12 @@ steps:
- name: Display recent CUDA activity from logs
command: |
cd docker
LOGS=$(docker compose logs --since=5m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=5m 2>&1)
fi
echo "=== Recent CUDA Activity ==="
echo "$LOGS" | grep -iE "(CUDA|cuda|device=CUDA)" | tail -5 || echo "No recent CUDA activity logged"

24
tests/testcases/inference/TC-INFERENCE-003.yml
View File

@@ -22,8 +22,12 @@ steps:
- name: Verify API requests logged successfully
command: |
cd docker
LOGS=$(docker compose logs --since=5m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=5m 2>&1)
fi
echo "=== API Request Log Verification ==="
@@ -40,8 +44,12 @@ steps:
- name: Check for API errors in logs
command: |
cd docker
LOGS=$(docker compose logs --since=5m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=5m 2>&1)
fi
echo "=== API Error Check ==="
@@ -64,8 +72,12 @@ steps:
- name: Display API response times from logs
command: |
cd docker
LOGS=$(docker compose logs --since=5m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=5m 2>&1)
fi
echo "=== API Response Times ==="

16
tests/testcases/inference/TC-INFERENCE-004.yml
View File

@@ -27,8 +27,12 @@ steps:
- name: Verify model loaded to GPU
command: |
cd docker
LOGS=$(docker compose logs --since=5m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=5m 2>&1)
fi
echo "=== Model Loading Check for gemma3:12b ==="
@@ -63,8 +67,12 @@ steps:
- name: Check for inference errors
command: |
cd docker
LOGS=$(docker compose logs --since=5m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=5m 2>&1)
fi
echo "=== Inference Error Check ==="

16
tests/testcases/inference/TC-INFERENCE-005.yml
View File

@@ -39,8 +39,12 @@ steps:
- name: Verify model loaded across GPUs
command: |
cd docker
LOGS=$(docker compose logs --since=10m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=10m 2>&1)
fi
echo "=== Model Loading Check for gemma3:27b ==="
@@ -96,8 +100,12 @@ steps:
- name: Check for inference errors
command: |
cd docker
LOGS=$(docker compose logs --since=10m 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs --since=10m 2>&1)
fi
echo "=== Inference Error Check ==="

15
tests/testcases/runtime/TC-RUNTIME-001.yml
View File

@@ -22,12 +22,21 @@ steps:
- name: Capture startup logs
command: |
cd docker && docker compose logs 2>&1 | head -100
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
head -100 /tmp/test-${TEST_ID}-logs.txt
else
cd docker && docker compose logs 2>&1 | head -100
fi
- name: Check for startup errors in logs
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
# Check for critical errors
if echo "$LOGS" | grep -qE "(level=ERROR|CUBLAS_STATUS_|CUDA error|cudaMalloc failed)"; then

24
tests/testcases/runtime/TC-RUNTIME-002.yml
View File

@@ -30,8 +30,12 @@ steps:
- name: Verify GPU detection in Ollama logs
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== GPU Detection Check ==="
@@ -60,8 +64,12 @@ steps:
- name: Check for GPU-related errors in logs
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== GPU Error Check ==="
@@ -82,8 +90,12 @@ steps:
- name: Display GPU memory status from logs
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== GPU Memory Status ==="
echo "$LOGS" | grep -E "gpu memory.*library=CUDA" | tail -4

24
tests/testcases/runtime/TC-RUNTIME-003.yml
View File

@@ -30,8 +30,12 @@ steps:
- name: Verify server listening in logs
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== Server Status Check ==="
@@ -46,8 +50,12 @@ steps:
- name: Check for runtime errors in logs
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== Runtime Error Check ==="
@@ -71,8 +79,12 @@ steps:
- name: Verify API request handling in logs
command: |
cd docker
LOGS=$(docker compose logs 2>&1)
# Use log collector file if available, fallback to docker compose logs
if [ -f "/tmp/test-${TEST_ID}-logs.txt" ]; then
LOGS=$(cat /tmp/test-${TEST_ID}-logs.txt)
else
LOGS=$(cd docker && docker compose logs 2>&1)
fi
echo "=== API Request Logs ==="