From fb01b8b1ca4f6fbdfacf5e0f9620466d69417e47 Mon Sep 17 00:00:00 2001 From: Shang Chieh Tseng Date: Mon, 15 Dec 2025 17:57:11 +0800 Subject: [PATCH] Split monolithic workflow into modular components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Separate workflows for flexibility: - build.yml: Build verification (standalone + reusable) - runtime.yml: Container & runtime tests with container lifecycle - inference.yml: Inference tests with optional container management - full-pipeline.yml: Orchestrates all stages with LLM judge Each workflow can be triggered independently for targeted testing, or run the full pipeline for complete validation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .github/workflows/build.yml | 51 +++++++++ .../{build-test.yml => full-pipeline.yml} | 101 +++++++----------- .github/workflows/inference.yml | 78 ++++++++++++++ .github/workflows/runtime.yml | 74 +++++++++++++ 4 files changed, 244 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/build.yml rename .github/workflows/{build-test.yml => full-pipeline.yml} (70%) create mode 100644 .github/workflows/inference.yml create mode 100644 .github/workflows/runtime.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..73adf952 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,51 @@ +name: Build Verification + +on: + workflow_dispatch: # Manual trigger + workflow_call: # Called by other workflows + outputs: + result: + description: "Build test result" + value: ${{ jobs.build.outputs.result }} + +jobs: + build: + name: Build Verification + runs-on: self-hosted + outputs: + result: ${{ steps.build-tests.outcome }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install test runner dependencies + run: cd tests && npm ci + + - name: Run build tests + id: build-tests + run: | + cd tests + # Progress goes to stderr (visible), JSON results go to file + npm run --silent dev -- run --suite build --no-llm --output json > /tmp/build-results.json || true + + echo "--- JSON Results ---" + cat /tmp/build-results.json + + # Check if any tests failed + if grep -q '"pass": false' /tmp/build-results.json; then + echo "Some build tests failed" + exit 1 + fi + + - name: Upload build results + uses: actions/upload-artifact@v4 + if: always() + with: + name: build-test-results + path: /tmp/build-results.json diff --git a/.github/workflows/build-test.yml b/.github/workflows/full-pipeline.yml similarity index 70% rename from .github/workflows/build-test.yml rename to .github/workflows/full-pipeline.yml index 6ce4483c..81c92d3d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/full-pipeline.yml @@ -1,82 +1,67 @@ -name: Build and Test +name: Full Pipeline on: - workflow_dispatch: # Manual trigger only + workflow_dispatch: # Manual trigger + inputs: + skip_llm_judge: + description: 'Skip LLM judge evaluation' + required: false + default: 'false' + type: choice + options: + - 'true' + - 'false' env: - TESTLINK_URL: http://localhost:8090 - TESTLINK_PROJECT_ID: "1" OLLAMA_HOST: http://localhost:11434 jobs: build: name: Build Verification - runs-on: self-hosted + uses: ./.github/workflows/build.yml - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Install test runner dependencies - run: cd tests && npm ci - - - name: Run build tests - id: build-tests - run: | - cd tests - # Progress goes to stderr (visible), JSON results go to file - npm run --silent dev -- run --suite build --no-llm --output json > /tmp/build-results.json || true - - echo "--- JSON Results ---" - cat /tmp/build-results.json - - # Check if any tests failed - if grep -q '"passed": false' /tmp/build-results.json; then - echo "Some build tests failed" - exit 1 - fi - - - name: Upload build results - uses: actions/upload-artifact@v4 - if: always() - with: - name: build-test-results - path: /tmp/build-results.json - - runtime: - name: Container & Runtime Tests + start-container: + name: Start Container runs-on: self-hosted needs: build - steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Install test runner dependencies - run: cd tests && npm ci - - name: Start container run: | cd docker docker compose down 2>/dev/null || true docker compose up -d + echo "Waiting for container to be ready..." sleep 10 + - name: Verify container health + run: | + docker ps + curl -s http://localhost:11434/api/tags || echo "Ollama not ready yet, continuing..." + + runtime: + name: Runtime Tests + runs-on: self-hosted + needs: start-container + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install test runner dependencies + run: cd tests && npm ci + - name: Run runtime tests id: runtime-tests run: | cd tests - # Progress goes to stderr (visible), JSON results go to file npm run --silent dev -- run --suite runtime --no-llm --output json > /tmp/runtime-results.json || true echo "--- JSON Results ---" @@ -110,7 +95,6 @@ jobs: id: inference-tests run: | cd tests - # Progress goes to stderr (visible), JSON results go to file npm run --silent dev -- run --suite inference --no-llm --output json > /tmp/inference-results.json || true echo "--- JSON Results ---" @@ -127,7 +111,7 @@ jobs: name: LLM Judge Evaluation runs-on: self-hosted needs: [build, runtime, inference] - if: always() + if: ${{ inputs.skip_llm_judge != 'true' }} steps: - name: Checkout @@ -150,9 +134,6 @@ jobs: run: | cd tests echo "Running LLM judge evaluation..." - - # Re-run all tests with LLM judge using local Ollama - # Progress goes to stderr (visible), JSON results go to file npm run --silent dev -- run --output json > /tmp/llm-judged-results.json || true echo "--- JSON Results ---" @@ -183,13 +164,13 @@ jobs: - name: Summary run: | - echo "## Build and Test Summary" >> $GITHUB_STEP_SUMMARY + echo "## Full Pipeline Summary" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Stage | Status |" >> $GITHUB_STEP_SUMMARY echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY echo "| Build Verification | ${{ needs.build.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Container & Runtime | ${{ needs.runtime.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Runtime Tests | ${{ needs.runtime.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Inference Tests | ${{ needs.inference.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| LLM Judge Evaluation | ${{ needs.llm-judge.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| LLM Judge | ${{ needs.llm-judge.result }} |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "Commit: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/inference.yml b/.github/workflows/inference.yml new file mode 100644 index 00000000..173bad8d --- /dev/null +++ b/.github/workflows/inference.yml @@ -0,0 +1,78 @@ +name: Inference Tests + +on: + workflow_dispatch: # Manual trigger + inputs: + use_existing_container: + description: 'Use existing running container' + required: false + default: 'false' + type: choice + options: + - 'true' + - 'false' + workflow_call: # Called by other workflows + inputs: + use_existing_container: + description: 'Container is already running' + required: false + default: false + type: boolean + outputs: + result: + description: "Inference test result" + value: ${{ jobs.inference.outputs.result }} + +env: + OLLAMA_HOST: http://localhost:11434 + +jobs: + inference: + name: Inference Tests + runs-on: self-hosted + outputs: + result: ${{ steps.inference-tests.outcome }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install test runner dependencies + run: cd tests && npm ci + + - name: Start container (if needed) + if: ${{ inputs.use_existing_container != 'true' && inputs.use_existing_container != true }} + run: | + cd docker + docker compose down 2>/dev/null || true + docker compose up -d + sleep 10 + + - name: Run inference tests + id: inference-tests + run: | + cd tests + # Progress goes to stderr (visible), JSON results go to file + npm run --silent dev -- run --suite inference --no-llm --output json > /tmp/inference-results.json || true + + echo "--- JSON Results ---" + cat /tmp/inference-results.json + + - name: Upload inference results + uses: actions/upload-artifact@v4 + if: always() + with: + name: inference-test-results + path: /tmp/inference-results.json + + - name: Stop container (if we started it) + if: ${{ always() && inputs.use_existing_container != 'true' && inputs.use_existing_container != true }} + run: | + cd docker + docker compose down || true + echo "Container stopped" diff --git a/.github/workflows/runtime.yml b/.github/workflows/runtime.yml new file mode 100644 index 00000000..fde739c3 --- /dev/null +++ b/.github/workflows/runtime.yml @@ -0,0 +1,74 @@ +name: Runtime Tests + +on: + workflow_dispatch: # Manual trigger + inputs: + keep_container: + description: 'Keep container running after tests' + required: false + default: 'false' + type: choice + options: + - 'true' + - 'false' + workflow_call: # Called by other workflows + inputs: + keep_container: + description: 'Keep container running for subsequent jobs' + required: false + default: false + type: boolean + outputs: + result: + description: "Runtime test result" + value: ${{ jobs.runtime.outputs.result }} + +jobs: + runtime: + name: Container & Runtime Tests + runs-on: self-hosted + outputs: + result: ${{ steps.runtime-tests.outcome }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install test runner dependencies + run: cd tests && npm ci + + - name: Start container + run: | + cd docker + docker compose down 2>/dev/null || true + docker compose up -d + sleep 10 + + - name: Run runtime tests + id: runtime-tests + run: | + cd tests + # Progress goes to stderr (visible), JSON results go to file + npm run --silent dev -- run --suite runtime --no-llm --output json > /tmp/runtime-results.json || true + + echo "--- JSON Results ---" + cat /tmp/runtime-results.json + + - name: Upload runtime results + uses: actions/upload-artifact@v4 + if: always() + with: + name: runtime-test-results + path: /tmp/runtime-results.json + + - name: Stop container + if: ${{ inputs.keep_container != 'true' && inputs.keep_container != true }} + run: | + cd docker + docker compose down || true + echo "Container stopped"