diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 33c66991..6ce4483c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -29,7 +29,10 @@ jobs: id: build-tests run: | cd tests - npm run dev -- run --suite build --no-llm --output json > /tmp/build-results.json 2>&1 || true + # Progress goes to stderr (visible), JSON results go to file + npm run --silent dev -- run --suite build --no-llm --output json > /tmp/build-results.json || true + + echo "--- JSON Results ---" cat /tmp/build-results.json # Check if any tests failed @@ -73,7 +76,10 @@ jobs: id: runtime-tests run: | cd tests - npm run dev -- run --suite runtime --no-llm --output json > /tmp/runtime-results.json 2>&1 || true + # Progress goes to stderr (visible), JSON results go to file + npm run --silent dev -- run --suite runtime --no-llm --output json > /tmp/runtime-results.json || true + + echo "--- JSON Results ---" cat /tmp/runtime-results.json - name: Upload runtime results @@ -104,7 +110,10 @@ jobs: id: inference-tests run: | cd tests - npm run dev -- run --suite inference --no-llm --output json > /tmp/inference-results.json 2>&1 || true + # Progress goes to stderr (visible), JSON results go to file + npm run --silent dev -- run --suite inference --no-llm --output json > /tmp/inference-results.json || true + + echo "--- JSON Results ---" cat /tmp/inference-results.json - name: Upload inference results @@ -143,7 +152,10 @@ jobs: echo "Running LLM judge evaluation..." # Re-run all tests with LLM judge using local Ollama - npm run dev -- run --output json > /tmp/llm-judged-results.json 2>&1 || true + # Progress goes to stderr (visible), JSON results go to file + npm run --silent dev -- run --output json > /tmp/llm-judged-results.json || true + + echo "--- JSON Results ---" cat /tmp/llm-judged-results.json - name: Upload final results diff --git a/tests/src/cli.ts b/tests/src/cli.ts index 85463545..fb4e3256 100644 --- a/tests/src/cli.ts +++ b/tests/src/cli.ts @@ -13,6 +13,9 @@ import { RunnerOptions } from './types.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const defaultTestcasesDir = path.join(__dirname, '..', 'testcases') +// Progress output to stderr (visible in console even when stdout is redirected) +const log = (msg: string) => process.stderr.write(msg + '\n') + const program = new Command() program @@ -36,66 +39,65 @@ program .option('--no-llm', 'Skip LLM judging, use simple exit code check') .option('--testcases-dir ', 'Test cases directory', defaultTestcasesDir) .action(async (options) => { - console.log('='.repeat(60)) - console.log('OLLAMA37 TEST RUNNER') - console.log('='.repeat(60)) + log('='.repeat(60)) + log('OLLAMA37 TEST RUNNER') + log('='.repeat(60)) const loader = new TestLoader(options.testcasesDir) const executor = new TestExecutor(path.join(__dirname, '..', '..')) const judge = new LLMJudge(options.ollamaUrl, options.ollamaModel) // Load test cases - console.log('\nLoading test cases...') + log('\nLoading test cases...') let testCases = await loader.loadAll() if (options.suite) { testCases = testCases.filter(tc => tc.suite === options.suite) - console.log(` Filtered by suite: ${options.suite}`) + log(` Filtered by suite: ${options.suite}`) } if (options.id) { testCases = testCases.filter(tc => tc.id === options.id) - console.log(` Filtered by ID: ${options.id}`) + log(` Filtered by ID: ${options.id}`) } // Sort by dependencies testCases = loader.sortByDependencies(testCases) - console.log(` Found ${testCases.length} test cases`) + log(` Found ${testCases.length} test cases`) if (testCases.length === 0) { - console.log('\nNo test cases found!') + log('\nNo test cases found!') process.exit(1) } // Dry run if (options.dryRun) { - console.log('\nDRY RUN - Would execute:') + log('\nDRY RUN - Would execute:') for (const tc of testCases) { - console.log(` ${tc.id}: ${tc.name}`) + log(` ${tc.id}: ${tc.name}`) for (const step of tc.steps) { - console.log(` - ${step.name}: ${step.command}`) + log(` - ${step.name}: ${step.command}`) } } process.exit(0) } - // Execute tests - console.log('\nExecuting tests...') + // Execute tests (progress goes to stderr via executor) const workers = parseInt(options.workers) const results = await executor.executeAll(testCases, workers) // Judge results - console.log('\nJudging results...') + log('\nJudging results...') let judgments if (options.llm === false) { - console.log(' Using simple exit code check (--no-llm)') + log(' Using simple exit code check (--no-llm)') judgments = results.map(r => judge.simpleJudge(r)) } else { try { judgments = await judge.judgeResults(results) } catch (error) { - console.error(' LLM judging failed, falling back to simple check:', error) + log(` LLM judging failed, falling back to simple check: ${error}`) judgments = results.map(r => judge.simpleJudge(r)) } } @@ -107,15 +109,14 @@ program switch (options.output) { case 'json': const json = Reporter.toJSON(reports) - console.log(json) - writeFileSync('test-results.json', json) - console.log('\nResults written to test-results.json') + // JSON goes to stdout (can be redirected to file) + process.stdout.write(json + '\n') break case 'junit': const junit = Reporter.toJUnit(reports) writeFileSync('test-results.xml', junit) - console.log('\nResults written to test-results.xml') + log('\nResults written to test-results.xml') break case 'console': @@ -124,6 +125,13 @@ program break } + // Summary + const passed = reports.filter(r => r.pass).length + const failed = reports.filter(r => !r.pass).length + log('\n' + '='.repeat(60)) + log(`SUMMARY: ${passed} passed, ${failed} failed, ${reports.length} total`) + log('='.repeat(60)) + // Report to TestLink if (options.reportTestlink && options.testlinkApiKey) { const testlinkReporter = new TestLinkReporter( @@ -132,11 +140,10 @@ program ) // Would need plan ID and build ID // await testlinkReporter.reportResults(reports, planId, buildId) - console.log('\nTestLink reporting not yet implemented') + log('\nTestLink reporting not yet implemented') } // Exit with appropriate code - const failed = reports.filter(r => !r.pass).length process.exit(failed > 0 ? 1 : 0) }) diff --git a/tests/src/executor.ts b/tests/src/executor.ts index fbaa46af..6b5bd440 100644 --- a/tests/src/executor.ts +++ b/tests/src/executor.ts @@ -6,11 +6,18 @@ const execAsync = promisify(exec) export class TestExecutor { private workingDir: string + private totalTests: number = 0 + private currentTest: number = 0 constructor(workingDir: string = process.cwd()) { this.workingDir = workingDir } + // Progress output goes to stderr (visible in console) + private progress(msg: string): void { + process.stderr.write(msg + '\n') + } + async executeStep(command: string, timeout: number): Promise { const startTime = Date.now() let stdout = '' @@ -47,11 +54,17 @@ export class TestExecutor { async executeTestCase(testCase: TestCase): Promise { const startTime = Date.now() const stepResults: StepResult[] = [] + const timestamp = new Date().toISOString().substring(11, 19) - console.log(` Executing: ${testCase.id} - ${testCase.name}`) + this.currentTest++ + this.progress(`[${timestamp}] [${this.currentTest}/${this.totalTests}] ${testCase.id}: ${testCase.name}`) - for (const step of testCase.steps) { - console.log(` Step: ${step.name}`) + for (let i = 0; i < testCase.steps.length; i++) { + const step = testCase.steps[i] + const stepTimestamp = new Date().toISOString().substring(11, 19) + + this.progress(` [${stepTimestamp}] Step ${i + 1}/${testCase.steps.length}: ${step.name}`) + this.progress(` Command: ${step.command.substring(0, 80)}${step.command.length > 80 ? '...' : ''}`) const timeout = step.timeout || testCase.timeout const result = await this.executeStep(step.command, timeout) @@ -59,11 +72,15 @@ export class TestExecutor { stepResults.push(result) - // Log step result - if (result.exitCode === 0) { - console.log(` Exit: ${result.exitCode} (${result.duration}ms)`) - } else { - console.log(` Exit: ${result.exitCode} (FAILED, ${result.duration}ms)`) + // Log step result with status indicator + const status = result.exitCode === 0 ? '✓' : '✗' + const duration = `${(result.duration / 1000).toFixed(1)}s` + this.progress(` ${status} Exit: ${result.exitCode} (${duration})`) + + // Show brief error output if failed + if (result.exitCode !== 0 && result.stderr) { + const errorPreview = result.stderr.split('\n')[0].substring(0, 100) + this.progress(` Error: ${errorPreview}`) } } @@ -95,6 +112,14 @@ ${r.stderr || '(empty)'} async executeAll(testCases: TestCase[], concurrency: number = 1): Promise { const results: TestResult[] = [] + // Set total for progress tracking + this.totalTests = testCases.length + this.currentTest = 0 + + const startTimestamp = new Date().toISOString().substring(11, 19) + this.progress(`\n[${startTimestamp}] Starting ${this.totalTests} test(s)...`) + this.progress('─'.repeat(60)) + if (concurrency === 1) { // Sequential execution for (const tc of testCases) { @@ -114,6 +139,11 @@ ${r.stderr || '(empty)'} results.push(...parallelResults) } + // Summary + const endTimestamp = new Date().toISOString().substring(11, 19) + this.progress('─'.repeat(60)) + this.progress(`[${endTimestamp}] Execution complete: ${results.length} test(s)`) + return results } }