diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index b7d8c2f6e..fdbf84510 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -40,11 +40,22 @@ export default async function (workerCount, selectedRuns, options) { output.print(`CodeceptJS v${Codecept.version()} ${output.standWithUkraine()}`) output.print(`Running tests in ${output.styles.bold(numberOfWorkers)} workers...`) - output.print() store.hasWorkers = true const workers = new Workers(numberOfWorkers, config) workers.overrideConfig(overrideConfigs) + + // Show test distribution after workers are initialized + await workers.bootstrapAll() + + const workerObjects = workers.getWorkers() + output.print() + output.print('Test distribution:') + workerObjects.forEach((worker, index) => { + const testCount = worker.tests.length + output.print(` Worker ${index + 1}: ${testCount} test${testCount !== 1 ? 's' : ''}`) + }) + output.print() workers.on(event.test.failed, test => { output.test.failed(test) @@ -68,7 +79,6 @@ export default async function (workerCount, selectedRuns, options) { if (options.verbose) { await getMachineInfo() } - await workers.bootstrapAll() await workers.run() } catch (err) { output.error(err) diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index f9d3d36ec..92246ba80 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -19,6 +19,34 @@ const stderr = '' const { options, tests, testRoot, workerIndex, poolMode } = workerData +// Global error handlers to catch critical errors but not test failures +process.on('uncaughtException', (err) => { + // Log to stderr to bypass stdout suppression + process.stderr.write(`[Worker ${workerIndex}] UNCAUGHT EXCEPTION: ${err.message}\n`) + process.stderr.write(`${err.stack}\n`) + + // Don't exit on test assertion errors - those are handled by mocha + if (err.name === 'AssertionError' || err.message?.includes('expected')) { + return + } + process.exit(1) +}) + +process.on('unhandledRejection', (reason, promise) => { + // Log to stderr to bypass stdout suppression + const msg = reason?.message || String(reason) + process.stderr.write(`[Worker ${workerIndex}] UNHANDLED REJECTION: ${msg}\n`) + if (reason?.stack) { + process.stderr.write(`${reason.stack}\n`) + } + + // Don't exit on test-related rejections + if (msg.includes('expected') || msg.includes('AssertionError')) { + return + } + process.exit(1) +}) + // hide worker output // In pool mode, only suppress output if debug is NOT enabled // In regular mode, hide result output but allow step output in verbose/debug @@ -26,6 +54,10 @@ if (poolMode && !options.debug) { // In pool mode without debug, allow test names and important output but suppress verbose details const originalWrite = process.stdout.write process.stdout.write = string => { + // Always allow Worker logs + if (string.includes('[Worker')) { + return originalWrite.call(process.stdout, string) + } // Allow test names (✔ or ✖), Scenario Steps, failures, and important markers if ( string.includes('✔') || @@ -45,7 +77,12 @@ if (poolMode && !options.debug) { return originalWrite.call(process.stdout, string) } } else if (!poolMode && !options.debug && !options.verbose) { + const originalWrite = process.stdout.write process.stdout.write = string => { + // Always allow Worker logs + if (string.includes('[Worker')) { + return originalWrite.call(process.stdout, string) + } stdout += string return true } @@ -82,6 +119,13 @@ let config // Load test and run initPromise = (async function () { try { + // Add staggered delay at the very start to prevent resource conflicts + // Longer delay for browser initialization conflicts + const delay = (workerIndex - 1) * 2000 // 0ms, 2s, 4s, etc. + if (delay > 0) { + await new Promise(resolve => setTimeout(resolve, delay)) + } + // Import modules dynamically to avoid ES Module loader race conditions in Node 22.x const eventModule = await import('../../event.js') const containerModule = await import('../../container.js') @@ -98,8 +142,16 @@ initPromise = (async function () { const overrideConfigs = tryOrDefault(() => JSON.parse(options.override), {}) - // IMPORTANT: await is required here since getConfig is async - const baseConfig = await getConfig(options.config || testRoot) + let baseConfig + try { + // IMPORTANT: await is required here since getConfig is async + baseConfig = await getConfig(options.config || testRoot) + } catch (configErr) { + process.stderr.write(`[Worker ${workerIndex}] FAILED loading config: ${configErr.message}\n`) + process.stderr.write(`${configErr.stack}\n`) + await new Promise(resolve => setTimeout(resolve, 100)) + process.exit(1) + } // important deep merge so dynamic things e.g. functions on config are not overridden config = deepMerge(baseConfig, overrideConfigs) @@ -107,7 +159,15 @@ initPromise = (async function () { // Pass workerIndex as child option for output.process() to display worker prefix const optsWithChild = { ...options, child: workerIndex } codecept = new Codecept(config, optsWithChild) - await codecept.init(testRoot) + + try { + await codecept.init(testRoot) + } catch (initErr) { + process.stderr.write(`[Worker ${workerIndex}] FAILED during codecept.init(): ${initErr.message}\n`) + process.stderr.write(`${initErr.stack}\n`) + process.exit(1) + } + codecept.loadTests() mocha = container.mocha() @@ -126,10 +186,12 @@ initPromise = (async function () { await runTests() } else { // No tests to run, close the worker + console.error(`[Worker ${workerIndex}] ERROR: No tests found after filtering! Assigned ${tests.length} UIDs but none matched.`) parentPort?.close() } } catch (err) { - console.error('Error in worker initialization:', err) + process.stderr.write(`[Worker ${workerIndex}] FATAL ERROR: ${err.message}\n`) + process.stderr.write(`${err.stack}\n`) process.exit(1) } })() @@ -147,8 +209,14 @@ async function runTests() { disablePause() try { await codecept.run() + } catch (err) { + throw err } finally { - await codecept.teardown() + try { + await codecept.teardown() + } catch (err) { + // Ignore teardown errors + } } } @@ -336,8 +404,16 @@ function filterTests() { mocha.files = files mocha.loadFiles() - for (const suite of mocha.suite.suites) { + // Recursively filter tests in all suites (including nested ones) + const filterSuiteTests = (suite) => { suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0) + for (const childSuite of suite.suites) { + filterSuiteTests(childSuite) + } + } + + for (const suite of mocha.suite.suites) { + filterSuiteTests(suite) } } diff --git a/lib/helper/Playwright.js b/lib/helper/Playwright.js index 1d5e608dd..5ab67f98e 100644 --- a/lib/helper/Playwright.js +++ b/lib/helper/Playwright.js @@ -912,7 +912,7 @@ class Playwright extends Helper { } async _finishTest() { - if ((restartsSession() || restartsContext() || restartsBrowser()) && this.isRunning) { + if (this.isRunning) { try { await Promise.race([this._stopBrowser(), new Promise((_, reject) => setTimeout(() => reject(new Error('Test finish timeout')), 10000))]) } catch (e) { diff --git a/lib/listener/helpers.js b/lib/listener/helpers.js index ed38daa61..50256d392 100644 --- a/lib/listener/helpers.js +++ b/lib/listener/helpers.js @@ -73,30 +73,18 @@ export default function () { }) event.dispatcher.on(event.all.result, () => { - // Skip _finishTest for all helpers if any browser helper restarts to avoid double cleanup - const hasBrowserRestart = Object.values(helpers).some(helper => - (helper.config && (helper.config.restart === 'browser' || helper.config.restart === 'context' || helper.config.restart === true)) || - (helper.options && (helper.options.restart === 'browser' || helper.options.restart === 'context' || helper.options.restart === true)) - ) - Object.keys(helpers).forEach(key => { const helper = helpers[key] - if (helper._finishTest && !hasBrowserRestart) { + if (helper._finishTest) { recorder.add(`hook ${key}._finishTest()`, () => helper._finishTest(), true, false) } }) }) event.dispatcher.on(event.all.after, () => { - // Skip _cleanup for all helpers if any browser helper restarts to avoid double cleanup - const hasBrowserRestart = Object.values(helpers).some(helper => - (helper.config && (helper.config.restart === 'browser' || helper.config.restart === 'context' || helper.config.restart === true)) || - (helper.options && (helper.options.restart === 'browser' || helper.options.restart === 'context' || helper.options.restart === true)) - ) - Object.keys(helpers).forEach(key => { const helper = helpers[key] - if (helper._cleanup && !hasBrowserRestart) { + if (helper._cleanup) { recorder.add(`hook ${key}._cleanup()`, () => helper._cleanup(), true, false) } }) diff --git a/lib/utils/typescript.js b/lib/utils/typescript.js index 11d9fea05..3f134c7ea 100644 --- a/lib/utils/typescript.js +++ b/lib/utils/typescript.js @@ -119,7 +119,7 @@ const __dirname = __dirname_fn(__filename); let jsContent = transpileTS(filePath) // Find all relative TypeScript imports in this file - const importRegex = /from\s+['"](\..+?)(?:\.ts)?['"]/g + const importRegex = /from\s+['"](\.[^'"]+?)(?:\.ts)?['"]/g let match const imports = [] @@ -170,7 +170,7 @@ const __dirname = __dirname_fn(__filename); // After all dependencies are transpiled, rewrite imports in this file jsContent = jsContent.replace( - /from\s+['"](\..+?)(?:\.ts)?['"]/g, + /from\s+['"](\.[^'"]+?)(?:\.ts)?['"]/g, (match, importPath) => { let resolvedPath = path.resolve(fileBaseDir, importPath) const originalExt = path.extname(importPath) diff --git a/lib/workers.js b/lib/workers.js index 8fa9e9dd6..4e77fc6a8 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -370,6 +370,9 @@ class Workers extends EventEmitter { // If Codecept isn't initialized yet, return empty groups as a safe fallback if (!this.codecept) return populateGroups(numberOfWorkers) const files = this.codecept.testFiles + + // Create a fresh mocha instance to avoid state pollution + Container.createMocha(this.codecept.config.mocha || {}, this.options) const mocha = Container.mocha() mocha.files = files mocha.loadFiles() @@ -384,6 +387,10 @@ class Workers extends EventEmitter { groupCounter++ } }) + + // Clean up after collecting test UIDs + mocha.unloadFiles() + return groups } @@ -452,9 +459,12 @@ class Workers extends EventEmitter { const files = this.codecept.testFiles const groups = populateGroups(numberOfWorkers) + // Create a fresh mocha instance to avoid state pollution + Container.createMocha(this.codecept.config.mocha || {}, this.options) const mocha = Container.mocha() mocha.files = files mocha.loadFiles() + mocha.suite.suites.forEach(suite => { const i = indexOfSmallestElement(groups) suite.tests.forEach(test => { @@ -463,6 +473,10 @@ class Workers extends EventEmitter { } }) }) + + // Clean up after collecting test UIDs + mocha.unloadFiles() + return groups } @@ -504,8 +518,24 @@ class Workers extends EventEmitter { // Workers are already running, this is just a placeholder step }) + // Add overall timeout to prevent infinite hanging + const overallTimeout = setTimeout(() => { + console.error('[Main] Overall timeout reached (10 minutes). Force terminating remaining workers...') + workerThreads.forEach(w => { + try { + w.terminate() + } catch (e) { + // ignore + } + }) + this._finishRun() + }, 600000) // 10 minutes + return new Promise(resolve => { - this.on('end', resolve) + this.on('end', () => { + clearTimeout(overallTimeout) + resolve() + }) }) } @@ -528,8 +558,32 @@ class Workers extends EventEmitter { if (this.isPoolMode) { this.activeWorkers.set(worker, { available: true, workerIndex: null }) } + + // Track last activity time to detect hanging workers + let lastActivity = Date.now() + let currentTest = null + const workerTimeout = 300000 // 5 minutes + + const timeoutChecker = setInterval(() => { + const elapsed = Date.now() - lastActivity + if (elapsed > workerTimeout) { + console.error(`[Main] Worker appears to be hanging (no activity for ${Math.floor(elapsed/1000)}s). Terminating...`) + if (currentTest) { + console.error(`[Main] Last test: ${currentTest}`) + } + clearInterval(timeoutChecker) + worker.terminate() + } + }, 30000) // Check every 30 seconds worker.on('message', message => { + lastActivity = Date.now() // Update activity timestamp + + // Track current test + if (message.event === event.test.started && message.data) { + currentTest = message.data.title || message.data.fullTitle + } + output.process(message.workerIndex) // Handle test requests for pool mode @@ -646,11 +700,25 @@ class Workers extends EventEmitter { }) worker.on('error', err => { + console.error(`[Main] Worker error:`, err.message || err) + if (currentTest) { + console.error(`[Main] Failed during test: ${currentTest}`) + } this.errors.push(err) }) - worker.on('exit', () => { + worker.on('exit', (code) => { + clearInterval(timeoutChecker) this.closedWorkers += 1 + + if (code !== 0) { + console.error(`[Main] Worker exited with code ${code}`) + if (currentTest) { + console.error(`[Main] Last test running: ${currentTest}`) + } + // Mark as failed + process.exitCode = 1 + } if (this.isPoolMode) { // Pool mode: finish when all workers have exited and no more tests @@ -666,7 +734,7 @@ class Workers extends EventEmitter { _finishRun() { event.dispatcher.emit(event.workers.after, { tests: this.workers.map(worker => worker.tests) }) - if (Container.result().hasFailed) { + if (Container.result().hasFailed || this.errors.length > 0) { process.exitCode = 1 } else { process.exitCode = 0 diff --git a/package.json b/package.json index 14458f905..49e538dc2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeceptjs", - "version": "4.0.1-beta.9", + "version": "4.0.2-beta.17", "type": "module", "description": "Supercharged End 2 End Testing Framework for NodeJS", "keywords": [ diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 1fd2f19a2..e7b03f268 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -345,7 +345,8 @@ describe('CodeceptJS Workers Runner', function () { if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') // Run regular workers mode first to get baseline counts exec(`${codecept_run} 2`, (err, stdout) => { - const regularStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/) + // Match only the final summary line (starts with spaces, not [Worker]) + const regularStats = stdout.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/m) if (!regularStats) return done(new Error('Could not parse regular mode statistics')) const expectedPassed = parseInt(regularStats[2]) @@ -357,8 +358,8 @@ describe('CodeceptJS Workers Runner', function () { expect(stdout2).toContain('CodeceptJS') expect(stdout2).toContain('Running tests in 2 workers') - // Extract pool mode statistics - const poolStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/) + // Match only the final summary line + const poolStats = stdout2.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/m) expect(poolStats).toBeTruthy() const actualPassed = parseInt(poolStats[2]) @@ -381,7 +382,8 @@ describe('CodeceptJS Workers Runner', function () { if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') // Run regular workers mode with grep first exec(`${codecept_run} 2 --grep "grep"`, (err, stdout) => { - const regularStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + // Match only the final summary line (starts with spaces, not [Worker]) + const regularStats = stdout.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/m) if (!regularStats) return done(new Error('Could not parse regular mode grep statistics')) const expectedPassed = parseInt(regularStats[2]) @@ -389,7 +391,8 @@ describe('CodeceptJS Workers Runner', function () { // Now run pool mode with grep and compare exec(`${codecept_run} 2 --by pool --grep "grep"`, (err2, stdout2) => { - const poolStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + // Match only the final summary line + const poolStats = stdout2.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/m) expect(poolStats).toBeTruthy() const actualPassed = parseInt(poolStats[2]) @@ -408,7 +411,8 @@ describe('CodeceptJS Workers Runner', function () { if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') // Run pool mode with 1 worker exec(`${codecept_run} 1 --by pool --grep "grep"`, (err, stdout) => { - const singleStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + // Match only the final summary line (starts with spaces, not [Worker]) + const singleStats = stdout.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/m) if (!singleStats) return done(new Error('Could not parse single worker statistics')) const singlePassed = parseInt(singleStats[2]) @@ -416,7 +420,8 @@ describe('CodeceptJS Workers Runner', function () { // Run pool mode with multiple workers exec(`${codecept_run} 3 --by pool --grep "grep"`, (err2, stdout2) => { - const multiStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + // Match only the final summary line + const multiStats = stdout2.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/m) expect(multiStats).toBeTruthy() const multiPassed = parseInt(multiStats[2]) @@ -462,8 +467,8 @@ describe('CodeceptJS Workers Runner', function () { expect(stdout).toContain('CodeceptJS') expect(stdout).toContain('Running tests in 2 workers') - // Should have some passing and some failing tests - const stats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/) + // Match only the final summary line (starts with spaces, not [Worker]) + const stats = stdout.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?(?:,\s+(\d+) failedHooks)?/m) expect(stats).toBeTruthy() const passed = parseInt(stats[2]) @@ -482,7 +487,8 @@ describe('CodeceptJS Workers Runner', function () { if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') // Run pool mode first time exec(`${codecept_run} 2 --by pool --grep "grep"`, (err, stdout) => { - const firstStats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + // Match only the final summary line (starts with spaces, not [Worker]) + const firstStats = stdout.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/m) if (!firstStats) return done(new Error('Could not parse first run statistics')) const firstPassed = parseInt(firstStats[2]) @@ -490,7 +496,8 @@ describe('CodeceptJS Workers Runner', function () { // Run pool mode second time exec(`${codecept_run} 2 --by pool --grep "grep"`, (err2, stdout2) => { - const secondStats = stdout2.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + // Match only the final summary line + const secondStats = stdout2.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/m) expect(secondStats).toBeTruthy() const secondPassed = parseInt(secondStats[2]) @@ -512,7 +519,8 @@ describe('CodeceptJS Workers Runner', function () { expect(stdout).toContain('CodeceptJS') expect(stdout).toContain('Running tests in 8 workers') - const stats = stdout.match(/(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/) + // Match only the final summary line (starts with spaces, not [Worker]) + const stats = stdout.match(/^\s{2}(FAIL|OK)\s+\|\s+(\d+) passed(?:,\s+(\d+) failed)?/m) expect(stats).toBeTruthy() const passed = parseInt(stats[2])