diff --git a/.gitignore b/.gitignore index 3d1f3f1..3c573a8 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,3 @@ example/**/* dist/ dist/* -.DS_store diff --git a/.talismanrc b/.talismanrc index 52506d1..8f0d67b 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,13 @@ fileignoreconfig: -- filename: package-lock.json - checksum: 009fb6f2f26eda48369458fddb51a255ebbb4c73bd66d247d428c3a0faf2ec1b -version: "" +- filename: .github/workflows/secrets-scan.yml + ignore_detectors: + - filecontent +- filename: .github/workflows/check-version-bump.yml + ignore_detectors: +- filename: skills/dev-workflow/SKILL.md + checksum: c9fa42c57463f3d00aa34ab6c8b4e58b984cd6a3a5878d25596e6e66abb4a129 +- filename: skills/contentstack-datasync/SKILL.md + checksum: 62cf62a46c4dcdd9603b1e6ef910e0d77dfa39d2350db992a6a45d3a2791eaee +- filename: skills/testing/SKILL.md + checksum: ab1f82b284189e2779570d4fd5edcde93494c1ee01110da9f0be6f1f5cf5177b +version: "1.0" diff --git a/jest.config.js b/jest.config.js index bf396ef..14801e0 100644 --- a/jest.config.js +++ b/jest.config.js @@ -102,8 +102,6 @@ module.exports = { // The test environment that will be used for testing testEnvironment: 'node', - setupFilesAfterEnv: ['/jest.setup.js'], - // Options that will be passed to the testEnvironment // testEnvironmentOptions: {}, testPathIgnorePatterns: [ diff --git a/jest.setup.js b/jest.setup.js deleted file mode 100644 index 5fc0995..0000000 --- a/jest.setup.js +++ /dev/null @@ -1,10 +0,0 @@ -/* eslint-env jest */ -// marked ships ESM; Jest/ts-jest need a stub for tests that load the app entry. -jest.mock('marked', () => ({ - __esModule: true, - default: { - parse: jest.fn(() => ''), - setOptions: jest.fn(), - use: jest.fn(), - }, -})) diff --git a/package-lock.json b/package-lock.json index f60facd..ec614f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,29 +1,29 @@ { "name": "@contentstack/datasync-manager", - "version": "2.4.0-beta.3", + "version": "2.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@contentstack/datasync-manager", - "version": "2.4.0-beta.3", + "version": "2.3.1", "license": "MIT", "dependencies": { "@braintree/sanitize-url": "^7.1.2", "debug": "^4.4.3", "dns-socket": "^4.2.2", "lodash": "^4.18.1", - "marked": "^17.0.6", + "marked": "^17.0.5", "write-file-atomic": "7.0.1" }, "devDependencies": { "@semantic-release/commit-analyzer": "^9.0.2", "@semantic-release/git": "^10.0.1", - "@semantic-release/npm": "^12.0.2", + "@semantic-release/npm": "^12.0.1", "@semantic-release/release-notes-generator": "^10.0.3", "@types/debug": "0.0.31", "@types/jest": "23.3.14", - "@types/lodash": "4.17.24", + "@types/lodash": "4.17.15", "@types/marked": "^4.3.2", "@types/mkdirp": "0.5.2", "@types/nock": "9.3.1", @@ -2024,9 +2024,9 @@ "license": "MIT" }, "node_modules/@types/lodash": { - "version": "4.17.24", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.24.tgz", - "integrity": "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==", + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-w/P33JFeySuhN6JLkysYUK2gEmy9kHHFN7E8ro0tkfmlDOgxBDzWEZ/J8cWA+fHqFevpswDTFZnDx+R9lbL6xw==", "dev": true, "license": "MIT" }, diff --git a/package.json b/package.json index edebe97..a2d5e01 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/datasync-manager", "author": "Contentstack LLC ", - "version": "2.4.0-beta.3", + "version": "2.3.1", "description": "The primary module of Contentstack DataSync. Syncs Contentstack data with your server using Contentstack Sync API", "main": "dist/index.js", "dependencies": { @@ -9,17 +9,17 @@ "debug": "^4.4.3", "dns-socket": "^4.2.2", "lodash": "^4.18.1", - "marked": "^17.0.6", + "marked": "^17.0.5", "write-file-atomic": "7.0.1" }, "devDependencies": { "@semantic-release/commit-analyzer": "^9.0.2", "@semantic-release/git": "^10.0.1", - "@semantic-release/npm": "^12.0.2", + "@semantic-release/npm": "^12.0.1", "@semantic-release/release-notes-generator": "^10.0.3", "@types/debug": "0.0.31", "@types/jest": "23.3.14", - "@types/lodash": "4.17.24", + "@types/lodash": "4.17.15", "@types/marked": "^4.3.2", "@types/mkdirp": "0.5.2", "@types/nock": "9.3.1", @@ -48,9 +48,6 @@ "start": "dist", "tslint": "npx tslint -c tslint.json 'src/**/*.ts' --fix", "test": "PLUGIN_PATH=./test/dummy jest --colors --coverage --verbose", - "mock:sync-api": "node scripts/sync-api-mock-server/server.js", - "mock:run-all-scenarios": "node scripts/sync-api-mock-server/run-all-scenarios.js", - "demo:mock": "node scripts/demo-with-mock.js", "lint": "eslint", "semantic-release": "semantic-release", "husky-check": "npx husky && chmod +x .husky/pre-commit" diff --git a/src/api.ts b/src/api.ts index 515cbfe..a4689fe 100644 --- a/src/api.ts +++ b/src/api.ts @@ -60,7 +60,7 @@ import { MESSAGES } from './util/messages' const debug = Debug('api') let MAX_RETRY_LIMIT let RETRY_DELAY_BASE = 200 // Default base delay in milliseconds -let TIMEOUT = 60000 // Increased from 30000 to 60000 (60 seconds) for large stack syncs +let TIMEOUT = 30000 // Default timeout in milliseconds let Contentstack /** @@ -171,51 +171,8 @@ export const get = (req, RETRY = 1) => { .catch(reject) }, timeDelay) } else { - // Enhanced error handling for Error 141 (Invalid sync_token) - try { - const errorBody = JSON.parse(body) - - // Validate error response structure and check for Error 141 - if (errorBody && typeof errorBody === 'object' && errorBody.error_code === 141 && errorBody.errors && typeof errorBody.errors === 'object' && errorBody.errors.sync_token) { - - debug('Error 141 detected: Invalid sync_token. Triggering auto-recovery with init=true') - - // Ensure req.qs exists before modifying - if (!req.qs) { - req.qs = {} - } - - // Clear the invalid token parameters and reinitialize - delete req.qs.sync_token - delete req.qs.pagination_token - req.qs.init = true - // Reset req.path so it gets rebuilt from Contentstack.apis.sync - // (req.path has the old query string baked in from line 109) - delete req.path - - // Mark this as a recovery attempt to prevent infinite loops - if (!req._error141Recovery) { - req._error141Recovery = true - debug('Retrying with init=true after Error 141') - // Use delayed retry - timeDelay = Math.pow(Math.SQRT2, RETRY) * RETRY_DELAY_BASE - debug(`Error 141 recovery: waiting ${timeDelay}ms before retry`) - - return setTimeout(() => { - return get(req, RETRY) - .then(resolve) - .catch(reject) - }, timeDelay) - } else { - debug('Error 141 recovery already attempted, failing to prevent infinite loop') - } - } - } catch (parseError) { - // Body is not JSON or parsing failed, continue with normal error handling - debug('Error response parsing failed:', parseError) - } - debug(MESSAGES.API.REQUEST_FAILED(options)) + return reject(body) } }) @@ -225,36 +182,17 @@ export const get = (req, RETRY = 1) => { httpRequest.setTimeout(options.timeout, () => { debug(MESSAGES.API.REQUEST_TIMEOUT(options.path)) httpRequest.destroy() - const timeoutError = Object.assign(new Error('Request timeout'), { - code: 'ETIMEDOUT', - }) as Error & { code: string } - reject(timeoutError) + reject(new Error('Request timeout')) }) - // Enhanced error handling for network and connection errors + // Enhanced error handling for socket hang ups and connection resets httpRequest.on('error', (error: any) => { debug(MESSAGES.API.REQUEST_ERROR(options.path, error?.message, error?.code)) - // List of retryable network error codes - const retryableErrors = [ - 'ECONNRESET', // Connection reset by peer - 'ETIMEDOUT', // Connection timeout - 'ECONNREFUSED', // Connection refused - 'ENOTFOUND', // DNS lookup failed - 'ENETUNREACH', // Network unreachable - 'EAI_AGAIN', // DNS lookup timeout - 'EPIPE', // Broken pipe - 'EHOSTUNREACH', // Host unreachable - ] - - // Check if error is retryable - const isRetryable = retryableErrors.includes(error?.code) || - error?.message?.includes('socket hang up') || - error?.message?.includes('ETIMEDOUT') - - if (isRetryable && RETRY <= MAX_RETRY_LIMIT) { + // Handle socket hang up and connection reset errors with retry + if ((error?.code === 'ECONNRESET' || error?.message?.includes('socket hang up')) && RETRY <= MAX_RETRY_LIMIT) { timeDelay = Math.pow(Math.SQRT2, RETRY) * RETRY_DELAY_BASE - debug(`Network error ${error?.code || error?.message}: waiting ${timeDelay}ms before retry ${RETRY}/${MAX_RETRY_LIMIT}`) + debug(MESSAGES.API.SOCKET_HANGUP_RETRY(options.path, timeDelay, RETRY, MAX_RETRY_LIMIT)) RETRY++ return setTimeout(() => { diff --git a/src/core/index.ts b/src/core/index.ts index b0c4bef..c59f33c 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -4,6 +4,7 @@ * MIT Licensed */ import * as fs from 'fs' +import * as path from 'path' import Debug from 'debug' import { EventEmitter } from 'events' import { cloneDeep, remove } from 'lodash' @@ -17,6 +18,7 @@ import { map } from '../util/promise.map' import { netConnectivityIssues } from './inet' import { Q as Queue } from './q' import { getToken, saveCheckpoint } from './token-management' +import { sanitizePath } from '../plugins/helper' interface IQueryString { init?: true, @@ -121,9 +123,18 @@ export const init = (contentStore, assetStore) => { const loadCheckpoint = (checkPointConfig: ICheckpoint, paths: any): void => { if (!checkPointConfig?.enabled) return; - // Read checkpoint from configured path only + // Try reading checkpoint from primary path let checkpoint = readHiddenFile(paths.checkpoint); + // Fallback to filePath in config if not found + if (!checkpoint) { + const fallbackPath = path.join( + sanitizePath(__dirname), + sanitizePath(checkPointConfig.filePath || ".checkpoint") + ); + checkpoint = readHiddenFile(fallbackPath); + } + // Set sync token if checkpoint is found if (checkpoint) { debug(MESSAGES.SYNC_CORE.TOKEN_FOUND, checkpoint); @@ -212,16 +223,9 @@ const check = async () => { const sync = async () => { try { debug(MESSAGES.SYNC_CORE.SYNC_STARTED); - let tokenObject: IToken - try { - tokenObject = await getToken() as IToken - } catch (tokenError) { - // check() sets SQ before sync(); fire() is never entered — release the gate - flag.SQ = false - throw tokenError - } + const tokenObject = await getToken(); debug(MESSAGES.SYNC_CORE.SYNC_TOKEN_OBJECT, tokenObject); - const token: IToken = tokenObject + const token: IToken = (tokenObject as IToken) const request: any = { qs: { environment: process.env.SYNC_ENV || Contentstack.environment || 'development', @@ -258,7 +262,7 @@ export const unlock = (refire?: boolean) => { .catch(flag.requestCache.reject) } } - return check() + check() } /** @@ -372,23 +376,10 @@ const fire = (req: IApiRequest) => { .catch(reject) }).catch((error) => { debug(MESSAGES.SYNC_CORE.ERROR_FIRE, error); - - // Check if this is an Error 141 (outdated token) - // Note: api.ts already handles recovery by retrying with init=true - try { - const parsedError = typeof error === 'string' ? JSON.parse(error) : error - if (parsedError.error_code === 141) { - logger.error(MESSAGES.SYNC_CORE.OUTDATED_SYNC_TOKEN) - logger.info(MESSAGES.SYNC_CORE.SYNC_TOKEN_RENEWAL) - // Reset sync_token so next sync starts fresh with init=true - Contentstack.sync_token = undefined - } - } catch (parseError) { - // Not a JSON error or not Error 141, continue with normal handling + if (netConnectivityIssues(error)) { + flag.SQ = false } - - // Any failed sync API attempt must release the gate so the next notify/poke can run - flag.SQ = false + // do something return reject(error) }) diff --git a/src/core/inet.ts b/src/core/inet.ts index 4573491..424e228 100644 --- a/src/core/inet.ts +++ b/src/core/inet.ts @@ -81,23 +81,10 @@ export const checkNetConnectivity = () => { } export const netConnectivityIssues = (error) => { - // Align with retryable codes in api.ts plus client-side timeout (no .code on Request timeout) - const networkErrorCodes = [ - 'ENOTFOUND', - 'ETIMEDOUT', - 'ECONNRESET', - 'EPIPE', - 'EHOSTUNREACH', - 'ECONNREFUSED', - 'ENETUNREACH', - 'EAI_AGAIN', - ] - const msg = typeof error?.message === 'string' ? error.message : '' - - if (networkErrorCodes.includes(error?.code)) { - return true - } - if (msg.includes('socket hang up') || msg.includes('Request timeout')) { + // Include socket hang up and connection reset errors as network connectivity issues + const networkErrorCodes = ['ENOTFOUND', 'ETIMEDOUT', 'ECONNRESET', 'EPIPE', 'EHOSTUNREACH'] + + if (networkErrorCodes.includes(error.code) || error.message?.includes('socket hang up')) { return true } diff --git a/src/plugins/helper.js b/src/plugins/helper.js index f54d679..b7e30a4 100644 --- a/src/plugins/helper.js +++ b/src/plugins/helper.js @@ -1,4 +1,5 @@ const { cloneDeep } = require('lodash') +const { getConfig } = require('../index') const fieldType = { @@ -183,9 +184,6 @@ const checkReferences = (schema, key) => { } exports.buildAssetObject = (asset, locale, entry_uid, content_type_uid) => { - // Lazy-load getConfig at runtime to avoid circular dependency - // (helper.js is loaded during plugin init before index.ts finishes exporting) - const { getConfig } = require('../index') const { contentstack } = getConfig() // add locale key to inside of asset asset.locale = locale diff --git a/src/util/messages.ts b/src/util/messages.ts index 54e87e8..8b41966 100644 --- a/src/util/messages.ts +++ b/src/util/messages.ts @@ -41,9 +41,6 @@ export const MESSAGES = { `Request error for ${path || 'unknown'}: ${message || 'Unknown error'} (${code || 'NO_CODE'})`, SOCKET_HANGUP_RETRY: (path: string, delay: number, attempt: number, max: number) => `Socket hang up detected. Retrying ${path || 'unknown'} with ${delay} ms delay (attempt ${attempt}/${max})`, - ERROR_141_DETECTED: 'Error 141: Invalid sync_token detected. Token is no longer valid.', - ERROR_141_RECOVERY: 'Attempting automatic recovery with init=true to get fresh token.', - ERROR_141_RETRY: 'Retrying sync operation with fresh initialization after Error 141.', }, // Plugin messages (plugins.ts) @@ -88,8 +85,6 @@ export const MESSAGES = { ERROR_FIRE: 'Error during fire operation.', REFIRE_CALLED: (req: any) => `Re-fire operation triggered with: ${JSON.stringify(req)}`, CHECKPOINT_LOCKDOWN: 'Checkpoint: lockdown has been invoked', - OUTDATED_SYNC_TOKEN: 'Sync token is outdated and no longer accepted by the API. Recovering...', - SYNC_TOKEN_RENEWAL: 'Renewing sync token. This typically happens after network interruptions or long inactivity.', }, // Main index messages (index.ts) diff --git a/test/api.ts b/test/api.ts index 733588c..161de39 100644 --- a/test/api.ts +++ b/test/api.ts @@ -150,5 +150,3 @@ describe('test api - get()', () => { // }) }) - -// Socket timeout + ETIMEDOUT: covered by test/core/inet.ts, scripts/sync-api-mock-server (MOCK_SCENARIO=hang). diff --git a/test/core/index.ts b/test/core/index.ts index b2ea338..4c51903 100644 --- a/test/core/index.ts +++ b/test/core/index.ts @@ -13,7 +13,6 @@ describe('check lock-unlock', () => { }) test('lock-unlock', () => { expect(lock()).toBeUndefined() - // unlock(true) would run check() -> sync() and requires init(); use unlock(false) to only exercise the gate - expect(unlock(false)).toBeUndefined() + expect(unlock(true)).toBeUndefined() }) }) diff --git a/test/core/inet.ts b/test/core/inet.ts index 8468b01..66cb183 100644 --- a/test/core/inet.ts +++ b/test/core/inet.ts @@ -20,26 +20,4 @@ describe('# inet', () => { expect(netConnectivityIssues({})) .toEqual(false) }) - - describe('netConnectivityIssues', () => { - test('returns false for unrelated errors', () => { - expect(netConnectivityIssues({ message: 'business logic failed' })).toBe(false) - expect(netConnectivityIssues({ code: 'ICTC' })).toBe(false) - }) - - test('returns true for Request timeout message', () => { - expect(netConnectivityIssues({ message: 'Request timeout' })).toBe(true) - }) - - test('returns true for ETIMEDOUT and other retryable codes', () => { - expect(netConnectivityIssues({ code: 'ETIMEDOUT' })).toBe(true) - expect(netConnectivityIssues({ code: 'ECONNREFUSED' })).toBe(true) - expect(netConnectivityIssues({ code: 'ENETUNREACH' })).toBe(true) - expect(netConnectivityIssues({ code: 'EAI_AGAIN' })).toBe(true) - }) - - test('returns true for socket hang up in message', () => { - expect(netConnectivityIssues({ message: 'socket hang up' })).toBe(true) - }) - }) }) diff --git a/test/core/sync-resume.ts b/test/core/sync-resume.ts deleted file mode 100644 index a72e153..0000000 --- a/test/core/sync-resume.ts +++ /dev/null @@ -1,50 +0,0 @@ -/*! - * Ensures a failed sync (e.g. Request timeout) does not block a later poke(). - */ -import { join } from 'path' -import { cloneDeep, merge } from 'lodash' -import { setConfig } from '../../src' -import { config as internalConfig } from '../../src/config' -import { init as initCore, poke } from '../../src/core' -import { buildConfigPaths } from '../../src/util/build-paths' -import { setLogger } from '../../src/util/logger' -import { config as mockConfig } from '../dummy/config' -import { assetConnector, contentConnector } from '../dummy/connector-listener-instances' -import { get, init as initApi } from '../../src/api' - -jest.mock('../../src/api', () => { - const actual = jest.requireActual('../../src/api') - - return { - ...actual, - get: jest.fn(), - } -}) - -const configs: any = cloneDeep(merge({}, internalConfig, mockConfig)) - -describe('poke after sync failure', () => { - beforeAll(async () => { - configs.paths = buildConfigPaths() - const testRoot = join(process.cwd(), 'test', 'dummy') - configs.paths.checkpoint = join(testRoot, '.checkpoint') - configs.paths.token = join(testRoot, '.token') - setConfig(configs) - setLogger() - initApi(configs.contentstack) - ;(get as jest.Mock).mockResolvedValue({ items: [], sync_token: 'init-token' }) - await initCore(contentConnector, assetConnector) - }) - - test('second poke calls get again after Request timeout', async () => { - const g = get as jest.Mock - g.mockReset() - g.mockRejectedValueOnce(Object.assign(new Error('Request timeout'), { code: 'ETIMEDOUT' })) - g.mockResolvedValueOnce({ items: [], sync_token: 'recover-token' }) - - await expect(poke()).rejects.toThrow('Request timeout') - await poke() - - expect(g.mock.calls.length).toBe(2) - }) -}) diff --git a/test/core/sync.ts b/test/core/sync.ts index 8098dd6..e549378 100644 --- a/test/core/sync.ts +++ b/test/core/sync.ts @@ -1,37 +1,19 @@ -import { join } from 'path' import { cloneDeep, merge } from 'lodash' import { setConfig } from '../../src' import { config as internalConfig } from '../../src/config' -import { init as initCore, poke } from '../../src/core' +import { poke } from '../../src/core' import { buildConfigPaths } from '../../src/util/build-paths' import { setLogger } from '../../src/util/logger' import { config } from '../dummy/config' -import { assetConnector, contentConnector } from '../dummy/connector-listener-instances' -import { get, init as initApi } from '../../src/api' - -jest.mock('../../src/api', () => { - const actual = jest.requireActual('../../src/api') - - return { - ...actual, - get: jest.fn(), - } -}) const configs: any = cloneDeep(merge({}, internalConfig, config)) -beforeAll(async () => { - configs.paths = buildConfigPaths() - const testRoot = join(process.cwd(), 'test', 'dummy') - configs.paths.checkpoint = join(testRoot, '.checkpoint') - configs.paths.token = join(testRoot, '.token') +beforeAll(() => { setConfig(configs) + configs.paths = buildConfigPaths() setLogger() - initApi(configs.contentstack) - ;(get as jest.Mock).mockResolvedValue({ items: [], sync_token: 'poke-test' }) - await initCore(contentConnector, assetConnector) }) -test('Poke should work without errors', async () => { - await expect(poke()).resolves.toBeUndefined() +test('Poke should work without errors', () => { + expect(poke()).toBeUndefined() })