diff --git a/.talismanrc b/.talismanrc index 58ae8ba6b..63426bbfe 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,5 @@ fileignoreconfig: + - filename: pnpm-lock.yaml - checksum: 31e333d6769adbaae042c92ea0930fab168a0e06fc1bda406d49fd1042a7a9c7 -version: '1.0' + checksum: 3d37dc1eb3401e49396b5893da28562f81c290d4b778e9276016e02dda9bb908 + version: '1.0' diff --git a/packages/contentstack-audit/src/audit-base-command.ts b/packages/contentstack-audit/src/audit-base-command.ts index b78650a31..d86efb057 100644 --- a/packages/contentstack-audit/src/audit-base-command.ts +++ b/packages/contentstack-audit/src/audit-base-command.ts @@ -94,6 +94,8 @@ export abstract class AuditBaseCommand extends BaseCommand; missingEnvLocale?: Record; missingMultipleFields?: Record; + missingAssetRefs?: Record; } = {}, missingMandatoryFields, missingTitleFields, @@ -215,6 +228,8 @@ export abstract class AuditBaseCommand extends BaseCommand, ): Promise { log.debug(`Preparing report for module: ${moduleName}`, this.auditContext); @@ -656,7 +682,8 @@ export abstract class AuditBaseCommand extends BaseCommand, ): Promise { if (Object.keys(config.moduleConfig).includes(moduleName) || config.feild_level_modules.includes(moduleName)) { diff --git a/packages/contentstack-audit/src/config/index.ts b/packages/contentstack-audit/src/config/index.ts index e291554fa..b3f0e2553 100644 --- a/packages/contentstack-audit/src/config/index.ts +++ b/packages/contentstack-audit/src/config/index.ts @@ -55,6 +55,9 @@ const config = { name: 'assets', dirName: 'assets', fileName: 'assets.json', + // Asset scan statuses that must block import/reference; any other value (including + // 'not_scanned', 'clean', or the field being absent) is treated as safe. + blockingScanStatuses: ['pending', 'quarantined'], }, environments: { name: 'environments', @@ -110,6 +113,8 @@ const config = { 'publish_locale', 'publish_environment', 'asset_uid', + 'scan_status', + 'mandatory', 'selectedValue', 'ct_uid', 'action', @@ -129,6 +134,7 @@ const config = { Entry_Missing_Locale_and_Env: 'Entry_Missing_Locale_and_Env', Entry_Missing_Locale_and_Env_in_Publish_Details: 'Entry_Missing_Locale_and_Env_in_Publish_Details', Entry_Multiple_Fields: 'Entry_Multiple_Fields', + Entries_Asset_field: 'Entries_Asset_field', }, feild_level_modules: [ 'Entries_Title_field', @@ -137,6 +143,8 @@ const config = { 'Entry_Missing_Locale_and_Env_in_Publish_Details', 'field-rules', 'Entry_Multiple_Fields', + 'Entries_Asset_field', + 'asset-scan-status', 'Summary', ], fixSelectField: false, diff --git a/packages/contentstack-audit/src/messages/index.ts b/packages/contentstack-audit/src/messages/index.ts index a2b654128..dac9136f9 100644 --- a/packages/contentstack-audit/src/messages/index.ts +++ b/packages/contentstack-audit/src/messages/index.ts @@ -49,6 +49,8 @@ const auditMsg = { FIELD_RULE_TARGET_ABSENT: `The target field '{target_field}' is not present in the schema of the content-type {ctUid}`, FIELD_RULE_CONDITION_SCAN_MESSAGE: `Completed Scanning of Field Rule '{num}' condition of Content-type '{ctUid}'`, FIELD_RULE_TARGET_SCAN_MESSAGE: `Completed Scanning of Field Rule '{num}' target of Content-type '{ctUid}'`, + SCAN_ASSET_QUARANTINE_MSG: `Asset with UID '{uid}' has a non-clean scan status ('{status}') and will be excluded from import.`, + ENTRY_ASSET_REF_WARN_MSG: `Entry '{uid}' field '{field}' references asset '{asset_uid}' with scan status '{status}'; it will be removed on fix.`, }; const auditFixMsg = { @@ -63,6 +65,8 @@ const auditFixMsg = { ENTRY_SELECT_FIELD_FIX: `Adding the value '{value}' in the select field of entry UID '{uid}'...`, ASSET_FIX: `Fixed publish detials for Asset with UID '{uid}'`, FIELD_RULE_FIX_MESSAGE: `Fixed Field Rule '{num}' target of Content-type '{ctUid}`, + ASSET_SCAN_STATUS_FIX: `Removed asset with UID '{uid}' (scan status: '{status}') from assets.json.`, + ENTRY_ASSET_REF_FIX: `Removed reference to asset '{asset_uid}' from entry '{uid}' field '{field}'.`, }; const messages: typeof errors & diff --git a/packages/contentstack-audit/src/modules/assets.ts b/packages/contentstack-audit/src/modules/assets.ts index af00aa5bb..022af13f9 100644 --- a/packages/contentstack-audit/src/modules/assets.ts +++ b/packages/contentstack-audit/src/modules/assets.ts @@ -27,6 +27,7 @@ export default class Assets { public environments: string[] = []; protected schema: ContentTypeStruct[] = []; protected missingEnvLocales: Record = {}; + public missingScanStatusAssets: Record = {}; public moduleName: keyof typeof auditConfig.moduleConfig; constructor({ fix, config, moduleName }: ModuleConstructorParam & CtConstructorParam) { @@ -184,7 +185,24 @@ export default class Assets { for (const assetUid in assets) { log.debug(`Processing asset: ${assetUid}`, this.config.auditContext); - + + const scanStatus = this.assets[assetUid]?._asset_scan_status; + if (this.config.moduleConfig.assets.blockingScanStatuses.includes(scanStatus)) { + log.debug(`Asset ${assetUid} has a non-clean scan status: ${scanStatus}`, this.config.auditContext); + cliux.print($t(auditMsg.SCAN_ASSET_QUARANTINE_MSG, { uid: assetUid, status: scanStatus }), { + color: 'yellow', + }); + this.missingScanStatusAssets[assetUid] = [ + { asset_uid: assetUid, filename: this.assets[assetUid].filename, scan_status: scanStatus }, + ]; + + if (this.fix) { + log.info($t(auditFixMsg.ASSET_SCAN_STATUS_FIX, { uid: assetUid, status: scanStatus }), this.config.auditContext); + delete this.assets[assetUid]; + continue; + } + } + if (this.assets[assetUid]?.publish_details && !Array.isArray(this.assets[assetUid].publish_details)) { log.debug(`Asset ${assetUid} has invalid publish_details format`, this.config.auditContext); cliux.print($t(auditMsg.ASSET_NOT_EXIST, { uid: assetUid }), { color: 'red' }); @@ -226,15 +244,19 @@ export default class Assets { const remainingPublishDetails = this.assets[assetUid].publish_details?.length || 0; log.debug(`Asset ${assetUid} now has ${remainingPublishDetails} valid publish details`, this.config.auditContext); - + if (this.fix) { log.debug(`Fixing asset ${assetUid}`, this.config.auditContext); log.info($t(auditFixMsg.ASSET_FIX, { uid: assetUid }), this.config.auditContext); - await this.writeFixContent(`${basePath}/${indexer[fileIndex]}`, this.assets); } } + + if (this.fix) { + log.debug(`Writing fixed assets chunk to: ${basePath}/${indexer[fileIndex]}`, this.config.auditContext); + await this.writeFixContent(`${basePath}/${indexer[fileIndex]}`, this.assets); + } } - + log.debug(`Asset reference validation completed. Processed ${Object.keys(this.missingEnvLocales).length} assets with issues`, this.config.auditContext); } } diff --git a/packages/contentstack-audit/src/modules/entries.ts b/packages/contentstack-audit/src/modules/entries.ts index 02697d50b..6fc0ee144 100644 --- a/packages/contentstack-audit/src/modules/entries.ts +++ b/packages/contentstack-audit/src/modules/entries.ts @@ -58,8 +58,11 @@ export default class Entries { protected missingTitleFields: Record = {}; protected missingEnvLocale: Record = {}; protected missingMultipleField: Record = {}; + protected missingAssetRefs: Record = {}; public environments: string[] = []; public entryMetaData: Record[] = []; + public assetMetaData: Record = {}; + public assetsDataAvailable = false; public moduleName: keyof typeof auditConfig.moduleConfig = 'entries'; constructor({ fix, config, moduleName, ctSchema, gfSchema }: ModuleConstructorParam & CtConstructorParam) { @@ -154,6 +157,10 @@ export default class Entries { await this.prepareEntryMetaData(); log.debug(`Entry metadata prepared: ${this.entryMetaData.length} entries found`, this.config.auditContext); + log.debug('Preparing asset metadata', this.config.auditContext); + await this.prepareAssetMetaData(); + log.debug(`Asset metadata prepared: ${Object.keys(this.assetMetaData).length} assets found`, this.config.auditContext); + log.debug('Fixing prerequisite data', this.config.auditContext); await this.fixPrerequisiteData(); log.debug('Prerequisite data fix completed', this.config.auditContext); @@ -198,6 +205,10 @@ export default class Entries { if (!this.missingMandatoryFields[this.currentUid]) { this.missingMandatoryFields[this.currentUid] = []; } + + if (!this.missingAssetRefs[this.currentUid]) { + this.missingAssetRefs[this.currentUid] = []; + } if (this.fix) { log.debug(`Removing missing keys from entry ${uid}`, this.config.auditContext); this.removeMissingKeysOnEntry(ctSchema.schema as ContentTypeSchemaType[], this.entries[entryUid]); @@ -234,6 +245,14 @@ export default class Entries { }); } + if (this.missingAssetRefs[this.currentUid]?.length) { + log.debug(`Found ${this.missingAssetRefs[this.currentUid].length} missing/quarantined asset references for entry ${uid}`, this.config.auditContext); + this.missingAssetRefs[this.currentUid].forEach((entry: any) => { + entry.ct = ctSchema.uid; + entry.locale = code; + }); + } + const fields = this.missingMandatoryFields[uid]; const isPublished = entry.publish_details?.length > 0; log.debug(`Entry ${uid} published status: ${isPublished}, missing mandatory fields: ${fields?.length || 0}`, this.config.auditContext); @@ -343,6 +362,7 @@ export default class Entries { missingTitleFields: this.missingTitleFields, missingEnvLocale: this.missingEnvLocale, missingMultipleFields: this.missingMultipleField, + missingAssetRefs: this.missingAssetRefs, }; log.debug(`Entries audit completed. Found issues:`, this.config.auditContext); @@ -388,8 +408,17 @@ export default class Entries { removedMandatoryFields++; } } - - log.debug(`Cleanup completed: removed ${removedRefs} empty refs, ${removedSelectFields} empty select fields, ${removedMandatoryFields} empty mandatory fields`, this.config.auditContext); + + let removedAssetRefs = 0; + for (let propName in this.missingAssetRefs) { + if (!this.missingAssetRefs[propName].length) { + log.debug(`Removing empty missing asset references for entry: ${propName}`, this.config.auditContext); + delete this.missingAssetRefs[propName]; + removedAssetRefs++; + } + } + + log.debug(`Cleanup completed: removed ${removedRefs} empty refs, ${removedSelectFields} empty select fields, ${removedMandatoryFields} empty mandatory fields, ${removedAssetRefs} empty asset refs`, this.config.auditContext); } /** @@ -624,6 +653,16 @@ export default class Entries { entry[uid] as EntryGroupFieldDataType[], ); break; + case 'file': + log.debug(`Validating file/asset field: ${display_name}`, this.config.auditContext); + const assetRefResults = this.validateFileField( + [...tree, { uid: child.uid, name: child.display_name, field: uid }], + child, + entry[uid], + ); + this.missingAssetRefs[this.currentUid].push(...assetRefResults); + log.debug(`Found ${assetRefResults.length} quarantined/missing asset references in field: ${display_name}`, this.config.auditContext); + break; case 'text': case 'number': if (child.hasOwnProperty('display_type')) { @@ -888,6 +927,60 @@ export default class Entries { log.debug(`Group field validation completed for: ${fieldStructure.display_name}`); } + /** + * Returns true when the given asset uid should be treated as unusable — either it doesn't + * exist in the exported assets.json at all, or it exists but its scan status is one of + * config.moduleConfig.assets.blockingScanStatuses (e.g. 'pending'/'quarantined'). Any other + * status — including 'clean', 'not_scanned' (org has asset scanning disabled), or the field + * being absent — is not a blocking condition. Returns false (never flag) when asset metadata + * wasn't available at all, since we can't validate what we don't have data for. + */ + isAssetBad(uid?: string): boolean { + if (!this.assetsDataAvailable || !uid) return false; + const assetRecord = this.assetMetaData[uid]; + if (!assetRecord) return true; + return this.config.moduleConfig.assets.blockingScanStatuses.includes(assetRecord._asset_scan_status ?? ''); + } + + /** + * The function `validateFileField` checks a `data_type: 'file'` (asset reference) field's + * value(s) against the asset metadata index and returns an issue when any referenced asset is + * missing or has a non-clean scan status. + */ + validateFileField( + tree: Record[], + fieldStructure: { uid: string; data_type: string; display_name: string; mandatory?: boolean }, + field: any, + ): EntryRefErrorReturnType[] { + log.debug(`Validating file/asset field: ${fieldStructure.display_name}`, this.config.auditContext); + + const values = Array.isArray(field) ? field : field ? [field] : []; + const missingRefs = values + .filter((ref: any) => this.isAssetBad(ref?.uid)) + .map((ref: any) => ({ asset_uid: ref?.uid, filename: ref?.filename })); + + if (isEmpty(missingRefs)) { + log.debug('File/asset field validation completed: no issues found', this.config.auditContext); + return []; + } + + return [ + { + tree, + missingRefs, + uid: this.currentUid, + name: this.currentTitle, + data_type: fieldStructure.data_type, + display_name: fieldStructure.display_name, + mandatory: fieldStructure.mandatory, + treeStr: tree + .map(({ name }) => name) + .filter((val) => val) + .join(' ➜ '), + } as unknown as EntryRefErrorReturnType, + ]; + } + /** * The function `validateReferenceValues` checks if the references in a given field exist in the * provided tree and returns any missing references. @@ -1125,6 +1218,14 @@ export default class Entries { entry[uid] as EntryGroupFieldDataType[], ) as EntryGroupFieldDataType; break; + case 'file': + log.debug(`Fixing file/asset field: ${uid}`); + this.fixFileFieldReferences( + [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], + field, + entry, + ); + break; case 'text': case 'number': if (field.hasOwnProperty('display_type')) { @@ -1574,6 +1675,65 @@ export default class Entries { return field; } + /** + * The function `fixFileFieldReferences` strips references to missing/quarantined/pending-scan + * assets from a `data_type: 'file'` field. A single-object value with a bad asset uid is deleted + * from the entry entirely; a `multiple: true` array value has its bad entries filtered out, and + * the key itself is deleted if the array becomes empty as a result. + */ + fixFileFieldReferences( + tree: Record[], + field: { uid: string; data_type: string; display_name: string; mandatory?: boolean }, + entry: Record, + ) { + log.debug(`Fixing file/asset field: ${field.display_name}`); + const { uid, display_name, data_type, mandatory } = field; + const value = entry[uid]; + + if (value == null) { + return entry; + } + + const missingRefs: Record[] = []; + + if (Array.isArray(value)) { + entry[uid] = value.filter((ref: any) => { + if (this.isAssetBad(ref?.uid)) { + missingRefs.push({ asset_uid: ref?.uid, filename: ref?.filename }); + return false; + } + return true; + }); + if (!entry[uid].length) { + delete entry[uid]; + } + } else if (this.isAssetBad(value?.uid)) { + missingRefs.push({ asset_uid: value?.uid, filename: value?.filename }); + delete entry[uid]; + } + + if (!isEmpty(missingRefs)) { + log.debug(`Recording asset reference fix for entry: ${this.currentUid}`); + this.missingAssetRefs[this.currentUid].push({ + tree, + data_type, + missingRefs, + display_name, + mandatory, + fixStatus: 'Fixed', + uid: this.currentUid, + name: this.currentTitle, + treeStr: tree + .map(({ name }) => name) + .filter((val) => val) + .join(' ➜ '), + }); + } + + log.debug(`File/asset fix completed for: ${field.display_name}`); + return entry; + } + /** * The function `fixGroupField` takes in a tree, a field, and an entry, and if the field has a * schema, it runs a fix on the schema and returns the updated entry, otherwise it returns the @@ -1991,4 +2151,45 @@ export default class Entries { log.debug(`Entry metadata preparation completed: ${this.entryMetaData.length} entries processed`, this.config.auditContext); log.debug(`Missing title fields found: ${Object.keys(this.missingTitleFields).length}`, this.config.auditContext); } + + /** + * Builds an index of asset uid -> { uid, filename, _asset_scan_status } from the exported + * assets/assets.json, mirroring the FsUtility chunk-read pattern used in the Assets module + * (src/modules/assets.ts). If the assets data isn't present in this export at all, + * `assetsDataAvailable` stays false and `isAssetBad()` never flags anything — absence of data + * must not be treated as every file-field reference being broken. + */ + async prepareAssetMetaData() { + log.debug('Starting asset metadata preparation', this.config.auditContext); + + const assetsBasePath = resolve( + sanitizePath(this.config.basePath), + sanitizePath(this.config.moduleConfig.assets.dirName), + ); + const assetsIndexPath = join(assetsBasePath, this.config.moduleConfig.assets.fileName); + + if (!existsSync(assetsIndexPath)) { + log.debug(`No assets data found at: ${assetsIndexPath}`, this.config.auditContext); + this.assetsDataAvailable = false; + return; + } + + this.assetsDataAvailable = true; + const fsUtility = new FsUtility({ basePath: assetsBasePath, indexFileName: 'assets.json' }); + const indexer = fsUtility.indexFileContent; + log.debug(`Found ${Object.keys(indexer).length} asset files to process`, this.config.auditContext); + + for (const _ in indexer) { + const assets = (await fsUtility.readChunkFiles.next()) as Record; + for (const assetUid in assets) { + this.assetMetaData[assetUid] = { + uid: assetUid, + filename: assets[assetUid]?.filename, + _asset_scan_status: assets[assetUid]?._asset_scan_status, + }; + } + } + + log.debug(`Asset metadata preparation completed: ${Object.keys(this.assetMetaData).length} assets processed`, this.config.auditContext); + } } diff --git a/packages/contentstack-audit/src/types/content-types.ts b/packages/contentstack-audit/src/types/content-types.ts index 2a27e92af..bd723c42e 100644 --- a/packages/contentstack-audit/src/types/content-types.ts +++ b/packages/contentstack-audit/src/types/content-types.ts @@ -162,6 +162,8 @@ enum OutputColumn { 'publish_locale' = 'publish_locale', 'publish_environment' = 'publish_environment', 'asset_uid' = 'asset_uid', + 'scan_status' = 'scan_status', + 'mandatory' = 'mandatory', 'selectedValue' = 'selectedValue', 'fixStatus' = 'fixStatus', 'Content_type_uid' = 'ct_uid', diff --git a/packages/contentstack-audit/test/unit/mock/contents/assets/assets.json b/packages/contentstack-audit/test/unit/mock/contents/assets/assets.json new file mode 100644 index 000000000..825a196ea --- /dev/null +++ b/packages/contentstack-audit/test/unit/mock/contents/assets/assets.json @@ -0,0 +1 @@ +{"1":"chunk1-assets.json"} diff --git a/packages/contentstack-audit/test/unit/mock/contents/assets/chunk1-assets.json b/packages/contentstack-audit/test/unit/mock/contents/assets/chunk1-assets.json new file mode 100644 index 000000000..e3ddd99ba --- /dev/null +++ b/packages/contentstack-audit/test/unit/mock/contents/assets/chunk1-assets.json @@ -0,0 +1,41 @@ +{ + "blt-clean-asset": { + "uid": "blt-clean-asset", + "filename": "clean.jpg", + "url": "https://images.contentstack.io/v3/assets/blt/blt-clean-asset/clean.jpg", + "_version": 1, + "_asset_scan_status": "clean", + "publish_details": [] + }, + "blt-pending-asset": { + "uid": "blt-pending-asset", + "filename": "pending.zip", + "url": "https://assets.contentstack.io/v3/assets/blt/blt-pending-asset/pending.zip", + "_version": 1, + "_asset_scan_status": "pending", + "publish_details": [] + }, + "blt-quarantined-asset": { + "uid": "blt-quarantined-asset", + "filename": "quarantined.zip", + "url": "https://assets.contentstack.io/v3/assets/blt/blt-quarantined-asset/quarantined.zip", + "_version": 1, + "_asset_scan_status": "quarantined", + "publish_details": [] + }, + "blt-no-status-asset": { + "uid": "blt-no-status-asset", + "filename": "no-status.png", + "url": "https://images.contentstack.io/v3/assets/blt/blt-no-status-asset/no-status.png", + "_version": 1, + "publish_details": [] + }, + "blt-not-scanned-asset": { + "uid": "blt-not-scanned-asset", + "filename": "not-scanned.png", + "url": "https://images.contentstack.io/v3/assets/blt/blt-not-scanned-asset/not-scanned.png", + "_version": 1, + "_asset_scan_status": "not_scanned", + "publish_details": [] + } +} diff --git a/packages/contentstack-audit/test/unit/modules/assets.test.ts b/packages/contentstack-audit/test/unit/modules/assets.test.ts new file mode 100644 index 000000000..82409db87 --- /dev/null +++ b/packages/contentstack-audit/test/unit/modules/assets.test.ts @@ -0,0 +1,76 @@ +import fs from 'fs'; +import { resolve } from 'path'; +import { expect } from 'chai'; +import fancy from 'fancy-test'; +import Sinon from 'sinon'; +import config from '../../../src/config'; +import { Assets } from '../../../src/modules'; +import { ModuleConstructorParam, CtConstructorParam } from '../../../src/types'; +import { mockLogger } from '../mock-logger'; + +describe('Assets module', () => { + let constructorParam: ModuleConstructorParam & CtConstructorParam; + + beforeEach(() => { + constructorParam = { + moduleName: 'assets', + ctSchema: [], + gfSchema: [], + config: Object.assign(config, { basePath: resolve(__dirname, '..', 'mock', 'contents'), flags: {} }), + }; + + Sinon.stub(require('@contentstack/cli-utilities'), 'log').value(mockLogger); + }); + + afterEach(() => { + Sinon.restore(); + }); + + describe('lookForReference method (scan status)', () => { + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('flags assets with a non-clean scan status and leaves clean/no-status assets alone', async () => { + const assetsInstance = new Assets(constructorParam); + await assetsInstance.prerequisiteData(); + await assetsInstance.lookForReference(); + + expect(Object.keys(assetsInstance.missingScanStatusAssets)).to.have.members([ + 'blt-pending-asset', + 'blt-quarantined-asset', + ]); + expect(assetsInstance.missingScanStatusAssets['blt-pending-asset'][0]).to.deep.include({ + asset_uid: 'blt-pending-asset', + scan_status: 'pending', + }); + expect(assetsInstance.missingScanStatusAssets['blt-quarantined-asset'][0]).to.deep.include({ + asset_uid: 'blt-quarantined-asset', + scan_status: 'quarantined', + }); + expect(assetsInstance.missingScanStatusAssets).to.not.have.property('blt-clean-asset'); + expect(assetsInstance.missingScanStatusAssets).to.not.have.property('blt-no-status-asset'); + expect(assetsInstance.missingScanStatusAssets).to.not.have.property('blt-not-scanned-asset'); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .stub(fs, 'writeFileSync', () => {}) + .it('removes non-clean assets from the written-back chunk when fix mode is on', async () => { + const writeFileSyncStub = Sinon.spy(fs, 'writeFileSync'); + const assetsInstance = new Assets({ + ...constructorParam, + fix: true, + config: { ...constructorParam.config, flags: { yes: true } }, + }); + await assetsInstance.prerequisiteData(); + await assetsInstance.lookForReference(); + + expect(writeFileSyncStub.called).to.be.true; + const writtenContent = JSON.parse(writeFileSyncStub.firstCall.args[1] as string); + expect(writtenContent).to.not.have.property('blt-pending-asset'); + expect(writtenContent).to.not.have.property('blt-quarantined-asset'); + expect(writtenContent).to.have.property('blt-clean-asset'); + expect(writtenContent).to.have.property('blt-no-status-asset'); + expect(writtenContent).to.have.property('blt-not-scanned-asset'); + }); + }); +}); diff --git a/packages/contentstack-audit/test/unit/modules/entries.test.ts b/packages/contentstack-audit/test/unit/modules/entries.test.ts index 911ba3316..4c486db58 100644 --- a/packages/contentstack-audit/test/unit/modules/entries.test.ts +++ b/packages/contentstack-audit/test/unit/modules/entries.test.ts @@ -1540,4 +1540,254 @@ describe('Entries module', () => { expect(callHelper('sys_assets', ['ct1'])).to.be.true; }); }); + + describe('prepareAssetMetaData method', () => { + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('builds an asset metadata index from assets/assets.json when it exists', async () => { + const ctInstance = new Entries(constructorParam); + await ctInstance.prepareAssetMetaData(); + + expect(ctInstance.assetsDataAvailable).to.be.true; + expect(ctInstance.assetMetaData['blt-clean-asset']).to.deep.include({ + uid: 'blt-clean-asset', + _asset_scan_status: 'clean', + }); + expect(ctInstance.assetMetaData['blt-pending-asset']).to.deep.include({ + uid: 'blt-pending-asset', + _asset_scan_status: 'pending', + }); + expect(ctInstance.assetMetaData['blt-not-scanned-asset']).to.deep.include({ + uid: 'blt-not-scanned-asset', + _asset_scan_status: 'not_scanned', + }); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('leaves assetsDataAvailable false when no assets data exists in this export', async () => { + const ctInstance = new Entries({ + ...constructorParam, + config: { ...constructorParam.config, basePath: resolve(__dirname, '..', 'mock', 'contents-1') }, + }); + await ctInstance.prepareAssetMetaData(); + + expect(ctInstance.assetsDataAvailable).to.be.false; + expect(ctInstance.assetMetaData).to.deep.equal({}); + }); + }); + + describe('isAssetBad method', () => { + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns false when assets data is unavailable', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = false; + (ctInstance as any).assetMetaData = {}; + expect(ctInstance.isAssetBad('blt-unknown')).to.be.false; + }); + + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns false when uid is falsy', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = true; + expect(ctInstance.isAssetBad(undefined)).to.be.false; + }); + + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns true when the asset uid is not found in the index', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = {}; + expect(ctInstance.isAssetBad('blt-unknown')).to.be.true; + }); + + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns true when the asset scan status is non-clean', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { 'blt-1': { uid: 'blt-1', _asset_scan_status: 'quarantined' } }; + expect(ctInstance.isAssetBad('blt-1')).to.be.true; + }); + + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns false when the asset is clean or has no scan status', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { + 'blt-clean': { uid: 'blt-clean', _asset_scan_status: 'clean' }, + 'blt-legacy': { uid: 'blt-legacy' }, + }; + expect(ctInstance.isAssetBad('blt-clean')).to.be.false; + expect(ctInstance.isAssetBad('blt-legacy')).to.be.false; + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('returns false when the asset scan status is not_scanned (org has asset scanning disabled)', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { + 'blt-not-scanned': { uid: 'blt-not-scanned', _asset_scan_status: 'not_scanned' }, + }; + expect(ctInstance.isAssetBad('blt-not-scanned')).to.be.false; + }); + }); + + describe('validateFileField method', () => { + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('flags a single file-field value that references a quarantined asset', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).currentTitle = 'Test Entry'; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { + 'blt-quarantined': { uid: 'blt-quarantined', _asset_scan_status: 'quarantined' }, + }; + + const fieldStructure = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const value = { uid: 'blt-quarantined', filename: 'bad.zip' }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.validateFileField(tree, fieldStructure, value); + + expect(result).to.have.length(1); + expect(result[0].missingRefs).to.deep.equal([{ asset_uid: 'blt-quarantined', filename: 'bad.zip' }]); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('does not flag a single file-field value that references a clean asset', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { 'blt-clean': { uid: 'blt-clean', _asset_scan_status: 'clean' } }; + + const fieldStructure = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const value = { uid: 'blt-clean', filename: 'ok.jpg' }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.validateFileField(tree, fieldStructure, value); + + expect(result).to.have.length(0); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('flags only the bad entries within a multiple:true file-field array', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { 'blt-clean': { uid: 'blt-clean', _asset_scan_status: 'clean' } }; + + const fieldStructure = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const value = [ + { uid: 'blt-clean', filename: 'ok.jpg' }, + { uid: 'blt-pending', filename: 'bad.zip' }, + ]; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.validateFileField(tree, fieldStructure, value); + + expect(result).to.have.length(1); + expect(result[0].missingRefs).to.deep.equal([{ asset_uid: 'blt-pending', filename: 'bad.zip' }]); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('does not flag anything when asset metadata is unavailable', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).assetsDataAvailable = false; + + const fieldStructure = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const value = { uid: 'blt-unknown', filename: 'unknown.jpg' }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.validateFileField(tree, fieldStructure, value); + + expect(result).to.have.length(0); + }); + }); + + describe('fixFileFieldReferences method', () => { + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('deletes the field key when a single file-field value references a bad asset', () => { + const ctInstance = new Entries({ ...constructorParam, fix: true }); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).currentTitle = 'Test Entry'; + (ctInstance as any).missingAssetRefs = { 'test-entry': [] }; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { + 'blt-quarantined': { uid: 'blt-quarantined', _asset_scan_status: 'quarantined' }, + }; + + const field = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const entry: Record = { file_field: { uid: 'blt-quarantined', filename: 'bad.zip' } }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.fixFileFieldReferences(tree, field, entry); + + expect(result).to.not.have.property('file_field'); + expect((ctInstance as any).missingAssetRefs['test-entry']).to.have.length(1); + expect((ctInstance as any).missingAssetRefs['test-entry'][0].fixStatus).to.equal('Fixed'); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('leaves a single file-field value referencing a clean asset untouched', () => { + const ctInstance = new Entries({ ...constructorParam, fix: true }); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).missingAssetRefs = { 'test-entry': [] }; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { 'blt-clean': { uid: 'blt-clean', _asset_scan_status: 'clean' } }; + + const field = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const entry: Record = { file_field: { uid: 'blt-clean', filename: 'ok.jpg' } }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + ctInstance.fixFileFieldReferences(tree, field, entry); + + expect(entry).to.have.property('file_field'); + expect((ctInstance as any).missingAssetRefs['test-entry']).to.have.length(0); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('filters bad entries out of a multiple:true array and deletes the key if it empties out', () => { + const ctInstance = new Entries({ ...constructorParam, fix: true }); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).missingAssetRefs = { 'test-entry': [] }; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = {}; + + const field = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const entry: Record = { + file_field: [ + { uid: 'blt-1', filename: 'one.zip' }, + { uid: 'blt-2', filename: 'two.zip' }, + ], + }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.fixFileFieldReferences(tree, field, entry); + + expect(result).to.not.have.property('file_field'); + expect((ctInstance as any).missingAssetRefs['test-entry'][0].missingRefs).to.have.length(2); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('records mandatory:true on the fix issue so a stripped required field stays visible in the report', () => { + const ctInstance = new Entries({ ...constructorParam, fix: true }); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).missingAssetRefs = { 'test-entry': [] }; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = {}; + + const field = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: true }; + const entry: Record = { file_field: { uid: 'blt-missing', filename: 'gone.zip' } }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + ctInstance.fixFileFieldReferences(tree, field, entry); + + expect((ctInstance as any).missingAssetRefs['test-entry'][0].mandatory).to.be.true; + }); + }); }); diff --git a/packages/contentstack-bulk-publish/.mocharc.json b/packages/contentstack-bulk-publish/.mocharc.json index 50e4a6804..3d76d7307 100644 --- a/packages/contentstack-bulk-publish/.mocharc.json +++ b/packages/contentstack-bulk-publish/.mocharc.json @@ -5,7 +5,8 @@ "test/unit/commands/assets/unpublish.test.js", "test/unit/commands/bulk-publish/cross-publish.test.js", "test/unit/commands/entries/publish.test.js", - "test/unit/commands/entries/unpublish.test.js" + "test/unit/commands/entries/unpublish.test.js", + "test/unit/util/asset-scan.test.js" ], "reporter": "dot", "timeout": 60000, diff --git a/packages/contentstack-bulk-publish/src/commands/cm/assets/publish.js b/packages/contentstack-bulk-publish/src/commands/cm/assets/publish.js index a032678c0..b121906c9 100644 --- a/packages/contentstack-bulk-publish/src/commands/cm/assets/publish.js +++ b/packages/contentstack-bulk-publish/src/commands/cm/assets/publish.js @@ -15,10 +15,12 @@ class AssetsPublishCommand extends Command { assetsFlags.folderUid = assetsFlags['folder-uid'] || assetsFlags.folderUid; assetsFlags.bulkPublish = assetsFlags['bulk-publish'] || assetsFlags.bulkPublish; assetsFlags.apiVersion = assetsFlags['api-version'] || '3'; // setting default value for apiVersion + assetsFlags.backupDir = assetsFlags['backup-dir'] || assetsFlags.backupDir; delete assetsFlags['api-version']; delete assetsFlags['retry-failed']; delete assetsFlags['folder-uid']; delete assetsFlags['bulk-publish']; + delete assetsFlags['backup-dir']; let updatedFlags; try { @@ -111,12 +113,18 @@ class AssetsPublishCommand extends Command { } } - validate({ environments, retryFailed, locales, 'source-env': sourceEnv, 'delivery-token': deliveryToken }) { + validate({ environments, retryFailed, locales, backupDir, 'source-env': sourceEnv, 'delivery-token': deliveryToken }) { let missing = []; if (retryFailed) { return true; } + // In backup-dir mode, environments and locales are derived per-asset from the + // backup publish_details, so they are not required on the command line. + if (backupDir) { + return true; + } + if (sourceEnv && !deliveryToken) { this.error('Specify the source environment delivery token. Run --help for more details.', { exit: 2 }); } @@ -181,6 +189,11 @@ AssetsPublishCommand.flags = { '(optional) The UID of the Assets’ folder from which the assets need to be published. The default value is cs_root.', exclusive: ['source-env'], }), + 'backup-dir': flags.string({ + description: + '(optional) Path to the import backup directory. When set, each imported asset is published only to the environments and locales it was published to in the source stack (read from the backup’s publish details and asset UID mapping), with asset-scan gating applied. Intended for the post-import publish flow.', + exclusive: ['source-env', 'folder-uid', 'environments', 'locales'], + }), 'bulk-publish': flags.string({ description: 'Set this flag to use Contentstack’s Bulk Publish APIs. It is true, by default.', default: 'true', @@ -259,11 +272,14 @@ AssetsPublishCommand.examples = [ '', 'Using --stack-api-key flag', 'csdx cm:assets:publish --environments [ENVIRONMENT 1] [ENVIRONMENT 2] --locales [LOCALE] --stack-api-key [STACK API KEY]', + '', + 'Using --backup-dir flag (publish imported assets to their original environments after asset scanning)', + 'csdx cm:assets:publish --backup-dir [PATH TO IMPORT BACKUP DIR] --stack-api-key [STACK API KEY]', ]; AssetsPublishCommand.aliases = ['cm:bulk-publish:assets']; AssetsPublishCommand.usage = - 'cm:assets:publish [-a ] [--retry-failed ] [-e ] [--folder-uid ] [--bulk-publish ] [-c ] [-y] [--locales ] [--branch ] [--delivery-token ] [--source-env ]'; + 'cm:assets:publish [-a ] [--retry-failed ] [-e ] [--folder-uid ] [--backup-dir ] [--bulk-publish ] [-c ] [-y] [--locales ] [--branch ] [--delivery-token ] [--source-env ]'; module.exports = AssetsPublishCommand; diff --git a/packages/contentstack-bulk-publish/src/consumer/publish.js b/packages/contentstack-bulk-publish/src/consumer/publish.js index f466b250c..cea6392fd 100644 --- a/packages/contentstack-bulk-publish/src/consumer/publish.js +++ b/packages/contentstack-bulk-publish/src/consumer/publish.js @@ -61,12 +61,12 @@ function displayEntriesDetails(sanitizedData, action, mapping = []) { function displayAssetsDetails(sanitizedData, action, mapping) { if (action === 'bulk_publish') { sanitizedData.forEach((asset) => { - asset?.publish_details.forEach((pd) => { + asset?.publish_details?.forEach((pd) => { if (Object.keys(mapping).includes(pd.environment)) { console.log( chalk.green( `Asset UID: '${asset.uid}'${pd.version ? `, Version: '${pd.version}'` : ''}${ - asset.locale ? `, Locale: '${asset.locale}'` : '' + asset.locale ? `, Locale: '${asset.locale}'` : ''} }, Environment: ${pd.environment}`, ), ); diff --git a/packages/contentstack-bulk-publish/src/producer/publish-assets.js b/packages/contentstack-bulk-publish/src/producer/publish-assets.js index a9afbdd7c..f35f2d751 100644 --- a/packages/contentstack-bulk-publish/src/producer/publish-assets.js +++ b/packages/contentstack-bulk-publish/src/producer/publish-assets.js @@ -1,7 +1,10 @@ /* eslint-disable no-console */ /* eslint-disable new-cap */ /* eslint-disable camelcase */ -const { cliux } = require('@contentstack/cli-utilities'); +const path = require('path'); +const { existsSync } = require('fs'); +const chalk = require('chalk'); +const { cliux, FsUtility } = require('@contentstack/cli-utilities'); const { getQueue } = require('../util/queue'); const { performBulkPublish, publishAsset, initializeLogger } = require('../consumer/publish'); const retryFailedLogs = require('../util/retryfailed'); @@ -9,14 +12,27 @@ const { validateFile } = require('../util/fs'); const { isEmpty } = require('../util'); const { fetchBulkPublishLimit } = require('../util/common-utility'); const { generateBulkPublishStatusUrl } = require('../util/generate-bulk-publish-url'); +const { resolveInQueueAssets, fetchScanStatusBatch, ASSET_SCAN_STATUS } = require('../util/asset-scan'); const queue = getQueue(); let logFileName; let bulkPublishSet = []; +let pendingAssetsForRetry = []; +let scanSummary = { clean: 0, quarantined: 0, inQueue: 0, noStatus: 0 }; let filePath; /* eslint-disable no-param-reassign */ +function printScanSummary({ clean, noStatus, inQueue, quarantined }) { + const total = clean + noStatus + inQueue + quarantined; + if (total === 0) return; + console.log(chalk.bold(`\nAsset scan summary (${total} total):`)); + console.log(chalk.green(` ✓ Clean (publishing): ${clean}`)); + if (noStatus > 0) console.log(chalk.green(` ✓ No scan status (publishing): ${noStatus}`)); + if (inQueue > 0) console.log(chalk.yellow(` ⧖ In queue (retrying): ${inQueue}`)); + if (quarantined > 0) console.log(chalk.red(` ✗ Quarantined (skipped): ${quarantined}`)); +} + async function getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, bulkPublishLimit, skip = 0) { return new Promise((resolve, reject) => { let queryParams = { @@ -25,6 +41,7 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe include_count: true, include_folders: true, include_publish_details: true, + include_asset_scan_status: true, }; stack .asset() @@ -34,7 +51,8 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe if (assetResponse && assetResponse.items.length > 0) { skip += assetResponse.items.length; let assets = assetResponse.items; - for (let index = 0; index < assetResponse.items.length; index++) { + + for (let index = 0; index < assets.length; index++) { if (assets[index].is_dir === true) { await getAssets( stack, @@ -48,6 +66,35 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe ); continue; } + + const scanStatus = assets[index]._asset_scan_status; + + // Quarantined assets are skipped permanently + if (scanStatus === ASSET_SCAN_STATUS.QUARANTINE) { + scanSummary.quarantined++; + console.log(chalk.yellow(`Skipped (quarantined): Asset UID '${assets[index].uid}'`)); + continue; + } + + // In-queue assets are deferred for retry after all pages are processed + if (scanStatus === ASSET_SCAN_STATUS.IN_QUEUE) { + scanSummary.inQueue++; + pendingAssetsForRetry.push({ + uid: assets[index].uid, + locale, + publish_details: assets[index].publish_details || [], + environments, + }); + continue; + } + + // Ready (clean) or no scan status — enqueue for publish + if (scanStatus === ASSET_SCAN_STATUS.READY) { + scanSummary.clean++; + } else { + scanSummary.noStatus++; + } + if (bulkPublish) { if (bulkPublishSet.length < bulkPublishLimit) { bulkPublishSet.push({ @@ -67,22 +114,6 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe }); bulkPublishSet = []; } - - if ( - assetResponse.items.length - 1 === index && - bulkPublishSet.length > 0 && - bulkPublishSet.length < bulkPublishLimit - ) { - await queue.Enqueue({ - assets: bulkPublishSet, - Type: 'asset', - environments: environments, - locale, - stack: stack, - apiVersion, - }); - bulkPublishSet = []; - } } else { await queue.Enqueue({ assetUid: assets[index].uid, @@ -94,6 +125,23 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe }); } } + + // Flush any partial bulk batch at the end of the page. + // Done outside the for-loop so it fires correctly even when some assets + // were skipped (quarantined/in-queue) and the last non-skipped asset is + // not at the final array index. + if (bulkPublish && bulkPublishSet.length > 0) { + await queue.Enqueue({ + assets: bulkPublishSet, + Type: 'asset', + environments: environments, + locale, + stack: stack, + apiVersion, + }); + bulkPublishSet = []; + } + if (skip === assetResponse.count) { return resolve(true); } @@ -109,6 +157,291 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe }); } +/** + * After all pages/locales are scanned, retry any assets that were in-queue. + * Takes pendingItems explicitly — does not read from module-level state. + * Uses incremental backoff (see asset-scan.js SCAN_RETRY config). + */ +async function processPendingAssets(pendingItems, stack, bulkPublish, environments, apiVersion, bulkPublishLimit) { + if (pendingItems.length === 0) return; + + // Deduplicate UIDs across locales — scan status is per-asset, not per-locale. + // Resolving once avoids redundant retry loops for multi-locale runs. + const allUids = [...new Set(pendingItems.map((a) => a.uid))]; + const resolvedUids = await resolveInQueueAssets(stack, allUids); + + if (resolvedUids.length === 0) { + console.log(chalk.yellow('No in-queue assets resolved after retries.')); + return; + } + + const resolvedSet = new Set(resolvedUids); + + // Group resolved items by locale for correct enqueue context + const byLocale = {}; + for (const item of pendingItems) { + if (!resolvedSet.has(item.uid)) continue; + if (!byLocale[item.locale]) byLocale[item.locale] = []; + byLocale[item.locale].push(item); + } + + for (const locale of Object.keys(byLocale)) { + const resolvedItems = byLocale[locale]; + + if (bulkPublish) { + let batchSet = []; + for (const item of resolvedItems) { + batchSet.push({ uid: item.uid, locale, publish_details: item.publish_details }); + if (batchSet.length === bulkPublishLimit) { + await queue.Enqueue({ + assets: batchSet, + Type: 'asset', + environments, + locale, + stack, + apiVersion, + }); + batchSet = []; + } + } + if (batchSet.length > 0) { + await queue.Enqueue({ + assets: batchSet, + Type: 'asset', + environments, + locale, + stack, + apiVersion, + }); + } + } else { + for (const item of resolvedItems) { + await queue.Enqueue({ + assetUid: item.uid, + publish_details: item.publish_details, + environments, + Type: 'asset', + locale, + stack, + }); + } + } + } +} + +/** + * Publish assets from an import backup directory (post-import flow). + * + * Unlike getAssets (live folder scan), this drives publishing from the backup: + * each imported asset is published ONLY to the environments/locales it was + * published to in the source stack (from its publish_details), remapped to the + * target stack. Scan-status gating is applied to the target asset UIDs. + * + * Mirrors the publish_details/env-name resolution of contentstack-import's + * assets `publish()` (the bulk publish API resolves environment NAMES against + * the target stack, and import preserves env names, so source name == target + * name), and adds the clean/quarantined/in-queue scan gating that import skips. + * + * Source of truth split: + * - publish_details + environments come from the BACKUP (post-import flow): an + * asset's target environments are its source publish_details, gated by the + * environment uid-mapping (only environments actually imported into the target + * are publishable) — avoids doomed publish calls to envs never created there. + * - scan status comes from the LIVE target API (it is a runtime property of the + * freshly-imported assets and cannot exist in the backup). + * + * Streaming: asset chunks are processed and released one at a time and scan-gated + * per chunk, so memory does not scale with total asset count. The only structures + * retained across chunks are bounded (partial publish batches + the in-queue + * subset). The single in-memory floor is the asset uid-mapping file itself (same + * as import's publish()); for very large stacks raise Node's --max-old-space-size. + */ +async function getAssetsFromBackup(stack, backupDir, bulkPublish, apiVersion, bulkPublishLimit) { + const assetsPath = path.join(backupDir, 'assets'); + const assetsIndexPath = path.join(assetsPath, 'assets.json'); + const assetUidMapperPath = path.join(backupDir, 'mapper', 'assets', 'uid-mapping.json'); + const envUidMapperPath = path.join(backupDir, 'mapper', 'environments', 'uid-mapping.json'); + const environmentsPath = path.join(backupDir, 'environments', 'environments.json'); + + // A backup with no assets is a legitimate outcome of a successful import that + // had 0 assets (assets.json is never produced) — exit cleanly, not as a crash. + // A genuinely wrong --backup-dir surfaces via the uid-mapping/environments guards below. + if (!existsSync(assetsPath) || !existsSync(assetsIndexPath)) { + console.log(chalk.yellow('No assets found in backup — nothing to publish.')); + return; + } + if (!existsSync(assetUidMapperPath)) { + throw new Error( + `Asset UID mapping not found at '${assetUidMapperPath}'. Run import against this data dir before publishing.`, + ); + } + if (!existsSync(environmentsPath)) { + throw new Error(`Environments not found at '${environmentsPath}'. Cannot resolve target environments.`); + } + + const fsUtil = new FsUtility({ basePath: assetsPath, indexFileName: 'assets.json' }); + const assetUidMap = fsUtil.readFile(assetUidMapperPath, true) || {}; + // environments.json: { [sourceEnvUid]: { name, ... } } — source env definitions. + const environments = fsUtil.readFile(environmentsPath, true) || {}; + // uid-mapping.json: { [sourceEnvUid]: targetEnvUid } — only environments actually + // imported into the target. Used as the "is publishable" gate. Optional: older + // backups (or runs with no imported environments) may not have it. + const envUidMapping = existsSync(envUidMapperPath) ? fsUtil.readFile(envUidMapperPath, true) || {} : null; + if (!envUidMapping) { + console.log( + chalk.yellow( + `Environment UID mapping not found at '${envUidMapperPath}'. Falling back to environment names from ` + + `environments.json — ensure the target stack has environments with matching names.`, + ), + ); + } + const isEnvImported = (sourceEnvUid) => + !envUidMapping || Object.prototype.hasOwnProperty.call(envUidMapping, sourceEnvUid); + + // Resolve an asset's deduped, env-gated target (envName, locale) pairs from its + // publish_details. Env name comes from the backup; only environments actually + // imported into the target (per the env uid-mapping) are publishable. + const resolvePairs = (asset) => { + const seen = new Set(); + const pairs = []; + for (const pd of asset.publish_details) { + const env = environments[pd.environment]; + if (!env || !env.name) continue; // env not in the data dir — cannot resolve a name + if (!isEnvImported(pd.environment)) continue; // env not imported into target — skip + const key = `${env.name}||${pd.locale}`; + if (seen.has(key)) continue; + seen.add(key); + pairs.push({ envName: env.name, locale: pd.locale }); + } + return pairs; + }; + + // Bounded cross-chunk state only — nothing scales with total asset count: + // - `buffers`: partial publish batches, capped at envCount x localeCount x bulkPublishLimit. + // - `pending`: the in-queue (scanning) subset awaiting retry. + // The full asset universe is never held in memory; chunks are processed and + // released one at a time (same streaming shape as contentstack-import's publish()). + const buffers = new Map(); // "envName||locale" -> { envName, locale, uids: [] } + const pending = []; // { targetUid, pairs } for assets whose scan is still in queue + let skippedNoUidMapping = 0; // source asset was not imported (no asset uid mapping) + let skippedNoMappableEnv = 0; // asset has publish details, but none of its envs were imported + let publishableAssets = 0; // assets enqueued for publish (across all env/locale pairs) + + const enqueueBatch = async (envName, locale, uids) => { + if (uids.length === 0) return; + if (bulkPublish) { + const assets = uids.map((uid) => ({ uid, locale })); + await queue.Enqueue({ assets, Type: 'asset', environments: [envName], locale, stack, apiVersion }); + } else { + for (const uid of uids) { + await queue.Enqueue({ assetUid: uid, environments: [envName], Type: 'asset', locale, stack }); + } + } + }; + + // Add a publishable asset to its (env, locale) buffers, flushing any that fill up. + // Grouping by the exact pair keeps the bulk API from publishing an asset to a + // combo it was not published to in source. + const bufferAsset = async (targetUid, pairs) => { + publishableAssets++; + for (const { envName, locale } of pairs) { + const key = `${envName}||${locale}`; + let buf = buffers.get(key); + if (!buf) { + buf = { envName, locale, uids: [] }; + buffers.set(key, buf); + } + buf.uids.push(targetUid); + if (buf.uids.length >= bulkPublishLimit) { + await enqueueBatch(envName, locale, buf.uids); + buf.uids = []; + } + } + }; + + const indexer = fsUtil.indexFileContent; + + // NOTE: one readChunkFiles.next() call per index entry — the iteration count must + // equal the number of chunk files (same contract as contentstack-import's publish()). + for (const _index in indexer) { + const chunk = await fsUtil.readChunkFiles.next(); + const assetsArr = Object.values(chunk || {}); + + // Resolve this chunk's assets to publish targets (bounded by chunk size). + const resolved = []; + for (const asset of assetsArr) { + if (!asset || !Array.isArray(asset.publish_details) || asset.publish_details.length === 0) { + continue; + } + const targetUid = assetUidMap[asset.uid]; + if (!targetUid) { + skippedNoUidMapping++; + continue; + } + const pairs = resolvePairs(asset); + if (pairs.length === 0) { + skippedNoMappableEnv++; + continue; + } + resolved.push({ targetUid, pairs }); + } + if (resolved.length === 0) continue; + + // Scan status is a target-stack property of the freshly-imported assets, so it + // is fetched live (one batched read per chunk) — it is not in the backup. + const statusMap = await fetchScanStatusBatch( + stack, + resolved.map((r) => r.targetUid), + ); + + for (const { targetUid, pairs } of resolved) { + const status = statusMap.get(targetUid); + if (status === ASSET_SCAN_STATUS.QUARANTINE) { + scanSummary.quarantined++; + console.log(chalk.yellow(`Skipped (quarantined): Asset UID '${targetUid}'`)); + } else if (status === ASSET_SCAN_STATUS.IN_QUEUE) { + scanSummary.inQueue++; + pending.push({ targetUid, pairs }); + } else { + if (status === ASSET_SCAN_STATUS.READY) scanSummary.clean++; + else scanSummary.noStatus++; + await bufferAsset(targetUid, pairs); + } + } + } + + // Resolve in-queue assets once (incremental backoff); publish those that turn clean. + if (pending.length > 0) { + const resolvedUids = await resolveInQueueAssets( + stack, + pending.map((p) => p.targetUid), + ); + const resolvedSet = new Set(resolvedUids); + for (const { targetUid, pairs } of pending) { + if (resolvedSet.has(targetUid)) await bufferAsset(targetUid, pairs); + } + } + + // Flush remaining partial (env, locale) batches. + for (const { envName, locale, uids } of buffers.values()) { + await enqueueBatch(envName, locale, uids); + } + + if (skippedNoUidMapping > 0) { + console.log(chalk.yellow(`Skipped ${skippedNoUidMapping} asset(s): no UID mapping (not imported into target).`)); + } + if (skippedNoMappableEnv > 0) { + console.log( + chalk.yellow( + `Skipped ${skippedNoMappableEnv} asset(s): none of their published environments were imported into the target.`, + ), + ); + } + if (publishableAssets === 0) { + console.log(chalk.yellow('No publishable assets found in backup (no mapped assets with publishable environments).')); + } +} + function setConfig(conf, bp) { if (bp) { queue.consumer = performBulkPublish; @@ -120,10 +453,15 @@ function setConfig(conf, bp) { config = conf; queue.config = conf; filePath = initializeLogger(logFileName); + pendingAssetsForRetry = []; + scanSummary = { clean: 0, quarantined: 0, inQueue: 0, noStatus: 0 }; } -async function start({ retryFailed, bulkPublish, environments, folderUid, locales, apiVersion }, stack, config) { +async function start({ retryFailed, bulkPublish, environments, folderUid, locales, apiVersion, backupDir }, stack, config) { process.on('beforeExit', async () => { + // Print the scan summary here (not inline after enqueueing): + printScanSummary(scanSummary); + const isErrorLogEmpty = await isEmpty(`${filePath}.error`); const isSuccessLogEmpty = await isEmpty(`${filePath}.success`); if (!isErrorLogEmpty) { @@ -131,7 +469,7 @@ async function start({ retryFailed, bulkPublish, environments, folderUid, locale } else if (!isSuccessLogEmpty) { console.log(`The success log for this session is stored at ${filePath}.success`); } - + // Generate and display the bulk publish status link if (bulkPublish && stack && config) { const statusUrl = generateBulkPublishStatusUrl(stack, config); @@ -142,11 +480,12 @@ async function start({ retryFailed, bulkPublish, environments, folderUid, locale process.stdout.write('\n'); } } - + process.exit(0); }); if (retryFailed) { + console.log(chalk.yellow('Note: --retry-failed replays from log and skips asset scan status checks.')); if (!validateFile(retryFailed, ['publish-assets', 'bulk-publish-assets'])) { return false; } @@ -159,17 +498,31 @@ async function start({ retryFailed, bulkPublish, environments, folderUid, locale } else { await retryFailedLogs(retryFailed, { assetQueue: queue }, 'publish'); } + } else if (backupDir) { + // Post-import flow: publish each imported asset only to its original + // environments/locales (from backup publish_details), scan-gated. + setConfig(config, bulkPublish); + const bulkPublishLimit = fetchBulkPublishLimit(stack?.org_uid); + await getAssetsFromBackup(stack, backupDir, bulkPublish, apiVersion, bulkPublishLimit); } else if (folderUid) { setConfig(config, bulkPublish); const bulkPublishLimit = fetchBulkPublishLimit(stack?.org_uid); for (const locale of locales) { await getAssets(stack, folderUid, bulkPublish, environments, locale, apiVersion, bulkPublishLimit); } + + // Resolve in-queue assets with incremental retry; pass pendingAssetsForRetry explicitly + if (pendingAssetsForRetry.length > 0) { + await processPendingAssets(pendingAssetsForRetry, stack, bulkPublish, environments, apiVersion, bulkPublishLimit); + pendingAssetsForRetry = []; + } } } module.exports = { getAssets, + getAssetsFromBackup, setConfig, start, + processPendingAssets, }; diff --git a/packages/contentstack-bulk-publish/src/util/asset-scan.js b/packages/contentstack-bulk-publish/src/util/asset-scan.js new file mode 100644 index 000000000..27c3ffe77 --- /dev/null +++ b/packages/contentstack-bulk-publish/src/util/asset-scan.js @@ -0,0 +1,121 @@ +/* eslint-disable no-console */ +const chalk = require('chalk'); + +const ASSET_SCAN_STATUS = { + READY: 'clean', + QUARANTINE: 'quarantined', + IN_QUEUE: 'pending', +}; + +const SCAN_RETRY = { + MAX_RETRIES: 0, + INITIAL_WAIT_MS: 5000, + BACKOFF_FACTOR: 2, +}; + +function getIncrementalWaitMs(attempt) { + return SCAN_RETRY.INITIAL_WAIT_MS * Math.pow(SCAN_RETRY.BACKOFF_FACTOR, attempt); +} + +/** + * Batch-fetch asset scan statuses for a list of UIDs. + * Returns a Map. UIDs with no scan data map to undefined. + * Throws on API error — callers must not silently treat failures as "ready". + */ +async function fetchScanStatusBatch(stack, uids) { + const statusMap = new Map(); + if (!uids || uids.length === 0) return statusMap; + + const BATCH_SIZE = 100; + for (let i = 0; i < uids.length; i += BATCH_SIZE) { + const batch = uids.slice(i, i + BATCH_SIZE); + const response = await stack + .asset() + .query({ uid: { $in: batch }, include_asset_scan_status: true, limit: BATCH_SIZE }) + .find(); + for (const asset of response.items || []) { + statusMap.set(asset.uid, asset._asset_scan_status); + } + } + + return statusMap; +} + +/** + * Retry pending (in-queue) assets with incremental backoff until they become + * clean or max retries is reached. + * + * Wait series: 5s, 10s, 20s, 40s, 80s (5 attempts total, max 155s). + * + * @param {object} stack - Management SDK stack instance + * @param {string[]} pendingUids - UIDs currently in scan queue + * @returns {string[]} UIDs that became clean and are safe to publish + */ +async function resolveInQueueAssets(stack, pendingUids) { + if (!pendingUids || pendingUids.length === 0) return []; + + const totalWaitSec = + Array.from({ length: SCAN_RETRY.MAX_RETRIES }, (_, i) => getIncrementalWaitMs(i)).reduce( + (a, b) => a + b, + 0, + ) / 1000; + console.log( + chalk.yellow( + `Resolving ${pendingUids.length} in-queue asset(s). Max wait: ${totalWaitSec}s over ${SCAN_RETRY.MAX_RETRIES} retries.`, + ), + ); + + let remaining = [...pendingUids]; + const resolvedUids = []; + + for (let attempt = 0; attempt < SCAN_RETRY.MAX_RETRIES && remaining.length > 0; attempt++) { + const waitMs = getIncrementalWaitMs(attempt); + console.log( + chalk.yellow( + `Asset scan: ${remaining.length} asset(s) in queue. Waiting ${waitMs / 1000}s before retry ${attempt + 1}/${ + SCAN_RETRY.MAX_RETRIES + }...`, + ), + ); + + await new Promise((resolve) => setTimeout(resolve, waitMs)); + + const statusMap = await fetchScanStatusBatch(stack, remaining); + const stillPending = []; + + for (const uid of remaining) { + const status = statusMap.get(uid); + if (status === ASSET_SCAN_STATUS.QUARANTINE) { + console.log(chalk.red(`Skipped (quarantined after retry): Asset UID '${uid}'`)); + } else if (status === ASSET_SCAN_STATUS.IN_QUEUE) { + stillPending.push(uid); + } else { + // clean or undefined (scanning disabled) — publishable + resolvedUids.push(uid); + } + } + + remaining = stillPending; + } + + if (remaining.length > 0) { + console.warn( + chalk.red( + `Asset scan: ${remaining.length} asset(s) remained in queue after ${SCAN_RETRY.MAX_RETRIES} retries and will be skipped.`, + ), + ); + for (const uid of remaining) { + console.warn(chalk.red(`Skipped (max retries exceeded): Asset UID '${uid}'`)); + } + } + + return resolvedUids; +} + +module.exports = { + ASSET_SCAN_STATUS, + SCAN_RETRY, + getIncrementalWaitMs, + fetchScanStatusBatch, + resolveInQueueAssets, +}; diff --git a/packages/contentstack-bulk-publish/test/unit/util/asset-scan.test.js b/packages/contentstack-bulk-publish/test/unit/util/asset-scan.test.js new file mode 100644 index 000000000..83b430bde --- /dev/null +++ b/packages/contentstack-bulk-publish/test/unit/util/asset-scan.test.js @@ -0,0 +1,162 @@ +'use strict'; + +const { describe, it, beforeEach, afterEach } = require('mocha'); +const { expect } = require('chai'); + +const { + ASSET_SCAN_STATUS, + SCAN_RETRY, + getIncrementalWaitMs, + fetchScanStatusBatch, + resolveInQueueAssets, +} = require('../../../src/util/asset-scan'); + +// Minimal mock stack factory +function makeStack(items) { + return { + asset() { + return { + query() { + return { + find: async () => ({ items: items || [] }), + }; + }, + }; + }, + }; +} + +// Stack that throws on query +function makeErrorStack(errorMsg) { + return { + asset() { + return { + query() { + return { + find: async () => { + throw new Error(errorMsg); + }, + }; + }, + }; + }, + }; +} + +describe('asset-scan utilities', () => { + // ─── getIncrementalWaitMs ──────────────────────────────────────────────── + + describe('getIncrementalWaitMs', () => { + it('returns INITIAL_WAIT_MS for attempt 0', () => { + expect(getIncrementalWaitMs(0)).to.equal(SCAN_RETRY.INITIAL_WAIT_MS); + }); + + it('doubles on each subsequent attempt', () => { + const seq = [0, 1, 2, 3, 4].map(getIncrementalWaitMs); + for (let i = 1; i < seq.length; i++) { + expect(seq[i]).to.equal(seq[i - 1] * SCAN_RETRY.BACKOFF_FACTOR); + } + }); + + it('produces the correct 5-attempt sequence', () => { + const expected = [5000, 10000, 20000, 40000, 80000]; + expected.forEach((ms, attempt) => { + expect(getIncrementalWaitMs(attempt)).to.equal(ms); + }); + }); + }); + + // ─── fetchScanStatusBatch ─────────────────────────────────────────────── + + describe('fetchScanStatusBatch', () => { + it('returns empty Map when called with empty uid array', async () => { + const map = await fetchScanStatusBatch(makeStack([]), []); + expect(map.size).to.equal(0); + }); + + it('maps UIDs to their scan statuses', async () => { + const items = [ + { uid: 'a1', _asset_scan_status: 'clean' }, + { uid: 'a2', _asset_scan_status: 'quarantined' }, + { uid: 'a3', _asset_scan_status: 'pending' }, + ]; + const map = await fetchScanStatusBatch(makeStack(items), ['a1', 'a2', 'a3']); + expect(map.get('a1')).to.equal(ASSET_SCAN_STATUS.READY); + expect(map.get('a2')).to.equal(ASSET_SCAN_STATUS.QUARANTINE); + expect(map.get('a3')).to.equal(ASSET_SCAN_STATUS.IN_QUEUE); + }); + + it('maps UIDs with no scan field to undefined', async () => { + const items = [{ uid: 'a1' }]; + const map = await fetchScanStatusBatch(makeStack(items), ['a1']); + expect(map.get('a1')).to.equal(undefined); + }); + + it('throws on API error (fail fast — do not silently treat as ready)', async () => { + try { + await fetchScanStatusBatch(makeErrorStack('Network error'), ['a1']); + expect.fail('Expected fetchScanStatusBatch to throw'); + } catch (error) { + expect(error.message).to.equal('Network error'); + } + }); + }); + + // ─── resolveInQueueAssets ─────────────────────────────────────────────── + + describe('resolveInQueueAssets', () => { + let originalSetTimeout; + + beforeEach(() => { + // Replace setTimeout with an immediate resolver to avoid real waits + originalSetTimeout = global.setTimeout; + global.setTimeout = (fn) => fn(); + }); + + afterEach(() => { + global.setTimeout = originalSetTimeout; + }); + + it('returns empty array for empty input without calling stack', async () => { + const result = await resolveInQueueAssets(makeStack([]), []); + expect(result).to.deep.equal([]); + }); + + it('resolves UIDs that become clean on the first retry', async () => { + const items = [{ uid: 'a1', _asset_scan_status: 'clean' }]; + const result = await resolveInQueueAssets(makeStack(items), ['a1']); + expect(result).to.include('a1'); + }); + + it('excludes UIDs that become quarantined during retry', async () => { + const items = [{ uid: 'a1', _asset_scan_status: 'quarantined' }]; + const result = await resolveInQueueAssets(makeStack(items), ['a1']); + expect(result).to.not.include('a1'); + }); + + it('resolves UIDs with no scan status (scanning disabled)', async () => { + const items = [{ uid: 'a1' }]; // no _asset_scan_status field + const result = await resolveInQueueAssets(makeStack(items), ['a1']); + expect(result).to.include('a1'); + }); + + it('drops UIDs still pending after MAX_RETRIES', async () => { + // Always returns pending status + const items = [{ uid: 'a1', _asset_scan_status: 'pending' }]; + const result = await resolveInQueueAssets(makeStack(items), ['a1']); + expect(result).to.deep.equal([]); + }); + + it('handles mixed outcomes: clean, quarantined, and pending exhausted', async () => { + const items = [ + { uid: 'clean1', _asset_scan_status: 'clean' }, + { uid: 'quar1', _asset_scan_status: 'quarantined' }, + { uid: 'pend1', _asset_scan_status: 'pending' }, + ]; + const result = await resolveInQueueAssets(makeStack(items), ['clean1', 'quar1', 'pend1']); + expect(result).to.include('clean1'); + expect(result).to.not.include('quar1'); + expect(result).to.not.include('pend1'); + }); + }); +}); diff --git a/packages/contentstack-export/messages/index.json b/packages/contentstack-export/messages/index.json index 21a72c884..73925c5fd 100644 --- a/packages/contentstack-export/messages/index.json +++ b/packages/contentstack-export/messages/index.json @@ -10,6 +10,8 @@ "ASSET_QUERY_FAILED": "Failed to query asset data from the API", "ASSET_VERSIONED_QUERY_FAILED": "Failed to query versioned asset data from the API", "ASSET_COUNT_QUERY_FAILED": "Failed to retrieve total asset count", + "ASSET_SCAN_SKIPPED": "Skipping download of asset '%s' (UID: %s) — scan status: %s", + "ASSET_SCAN_SKIP_SUMMARY": "%s asset(s) skipped due to a non-clean scan status (pending/quarantined). Re-run the export once scanning completes to download them.", "CONTENT_TYPE_EXPORT_COMPLETE": "Content types exported successfully", "CONTENT_TYPE_NO_TYPES": "No content types found", diff --git a/packages/contentstack-export/package.json b/packages/contentstack-export/package.json index db0a69ad0..768acdd66 100644 --- a/packages/contentstack-export/package.json +++ b/packages/contentstack-export/package.json @@ -93,7 +93,8 @@ "shortCommandName": { "cm:stacks:export": "EXPRT", "cm:export": "O-EXPRT" - } + }, + "planProtectedFeatures": ["assetsScan"] }, "repository": "https://github.com/contentstack/cli" -} +} \ No newline at end of file diff --git a/packages/contentstack-export/src/commands/cm/stacks/export.ts b/packages/contentstack-export/src/commands/cm/stacks/export.ts index 15f88c573..d7dd749e5 100644 --- a/packages/contentstack-export/src/commands/cm/stacks/export.ts +++ b/packages/contentstack-export/src/commands/cm/stacks/export.ts @@ -120,7 +120,7 @@ export default class ExportCommand extends Command { let exportDir: string = pathValidator('logs'); try { const { flags } = await this.parse(ExportCommand); - const exportConfig = await setupExportConfig(flags); + const exportConfig = await setupExportConfig(flags, this.context); // Store apiKey in configHandler for session.json (return value not needed) createLogContext( diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 556764767..218ee56a8 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -118,6 +118,9 @@ const config: DefaultConfig = { displayExecutionTime: false, enableDownloadStatus: false, includeVersionedAssets: false, + // Asset scan statuses that must block download; any other value (including 'not_scanned', + // 'clean', or the field being absent) is treated as safe to download. + blockingScanStatuses: ['pending', 'quarantined'], }, content_types: { dirName: 'content_types', diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 1a95d5f25..b3a37df90 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -135,6 +135,7 @@ export default class ExportAssets extends BaseClass { const queryParam = { ...this.commonQueryParam, include_publish_details: true, + include_asset_scan_status: true, except: { BASE: this.assetConfig.invalidKeys }, }; this.applyQueryFilters(queryParam, 'assets'); @@ -168,7 +169,10 @@ export default class ExportAssets extends BaseClass { indexFileName: 'assets.json', basePath: this.assetsRootPath, chunkFileSize: this.assetConfig.chunkFileSize, - metaPickKeys: merge(['uid', 'url', 'filename', 'parent_uid'], this.assetConfig.assetsMetaKeys), + metaPickKeys: merge( + ['uid', 'url', 'filename', 'parent_uid', '_asset_scan_status'], + this.assetConfig.assetsMetaKeys, + ), }); } if (!isEmpty(items)) { @@ -204,6 +208,7 @@ export default class ExportAssets extends BaseClass { const queryParam = { ...this.commonQueryParam, include_publish_details: true, + include_asset_scan_status: true, except: { BASE: this.assetConfig.invalidKeys }, }; @@ -243,7 +248,10 @@ export default class ExportAssets extends BaseClass { indexFileName: 'versioned-assets.json', chunkFileSize: this.assetConfig.chunkFileSize, basePath: pResolve(this.assetsRootPath, 'versions'), - metaPickKeys: merge(['uid', 'url', 'filename', '_version', 'parent_uid'], this.assetConfig.assetsMetaKeys), + metaPickKeys: merge( + ['uid', 'url', 'filename', '_version', 'parent_uid', '_asset_scan_status'], + this.assetConfig.assetsMetaKeys, + ), }); } if (!isEmpty(response)) { @@ -329,6 +337,19 @@ export default class ExportAssets extends BaseClass { listOfAssets = uniqBy(listOfAssets, 'url'); log.debug(`Total unique assets to download: ${listOfAssets.length}`, this.exportConfig.context); + const isNotClean = (asset: any) => this.assetConfig.blockingScanStatuses.includes(asset._asset_scan_status); + const skippedAssets = filter(listOfAssets, isNotClean); + listOfAssets = filter(listOfAssets, (asset: any) => !isNotClean(asset)); + + if (!isEmpty(skippedAssets)) { + for (const asset of skippedAssets) { + log.warn( + messageHandler.parse('ASSET_SCAN_SKIPPED', asset.filename, asset.uid, asset._asset_scan_status), + this.exportConfig.context, + ); + } + } + const apiBatches: Array = chunk(listOfAssets, this.assetConfig.downloadLimit); const downloadedAssetsDirs = await getDirectories(pResolve(this.assetsRootPath, 'files')); @@ -413,6 +434,9 @@ export default class ExportAssets extends BaseClass { promisifyHandler, ).then(() => { log.success(messageHandler.parse('ASSET_DOWNLOAD_COMPLETE'), this.exportConfig.context); + if (!isEmpty(skippedAssets)) { + log.warn(messageHandler.parse('ASSET_SCAN_SKIP_SUMMARY', skippedAssets.length), this.exportConfig.context); + } }); } } diff --git a/packages/contentstack-export/src/export/modules/base-class.ts b/packages/contentstack-export/src/export/modules/base-class.ts index 6379669e1..68fb1adb9 100644 --- a/packages/contentstack-export/src/export/modules/base-class.ts +++ b/packages/contentstack-export/src/export/modules/base-class.ts @@ -5,7 +5,7 @@ import chunk from 'lodash/chunk'; import isEmpty from 'lodash/isEmpty'; import entries from 'lodash/entries'; import isEqual from 'lodash/isEqual'; -import { log } from '@contentstack/cli-utilities'; +import { log, handleAndLogError } from '@contentstack/cli-utilities'; import { ExportConfig, ModuleClassParams } from '../../types'; @@ -115,7 +115,16 @@ export default abstract class BaseClass { } /* eslint-disable no-await-in-loop */ - await Promise.allSettled(allPromise); + const settledResults = await Promise.allSettled(allPromise); + settledResults.forEach((result) => { + if (result.status === 'rejected') { + handleAndLogError( + result.reason, + { ...this.exportConfig.context }, + `Unhandled rejection in '${module}' batch ${batchNo}`, + ); + } + }); /* eslint-disable no-await-in-loop */ await this.logMsgAndWaitIfRequired(module, start, batchNo); @@ -150,6 +159,30 @@ export default abstract class BaseClass { if (exeTime < 1000) await this.delay(1000 - exeTime); } + /** + * Wraps a caller-supplied resolve/reject callback so that if the callback itself throws + * (e.g. a bug while writing/logging the result), the failure is caught right here and + * always recorded via handleAndLogError, instead of turning into a rejected promise that + * a caller further up the chain may or may not notice. + */ + private guardCallback( + callback: (value: any) => void, + moduleName: ApiModuleType, + context: Record, + ): (value: any) => void { + return (value: any) => { + try { + callback(value); + } catch (error) { + handleAndLogError( + error, + { ...this.exportConfig.context, ...context }, + `Unhandled error while processing '${moduleName}' API result`, + ); + } + }; + } + /** * @method makeAPICall * @param {Record} options - Api related params @@ -160,32 +193,35 @@ export default abstract class BaseClass { { module: moduleName, reject, resolve, url = '', uid = '', additionalInfo, queryParam = {} }: ApiOptions, isLastRequest = false, ): Promise { + const safeResolve = this.guardCallback(resolve, moduleName, { uid, additionalInfo }); + const safeReject = this.guardCallback(reject, moduleName, { uid, additionalInfo }); + switch (moduleName) { case 'asset': return this.stack .asset(uid) .fetch(queryParam) - .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) - .catch((error: Error) => reject({ error, isLastRequest, additionalInfo })); + .then((response: any) => safeResolve({ response, isLastRequest, additionalInfo })) + .catch((error: Error) => safeReject({ error, isLastRequest, additionalInfo })); case 'assets': return this.stack .asset() .query(queryParam) .find() - .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) - .catch((error: Error) => reject({ error, isLastRequest, additionalInfo })); + .then((response: any) => safeResolve({ response, isLastRequest, additionalInfo })) + .catch((error: Error) => safeReject({ error, isLastRequest, additionalInfo })); case 'download-asset': return this.stack .asset() .download({ url, responseType: 'stream' }) - .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) - .catch((error: any) => reject({ error, isLastRequest, additionalInfo })); + .then((response: any) => safeResolve({ response, isLastRequest, additionalInfo })) + .catch((error: any) => safeReject({ error, isLastRequest, additionalInfo })); case 'export-taxonomy': return this.stack .taxonomy(uid) .export(queryParam) - .then((response: any) => resolve({ response, uid })) - .catch((error: any) => reject({ error, uid })); + .then((response: any) => safeResolve({ response, uid })) + .catch((error: any) => safeReject({ error, uid })); default: return Promise.resolve(); } diff --git a/packages/contentstack-export/src/types/default-config.ts b/packages/contentstack-export/src/types/default-config.ts index 01cb84c89..3ea16980b 100644 --- a/packages/contentstack-export/src/types/default-config.ts +++ b/packages/contentstack-export/src/types/default-config.ts @@ -102,6 +102,7 @@ export default interface DefaultConfig { displayExecutionTime: boolean; enableDownloadStatus: boolean; includeVersionedAssets: boolean; + blockingScanStatuses: string[]; dependencies?: Modules[]; }; content_types: { diff --git a/packages/contentstack-export/src/types/export-config.ts b/packages/contentstack-export/src/types/export-config.ts index 8b0e1b37b..a6fd5fe4c 100644 --- a/packages/contentstack-export/src/types/export-config.ts +++ b/packages/contentstack-export/src/types/export-config.ts @@ -1,3 +1,4 @@ +import { FeatureStatus } from '@contentstack/cli-utilities'; import { Context, Modules, Region } from '.'; import DefaultConfig from './default-config'; @@ -36,6 +37,7 @@ export default interface ExportConfig extends DefaultConfig { skipStackSettings?: boolean; skipDependencies?: boolean; authenticationMethod?: string; + planStatus?: Record; } type branch = { diff --git a/packages/contentstack-export/src/utils/export-config-handler.ts b/packages/contentstack-export/src/utils/export-config-handler.ts index c67b6c12b..bf984f3da 100644 --- a/packages/contentstack-export/src/utils/export-config-handler.ts +++ b/packages/contentstack-export/src/utils/export-config-handler.ts @@ -1,6 +1,14 @@ import merge from 'merge'; import * as path from 'path'; -import { configHandler, isAuthenticated,cliux, sanitizePath, log } from '@contentstack/cli-utilities'; +import { + configHandler, + isAuthenticated, + cliux, + sanitizePath, + log, + FeatureCtx, + isFeatureEnabled, +} from '@contentstack/cli-utilities'; import defaultConfig from '../config'; import { readFile } from './file-helper'; import { askExportDir, askAPIKey } from './interactive'; @@ -8,7 +16,7 @@ import login from './basic-login'; import { filter, includes } from 'lodash'; import { ExportConfig } from '../types'; -const setupConfig = async (exportCmdFlags: any): Promise => { +const setupConfig = async (exportCmdFlags: any, context?: any): Promise => { let config = merge({}, defaultConfig); // Track authentication method @@ -97,7 +105,7 @@ const setupConfig = async (exportCmdFlags: any): Promise => { if (exportCmdFlags['branch-alias']) { config.branchAlias = exportCmdFlags['branch-alias']; - } + } if (exportCmdFlags['branch']) { config.branchName = exportCmdFlags['branch']; } @@ -133,10 +141,36 @@ const setupConfig = async (exportCmdFlags: any): Promise => { } } - // Add authentication details to config for context tracking + // Add authentication details to config for context tracking config.authenticationMethod = authenticationMethod; log.debug('Export configuration setup completed.', { ...config }); + // Deferred plan check — credentials now available after setupExportConfig + const deferredFeatures: string[] = context?.planCheckRequired ?? []; + if (deferredFeatures.length > 0) { + const planCtx: FeatureCtx = { + apiKey: config.apiKey, + managementToken: config.management_token, + authToken: config.auth_token, + }; + for (const featureUid of deferredFeatures) { + try { + const status = await isFeatureEnabled(featureUid, planCtx); + if (context) { + context.planStatus[featureUid] = status; + } + + log.debug(`[export] Deferred plan status fetched for "${featureUid}".`); + } catch (error) { + log.warn(`[export] Could not fetch deferred plan status for "${featureUid}": ${(error as Error).message}`); + } + } + } + + if (context?.planStatus) { + config.planStatus = context.planStatus; + } + return config; }; diff --git a/packages/contentstack-export/test/unit/export/modules/assets.test.ts b/packages/contentstack-export/test/unit/export/modules/assets.test.ts index 5509682a4..61a1375c9 100644 --- a/packages/contentstack-export/test/unit/export/modules/assets.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/assets.test.ts @@ -142,6 +142,7 @@ describe('ExportAssets', () => { displayExecutionTime: false, enableDownloadStatus: false, includeVersionedAssets: false, + blockingScanStatuses: ['pending', 'quarantined'], }, content_types: { dirName: 'content_types', @@ -564,6 +565,46 @@ describe('ExportAssets', () => { expect(makeConcurrentCallStub.called).to.be.true; }); + + it('should skip assets with a non-clean scan status and download the rest', async () => { + getPlainMetaStub.returns({ + 'file-1': [ + { uid: 'clean-1', url: 'https://test.io/assets/clean-1.jpeg', filename: 'clean-1.jpeg', _asset_scan_status: 'clean' }, + { uid: 'pending-1', url: 'https://test.io/assets/pending-1.zip', filename: 'pending-1.zip', _asset_scan_status: 'pending' }, + { uid: 'quarantined-1', url: 'https://test.io/assets/quarantined-1.zip', filename: 'quarantined-1.zip', _asset_scan_status: 'quarantined' }, + ], + }); + + await exportAssets.downloadAssets(); + + expect(makeConcurrentCallStub.called).to.be.true; + // Only the 'clean' asset should be handed off for download; pending/quarantined are skipped. + expect(makeConcurrentCallStub.firstCall.args[0].totalCount).to.equal(1); + }); + + it('should download assets with no scan status field (stacks without asset scanning enabled)', async () => { + getPlainMetaStub.returns({ + 'file-1': [{ uid: 'legacy-1', url: 'https://test.io/assets/legacy-1.jpeg', filename: 'legacy-1.jpeg' }], + }); + + await exportAssets.downloadAssets(); + + // Missing _asset_scan_status must not be treated as non-clean, or every export would break. + expect(makeConcurrentCallStub.firstCall.args[0].totalCount).to.equal(1); + }); + + it('should download assets with a not_scanned status (org has asset scanning disabled)', async () => { + getPlainMetaStub.returns({ + 'file-1': [ + { uid: 'not-scanned-1', url: 'https://test.io/assets/not-scanned-1.jpeg', filename: 'not-scanned-1.jpeg', _asset_scan_status: 'not_scanned' }, + ], + }); + + await exportAssets.downloadAssets(); + + // 'not_scanned' means scanning is off for the org, not that the asset is unsafe. + expect(makeConcurrentCallStub.firstCall.args[0].totalCount).to.equal(1); + }); }); describe('Edge Cases', () => { diff --git a/packages/contentstack-export/test/unit/export/modules/base-class.test.ts b/packages/contentstack-export/test/unit/export/modules/base-class.test.ts index 7fbdc6f74..2741963a0 100644 --- a/packages/contentstack-export/test/unit/export/modules/base-class.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/base-class.test.ts @@ -160,6 +160,7 @@ describe('BaseClass', () => { displayExecutionTime: false, enableDownloadStatus: false, includeVersionedAssets: false, + blockingScanStatuses: ['pending', 'quarantined'], }, content_types: { dirName: 'content_types', diff --git a/packages/contentstack-export/test/unit/export/modules/stack.test.ts b/packages/contentstack-export/test/unit/export/modules/stack.test.ts index 8fa749c72..62bef40cb 100644 --- a/packages/contentstack-export/test/unit/export/modules/stack.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/stack.test.ts @@ -151,7 +151,8 @@ describe('ExportStack', () => { securedAssets: false, displayExecutionTime: false, enableDownloadStatus: false, - includeVersionedAssets: false + includeVersionedAssets: false, + blockingScanStatuses: ['pending', 'quarantined'] }, content_types: { dirName: 'content_types', diff --git a/packages/contentstack-external-migrate/tsconfig.tsbuildinfo b/packages/contentstack-external-migrate/tsconfig.tsbuildinfo new file mode 100644 index 000000000..758731050 --- /dev/null +++ b/packages/contentstack-external-migrate/tsconfig.tsbuildinfo @@ -0,0 +1 @@ +{"root":["./src/index.ts","./src/adapters/registry.ts","./src/adapters/types.ts","./src/adapters/contentful/convert.ts","./src/adapters/contentful/export.ts","./src/adapters/contentful/index.ts","./src/adapters/contentful/validator.ts","./src/commands/migrate/audit.ts","./src/commands/migrate/convert.ts","./src/commands/migrate/create.ts","./src/commands/migrate/export.ts","./src/commands/migrate/import.ts","./src/commands/migrate/status.ts","./src/lib/bundle.ts","./src/lib/clear-import-state.ts","./src/lib/contentful-cli-spawn.ts","./src/lib/conversion-summary.ts","./src/lib/create-stack.ts","./src/lib/csdx-spawn.ts","./src/lib/helpers.ts","./src/lib/local-date.ts","./src/lib/log.ts","./src/lib/manifest.ts","./src/lib/parse-json-loose.ts","./src/services/contentful/config.ts","./src/services/contentful/constants.ts","./src/services/contentful/content-type-creator.ts","./src/services/contentful/contentful.service.ts","./src/services/contentful/extension.service.ts","./src/services/contentful/market-app.utils.ts","./src/services/contentful/marketplace.service.ts","./src/services/contentful/releases.ts","./src/services/contentful/scheduled.ts","./src/services/contentful/tasks.ts","./src/services/contentful/types.ts","./src/services/contentful/users.ts","./src/services/contentful/workflows.ts","./src/services/contentful/contentful/jsonrte.ts","./src/services/contentful/contentful/markdown.ts","./src/services/contentful/contentful/roles.ts","./src/services/contentful/contentful/taxonomy.service.ts","./src/services/contentful/mapper/write.ts","./src/services/contentful/migration-contentful/index.js","./src/services/contentful/migration-contentful/libs/contenttypemapper.js","./src/services/contentful/migration-contentful/libs/createinitialmapper.js","./src/services/contentful/migration-contentful/libs/extractcontenttypes.js","./src/services/contentful/migration-contentful/libs/extractlocale.js","./src/services/contentful/migration-contentful/libs/extracttaxonomy.js","./src/services/contentful/migration-contentful/utils/helper.js","./src/services/contentful/prompts/master-locale.ts","./src/services/contentful/utils/custom-logger.utils.ts","./src/services/contentful/utils/index.ts"],"version":"5.9.3"} \ No newline at end of file diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index 05a623db0..396725b14 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -85,7 +85,8 @@ "shortCommandName": { "cm:stacks:import": "IMPRT", "cm:import": "O-IMPRT" - } + }, + "planProtectedFeatures": ["assetsScan"] }, "repository": "https://github.com/contentstack/cli" -} +} \ No newline at end of file diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index 74217cf45..0d795b66b 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -154,7 +154,7 @@ export default class ImportCommand extends Command { let importConfig: ImportConfig; try { const { flags } = await this.parse(ImportCommand); - importConfig = await setupImportConfig(flags); + importConfig = await setupImportConfig(flags, this.context); // Prepare the context object createLogContext( this.context?.info?.command || 'cm:stacks:export', @@ -187,6 +187,23 @@ export default class ImportCommand extends Command { log.success(`The log has been stored at: ${getLogPath()}`, importConfig.context); log.info(`The backup content has been stored at: ${backupDir}`, importConfig.context); + + // Closing reminder: when assets were imported but not published inline + // (asset scanning enabled, or --skip-assets-publish), point the user to + // cm:assets:publish with the backup dir and stack pre-filled so the note + // isn't lost in the per-module logs above. + const assetsImported = importConfig.moduleName + ? importConfig.moduleName === 'assets' + : importConfig.modules?.types?.includes('assets'); + // Mirror the publish gate in assets.ts (`!skipAssetsPublish`): assets are + // left unpublished exactly when skipAssetsPublish is set — which also + // covers the scanning case, since detecting scanning sets skipAssetsPublish. + if (!result?.noSuccessMsg && assetsImported && importConfig.skipAssetsPublish) { + log.info( + `Note: assets were imported but not published asset scanning is enabled and must complete first. To publish them, run:\n csdx cm:assets:publish --backup-dir ${backupDir} --stack-api-key ${importConfig.apiKey}`, + importConfig.context, + ); + } } catch (error) { handleAndLogError(error); log.info(`The log has been stored at '${getLogPath()}'`); diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index e8b792f0b..27165fab1 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -53,11 +53,15 @@ export default class ImportAssets extends BaseClass { */ async start(): Promise { try { - // NOTE Step 1: Import folders and create uid mapping file + if (this.importConfig.assetScanningEnabled) { + log.info('Assets Scanning is enabled in this stack', this.importConfig.context); + log.warn('Assets publishing will be skipped', this.importConfig.context); + } + // NOTE Step 1: Import folders and create uid mapping file log.debug('Starting folder import process...', this.importConfig.context); await this.importFolders(); - // NOTE Step 2: Import versioned assets and create it mapping files (uid, url) + // NOTE Step 2: Import versioned assets and create it mapping files (uid, url) if (this.assetConfig.includeVersionedAssets) { const versionsPath = `${this.assetsPath}/versions`; if (existsSync(versionsPath)) { @@ -68,17 +72,31 @@ export default class ImportAssets extends BaseClass { } } - // NOTE Step 3: Import Assets and create it mapping files (uid, url) + // NOTE Step 3: Import Assets and create it mapping files (uid, url) log.debug('Starting assets import...', this.importConfig.context); await this.importAssets(); - // NOTE Step 4: Publish assets + // NOTE Step 4: Publish assets if (!this.importConfig.skipAssetsPublish) { log.debug('Starting assets publishing...', this.importConfig.context); await this.publish(); } log.success('Assets imported successfully!', this.importConfig.context); + + // Only surface the "publish later" guidance when assets were actually + // imported. With 0 assets, assetsUidMap is empty and the backup has no + // assets to publish — printing the guidance would send the user to run + // cm:assets:publish against an empty backup. + if (this.importConfig.assetScanningEnabled && !isEmpty(this.assetsUidMap)) { + log.info('Asset Scanning is enabled for this stack.', this.importConfig.context); + log.info('Assets cannot be published immediately — scanning must complete first.', this.importConfig.context); + log.info('Once scanning is done, publish your assets using:', this.importConfig.context); + log.info( + `csdx cm:assets:publish --backup-dir ${this.importConfig.backupDir} --stack-api-key [STACK API KEY]`, + this.importConfig.context, + ); + } } catch (error) { handleAndLogError(error, { ...this.importConfig.context }); } diff --git a/packages/contentstack-import/src/types/import-config.ts b/packages/contentstack-import/src/types/import-config.ts index 2c4c9a000..80805ae2c 100644 --- a/packages/contentstack-import/src/types/import-config.ts +++ b/packages/contentstack-import/src/types/import-config.ts @@ -1,3 +1,4 @@ +import { FeatureStatus } from '@contentstack/cli-utilities'; import { Context, Modules, Region } from '.'; import DefaultConfig from './default-config'; @@ -15,6 +16,7 @@ export default interface ImportConfig extends DefaultConfig, ExternalConfig { skipAssetsPublish?: boolean; skipEntriesPublish?: boolean; cliLogsPath: string; + assetScanningEnabled?: boolean; canCreatePrivateApp: boolean; contentDir: string; data: string; @@ -59,6 +61,7 @@ export default interface ImportConfig extends DefaultConfig, ExternalConfig { personalizeProjectName?: string; 'exclude-global-modules': false; context: Context; + planStatus?: Record; } type branch = { diff --git a/packages/contentstack-import/src/utils/import-config-handler.ts b/packages/contentstack-import/src/utils/import-config-handler.ts index 9df8a1bbd..e25a93034 100644 --- a/packages/contentstack-import/src/utils/import-config-handler.ts +++ b/packages/contentstack-import/src/utils/import-config-handler.ts @@ -7,6 +7,8 @@ import { cliux, sanitizePath, log, + isFeatureEnabled, + FeatureCtx, } from '@contentstack/cli-utilities'; import defaultConfig from '../config'; import { readFile, fileExistsSync } from './file-helper'; @@ -14,7 +16,7 @@ import { askContentDir, askAPIKey } from './interactive'; import login from './login-handler'; import { ImportConfig } from '../types'; -const setupConfig = async (importCmdFlags: any): Promise => { +const setupConfig = async (importCmdFlags: any, context?: any): Promise => { let config: ImportConfig = merge({}, defaultConfig); // Track authentication method let authenticationMethod = 'unknown'; @@ -139,6 +141,33 @@ const setupConfig = async (importCmdFlags: any): Promise => { config.authenticationMethod = authenticationMethod; log.debug('Import configuration setup completed.', { ...config }); + // Deferred plan check — credentials now available after setupImportConfig + const deferredFeatures: string[] = context?.planCheckRequired ?? []; + if (deferredFeatures.length > 0) { + const planCtx: FeatureCtx = { + apiKey: config.apiKey, + managementToken: config.management_token, + authToken: config.auth_token, + }; + for (const featureUid of deferredFeatures) { + try { + const status = await isFeatureEnabled(featureUid, planCtx); + if (context) context.planStatus[featureUid] = status; + log.debug(`[import] Deferred plan status fetched for "${featureUid}".`); + } catch (error) { + log.warn(`[import] Could not fetch deferred plan status for "${featureUid}": ${(error as Error).message}`); + } + } + } + + if (context?.planStatus) { + config.planStatus = context.planStatus; + if (config.planStatus['assetsScan']?.is_part_of_plan) { + config.assetScanningEnabled = true; + config.skipAssetsPublish = true; + } + } + return config; }; diff --git a/packages/contentstack-query-export/package.json b/packages/contentstack-query-export/package.json index 7f4f8dba7..8e111e94f 100644 --- a/packages/contentstack-query-export/package.json +++ b/packages/contentstack-query-export/package.json @@ -98,4 +98,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} +} \ No newline at end of file