From 19c2a36b1e3cf3234082538bf2b123900436f132 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 30 Jul 2026 12:25:07 +0530 Subject: [PATCH 1/2] handled assets in pending or quarantine states export failure --- .../contentstack-export/messages/index.json | 2 + .../src/export/modules/assets.ts | 28 +++++++++- .../src/export/modules/base-class.ts | 56 +++++++++++++++---- .../test/unit/export/modules/assets.test.ts | 27 +++++++++ 4 files changed, 101 insertions(+), 12 deletions(-) diff --git a/packages/contentstack-export/messages/index.json b/packages/contentstack-export/messages/index.json index 21a72c884..73925c5fd 100644 --- a/packages/contentstack-export/messages/index.json +++ b/packages/contentstack-export/messages/index.json @@ -10,6 +10,8 @@ "ASSET_QUERY_FAILED": "Failed to query asset data from the API", "ASSET_VERSIONED_QUERY_FAILED": "Failed to query versioned asset data from the API", "ASSET_COUNT_QUERY_FAILED": "Failed to retrieve total asset count", + "ASSET_SCAN_SKIPPED": "Skipping download of asset '%s' (UID: %s) — scan status: %s", + "ASSET_SCAN_SKIP_SUMMARY": "%s asset(s) skipped due to a non-clean scan status (pending/quarantined). Re-run the export once scanning completes to download them.", "CONTENT_TYPE_EXPORT_COMPLETE": "Content types exported successfully", "CONTENT_TYPE_NO_TYPES": "No content types found", diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 1a95d5f25..eb0b23a02 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -135,6 +135,7 @@ export default class ExportAssets extends BaseClass { const queryParam = { ...this.commonQueryParam, include_publish_details: true, + include_asset_scan_status: true, except: { BASE: this.assetConfig.invalidKeys }, }; this.applyQueryFilters(queryParam, 'assets'); @@ -168,7 +169,10 @@ export default class ExportAssets extends BaseClass { indexFileName: 'assets.json', basePath: this.assetsRootPath, chunkFileSize: this.assetConfig.chunkFileSize, - metaPickKeys: merge(['uid', 'url', 'filename', 'parent_uid'], this.assetConfig.assetsMetaKeys), + metaPickKeys: merge( + ['uid', 'url', 'filename', 'parent_uid', '_asset_scan_status'], + this.assetConfig.assetsMetaKeys, + ), }); } if (!isEmpty(items)) { @@ -204,6 +208,7 @@ export default class ExportAssets extends BaseClass { const queryParam = { ...this.commonQueryParam, include_publish_details: true, + include_asset_scan_status: true, except: { BASE: this.assetConfig.invalidKeys }, }; @@ -243,7 +248,10 @@ export default class ExportAssets extends BaseClass { indexFileName: 'versioned-assets.json', chunkFileSize: this.assetConfig.chunkFileSize, basePath: pResolve(this.assetsRootPath, 'versions'), - metaPickKeys: merge(['uid', 'url', 'filename', '_version', 'parent_uid'], this.assetConfig.assetsMetaKeys), + metaPickKeys: merge( + ['uid', 'url', 'filename', '_version', 'parent_uid', '_asset_scan_status'], + this.assetConfig.assetsMetaKeys, + ), }); } if (!isEmpty(response)) { @@ -329,6 +337,19 @@ export default class ExportAssets extends BaseClass { listOfAssets = uniqBy(listOfAssets, 'url'); log.debug(`Total unique assets to download: ${listOfAssets.length}`, this.exportConfig.context); + const isNotClean = (asset: any) => asset._asset_scan_status && asset._asset_scan_status !== 'clean'; + const skippedAssets = filter(listOfAssets, isNotClean); + listOfAssets = filter(listOfAssets, (asset: any) => !isNotClean(asset)); + + if (!isEmpty(skippedAssets)) { + for (const asset of skippedAssets) { + log.warn( + messageHandler.parse('ASSET_SCAN_SKIPPED', asset.filename, asset.uid, asset._asset_scan_status), + this.exportConfig.context, + ); + } + } + const apiBatches: Array = chunk(listOfAssets, this.assetConfig.downloadLimit); const downloadedAssetsDirs = await getDirectories(pResolve(this.assetsRootPath, 'files')); @@ -413,6 +434,9 @@ export default class ExportAssets extends BaseClass { promisifyHandler, ).then(() => { log.success(messageHandler.parse('ASSET_DOWNLOAD_COMPLETE'), this.exportConfig.context); + if (!isEmpty(skippedAssets)) { + log.warn(messageHandler.parse('ASSET_SCAN_SKIP_SUMMARY', skippedAssets.length), this.exportConfig.context); + } }); } } diff --git a/packages/contentstack-export/src/export/modules/base-class.ts b/packages/contentstack-export/src/export/modules/base-class.ts index 6379669e1..68fb1adb9 100644 --- a/packages/contentstack-export/src/export/modules/base-class.ts +++ b/packages/contentstack-export/src/export/modules/base-class.ts @@ -5,7 +5,7 @@ import chunk from 'lodash/chunk'; import isEmpty from 'lodash/isEmpty'; import entries from 'lodash/entries'; import isEqual from 'lodash/isEqual'; -import { log } from '@contentstack/cli-utilities'; +import { log, handleAndLogError } from '@contentstack/cli-utilities'; import { ExportConfig, ModuleClassParams } from '../../types'; @@ -115,7 +115,16 @@ export default abstract class BaseClass { } /* eslint-disable no-await-in-loop */ - await Promise.allSettled(allPromise); + const settledResults = await Promise.allSettled(allPromise); + settledResults.forEach((result) => { + if (result.status === 'rejected') { + handleAndLogError( + result.reason, + { ...this.exportConfig.context }, + `Unhandled rejection in '${module}' batch ${batchNo}`, + ); + } + }); /* eslint-disable no-await-in-loop */ await this.logMsgAndWaitIfRequired(module, start, batchNo); @@ -150,6 +159,30 @@ export default abstract class BaseClass { if (exeTime < 1000) await this.delay(1000 - exeTime); } + /** + * Wraps a caller-supplied resolve/reject callback so that if the callback itself throws + * (e.g. a bug while writing/logging the result), the failure is caught right here and + * always recorded via handleAndLogError, instead of turning into a rejected promise that + * a caller further up the chain may or may not notice. + */ + private guardCallback( + callback: (value: any) => void, + moduleName: ApiModuleType, + context: Record, + ): (value: any) => void { + return (value: any) => { + try { + callback(value); + } catch (error) { + handleAndLogError( + error, + { ...this.exportConfig.context, ...context }, + `Unhandled error while processing '${moduleName}' API result`, + ); + } + }; + } + /** * @method makeAPICall * @param {Record} options - Api related params @@ -160,32 +193,35 @@ export default abstract class BaseClass { { module: moduleName, reject, resolve, url = '', uid = '', additionalInfo, queryParam = {} }: ApiOptions, isLastRequest = false, ): Promise { + const safeResolve = this.guardCallback(resolve, moduleName, { uid, additionalInfo }); + const safeReject = this.guardCallback(reject, moduleName, { uid, additionalInfo }); + switch (moduleName) { case 'asset': return this.stack .asset(uid) .fetch(queryParam) - .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) - .catch((error: Error) => reject({ error, isLastRequest, additionalInfo })); + .then((response: any) => safeResolve({ response, isLastRequest, additionalInfo })) + .catch((error: Error) => safeReject({ error, isLastRequest, additionalInfo })); case 'assets': return this.stack .asset() .query(queryParam) .find() - .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) - .catch((error: Error) => reject({ error, isLastRequest, additionalInfo })); + .then((response: any) => safeResolve({ response, isLastRequest, additionalInfo })) + .catch((error: Error) => safeReject({ error, isLastRequest, additionalInfo })); case 'download-asset': return this.stack .asset() .download({ url, responseType: 'stream' }) - .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) - .catch((error: any) => reject({ error, isLastRequest, additionalInfo })); + .then((response: any) => safeResolve({ response, isLastRequest, additionalInfo })) + .catch((error: any) => safeReject({ error, isLastRequest, additionalInfo })); case 'export-taxonomy': return this.stack .taxonomy(uid) .export(queryParam) - .then((response: any) => resolve({ response, uid })) - .catch((error: any) => reject({ error, uid })); + .then((response: any) => safeResolve({ response, uid })) + .catch((error: any) => safeReject({ error, uid })); default: return Promise.resolve(); } diff --git a/packages/contentstack-export/test/unit/export/modules/assets.test.ts b/packages/contentstack-export/test/unit/export/modules/assets.test.ts index 5509682a4..e472ab13f 100644 --- a/packages/contentstack-export/test/unit/export/modules/assets.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/assets.test.ts @@ -564,6 +564,33 @@ describe('ExportAssets', () => { expect(makeConcurrentCallStub.called).to.be.true; }); + + it('should skip assets with a non-clean scan status and download the rest', async () => { + getPlainMetaStub.returns({ + 'file-1': [ + { uid: 'clean-1', url: 'https://test.io/assets/clean-1.jpeg', filename: 'clean-1.jpeg', _asset_scan_status: 'clean' }, + { uid: 'pending-1', url: 'https://test.io/assets/pending-1.zip', filename: 'pending-1.zip', _asset_scan_status: 'pending' }, + { uid: 'quarantined-1', url: 'https://test.io/assets/quarantined-1.zip', filename: 'quarantined-1.zip', _asset_scan_status: 'quarantined' }, + ], + }); + + await exportAssets.downloadAssets(); + + expect(makeConcurrentCallStub.called).to.be.true; + // Only the 'clean' asset should be handed off for download; pending/quarantined are skipped. + expect(makeConcurrentCallStub.firstCall.args[0].totalCount).to.equal(1); + }); + + it('should download assets with no scan status field (stacks without asset scanning enabled)', async () => { + getPlainMetaStub.returns({ + 'file-1': [{ uid: 'legacy-1', url: 'https://test.io/assets/legacy-1.jpeg', filename: 'legacy-1.jpeg' }], + }); + + await exportAssets.downloadAssets(); + + // Missing _asset_scan_status must not be treated as non-clean, or every export would break. + expect(makeConcurrentCallStub.firstCall.args[0].totalCount).to.equal(1); + }); }); describe('Edge Cases', () => { From 601947fc8258e259ec400cb5b3ea212e9a0f6a39 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 30 Jul 2026 13:52:20 +0530 Subject: [PATCH 2/2] added audit check and fix for assets in pending or quarantined state --- .talismanrc | 6 + .../src/audit-base-command.ts | 41 ++- .../contentstack-audit/src/config/index.ts | 5 + .../contentstack-audit/src/messages/index.ts | 4 + .../contentstack-audit/src/modules/assets.ts | 30 ++- .../contentstack-audit/src/modules/entries.ts | 204 ++++++++++++++- .../src/types/content-types.ts | 2 + .../unit/mock/contents/assets/assets.json | 1 + .../mock/contents/assets/chunk1-assets.json | 33 +++ .../test/unit/modules/assets.test.ts | 74 ++++++ .../test/unit/modules/entries.test.ts | 235 ++++++++++++++++++ 11 files changed, 622 insertions(+), 13 deletions(-) create mode 100644 packages/contentstack-audit/test/unit/mock/contents/assets/assets.json create mode 100644 packages/contentstack-audit/test/unit/mock/contents/assets/chunk1-assets.json create mode 100644 packages/contentstack-audit/test/unit/modules/assets.test.ts diff --git a/.talismanrc b/.talismanrc index c61a19cba..ab42eb8cd 100644 --- a/.talismanrc +++ b/.talismanrc @@ -2,4 +2,10 @@ fileignoreconfig: - filename: pnpm-lock.yaml checksum: 31e333d6769adbaae042c92ea0930fab168a0e06fc1bda406d49fd1042a7a9c7 +- filename: packages/contentstack-audit/src/audit-base-command.ts + checksum: 14275f241e4a194cfd1fb33d277df194011eba4458ccaab7e0d0bd3a1c9ccfe7 +- filename: packages/contentstack-audit/src/modules/entries.ts + checksum: b0fa5f7b390ef2d64bd4834a5b848780b902373e2c644833842b42c6b767d54e +- filename: packages/contentstack-audit/test/unit/modules/entries.test.ts + checksum: ff448c79d436d5e8b141efcc76f19892bfe96039ea9a60fae4e995b5ccbc2960 version: '1.0' diff --git a/packages/contentstack-audit/src/audit-base-command.ts b/packages/contentstack-audit/src/audit-base-command.ts index b78650a31..d86efb057 100644 --- a/packages/contentstack-audit/src/audit-base-command.ts +++ b/packages/contentstack-audit/src/audit-base-command.ts @@ -94,6 +94,8 @@ export abstract class AuditBaseCommand extends BaseCommand; missingEnvLocale?: Record; missingMultipleFields?: Record; + missingAssetRefs?: Record; } = {}, missingMandatoryFields, missingTitleFields, @@ -215,6 +228,8 @@ export abstract class AuditBaseCommand extends BaseCommand, ): Promise { log.debug(`Preparing report for module: ${moduleName}`, this.auditContext); @@ -656,7 +682,8 @@ export abstract class AuditBaseCommand extends BaseCommand, ): Promise { if (Object.keys(config.moduleConfig).includes(moduleName) || config.feild_level_modules.includes(moduleName)) { diff --git a/packages/contentstack-audit/src/config/index.ts b/packages/contentstack-audit/src/config/index.ts index e291554fa..eb583c690 100644 --- a/packages/contentstack-audit/src/config/index.ts +++ b/packages/contentstack-audit/src/config/index.ts @@ -110,6 +110,8 @@ const config = { 'publish_locale', 'publish_environment', 'asset_uid', + 'scan_status', + 'mandatory', 'selectedValue', 'ct_uid', 'action', @@ -129,6 +131,7 @@ const config = { Entry_Missing_Locale_and_Env: 'Entry_Missing_Locale_and_Env', Entry_Missing_Locale_and_Env_in_Publish_Details: 'Entry_Missing_Locale_and_Env_in_Publish_Details', Entry_Multiple_Fields: 'Entry_Multiple_Fields', + Entries_Asset_field: 'Entries_Asset_field', }, feild_level_modules: [ 'Entries_Title_field', @@ -137,6 +140,8 @@ const config = { 'Entry_Missing_Locale_and_Env_in_Publish_Details', 'field-rules', 'Entry_Multiple_Fields', + 'Entries_Asset_field', + 'asset-scan-status', 'Summary', ], fixSelectField: false, diff --git a/packages/contentstack-audit/src/messages/index.ts b/packages/contentstack-audit/src/messages/index.ts index a2b654128..dac9136f9 100644 --- a/packages/contentstack-audit/src/messages/index.ts +++ b/packages/contentstack-audit/src/messages/index.ts @@ -49,6 +49,8 @@ const auditMsg = { FIELD_RULE_TARGET_ABSENT: `The target field '{target_field}' is not present in the schema of the content-type {ctUid}`, FIELD_RULE_CONDITION_SCAN_MESSAGE: `Completed Scanning of Field Rule '{num}' condition of Content-type '{ctUid}'`, FIELD_RULE_TARGET_SCAN_MESSAGE: `Completed Scanning of Field Rule '{num}' target of Content-type '{ctUid}'`, + SCAN_ASSET_QUARANTINE_MSG: `Asset with UID '{uid}' has a non-clean scan status ('{status}') and will be excluded from import.`, + ENTRY_ASSET_REF_WARN_MSG: `Entry '{uid}' field '{field}' references asset '{asset_uid}' with scan status '{status}'; it will be removed on fix.`, }; const auditFixMsg = { @@ -63,6 +65,8 @@ const auditFixMsg = { ENTRY_SELECT_FIELD_FIX: `Adding the value '{value}' in the select field of entry UID '{uid}'...`, ASSET_FIX: `Fixed publish detials for Asset with UID '{uid}'`, FIELD_RULE_FIX_MESSAGE: `Fixed Field Rule '{num}' target of Content-type '{ctUid}`, + ASSET_SCAN_STATUS_FIX: `Removed asset with UID '{uid}' (scan status: '{status}') from assets.json.`, + ENTRY_ASSET_REF_FIX: `Removed reference to asset '{asset_uid}' from entry '{uid}' field '{field}'.`, }; const messages: typeof errors & diff --git a/packages/contentstack-audit/src/modules/assets.ts b/packages/contentstack-audit/src/modules/assets.ts index af00aa5bb..14858cc4c 100644 --- a/packages/contentstack-audit/src/modules/assets.ts +++ b/packages/contentstack-audit/src/modules/assets.ts @@ -27,6 +27,7 @@ export default class Assets { public environments: string[] = []; protected schema: ContentTypeStruct[] = []; protected missingEnvLocales: Record = {}; + public missingScanStatusAssets: Record = {}; public moduleName: keyof typeof auditConfig.moduleConfig; constructor({ fix, config, moduleName }: ModuleConstructorParam & CtConstructorParam) { @@ -184,7 +185,24 @@ export default class Assets { for (const assetUid in assets) { log.debug(`Processing asset: ${assetUid}`, this.config.auditContext); - + + const scanStatus = this.assets[assetUid]?._asset_scan_status; + if (scanStatus && scanStatus !== 'clean') { + log.debug(`Asset ${assetUid} has a non-clean scan status: ${scanStatus}`, this.config.auditContext); + cliux.print($t(auditMsg.SCAN_ASSET_QUARANTINE_MSG, { uid: assetUid, status: scanStatus }), { + color: 'yellow', + }); + this.missingScanStatusAssets[assetUid] = [ + { asset_uid: assetUid, filename: this.assets[assetUid].filename, scan_status: scanStatus }, + ]; + + if (this.fix) { + log.info($t(auditFixMsg.ASSET_SCAN_STATUS_FIX, { uid: assetUid, status: scanStatus }), this.config.auditContext); + delete this.assets[assetUid]; + continue; + } + } + if (this.assets[assetUid]?.publish_details && !Array.isArray(this.assets[assetUid].publish_details)) { log.debug(`Asset ${assetUid} has invalid publish_details format`, this.config.auditContext); cliux.print($t(auditMsg.ASSET_NOT_EXIST, { uid: assetUid }), { color: 'red' }); @@ -226,15 +244,19 @@ export default class Assets { const remainingPublishDetails = this.assets[assetUid].publish_details?.length || 0; log.debug(`Asset ${assetUid} now has ${remainingPublishDetails} valid publish details`, this.config.auditContext); - + if (this.fix) { log.debug(`Fixing asset ${assetUid}`, this.config.auditContext); log.info($t(auditFixMsg.ASSET_FIX, { uid: assetUid }), this.config.auditContext); - await this.writeFixContent(`${basePath}/${indexer[fileIndex]}`, this.assets); } } + + if (this.fix) { + log.debug(`Writing fixed assets chunk to: ${basePath}/${indexer[fileIndex]}`, this.config.auditContext); + await this.writeFixContent(`${basePath}/${indexer[fileIndex]}`, this.assets); + } } - + log.debug(`Asset reference validation completed. Processed ${Object.keys(this.missingEnvLocales).length} assets with issues`, this.config.auditContext); } } diff --git a/packages/contentstack-audit/src/modules/entries.ts b/packages/contentstack-audit/src/modules/entries.ts index 02697d50b..e7a1257dd 100644 --- a/packages/contentstack-audit/src/modules/entries.ts +++ b/packages/contentstack-audit/src/modules/entries.ts @@ -58,8 +58,11 @@ export default class Entries { protected missingTitleFields: Record = {}; protected missingEnvLocale: Record = {}; protected missingMultipleField: Record = {}; + protected missingAssetRefs: Record = {}; public environments: string[] = []; public entryMetaData: Record[] = []; + public assetMetaData: Record = {}; + public assetsDataAvailable = false; public moduleName: keyof typeof auditConfig.moduleConfig = 'entries'; constructor({ fix, config, moduleName, ctSchema, gfSchema }: ModuleConstructorParam & CtConstructorParam) { @@ -154,6 +157,10 @@ export default class Entries { await this.prepareEntryMetaData(); log.debug(`Entry metadata prepared: ${this.entryMetaData.length} entries found`, this.config.auditContext); + log.debug('Preparing asset metadata', this.config.auditContext); + await this.prepareAssetMetaData(); + log.debug(`Asset metadata prepared: ${Object.keys(this.assetMetaData).length} assets found`, this.config.auditContext); + log.debug('Fixing prerequisite data', this.config.auditContext); await this.fixPrerequisiteData(); log.debug('Prerequisite data fix completed', this.config.auditContext); @@ -198,6 +205,10 @@ export default class Entries { if (!this.missingMandatoryFields[this.currentUid]) { this.missingMandatoryFields[this.currentUid] = []; } + + if (!this.missingAssetRefs[this.currentUid]) { + this.missingAssetRefs[this.currentUid] = []; + } if (this.fix) { log.debug(`Removing missing keys from entry ${uid}`, this.config.auditContext); this.removeMissingKeysOnEntry(ctSchema.schema as ContentTypeSchemaType[], this.entries[entryUid]); @@ -234,6 +245,14 @@ export default class Entries { }); } + if (this.missingAssetRefs[this.currentUid]?.length) { + log.debug(`Found ${this.missingAssetRefs[this.currentUid].length} missing/quarantined asset references for entry ${uid}`, this.config.auditContext); + this.missingAssetRefs[this.currentUid].forEach((entry: any) => { + entry.ct = ctSchema.uid; + entry.locale = code; + }); + } + const fields = this.missingMandatoryFields[uid]; const isPublished = entry.publish_details?.length > 0; log.debug(`Entry ${uid} published status: ${isPublished}, missing mandatory fields: ${fields?.length || 0}`, this.config.auditContext); @@ -343,6 +362,7 @@ export default class Entries { missingTitleFields: this.missingTitleFields, missingEnvLocale: this.missingEnvLocale, missingMultipleFields: this.missingMultipleField, + missingAssetRefs: this.missingAssetRefs, }; log.debug(`Entries audit completed. Found issues:`, this.config.auditContext); @@ -388,8 +408,17 @@ export default class Entries { removedMandatoryFields++; } } - - log.debug(`Cleanup completed: removed ${removedRefs} empty refs, ${removedSelectFields} empty select fields, ${removedMandatoryFields} empty mandatory fields`, this.config.auditContext); + + let removedAssetRefs = 0; + for (let propName in this.missingAssetRefs) { + if (!this.missingAssetRefs[propName].length) { + log.debug(`Removing empty missing asset references for entry: ${propName}`, this.config.auditContext); + delete this.missingAssetRefs[propName]; + removedAssetRefs++; + } + } + + log.debug(`Cleanup completed: removed ${removedRefs} empty refs, ${removedSelectFields} empty select fields, ${removedMandatoryFields} empty mandatory fields, ${removedAssetRefs} empty asset refs`, this.config.auditContext); } /** @@ -624,6 +653,16 @@ export default class Entries { entry[uid] as EntryGroupFieldDataType[], ); break; + case 'file': + log.debug(`Validating file/asset field: ${display_name}`, this.config.auditContext); + const assetRefResults = this.validateFileField( + [...tree, { uid: child.uid, name: child.display_name, field: uid }], + child, + entry[uid], + ); + this.missingAssetRefs[this.currentUid].push(...assetRefResults); + log.debug(`Found ${assetRefResults.length} quarantined/missing asset references in field: ${display_name}`, this.config.auditContext); + break; case 'text': case 'number': if (child.hasOwnProperty('display_type')) { @@ -888,6 +927,59 @@ export default class Entries { log.debug(`Group field validation completed for: ${fieldStructure.display_name}`); } + /** + * Returns true when the given asset uid should be treated as unusable — either it doesn't + * exist in the exported assets.json at all, or it exists but its scan status is present and + * not 'clean' (e.g. 'pending'/'quarantined'). Returns false (never flag) when asset metadata + * wasn't available at all, since we can't validate what we don't have data for. + */ + isAssetBad(uid?: string): boolean { + if (!this.assetsDataAvailable || !uid) return false; + const assetRecord = this.assetMetaData[uid]; + if (!assetRecord) return true; + const scanStatus = assetRecord._asset_scan_status; + return Boolean(scanStatus) && scanStatus !== 'clean'; + } + + /** + * The function `validateFileField` checks a `data_type: 'file'` (asset reference) field's + * value(s) against the asset metadata index and returns an issue when any referenced asset is + * missing or has a non-clean scan status. + */ + validateFileField( + tree: Record[], + fieldStructure: { uid: string; data_type: string; display_name: string; mandatory?: boolean }, + field: any, + ): EntryRefErrorReturnType[] { + log.debug(`Validating file/asset field: ${fieldStructure.display_name}`, this.config.auditContext); + + const values = Array.isArray(field) ? field : field ? [field] : []; + const missingRefs = values + .filter((ref: any) => this.isAssetBad(ref?.uid)) + .map((ref: any) => ({ asset_uid: ref?.uid, filename: ref?.filename })); + + if (isEmpty(missingRefs)) { + log.debug('File/asset field validation completed: no issues found', this.config.auditContext); + return []; + } + + return [ + { + tree, + missingRefs, + uid: this.currentUid, + name: this.currentTitle, + data_type: fieldStructure.data_type, + display_name: fieldStructure.display_name, + mandatory: fieldStructure.mandatory, + treeStr: tree + .map(({ name }) => name) + .filter((val) => val) + .join(' ➜ '), + } as unknown as EntryRefErrorReturnType, + ]; + } + /** * The function `validateReferenceValues` checks if the references in a given field exist in the * provided tree and returns any missing references. @@ -1125,6 +1217,14 @@ export default class Entries { entry[uid] as EntryGroupFieldDataType[], ) as EntryGroupFieldDataType; break; + case 'file': + log.debug(`Fixing file/asset field: ${uid}`); + this.fixFileFieldReferences( + [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], + field, + entry, + ); + break; case 'text': case 'number': if (field.hasOwnProperty('display_type')) { @@ -1574,6 +1674,65 @@ export default class Entries { return field; } + /** + * The function `fixFileFieldReferences` strips references to missing/quarantined/pending-scan + * assets from a `data_type: 'file'` field. A single-object value with a bad asset uid is deleted + * from the entry entirely; a `multiple: true` array value has its bad entries filtered out, and + * the key itself is deleted if the array becomes empty as a result. + */ + fixFileFieldReferences( + tree: Record[], + field: { uid: string; data_type: string; display_name: string; mandatory?: boolean }, + entry: Record, + ) { + log.debug(`Fixing file/asset field: ${field.display_name}`); + const { uid, display_name, data_type, mandatory } = field; + const value = entry[uid]; + + if (value == null) { + return entry; + } + + const missingRefs: Record[] = []; + + if (Array.isArray(value)) { + entry[uid] = value.filter((ref: any) => { + if (this.isAssetBad(ref?.uid)) { + missingRefs.push({ asset_uid: ref?.uid, filename: ref?.filename }); + return false; + } + return true; + }); + if (!entry[uid].length) { + delete entry[uid]; + } + } else if (this.isAssetBad(value?.uid)) { + missingRefs.push({ asset_uid: value?.uid, filename: value?.filename }); + delete entry[uid]; + } + + if (!isEmpty(missingRefs)) { + log.debug(`Recording asset reference fix for entry: ${this.currentUid}`); + this.missingAssetRefs[this.currentUid].push({ + tree, + data_type, + missingRefs, + display_name, + mandatory, + fixStatus: 'Fixed', + uid: this.currentUid, + name: this.currentTitle, + treeStr: tree + .map(({ name }) => name) + .filter((val) => val) + .join(' ➜ '), + }); + } + + log.debug(`File/asset fix completed for: ${field.display_name}`); + return entry; + } + /** * The function `fixGroupField` takes in a tree, a field, and an entry, and if the field has a * schema, it runs a fix on the schema and returns the updated entry, otherwise it returns the @@ -1991,4 +2150,45 @@ export default class Entries { log.debug(`Entry metadata preparation completed: ${this.entryMetaData.length} entries processed`, this.config.auditContext); log.debug(`Missing title fields found: ${Object.keys(this.missingTitleFields).length}`, this.config.auditContext); } + + /** + * Builds an index of asset uid -> { uid, filename, _asset_scan_status } from the exported + * assets/assets.json, mirroring the FsUtility chunk-read pattern used in the Assets module + * (src/modules/assets.ts). If the assets data isn't present in this export at all, + * `assetsDataAvailable` stays false and `isAssetBad()` never flags anything — absence of data + * must not be treated as every file-field reference being broken. + */ + async prepareAssetMetaData() { + log.debug('Starting asset metadata preparation', this.config.auditContext); + + const assetsBasePath = resolve( + sanitizePath(this.config.basePath), + sanitizePath(this.config.moduleConfig.assets.dirName), + ); + const assetsIndexPath = join(assetsBasePath, this.config.moduleConfig.assets.fileName); + + if (!existsSync(assetsIndexPath)) { + log.debug(`No assets data found at: ${assetsIndexPath}`, this.config.auditContext); + this.assetsDataAvailable = false; + return; + } + + this.assetsDataAvailable = true; + const fsUtility = new FsUtility({ basePath: assetsBasePath, indexFileName: 'assets.json' }); + const indexer = fsUtility.indexFileContent; + log.debug(`Found ${Object.keys(indexer).length} asset files to process`, this.config.auditContext); + + for (const _ in indexer) { + const assets = (await fsUtility.readChunkFiles.next()) as Record; + for (const assetUid in assets) { + this.assetMetaData[assetUid] = { + uid: assetUid, + filename: assets[assetUid]?.filename, + _asset_scan_status: assets[assetUid]?._asset_scan_status, + }; + } + } + + log.debug(`Asset metadata preparation completed: ${Object.keys(this.assetMetaData).length} assets processed`, this.config.auditContext); + } } diff --git a/packages/contentstack-audit/src/types/content-types.ts b/packages/contentstack-audit/src/types/content-types.ts index 2a27e92af..bd723c42e 100644 --- a/packages/contentstack-audit/src/types/content-types.ts +++ b/packages/contentstack-audit/src/types/content-types.ts @@ -162,6 +162,8 @@ enum OutputColumn { 'publish_locale' = 'publish_locale', 'publish_environment' = 'publish_environment', 'asset_uid' = 'asset_uid', + 'scan_status' = 'scan_status', + 'mandatory' = 'mandatory', 'selectedValue' = 'selectedValue', 'fixStatus' = 'fixStatus', 'Content_type_uid' = 'ct_uid', diff --git a/packages/contentstack-audit/test/unit/mock/contents/assets/assets.json b/packages/contentstack-audit/test/unit/mock/contents/assets/assets.json new file mode 100644 index 000000000..825a196ea --- /dev/null +++ b/packages/contentstack-audit/test/unit/mock/contents/assets/assets.json @@ -0,0 +1 @@ +{"1":"chunk1-assets.json"} diff --git a/packages/contentstack-audit/test/unit/mock/contents/assets/chunk1-assets.json b/packages/contentstack-audit/test/unit/mock/contents/assets/chunk1-assets.json new file mode 100644 index 000000000..a4fb857d4 --- /dev/null +++ b/packages/contentstack-audit/test/unit/mock/contents/assets/chunk1-assets.json @@ -0,0 +1,33 @@ +{ + "blt-clean-asset": { + "uid": "blt-clean-asset", + "filename": "clean.jpg", + "url": "https://images.contentstack.io/v3/assets/blt/blt-clean-asset/clean.jpg", + "_version": 1, + "_asset_scan_status": "clean", + "publish_details": [] + }, + "blt-pending-asset": { + "uid": "blt-pending-asset", + "filename": "pending.zip", + "url": "https://assets.contentstack.io/v3/assets/blt/blt-pending-asset/pending.zip", + "_version": 1, + "_asset_scan_status": "pending", + "publish_details": [] + }, + "blt-quarantined-asset": { + "uid": "blt-quarantined-asset", + "filename": "quarantined.zip", + "url": "https://assets.contentstack.io/v3/assets/blt/blt-quarantined-asset/quarantined.zip", + "_version": 1, + "_asset_scan_status": "quarantined", + "publish_details": [] + }, + "blt-no-status-asset": { + "uid": "blt-no-status-asset", + "filename": "no-status.png", + "url": "https://images.contentstack.io/v3/assets/blt/blt-no-status-asset/no-status.png", + "_version": 1, + "publish_details": [] + } +} diff --git a/packages/contentstack-audit/test/unit/modules/assets.test.ts b/packages/contentstack-audit/test/unit/modules/assets.test.ts new file mode 100644 index 000000000..f2d26a02c --- /dev/null +++ b/packages/contentstack-audit/test/unit/modules/assets.test.ts @@ -0,0 +1,74 @@ +import fs from 'fs'; +import { resolve } from 'path'; +import { expect } from 'chai'; +import fancy from 'fancy-test'; +import Sinon from 'sinon'; +import config from '../../../src/config'; +import { Assets } from '../../../src/modules'; +import { ModuleConstructorParam, CtConstructorParam } from '../../../src/types'; +import { mockLogger } from '../mock-logger'; + +describe('Assets module', () => { + let constructorParam: ModuleConstructorParam & CtConstructorParam; + + beforeEach(() => { + constructorParam = { + moduleName: 'assets', + ctSchema: [], + gfSchema: [], + config: Object.assign(config, { basePath: resolve(__dirname, '..', 'mock', 'contents'), flags: {} }), + }; + + Sinon.stub(require('@contentstack/cli-utilities'), 'log').value(mockLogger); + }); + + afterEach(() => { + Sinon.restore(); + }); + + describe('lookForReference method (scan status)', () => { + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('flags assets with a non-clean scan status and leaves clean/no-status assets alone', async () => { + const assetsInstance = new Assets(constructorParam); + await assetsInstance.prerequisiteData(); + await assetsInstance.lookForReference(); + + expect(Object.keys(assetsInstance.missingScanStatusAssets)).to.have.members([ + 'blt-pending-asset', + 'blt-quarantined-asset', + ]); + expect(assetsInstance.missingScanStatusAssets['blt-pending-asset'][0]).to.deep.include({ + asset_uid: 'blt-pending-asset', + scan_status: 'pending', + }); + expect(assetsInstance.missingScanStatusAssets['blt-quarantined-asset'][0]).to.deep.include({ + asset_uid: 'blt-quarantined-asset', + scan_status: 'quarantined', + }); + expect(assetsInstance.missingScanStatusAssets).to.not.have.property('blt-clean-asset'); + expect(assetsInstance.missingScanStatusAssets).to.not.have.property('blt-no-status-asset'); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .stub(fs, 'writeFileSync', () => {}) + .it('removes non-clean assets from the written-back chunk when fix mode is on', async () => { + const writeFileSyncStub = Sinon.spy(fs, 'writeFileSync'); + const assetsInstance = new Assets({ + ...constructorParam, + fix: true, + config: { ...constructorParam.config, flags: { yes: true } }, + }); + await assetsInstance.prerequisiteData(); + await assetsInstance.lookForReference(); + + expect(writeFileSyncStub.called).to.be.true; + const writtenContent = JSON.parse(writeFileSyncStub.firstCall.args[1] as string); + expect(writtenContent).to.not.have.property('blt-pending-asset'); + expect(writtenContent).to.not.have.property('blt-quarantined-asset'); + expect(writtenContent).to.have.property('blt-clean-asset'); + expect(writtenContent).to.have.property('blt-no-status-asset'); + }); + }); +}); diff --git a/packages/contentstack-audit/test/unit/modules/entries.test.ts b/packages/contentstack-audit/test/unit/modules/entries.test.ts index 911ba3316..49de8480a 100644 --- a/packages/contentstack-audit/test/unit/modules/entries.test.ts +++ b/packages/contentstack-audit/test/unit/modules/entries.test.ts @@ -1540,4 +1540,239 @@ describe('Entries module', () => { expect(callHelper('sys_assets', ['ct1'])).to.be.true; }); }); + + describe('prepareAssetMetaData method', () => { + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('builds an asset metadata index from assets/assets.json when it exists', async () => { + const ctInstance = new Entries(constructorParam); + await ctInstance.prepareAssetMetaData(); + + expect(ctInstance.assetsDataAvailable).to.be.true; + expect(ctInstance.assetMetaData['blt-clean-asset']).to.deep.include({ + uid: 'blt-clean-asset', + _asset_scan_status: 'clean', + }); + expect(ctInstance.assetMetaData['blt-pending-asset']).to.deep.include({ + uid: 'blt-pending-asset', + _asset_scan_status: 'pending', + }); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('leaves assetsDataAvailable false when no assets data exists in this export', async () => { + const ctInstance = new Entries({ + ...constructorParam, + config: { ...constructorParam.config, basePath: resolve(__dirname, '..', 'mock', 'contents-1') }, + }); + await ctInstance.prepareAssetMetaData(); + + expect(ctInstance.assetsDataAvailable).to.be.false; + expect(ctInstance.assetMetaData).to.deep.equal({}); + }); + }); + + describe('isAssetBad method', () => { + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns false when assets data is unavailable', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = false; + (ctInstance as any).assetMetaData = {}; + expect(ctInstance.isAssetBad('blt-unknown')).to.be.false; + }); + + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns false when uid is falsy', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = true; + expect(ctInstance.isAssetBad(undefined)).to.be.false; + }); + + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns true when the asset uid is not found in the index', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = {}; + expect(ctInstance.isAssetBad('blt-unknown')).to.be.true; + }); + + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns true when the asset scan status is non-clean', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { 'blt-1': { uid: 'blt-1', _asset_scan_status: 'quarantined' } }; + expect(ctInstance.isAssetBad('blt-1')).to.be.true; + }); + + fancy.stdout({ print: process.env.PRINT === 'true' || false }).it('returns false when the asset is clean or has no scan status', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { + 'blt-clean': { uid: 'blt-clean', _asset_scan_status: 'clean' }, + 'blt-legacy': { uid: 'blt-legacy' }, + }; + expect(ctInstance.isAssetBad('blt-clean')).to.be.false; + expect(ctInstance.isAssetBad('blt-legacy')).to.be.false; + }); + }); + + describe('validateFileField method', () => { + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('flags a single file-field value that references a quarantined asset', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).currentTitle = 'Test Entry'; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { + 'blt-quarantined': { uid: 'blt-quarantined', _asset_scan_status: 'quarantined' }, + }; + + const fieldStructure = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const value = { uid: 'blt-quarantined', filename: 'bad.zip' }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.validateFileField(tree, fieldStructure, value); + + expect(result).to.have.length(1); + expect(result[0].missingRefs).to.deep.equal([{ asset_uid: 'blt-quarantined', filename: 'bad.zip' }]); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('does not flag a single file-field value that references a clean asset', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { 'blt-clean': { uid: 'blt-clean', _asset_scan_status: 'clean' } }; + + const fieldStructure = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const value = { uid: 'blt-clean', filename: 'ok.jpg' }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.validateFileField(tree, fieldStructure, value); + + expect(result).to.have.length(0); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('flags only the bad entries within a multiple:true file-field array', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { 'blt-clean': { uid: 'blt-clean', _asset_scan_status: 'clean' } }; + + const fieldStructure = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const value = [ + { uid: 'blt-clean', filename: 'ok.jpg' }, + { uid: 'blt-pending', filename: 'bad.zip' }, + ]; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.validateFileField(tree, fieldStructure, value); + + expect(result).to.have.length(1); + expect(result[0].missingRefs).to.deep.equal([{ asset_uid: 'blt-pending', filename: 'bad.zip' }]); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('does not flag anything when asset metadata is unavailable', () => { + const ctInstance = new Entries(constructorParam); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).assetsDataAvailable = false; + + const fieldStructure = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const value = { uid: 'blt-unknown', filename: 'unknown.jpg' }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.validateFileField(tree, fieldStructure, value); + + expect(result).to.have.length(0); + }); + }); + + describe('fixFileFieldReferences method', () => { + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('deletes the field key when a single file-field value references a bad asset', () => { + const ctInstance = new Entries({ ...constructorParam, fix: true }); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).currentTitle = 'Test Entry'; + (ctInstance as any).missingAssetRefs = { 'test-entry': [] }; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { + 'blt-quarantined': { uid: 'blt-quarantined', _asset_scan_status: 'quarantined' }, + }; + + const field = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const entry: Record = { file_field: { uid: 'blt-quarantined', filename: 'bad.zip' } }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.fixFileFieldReferences(tree, field, entry); + + expect(result).to.not.have.property('file_field'); + expect((ctInstance as any).missingAssetRefs['test-entry']).to.have.length(1); + expect((ctInstance as any).missingAssetRefs['test-entry'][0].fixStatus).to.equal('Fixed'); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('leaves a single file-field value referencing a clean asset untouched', () => { + const ctInstance = new Entries({ ...constructorParam, fix: true }); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).missingAssetRefs = { 'test-entry': [] }; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = { 'blt-clean': { uid: 'blt-clean', _asset_scan_status: 'clean' } }; + + const field = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const entry: Record = { file_field: { uid: 'blt-clean', filename: 'ok.jpg' } }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + ctInstance.fixFileFieldReferences(tree, field, entry); + + expect(entry).to.have.property('file_field'); + expect((ctInstance as any).missingAssetRefs['test-entry']).to.have.length(0); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('filters bad entries out of a multiple:true array and deletes the key if it empties out', () => { + const ctInstance = new Entries({ ...constructorParam, fix: true }); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).missingAssetRefs = { 'test-entry': [] }; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = {}; + + const field = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: false }; + const entry: Record = { + file_field: [ + { uid: 'blt-1', filename: 'one.zip' }, + { uid: 'blt-2', filename: 'two.zip' }, + ], + }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + const result = ctInstance.fixFileFieldReferences(tree, field, entry); + + expect(result).to.not.have.property('file_field'); + expect((ctInstance as any).missingAssetRefs['test-entry'][0].missingRefs).to.have.length(2); + }); + + fancy + .stdout({ print: process.env.PRINT === 'true' || false }) + .it('records mandatory:true on the fix issue so a stripped required field stays visible in the report', () => { + const ctInstance = new Entries({ ...constructorParam, fix: true }); + (ctInstance as any).currentUid = 'test-entry'; + (ctInstance as any).missingAssetRefs = { 'test-entry': [] }; + (ctInstance as any).assetsDataAvailable = true; + (ctInstance as any).assetMetaData = {}; + + const field = { uid: 'file_field', display_name: 'File Field', data_type: 'file', mandatory: true }; + const entry: Record = { file_field: { uid: 'blt-missing', filename: 'gone.zip' } }; + const tree = [{ uid: 'test-entry', name: 'Test Entry' }]; + + ctInstance.fixFileFieldReferences(tree, field, entry); + + expect((ctInstance as any).missingAssetRefs['test-entry'][0].mandatory).to.be.true; + }); + }); });