Merge pull request #294 from contentstack/feat/Asset-Scanning

Feature branch for Asset scanning v2

Author: naman-contentstack

.talismanrc

@@ -1,18 +1,13 @@
 fileignoreconfig:
+
 - filename: pnpm-lock.yaml
-  checksum: 693eb73aae594f391247f2feecc55b84447cd39658b6f70f25ef87df19b5c134
-- filename: packages/contentstack-export/src/utils/export-config-handler.ts
-  checksum: 3ff8e8ea60f92311f8224bce457f7aaab026194de61e13eebdc6ca5a39f66bc9
-- filename: packages/contentstack-asset-management/src/utils/export-helpers.ts
-  checksum: 726d1632110ebc203ce70dc1cbe7d4b3011f56349ed371b6645d0375e7818cad
-- filename: packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts
-  checksum: bc4a53f96be6a10786e00133245c7bdc43c965c8a98b753e3879e1110cf9c601
-- filename: packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts
-  checksum: 63c6bff4d51842d8fa3cce88545259d0a2c3cfe71df95d303d993f692cee883b
-- filename: packages/contentstack-migration/src/commands/cm/stacks/migration.ts
-  checksum: 529431473623cf6bbefb35bc898be2a60dcb453e0c9273ecb97923de658769b3
-- filename: packages/contentstack-seed/src/commands/cm/stacks/seed.ts
-  checksum: 9a3fd45a57c6ebf63d6c154657f2ebf799befbbacb9a255944c2d89ae89b22ed
-- filename: packages/contentstack-seed/test/commands/cm/stacks/seed.test.ts
-  checksum: b54f572a614e5d80f7fa27a51e679147cb537fd76ac922eed981d1156a261e50
-version: '1.0'
+  checksum: 649aae748e3ae8c157439dc6c7c9ea402cc19b4dab7ca64b7e4d839d1375df53
+- filename: packages/contentstack-audit/src/audit-base-command.ts
+  checksum: b706ea114a3cb5e4c0e7e163f32ceffa3356a6b0766bf5b8fe387c3c72f44c36
+- filename: packages/contentstack-audit/src/types/content-types.ts
+  checksum: d16a65415c3184f15a807d58e5858310aeb5633794fc9075e36f89c94da636c3
+- filename: packages/contentstack-audit/src/modules/entries.ts
+  checksum: 13072df0872f8c098d23f49b02f9e8f3e492ab2f483d67a5ea5e423ab406ad94
+- filename: packages/contentstack-audit/test/unit/modules/entries.test.ts
+  checksum: 31d9767d87e4e9db54f641e8cb9d3360f591acdaa8b089a83eaa4aacdf0c3a35
+  version: '1.0'

packages/contentstack-apps-cli/package.json

@@ -25,8 +25,8 @@
     "@contentstack/cli-command": "~2.0.0-beta.11",
     "@contentstack/cli-launch": "^1.11.1",
     "@contentstack/cli-utilities": "~2.0.0-beta.12",
-    "adm-zip": "^0.5.18",
     "chalk": "^5.6.2",
+    "adm-zip": "^0.6.0",
     "lodash": "^4.18.1",
     "shelljs": "^0.10.0",
     "tmp": "^0.2.7",
@@ -110,4 +110,4 @@
       "app:deploy": "APDP"
     }
   }
-}
+}

packages/contentstack-audit/README.md

@@ -19,7 +19,7 @@ $ npm install -g @contentstack/cli-audit
 $ csdx COMMAND
 running command...
 $ csdx (--version|-v)
-@contentstack/cli-audit/2.0.0-beta.15 darwin-arm64 node-v24.14.0
+@contentstack/cli-audit/1.21.0 darwin-arm64 node-v24.18.0
 $ csdx --help [COMMAND]
 USAGE
   $ csdx COMMAND

packages/contentstack-audit/package.json

@@ -80,4 +80,4 @@
   "keywords": [
     "oclif"
   ]
-}
+}

packages/contentstack-audit/src/audit-base-command.ts

@@ -15,7 +15,7 @@ import {
   clearProgressModuleSetting,
   readContentTypeSchemas,
   readGlobalFieldSchemas,
-  generateUid
+  generateUid,
 } from '@contentstack/cli-utilities';
 import { createWriteStream, existsSync, mkdirSync, writeFileSync, rmSync } from 'fs';
 import config from './config';
@@ -56,7 +56,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
         minWidth: 7,
         header: 'Fix Status',
         get: (row: any) => {
-          return row.fixStatus === 'Fixed' ? getChalk().greenBright(row.fixStatus) : getChalk().redBright(row.fixStatus);
+          return row.fixStatus === 'Fixed'
+            ? getChalk().greenBright(row.fixStatus)
+            : getChalk().redBright(row.fixStatus);
         },
       },
     };
@@ -104,6 +106,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingFieldRules,
       missingMultipleFields,
       missingRefsInComposableStudio,
+      missingScanStatusAssets,
+      missingAssetRefsInEntries,
     } = await this.scanAndFix();
 
     if (this.flags['show-console-output']) {
@@ -135,6 +139,12 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
         { module: 'Entries Changed Multiple Fields', missingRefs: missingMultipleFields },
       ]);
       this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Studio', missingRefs: missingRefsInComposableStudio }]);
+      this.showOutputOnScreenWorkflowsAndExtension([
+        { module: 'Assets Scan Status', missingRefs: missingScanStatusAssets },
+      ]);
+      this.showOutputOnScreenWorkflowsAndExtension([
+        { module: 'Entries Asset Field', missingRefs: missingAssetRefsInEntries },
+      ]);
     }
     this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Summary', missingRefs: this.summaryDataToPrint }]);
 
@@ -151,7 +161,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       !isEmpty(missingEnvLocalesInEntries) ||
       !isEmpty(missingFieldRules) ||
       !isEmpty(missingMultipleFields) ||
-      !isEmpty(missingRefsInComposableStudio)
+      !isEmpty(missingRefsInComposableStudio) ||
+      !isEmpty(missingScanStatusAssets) ||
+      !isEmpty(missingAssetRefsInEntries)
     ) {
       if (this.currentCommand === 'cm:stacks:audit') {
         log.warn(this.$t(auditMsg.FINAL_REPORT_PATH, { path: this.sharedConfig.reportPath }), this.auditContext);
@@ -192,7 +204,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       !isEmpty(missingEnvLocalesInAssets) ||
       !isEmpty(missingEnvLocalesInEntries) ||
       !isEmpty(missingFieldRules) ||
-      !isEmpty(missingRefsInComposableStudio)
+      !isEmpty(missingRefsInComposableStudio) ||
+      !isEmpty(missingScanStatusAssets) ||
+      !isEmpty(missingAssetRefsInEntries)
     );
   }
 
@@ -223,6 +237,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
         missingTitleFields?: Record<string, any>;
         missingEnvLocale?: Record<string, any>;
         missingMultipleFields?: Record<string, any>;
+        missingAssetRefs?: Record<string, any>;
       } = {},
       missingMandatoryFields,
       missingTitleFields,
@@ -231,6 +246,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingEnvLocalesInEntries,
       missingFieldRules,
       missingMultipleFields,
+      missingAssetRefsInEntries,
+      missingScanStatusAssets,
       missingRefsInComposableStudio;
 
     const constructorParam: ModuleConstructorParam & CtConstructorParam = {
@@ -270,17 +287,21 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       constructorParam['moduleName'] = module;
 
       switch (module) {
-        case 'assets':
+        case 'assets': {
           log.info('Executing assets audit', this.auditContext);
-          const assetsTotalCount = dataModuleWise['assets']?.Total || 0;
-          missingEnvLocalesInAssets = await new Assets(cloneDeep(constructorParam)).run(false, assetsTotalCount);
+          const assetsModule = new Assets(cloneDeep(constructorParam));
+          missingEnvLocalesInAssets = await assetsModule.run();
+          missingScanStatusAssets = assetsModule.missingScanStatusAssets ?? {};
           await this.prepareReport(module, missingEnvLocalesInAssets);
+          await this.prepareReport('asset-scan-status', missingScanStatusAssets);
           this.getAffectedData('assets', dataModuleWise['assets'], missingEnvLocalesInAssets);
+          this.getAffectedData('asset-scan-status', dataModuleWise['assets'], missingScanStatusAssets);
           log.success(
-            `Assets audit completed. Found ${Object.keys(missingEnvLocalesInAssets || {}).length} issues`,
+            `Assets audit completed. Found ${Object.keys(missingEnvLocalesInAssets || {}).length} issues, ${Object.keys(missingScanStatusAssets).length} non-clean scan status assets`,
             this.auditContext,
           );
           break;
+        }
         case 'content-types':
           log.info('Executing content-types audit', this.auditContext);
           const contentTypesTotalCount = dataModuleWise['content-types']?.Total || 0;
@@ -313,6 +334,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
           missingTitleFields = missingEntry.missingTitleFields ?? {};
           missingEnvLocalesInEntries = missingEntry.missingEnvLocale ?? {};
           missingMultipleFields = missingEntry.missingMultipleFields ?? {};
+          missingAssetRefsInEntries = missingEntry.missingAssetRefs ?? {};
           await this.prepareReport(module, missingEntryRefs);
 
           await this.prepareReport(`Entries_Select_field`, missingSelectFeild);
@@ -324,6 +346,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
           await this.prepareReport('Entry_Missing_Locale_and_Env_in_Publish_Details', missingEnvLocalesInEntries);
 
           await this.prepareReport('Entry_Multiple_Fields', missingMultipleFields);
+
+          await this.prepareReport('Entries_Asset_field', missingAssetRefsInEntries);
           this.getAffectedData('entries', dataModuleWise['entries'], missingEntry);
           log.success(
             `Entries audit completed. Found ${Object.keys(missingEntryRefs || {}).length} reference issues`,
@@ -382,14 +406,14 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
           // schema from moduleName, so invoke it once per schema (overriding moduleName per
           // instance) and merge the results. Merged object is keyed by schema uid; a content type
           // and global field sharing a uid would collide, which is not expected in practice.
-          const ctFieldRules = await new FieldRule(
-            cloneDeep({ ...constructorParam, moduleName: 'content-types' }),
-          ).run(data.ctSchema?.length || 0);
+          const ctFieldRules = await new FieldRule(cloneDeep({ ...constructorParam, moduleName: 'content-types' })).run(
+            data.ctSchema?.length || 0,
+          );
           let gfFieldRules: Record<string, any> = {};
           if (data.gfSchema?.length) {
-            gfFieldRules = await new FieldRule(
-              cloneDeep({ ...constructorParam, moduleName: 'global-fields' }),
-            ).run(data.gfSchema.length);
+            gfFieldRules = await new FieldRule(cloneDeep({ ...constructorParam, moduleName: 'global-fields' })).run(
+              data.gfSchema.length,
+            );
           }
           missingFieldRules = { ...ctFieldRules, ...gfFieldRules };
 
@@ -455,6 +479,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingFieldRules,
       missingMultipleFields,
       missingRefsInComposableStudio,
+      missingScanStatusAssets,
+      missingAssetRefsInEntries,
     };
   }
 
@@ -638,7 +664,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       | keyof typeof config.moduleConfig
       | keyof typeof config.ReportTitleForEntries
       | 'field-rules'
-      | 'Summary',
+      | 'Summary'
+      | 'asset-scan-status',
     listOfMissingRefs: Record<string, any>,
   ): Promise<void> {
     log.debug(`Preparing report for module: ${moduleName}`, this.auditContext);
@@ -682,7 +709,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       | keyof typeof config.moduleConfig
       | keyof typeof config.ReportTitleForEntries
       | 'field-rules'
-      | 'Summary',
+      | 'Summary'
+      | 'asset-scan-status',
     listOfMissingRefs: Record<string, any>,
   ): Promise<void> {
     if (Object.keys(config.moduleConfig).includes(moduleName) || config.field_level_modules.includes(moduleName)) {

packages/contentstack-audit/src/config/index.ts

@@ -55,6 +55,9 @@ const config = {
       name: 'assets',
       dirName: 'assets',
       fileName: 'assets.json',
+      // Asset scan statuses that must block import/reference; any other value (including
+      // 'not_scanned', 'clean', or the field being absent) is treated as safe.
+      blockingScanStatuses: ['pending', 'quarantined'],
     },
     environments: {
       name: 'environments',
@@ -111,6 +114,8 @@ const config = {
     'publish_environment',
     'asset_uid',
     'space_id',
+    'scan_status',
+    'mandatory',
     'selectedValue',
     'ct_uid',
     'action',
@@ -130,6 +135,7 @@ const config = {
     Entry_Missing_Locale_and_Env: 'Entry_Missing_Locale_and_Env',
     Entry_Missing_Locale_and_Env_in_Publish_Details: 'Entry_Missing_Locale_and_Env_in_Publish_Details',
     Entry_Multiple_Fields: 'Entry_Multiple_Fields',
+    Entries_Asset_field: 'Entries_Asset_field',
   },
   field_level_modules: [
     'Entries_Title_field',
@@ -138,6 +144,8 @@ const config = {
     'Entry_Missing_Locale_and_Env_in_Publish_Details',
     'field-rules',
     'Entry_Multiple_Fields',
+    'Entries_Asset_field',
+    'asset-scan-status',
     'Summary',
   ],
   fixSelectField: false,

packages/contentstack-audit/src/messages/index.ts

@@ -53,6 +53,8 @@ const auditMsg = {
   FIELD_RULE_TARGET_ABSENT: `The target field '{target_field}' is not present in the schema of the content-type {ctUid}`,
   FIELD_RULE_CONDITION_SCAN_MESSAGE: `Completed Scanning of Field Rule '{num}' condition of Content-type '{ctUid}'`,
   FIELD_RULE_TARGET_SCAN_MESSAGE: `Completed Scanning of Field Rule '{num}' target of Content-type '{ctUid}'`,
+  SCAN_ASSET_QUARANTINE_MSG: `Asset with UID '{uid}' has a non-clean scan status ('{status}') and will be excluded from import.`,
+  ENTRY_ASSET_REF_WARN_MSG: `Entry '{uid}' field '{field}' references asset '{asset_uid}' with scan status '{status}'; it will be removed on fix.`,
 };
 
 const auditFixMsg = {
@@ -66,7 +68,9 @@ const auditFixMsg = {
   ENTRY_MANDATORY_FIELD_FIX: `Removing the publish details from the entry with UID '{uid}' in Locale '{locale}'...`,
   ENTRY_SELECT_FIELD_FIX: `Adding the value '{value}' in the select field of entry UID '{uid}'...`,
   ASSET_FIX: `Fixed publish details for Asset with UID '{uid}'`,
-  FIELD_RULE_FIX_MESSAGE: `Fixed Field Rule '{num}' target of Content-type '{ctUid}'`,
+  FIELD_RULE_FIX_MESSAGE: `Fixed Field Rule '{num}' target of Content-type '{ctUid}`,
+  ASSET_SCAN_STATUS_FIX: `Removed asset with UID '{uid}' (scan status: '{status}') from assets.json.`,
+  ENTRY_ASSET_REF_FIX: `Removed reference to asset '{asset_uid}' from entry '{uid}' field '{field}'.`,
 };
 
 const messages: typeof errors &

packages/contentstack-audit/src/modules/assets.ts

@@ -35,6 +35,7 @@ export default class Assets extends BaseClass {
   public environments: string[] = [];
   protected schema: ContentTypeStruct[] = [];
   protected missingEnvLocales: Record<string, any> = {};
+  public missingScanStatusAssets: Record<string, any> = {};
   public moduleName: keyof typeof auditConfig.moduleConfig;
   private fixOverwriteConfirmed: boolean | null = null;
   /** API key of the exported stack; null when `stack/stack.json` is missing or unreadable. */
@@ -342,6 +343,30 @@ export default class Assets extends BaseClass {
         for (const assetUid in assets) {
           log.debug(`Processing asset: ${assetUid}`, this.config.auditContext);
 
+          const scanStatus = this.assets[assetUid]?._asset_scan_status;
+          if (this.config.moduleConfig.assets.blockingScanStatuses.includes(scanStatus)) {
+            log.debug(`Asset ${assetUid} has a non-clean scan status: ${scanStatus}`, this.config.auditContext);
+            cliux.print($t(auditMsg.SCAN_ASSET_QUARANTINE_MSG, { uid: assetUid, status: scanStatus }), {
+              color: 'yellow',
+            });
+            this.missingScanStatusAssets[assetUid] = [
+              { asset_uid: assetUid, filename: this.assets[assetUid].filename, scan_status: scanStatus, space_id: spaceId },
+            ];
+
+            if (this.fix) {
+              log.info(
+                $t(auditFixMsg.ASSET_SCAN_STATUS_FIX, { uid: assetUid, status: scanStatus }),
+                this.config.auditContext,
+              );
+              delete this.assets[assetUid];
+              chunkChanged = true;
+              if (this.progressManager) {
+                this.progressManager.tick(true, `asset: ${assetUid}`, null, spaceProcessName);
+              }
+              continue;
+            }
+          }
+
           if (this.assets[assetUid]?.publish_details && !Array.isArray(this.assets[assetUid].publish_details)) {
             log.debug(`Asset ${assetUid} has invalid publish_details format`, this.config.auditContext);
             cliux.print($t(auditMsg.ASSET_NOT_EXIST, { uid: assetUid }), { color: 'red' });