From de3413c918a3cf2515e64328a8a740325da17f39 Mon Sep 17 00:00:00 2001 From: raj pandey Date: Thu, 23 Apr 2026 12:42:15 +0530 Subject: [PATCH 01/61] enhc: request publish details on taxonomy list export --- .../src/export/modules/taxonomies.ts | 9 +- .../unit/export/modules/taxonomies.test.ts | 136 +++++++++++++++++- 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index 5075a54fe..27be33d4c 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -22,6 +22,7 @@ export default class ExportTaxonomies extends BaseClass { private isLocaleBasedExportSupported: boolean = true; // Flag to track if locale-based export is supported private qs: { include_count: boolean; + include_publish_details: boolean; skip: number; asc?: string; limit: number; @@ -29,6 +30,7 @@ export default class ExportTaxonomies extends BaseClass { branch?: string; include_fallback?: boolean; fallback_locale?: string; + query?: Record; }; public taxonomiesFolderPath: string; private localesFilePath: string; @@ -38,7 +40,12 @@ export default class ExportTaxonomies extends BaseClass { this.taxonomies = {}; this.taxonomiesByLocale = {}; this.taxonomiesConfig = exportConfig.modules.taxonomies; - this.qs = { include_count: true, limit: this.taxonomiesConfig.limit || 100, skip: 0 }; + this.qs = { + include_count: true, + include_publish_details: true, + limit: this.taxonomiesConfig.limit || 100, + skip: 0, + }; this.applyQueryFilters(this.qs, 'taxonomies'); this.exportConfig.context.module = MODULE_CONTEXTS.TAXONOMIES; diff --git a/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts b/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts index 71e7dd8ba..276e0281e 100644 --- a/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts @@ -66,7 +66,14 @@ describe('ExportTaxonomies', () => { taxonomies: { dirName: 'taxonomies', fileName: 'taxonomies.json', - invalidKeys: [], + invalidKeys: [ + 'updated_at', + 'created_by', + 'updated_by', + 'stackHeaders', + 'urlPath', + 'created_at', + ], limit: 100, }, locales: { @@ -107,6 +114,118 @@ describe('ExportTaxonomies', () => { sinon.restore(); }); + describe('include_publish_details (list query)', () => { + it('should set include_publish_details true on taxonomy list query params', () => { + const qs = (exportTaxonomies as any).qs; + expect(qs.include_publish_details).to.equal(true); + expect(qs.include_count).to.equal(true); + }); + + it('should pass include_publish_details true to taxonomy().query from fetchTaxonomies', async () => { + const querySpy = sinon.stub().returns({ + find: sinon.stub().resolves({ + items: [{ uid: 'taxonomy-1', name: 'Cat' }], + count: 1, + }), + }); + mockStackClient.taxonomy.returns({ query: querySpy }); + + await exportTaxonomies.fetchTaxonomies(); + + expect(querySpy.called).to.be.true; + expect(querySpy.firstCall.args[0]).to.have.property('include_publish_details', true); + }); + + it('should pass include_publish_details on every pagination request', async () => { + let callCount = 0; + const querySpy = sinon.stub().returns({ + find: sinon.stub().callsFake(() => { + callCount++; + if (callCount === 1) { + return Promise.resolve({ + items: Array(100).fill(null).map((_, i) => ({ uid: `taxonomy-${i}`, name: 'Test' })), + count: 150, + }); + } + return Promise.resolve({ + items: Array(50).fill(null).map((_, i) => ({ uid: `taxonomy-${100 + i}`, name: 'Test' })), + count: 150, + }); + }), + }); + mockStackClient.taxonomy.returns({ query: querySpy }); + + await exportTaxonomies.fetchTaxonomies(); + + expect(callCount).to.be.greaterThan(1); + querySpy.getCalls().forEach((call) => { + expect(call.args[0]).to.have.property('include_publish_details', true); + }); + }); + + it('should pass include_publish_details when fetching with locale code', async () => { + const querySpy = sinon.stub().returns({ + find: sinon.stub().resolves({ + items: [{ uid: 'taxonomy-1', name: 'Cat', locale: 'en-us' }], + count: 1, + }), + }); + mockStackClient.taxonomy.returns({ query: querySpy }); + + await exportTaxonomies.fetchTaxonomies('fr-fr'); + + expect(querySpy.called).to.be.true; + expect(querySpy.firstCall.args[0]).to.include({ + include_publish_details: true, + locale: 'fr-fr', + }); + }); + + it('should include include_publish_details on initializeExport count query', async () => { + const querySpy = sinon.stub().returns({ + find: sinon.stub().resolves({ items: [{ uid: 'taxonomy-1', name: 'Cat' }], count: 1 }), + }); + mockStackClient.taxonomy.returns({ query: querySpy }); + + const stubDetermine = sinon.stub(exportTaxonomies as any, 'determineExportStrategy').resolves(); + const stubFetchAll = sinon.stub(exportTaxonomies as any, 'fetchAllTaxonomies').resolves(); + const stubExportAll = sinon.stub(exportTaxonomies as any, 'exportAllTaxonomies').resolves(1); + const stubWriteMeta = sinon.stub(exportTaxonomies, 'writeTaxonomiesMetadata').resolves(); + const stubGetLocales = sinon.stub(exportTaxonomies, 'getLocalesToExport').returns(['en-us']); + + await exportTaxonomies.start(); + + expect(querySpy.called).to.be.true; + expect(querySpy.firstCall.args[0]).to.have.property('include_publish_details', true); + expect(querySpy.firstCall.args[0]).to.have.property('limit', 1); + + stubDetermine.restore(); + stubFetchAll.restore(); + stubExportAll.restore(); + stubWriteMeta.restore(); + stubGetLocales.restore(); + }); + + it('should merge exportConfig query.modules.taxonomies without removing root include_publish_details', () => { + const cfg = { + ...mockExportConfig, + query: { + modules: { + taxonomies: { branch: 'main' }, + }, + }, + } as any; + const instance = new ExportTaxonomies({ + exportConfig: cfg, + stackAPIClient: mockStackClient, + moduleName: 'taxonomies', + }); + const qs = (instance as any).qs; + expect(qs.include_publish_details).to.equal(true); + expect(qs.query).to.deep.equal({ branch: 'main' }); + }); + }); + describe('Constructor', () => { it('should initialize with correct parameters', () => { expect(exportTaxonomies).to.be.instanceOf(ExportTaxonomies); @@ -323,6 +442,21 @@ describe('ExportTaxonomies', () => { expect(Object.keys(exportTaxonomies.taxonomies).length).to.equal(0); }); + it('should retain publish_details when not listed in invalidKeys', () => { + const publishDetails = [{ environment: 'bltEnv1', time: '2026-04-21T12:00:00.000Z', user: 'bltUser1' }]; + const taxonomies = [ + { + uid: 'taxonomy-pub', + name: 'Pub', + publish_details: publishDetails, + }, + ]; + + exportTaxonomies.sanitizeTaxonomiesAttribs(taxonomies); + + expect(exportTaxonomies.taxonomies['taxonomy-pub'].publish_details).to.deep.equal(publishDetails); + }); + // const taxonomies = [ // { uid: 'taxonomy-1', name: 'Category' }, // { uid: 'taxonomy-2', name: 'Tag' } From 5925b9468316b7eedd71ee735cbb497ab4b4f79f Mon Sep 17 00:00:00 2001 From: raj pandey Date: Mon, 27 Apr 2026 11:05:58 +0530 Subject: [PATCH 02/61] enhc: Added taxonomy publish details for all taxonomy --- .../src/export/modules/taxonomies.ts | 74 +++++++++- .../unit/export/modules/taxonomies.test.ts | 132 ++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index 27be33d4c..fc50966ff 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -1,3 +1,4 @@ +import cloneDeep from 'lodash/cloneDeep'; import omit from 'lodash/omit'; import keys from 'lodash/keys'; import isEmpty from 'lodash/isEmpty'; @@ -16,8 +17,12 @@ import { import { ModuleClassParams, ExportConfig } from '../../types'; export default class ExportTaxonomies extends BaseClass { + private static readonly PUBLISH_DETAILS_DEFAULT_LOCALE = '_default'; + private taxonomies: Record>; private taxonomiesByLocale: Record>; + /** List API `publish_details` keyed by non-localized bucket or locale code, then taxonomy uid */ + private publishDetailsByLocale: Record>; private taxonomiesConfig: ExportConfig['modules']['taxonomies']; private isLocaleBasedExportSupported: boolean = true; // Flag to track if locale-based export is supported private qs: { @@ -39,6 +44,7 @@ export default class ExportTaxonomies extends BaseClass { super({ exportConfig, stackAPIClient }); this.taxonomies = {}; this.taxonomiesByLocale = {}; + this.publishDetailsByLocale = {}; this.taxonomiesConfig = exportConfig.modules.taxonomies; this.qs = { include_count: true, @@ -143,6 +149,7 @@ export default class ExportTaxonomies extends BaseClass { log.debug('Falling back to legacy export (non-localized)', this.exportConfig.context); this.taxonomies = {}; this.taxonomiesByLocale = {}; + this.publishDetailsByLocale = {}; } else { log.debug('Localization enabled, proceeding with locale-based export', this.exportConfig.context); } @@ -330,11 +337,21 @@ export default class ExportTaxonomies extends BaseClass { log.debug(`Processing ${taxonomies.length} taxonomies${localeInfo}`, this.exportConfig.context); for (const taxonomy of taxonomies) { + const taxonomyRow = taxonomy as Record; const taxonomyUID = taxonomy.uid; const taxonomyName = taxonomy.name; log.debug(`Processing taxonomy: ${taxonomyName} (${taxonomyUID})${localeInfo}`, this.exportConfig.context); + // Store list API publish_details for merge into per-uid export files (per locale or default bucket) + if (taxonomyRow.publish_details != null) { + const bucket = localeCode ?? ExportTaxonomies.PUBLISH_DETAILS_DEFAULT_LOCALE; + if (!this.publishDetailsByLocale[bucket]) { + this.publishDetailsByLocale[bucket] = {}; + } + this.publishDetailsByLocale[bucket][taxonomyUID] = taxonomyRow.publish_details; + } + // Store taxonomy metadata (only once per taxonomy) if (!this.taxonomies[taxonomyUID]) { this.taxonomies[taxonomyUID] = omit(taxonomy, this.taxonomiesConfig.invalidKeys); @@ -374,8 +391,9 @@ export default class ExportTaxonomies extends BaseClass { const onSuccess = ({ response, uid }: any) => { const taxonomyName = this.taxonomies[uid]?.name; const filePath = pResolve(exportFolderPath, `${uid}.json`); + const merged = this.mergeListPublishDetailsIntoExportPayload(response, uid, localeCode); log.debug(`Writing detailed taxonomy data to: ${filePath}`, this.exportConfig.context); - fsUtil.writeFile(filePath, response); + fsUtil.writeFile(filePath, merged); // Track progress for each exported taxonomy this.progressManager?.tick( @@ -463,6 +481,60 @@ export default class ExportTaxonomies extends BaseClass { return localesToExport; } + /** + * List `find` may include `publish_details` while `export` may not; we copy list data into the + * written file when export omits or has an empty `taxonomy.publish_details`. + */ + private getListPublishDetailsForExport(taxonomyUid: string, localeCode?: string): unknown | undefined { + const bucket = localeCode ?? ExportTaxonomies.PUBLISH_DETAILS_DEFAULT_LOCALE; + return this.publishDetailsByLocale[bucket]?.[taxonomyUid]; + } + + private isPublishDetailsValueEmpty(publishDetails: unknown): boolean { + if (publishDetails == null) { + return true; + } + if (Array.isArray(publishDetails)) { + return publishDetails.length === 0; + } + if (typeof publishDetails === 'object') { + return Object.keys(publishDetails as object).length === 0; + } + return false; + } + + private mergeListPublishDetailsIntoExportPayload( + response: any, + taxonomyUid: string, + localeCode?: string, + ): any { + const fromList = this.getListPublishDetailsForExport(taxonomyUid, localeCode); + if (fromList == null) { + return response; + } + + const merged = cloneDeep(response); + const applyToTaxonomyObject = (tax: Record | undefined | null) => { + if (!tax || typeof tax !== 'object') { + return; + } + if (this.isPublishDetailsValueEmpty(tax.publish_details)) { + tax.publish_details = fromList; + } + }; + + if (merged && typeof merged === 'object' && 'taxonomy' in merged && (merged as any).taxonomy) { + applyToTaxonomyObject((merged as any).taxonomy); + return merged; + } + + log.debug( + 'Taxonomy export response has no taxonomy object; skipping publish_details merge from list', + this.exportConfig.context, + ); + return merged; + } + private isLocalePlanLimitationError(error: any): boolean { return ( error?.status === 403 && diff --git a/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts b/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts index 276e0281e..b946d4649 100644 --- a/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts @@ -826,4 +826,136 @@ describe('ExportTaxonomies', () => { mockFetchTaxonomies.restore(); }); }); + + describe('Detail file: merge list publish_details into export payload', () => { + it('should write merged file with list publish_details when export response omits them (legacy)', async () => { + const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub; + const listPublish = [{ environment: 'bltEnv1', locale: 'en-us' }]; + + exportTaxonomies.taxonomies = { 'tax-1': { uid: 'tax-1', name: 'T' } }; + exportTaxonomies.publishDetailsByLocale = { + _default: { 'tax-1': listPublish }, + } as any; + exportTaxonomies.taxonomiesFolderPath = '/test/export/taxonomies'; + + const makeAPICallStub = sinon.stub(exportTaxonomies, 'makeAPICall').callsFake((opts: any) => { + return Promise.resolve( + opts.resolve({ + response: { taxonomy: { uid: 'tax-1', name: 'T' }, terms: {} }, + uid: 'tax-1', + }), + ); + }); + + await exportTaxonomies.exportTaxonomies(); + + const detailWrite = writeFileStub + .getCalls() + .find((c) => typeof c.args[0] === 'string' && c.args[0].endsWith('tax-1.json')); + expect(detailWrite, 'writeFile for tax-1.json').to.exist; + const payload = detailWrite!.args[1]; + expect(payload.taxonomy.publish_details).to.deep.equal(listPublish); + + makeAPICallStub.restore(); + }); + + it('should prefer export taxonomy.publish_details when already present and non-empty', async () => { + const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub; + const fromExport = [{ environment: 'from-export' }]; + + exportTaxonomies.taxonomies = { 'tax-1': { uid: 'tax-1' } }; + exportTaxonomies.publishDetailsByLocale = { + _default: { 'tax-1': [{ from: 'list' }] }, + } as any; + exportTaxonomies.taxonomiesFolderPath = '/test/export/taxonomies'; + + const makeAPICallStub = sinon.stub(exportTaxonomies, 'makeAPICall').callsFake((opts: any) => { + return Promise.resolve( + opts.resolve({ + response: { taxonomy: { uid: 'tax-1', publish_details: fromExport }, terms: {} }, + uid: 'tax-1', + }), + ); + }); + + await exportTaxonomies.exportTaxonomies(); + + const detailWrite = writeFileStub + .getCalls() + .find((c) => typeof c.args[0] === 'string' && c.args[0].endsWith('tax-1.json')); + const payload = detailWrite!.args[1]; + expect(payload.taxonomy.publish_details).to.deep.equal(fromExport); + + makeAPICallStub.restore(); + }); + + it('should fill from list when export has empty array publish_details', async () => { + const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub; + const fromList = [{ from: 'list' }]; + + exportTaxonomies.taxonomies = { 'tax-1': { uid: 'tax-1' } }; + exportTaxonomies.publishDetailsByLocale = { _default: { 'tax-1': fromList } } as any; + exportTaxonomies.taxonomiesFolderPath = '/test/export/taxonomies'; + + const makeAPICallStub = sinon.stub(exportTaxonomies, 'makeAPICall').callsFake((opts: any) => { + return Promise.resolve( + opts.resolve({ + response: { taxonomy: { uid: 'tax-1', publish_details: [] }, terms: {} }, + uid: 'tax-1', + }), + ); + }); + + await exportTaxonomies.exportTaxonomies(); + + const detailWrite = writeFileStub + .getCalls() + .find((c) => typeof c.args[0] === 'string' && c.args[0].endsWith('tax-1.json')); + expect(detailWrite!.args[1].taxonomy.publish_details).to.deep.equal(fromList); + + makeAPICallStub.restore(); + }); + + it('should use per-locale list publish_details when exporting locale folder', async () => { + const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub; + const frPublish = [{ locale: 'fr-fr' }]; + + exportTaxonomies.taxonomies = { 'tax-1': { uid: 'tax-1' } }; + exportTaxonomies.taxonomiesByLocale['fr-fr'] = new Set(['tax-1']); + exportTaxonomies.publishDetailsByLocale = { 'fr-fr': { 'tax-1': frPublish } } as any; + exportTaxonomies.taxonomiesFolderPath = '/test/export/taxonomies'; + + const makeAPICallStub = sinon.stub(exportTaxonomies, 'makeAPICall').callsFake((opts: any) => { + return Promise.resolve( + opts.resolve({ + response: { taxonomy: { uid: 'tax-1' }, terms: {} }, + uid: 'tax-1', + }), + ); + }); + + await exportTaxonomies.exportTaxonomies('fr-fr'); + + const detailWrite = writeFileStub + .getCalls() + .find((c) => typeof c.args[0] === 'string' && c.args[0].includes('fr-fr') && c.args[0].endsWith('tax-1.json')); + expect(detailWrite, 'fr-fr/tax-1.json').to.exist; + expect(detailWrite!.args[1].taxonomy.publish_details).to.deep.equal(frPublish); + + makeAPICallStub.restore(); + }); + + it('should store publish_details per locale bucket in sanitizeTaxonomiesAttribs', () => { + const listPd = [{ env: 'a' }]; + const taxonomies = [ + { uid: 't-loc', name: 'L', publish_details: listPd }, + ]; + exportTaxonomies.publishDetailsByLocale = {}; + exportTaxonomies.taxonomiesByLocale['es-es'] = new Set(); + + exportTaxonomies.sanitizeTaxonomiesAttribs(taxonomies, 'es-es'); + + expect((exportTaxonomies as any).publishDetailsByLocale['es-es']['t-loc']).to.deep.equal(listPd); + }); + }); }); From 68c32c7f6faa23c047bdac5aed75cf0c2e50c303 Mon Sep 17 00:00:00 2001 From: raj pandey Date: Sat, 2 May 2026 23:45:55 +0530 Subject: [PATCH 03/61] feat(import): publish taxonomies after import --- .../src/import/modules/base-class.ts | 11 + .../src/import/modules/taxonomies.ts | 261 ++++++++++++++++-- .../src/utils/constants.ts | 5 + .../unit/import/modules/base-class.test.ts | 54 ++++ .../unit/import/modules/taxonomies.test.ts | 141 ++++++++-- 5 files changed, 433 insertions(+), 39 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 55f8067e3..286485139 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -58,6 +58,7 @@ export type ApiModuleType = | 'create-entries' | 'update-entries' | 'publish-entries' + | 'publish-taxonomies' | 'delete-entries' | 'create-taxonomies' | 'create-terms' @@ -343,6 +344,8 @@ export default abstract class BaseClass { if ( !apiData || (entity === 'publish-entries' && !apiData.entryUid) || + (entity === 'publish-taxonomies' && + (!apiData.environments?.length || !apiData.locales?.length || !apiData.items?.length)) || (entity === 'update-extensions' && !apiData.uid) ) { return Promise.resolve(); @@ -489,6 +492,14 @@ export default abstract class BaseClass { }) .then(onSuccess) .catch(onReject); + case 'publish-taxonomies': { + const publishParams = this.importConfig.branchName ? { branch: this.importConfig.branchName } : {}; + return (this.stack as any) + .taxonomy() + .publish(apiData, '3.2', publishParams) + .then(onSuccess) + .catch(onReject); + } case 'delete-entries': return this.stack .contentType(apiData.cTUid) diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index bd9b1a87c..c57462a6c 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -1,7 +1,7 @@ import { join } from 'node:path'; import values from 'lodash/values'; import isEmpty from 'lodash/isEmpty'; -import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { log, handleAndLogError, CLIProgressManager } from '@contentstack/cli-utilities'; import { PATH_CONSTANTS } from '../../constants'; import BaseClass, { ApiOptions } from './base-class'; @@ -19,6 +19,8 @@ export default class ImportTaxonomies extends BaseClass { private termsSuccessPath: string; private termsFailsPath: string; private localesFilePath: string; + private envUidMapperPath: string; + private envUidMapper: Record = {}; private isLocaleBasedStructure: boolean = false; public createdTaxonomies: Record = {}; public failedTaxonomies: Record = {}; @@ -46,8 +48,16 @@ export default class ImportTaxonomies extends BaseClass { importConfig.modules.locales.dirName, importConfig.modules.locales.fileName, ); + this.envUidMapperPath = join( + importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ENVIRONMENTS, + PATH_CONSTANTS.FILES.UID_MAPPING, + ); } + // --- Lifecycle --- + /** * @method start * @returns {Promise} Promise @@ -56,7 +66,7 @@ export default class ImportTaxonomies extends BaseClass { try { log.debug('Starting taxonomies import process...', this.importConfig.context); - const [taxonomiesCount] = await this.analyzeTaxonomies(); + const [taxonomiesCount, publishJobCount] = await this.analyzeTaxonomies(); if (taxonomiesCount === 0) { log.info('No taxonomies found to import', this.importConfig.context); return; @@ -67,8 +77,12 @@ export default class ImportTaxonomies extends BaseClass { // Check if locale-based structure exists before import this.isLocaleBasedStructure = this.detectAndScanLocaleStructure(); - const progress = this.createSimpleProgress(this.currentModuleName, taxonomiesCount); - progress.updateStatus(PROCESS_STATUS[PROCESS_NAMES.TAXONOMIES_IMPORT].IMPORTING); + const progress = this.createNestedProgress(this.currentModuleName); + this.initializeTaxonomiesProgress(progress, taxonomiesCount, publishJobCount); + + progress + .startProcess(PROCESS_NAMES.TAXONOMIES_IMPORT) + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.TAXONOMIES_IMPORT].IMPORTING, PROCESS_NAMES.TAXONOMIES_IMPORT); log.debug('Starting taxonomies import', this.importConfig.context); if (this.isLocaleBasedStructure) { @@ -79,6 +93,19 @@ export default class ImportTaxonomies extends BaseClass { await this.importTaxonomiesLegacy(); } + progress.completeProcess(PROCESS_NAMES.TAXONOMIES_IMPORT, true); + + if (publishJobCount > 0) { + progress + .startProcess(PROCESS_NAMES.TAXONOMIES_PUBLISH) + .updateStatus( + PROCESS_STATUS[PROCESS_NAMES.TAXONOMIES_PUBLISH].PUBLISHING, + PROCESS_NAMES.TAXONOMIES_PUBLISH, + ); + await this.processTaxonomyPublishing(); + progress.completeProcess(PROCESS_NAMES.TAXONOMIES_PUBLISH, true); + } + this.createSuccessAndFailedFile(); this.completeProgressWithMessage(); } catch (error) { @@ -87,6 +114,8 @@ export default class ImportTaxonomies extends BaseClass { } } + // --- Import --- + /** * create taxonomy and enter success & failure related data into taxonomies mapper file * @method importTaxonomies @@ -344,6 +373,191 @@ export default class ImportTaxonomies extends BaseClass { return true; } + // --- Progress --- + + /** + * Registers nested progress for taxonomy import and optional taxonomy publish when publish jobs exist. + */ + initializeTaxonomiesProgress(progress: CLIProgressManager, taxonomyCount: number, publishJobCount: number): void { + progress.addProcess(PROCESS_NAMES.TAXONOMIES_IMPORT, taxonomyCount); + if (publishJobCount > 0) { + progress.addProcess(PROCESS_NAMES.TAXONOMIES_PUBLISH, publishJobCount); + } + } + + // --- Publish --- + + /** + * Reads source env UID → destination stack env UID map produced during environments import. + */ + private readEnvUidMapperSync(): Record { + if (!fileHelper.fileExistsSync(this.envUidMapperPath)) { + log.debug(`Environment UID mapper not found at ${this.envUidMapperPath}`, this.importConfig.context); + return {}; + } + + try { + const raw = fsUtil.readFile(this.envUidMapperPath, true) as Record; + const out: Record = {}; + for (const [k, v] of Object.entries(raw || {})) { + if (v !== undefined && v !== null && String(v).trim() !== '') { + out[k] = String(v); + } + } + return out; + } catch { + log.debug('Failed to read environment UID mapper', this.importConfig.context); + return {}; + } + } + + private countPublishEligibleTaxonomies(envMapper: Record): number { + let count = 0; + for (const key of Object.keys(this.taxonomies || {})) { + const meta = this.taxonomies[key] as Record; + const taxonomyUid = meta?.uid || key; + const filePath = this.findTaxonomyFilePath(taxonomyUid); + if (!filePath) continue; + + const details = this.loadTaxonomyFile(filePath); + const tax = details?.taxonomy as Record | undefined; + if (!tax?.publish_details?.length || !tax?.locale) continue; + + const hasMapped = (tax.publish_details as any[]).some( + (p: any) => p?.environment && envMapper[String(p.environment)], + ); + if (hasMapped) count++; + } + return count; + } + + private collectTaxonomyPublishJobs(): Array<{ taxonomy: Record }> { + const jobs: Array<{ taxonomy: Record }> = []; + const seen = new Set(); + + for (const key of Object.keys(this.taxonomies || {})) { + const meta = this.taxonomies[key] as Record; + const taxonomyUid = meta?.uid || key; + if (seen.has(taxonomyUid)) continue; + + const filePath = this.findTaxonomyFilePath(taxonomyUid); + if (!filePath) continue; + + const details = this.loadTaxonomyFile(filePath); + const tax = details?.taxonomy as Record | undefined; + if (!tax?.publish_details?.length || !tax?.locale) continue; + + seen.add(taxonomyUid); + jobs.push({ taxonomy: tax }); + } + + return jobs; + } + + private loadEnvUidMapper(): void { + this.envUidMapper = this.readEnvUidMapperSync(); + if (isEmpty(this.envUidMapper)) { + log.warn( + 'Environment UID mapper is empty; taxonomy publishing is skipped. Import environments first or ensure mapper/environments/uid-mapping.json exists.', + this.importConfig.context, + ); + } + } + + async processTaxonomyPublishing(): Promise { + this.loadEnvUidMapper(); + const jobs = this.collectTaxonomyPublishJobs(); + + if (jobs.length === 0) { + log.debug('No taxonomies with publish_details to publish', this.importConfig.context); + return; + } + + log.info('Starting taxonomy publishing process', this.importConfig.context); + + const onSuccess = ({ apiData }: any) => { + const taxonomyUid = apiData?.items?.[0]?.uid; + this.progressManager?.tick( + true, + `taxonomy published: ${taxonomyUid}`, + null, + PROCESS_NAMES.TAXONOMIES_PUBLISH, + ); + log.success(`Published taxonomy '${taxonomyUid}'`, this.importConfig.context); + }; + + const onReject = ({ error, apiData }: any) => { + const taxonomyUid = apiData?.items?.[0]?.uid; + handleAndLogError( + error, + { ...this.importConfig.context, taxonomyUid }, + `Failed to publish taxonomy '${taxonomyUid}'`, + ); + this.progressManager?.tick( + false, + `taxonomy publish: ${taxonomyUid}`, + (error as Error)?.message || `Failed to publish taxonomy '${taxonomyUid}'`, + PROCESS_NAMES.TAXONOMIES_PUBLISH, + ); + }; + + await this.makeConcurrentCall( + { + apiContent: jobs as unknown as Record[], + processName: 'publish taxonomies', + apiParams: { + serializeData: this.serializePublishTaxonomies.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'publish-taxonomies', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, + }, + undefined, + false, + ); + } + + /** + * Builds taxonomy publish payload: destination env UIDs from mapper, locales from taxonomy.locale, items: [{ uid }]. + */ + serializePublishTaxonomies(apiOptions: ApiOptions): ApiOptions { + const job = apiOptions.apiData as { taxonomy?: Record }; + const taxonomy = job?.taxonomy; + + if (!taxonomy?.publish_details?.length || !taxonomy?.locale) { + apiOptions.apiData = undefined; + return apiOptions; + } + + const environments: string[] = []; + for (const pub of taxonomy.publish_details as any[]) { + const sourceEnvUid = pub?.environment; + if (!sourceEnvUid) continue; + const destUid = this.envUidMapper[String(sourceEnvUid)]; + if (destUid && !environments.includes(destUid)) { + environments.push(destUid); + } + } + + if (environments.length === 0) { + apiOptions.apiData = undefined; + return apiOptions; + } + + const locales = [String(taxonomy.locale)]; + apiOptions.apiData = { + environments, + locales, + items: [{ uid: taxonomy.uid }], + }; + + return apiOptions; + } + + // --- Mapper output --- + /** * create taxonomies success and fail in (mapper/taxonomies) * create terms success and fail in (mapper/taxonomies/terms) @@ -396,25 +610,36 @@ export default class ImportTaxonomies extends BaseClass { } } - private async analyzeTaxonomies(): Promise<[number]> { + // --- Analyze & prepare --- + + private async analyzeTaxonomies(): Promise<[number, number]> { return this.withLoadingSpinner('TAXONOMIES: Analyzing import data...', async () => { log.debug('Checking for taxonomies folder existence', this.importConfig.context); - if (fileHelper.fileExistsSync(this.taxonomiesFolderPath)) { - log.debug(`Found taxonomies folder: ${this.taxonomiesFolderPath}`, this.importConfig.context); - - this.taxonomies = fsUtil.readFile(join(this.taxonomiesFolderPath, 'taxonomies.json'), true) as Record< - string, - unknown - >; - - const taxonomyCount = Object.keys(this.taxonomies || {}).length; - log.debug(`Loaded ${taxonomyCount} taxonomy items from file`, this.importConfig.context); - return [taxonomyCount]; - } else { + if (!fileHelper.fileExistsSync(this.taxonomiesFolderPath)) { log.info(`No Taxonomies Found! - '${this.taxonomiesFolderPath}'`, this.importConfig.context); - return [0]; + return [0, 0]; } + + log.debug(`Found taxonomies folder: ${this.taxonomiesFolderPath}`, this.importConfig.context); + + this.taxonomies = fsUtil.readFile(join(this.taxonomiesFolderPath, 'taxonomies.json'), true) as Record< + string, + unknown + >; + + this.isLocaleBasedStructure = this.detectAndScanLocaleStructure(); + + const taxonomyCount = Object.keys(this.taxonomies || {}).length; + const envMapper = this.readEnvUidMapperSync(); + const publishJobCount = this.countPublishEligibleTaxonomies(envMapper); + + log.debug( + `Loaded ${taxonomyCount} taxonomy items; ${publishJobCount} eligible for publish (mapped environments).`, + this.importConfig.context, + ); + + return [taxonomyCount, publishJobCount]; }); } diff --git a/packages/contentstack-import/src/utils/constants.ts b/packages/contentstack-import/src/utils/constants.ts index c751bcfc8..80e36f7b5 100644 --- a/packages/contentstack-import/src/utils/constants.ts +++ b/packages/contentstack-import/src/utils/constants.ts @@ -57,6 +57,7 @@ export const PROCESS_NAMES = { CONTENT_TYPES_EXT_UPDATE: 'Content Types Ext Update', WEBHOOKS_IMPORT: 'Webhooks Import', TAXONOMIES_IMPORT: 'Taxonomies Import', + TAXONOMIES_PUBLISH: 'Taxonomies Publish', PERSONALIZE_PROJECTS: 'Projects', } as const; @@ -267,6 +268,10 @@ export const PROCESS_STATUS = { IMPORTING: 'Importing taxonomies...', FAILED: 'Failed to import taxonomies.', }, + [PROCESS_NAMES.TAXONOMIES_PUBLISH]: { + PUBLISHING: 'Publishing taxonomies...', + FAILED: 'Failed to publish taxonomies.', + }, [PROCESS_NAMES.PERSONALIZE_PROJECTS]: { IMPORTING: 'Importing personalization projects...', FAILED: 'Failed to import personalization projects.', diff --git a/packages/contentstack-import/test/unit/import/modules/base-class.test.ts b/packages/contentstack-import/test/unit/import/modules/base-class.test.ts index 43ad5ac72..bc8ea9219 100644 --- a/packages/contentstack-import/test/unit/import/modules/base-class.test.ts +++ b/packages/contentstack-import/test/unit/import/modules/base-class.test.ts @@ -74,6 +74,7 @@ describe('BaseClass', () => { create: sinon.stub().resolves({ uid: 'term-123' }), }), import: sinon.stub().resolves({ uid: 'import-123' }), + publish: sinon.stub().resolves({ notice: 'queued' }), }), globalField: sinon.stub().returns({ create: sinon.stub().resolves({ uid: 'gf-123' }), @@ -858,6 +859,59 @@ describe('BaseClass', () => { expect(result).to.be.undefined; expect(mockStackClient.taxonomy().import.called).to.be.false; }); + + it('should handle publish-taxonomies with api version 3.2', async () => { + const payload = { + environments: ['blt-env-dest'], + locales: ['en-us'], + items: [{ uid: 'tax-uid' }], + }; + mockApiOptions.entity = 'publish-taxonomies' as any; + mockApiOptions.apiData = payload; + + await testClass.makeAPICall(mockApiOptions); + + expect(mockStackClient.taxonomy().publish.calledOnce).to.be.true; + expect(mockStackClient.taxonomy().publish.firstCall.args[0]).to.deep.equal(payload); + expect(mockStackClient.taxonomy().publish.firstCall.args[1]).to.equal('3.2'); + expect(mockStackClient.taxonomy().publish.firstCall.args[2]).to.deep.equal({}); + expect(mockApiOptions.resolve.calledOnce).to.be.true; + }); + + it('should pass branch to publish-taxonomies when branchName is set', async () => { + const payload = { + environments: ['e1'], + locales: ['en-us'], + items: [{ uid: 't1' }], + }; + mockImportConfig.branchName = 'main-branch'; + testClass = new TestBaseClass({ + importConfig: mockImportConfig, + stackAPIClient: mockStackClient, + }); + + mockApiOptions.entity = 'publish-taxonomies' as any; + mockApiOptions.apiData = payload; + + await testClass.makeAPICall(mockApiOptions); + + expect(mockStackClient.taxonomy().publish.firstCall.args[2]).to.deep.equal({ branch: 'main-branch' }); + delete mockImportConfig.branchName; + }); + + it('should skip publish-taxonomies when environments empty', async () => { + mockApiOptions.entity = 'publish-taxonomies' as any; + mockApiOptions.apiData = { + environments: [], + locales: ['en-us'], + items: [{ uid: 't' }], + }; + + await testClass.makeAPICall(mockApiOptions); + + expect(mockStackClient.taxonomy().publish.called).to.be.false; + expect(mockApiOptions.resolve.called).to.be.false; + }); }); }); diff --git a/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts b/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts index 4ef453fd9..6744a4f85 100644 --- a/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts +++ b/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts @@ -5,6 +5,16 @@ import values from 'lodash/values'; import ImportTaxonomies from '../../../../src/import/modules/taxonomies'; import { fsUtil, fileHelper } from '../../../../src/utils'; +function nestedProgressMock(sb: sinon.SinonSandbox) { + return { + addProcess: sb.stub().returnsThis(), + startProcess: sb.stub().returnsThis(), + updateStatus: sb.stub().returnsThis(), + completeProcess: sb.stub().returnsThis(), + getFailureCount: sb.stub().returns(0), + }; +} + describe('ImportTaxonomies', () => { let importTaxonomies: ImportTaxonomies; let mockStackClient: any; @@ -67,11 +77,8 @@ describe('ImportTaxonomies', () => { sandbox.stub(importTaxonomies as any, 'withLoadingSpinner').callsFake(async (msg: string, fn: () => Promise) => { return await fn(); }); - sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([1]); - const mockProgress = { - updateStatus: sandbox.stub() - }; - sandbox.stub(importTaxonomies as any, 'createSimpleProgress').returns(mockProgress); + sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([1, 0]); + sandbox.stub(importTaxonomies as any, 'createNestedProgress').returns(nestedProgressMock(sandbox)); sandbox.stub(importTaxonomies as any, 'prepareMapperDirectories').resolves(); sandbox.stub(importTaxonomies as any, 'createSuccessAndFailedFile').resolves(); sandbox.stub(importTaxonomies as any, 'completeProgress').resolves(); @@ -104,6 +111,9 @@ describe('ImportTaxonomies', () => { expect((importTaxonomies as any).localesFilePath).to.equal( join(testBackupDir, 'locales', 'locales.json'), ); + expect((importTaxonomies as any).envUidMapperPath).to.equal( + join(testBackupDir, 'mapper', 'environments', 'uid-mapping.json'), + ); }); it('should set context module to taxonomies', () => { @@ -135,9 +145,7 @@ describe('ImportTaxonomies', () => { sandbox.stub(importTaxonomies as any, 'withLoadingSpinner').callsFake(async (msg: string, fn: () => Promise) => { return await fn(); }); - sandbox.stub(importTaxonomies as any, 'createSimpleProgress').returns({ - updateStatus: sinon.stub() - }); + sandbox.stub(importTaxonomies as any, 'createNestedProgress').returns(nestedProgressMock(sandbox)); const prepareMapperDirectoriesStub = sandbox.stub(importTaxonomies as any, 'prepareMapperDirectories').resolves(); const importTaxonomiesStub = sandbox.stub(importTaxonomies as any, 'importTaxonomies').resolves(); sandbox.stub(importTaxonomies as any, 'createSuccessAndFailedFile').resolves(); @@ -189,7 +197,7 @@ describe('ImportTaxonomies', () => { sandbox.stub(importTaxonomies as any, 'withLoadingSpinner').callsFake(async (msg: string, fn: () => Promise) => { return await fn(); }); - const analyzeTaxonomiesStub = sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([0]); + const analyzeTaxonomiesStub = sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([0, 0]); sandbox.stub(importTaxonomies as any, 'completeProgress').resolves(); await importTaxonomies.start(); @@ -211,10 +219,8 @@ describe('ImportTaxonomies', () => { sandbox.stub(importTaxonomies as any, 'withLoadingSpinner').callsFake(async (msg: string, fn: () => Promise) => { return await fn(); }); - sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([0]); // 0 taxonomies - sandbox.stub(importTaxonomies as any, 'createSimpleProgress').returns({ - updateStatus: sinon.stub() - }); + sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([0, 0]); // 0 taxonomies + sandbox.stub(importTaxonomies as any, 'createNestedProgress').returns(nestedProgressMock(sandbox)); sandbox.stub(importTaxonomies as any, 'prepareMapperDirectories').resolves(); sandbox.stub(importTaxonomies as any, 'importTaxonomies').resolves(); sandbox.stub(importTaxonomies as any, 'createSuccessAndFailedFile').resolves(); @@ -241,10 +247,8 @@ describe('ImportTaxonomies', () => { sandbox.stub(importTaxonomies as any, 'withLoadingSpinner').callsFake(async (msg: string, fn: () => Promise) => { return await fn(); }); - sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([1]); // 1 taxonomy - sandbox.stub(importTaxonomies as any, 'createSimpleProgress').returns({ - updateStatus: sinon.stub() - }); + sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([1, 0]); // 1 taxonomy + sandbox.stub(importTaxonomies as any, 'createNestedProgress').returns(nestedProgressMock(sandbox)); sandbox.stub(importTaxonomies as any, 'prepareMapperDirectories').resolves(); sandbox.stub(importTaxonomies as any, 'importTaxonomies').callsFake(async () => { (importTaxonomies as any).createdTaxonomies = { 'taxonomy_1': { uid: 'taxonomy_1' } }; @@ -392,6 +396,103 @@ describe('ImportTaxonomies', () => { }); }); + describe('serializePublishTaxonomies', () => { + it('maps source environment UIDs to destination UIDs and sets items to uid only', () => { + (importTaxonomies as any).envUidMapper = { bltSrc: 'bltDest' }; + const apiOptions: any = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { + uid: 'tax1', + locale: 'en-us', + publish_details: [{ environment: 'bltSrc', time: '', user: '' }], + }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + const result = (importTaxonomies as any).serializePublishTaxonomies(apiOptions); + + expect(result.apiData).to.deep.equal({ + environments: ['bltDest'], + locales: ['en-us'], + items: [{ uid: 'tax1' }], + }); + }); + + it('dedupes multiple publish_details environments', () => { + (importTaxonomies as any).envUidMapper = { e1: 'd1', e2: 'd2' }; + const apiOptions: any = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { + uid: 'tax2', + locale: 'fr-fr', + publish_details: [{ environment: 'e1' }, { environment: 'e2' }, { environment: 'e1' }], + }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + const result = (importTaxonomies as any).serializePublishTaxonomies(apiOptions); + + expect(result.apiData.environments).to.deep.equal(['d1', 'd2']); + expect(result.apiData.locales).to.deep.equal(['fr-fr']); + expect(result.apiData.items).to.deep.equal([{ uid: 'tax2' }]); + }); + + it('returns undefined when publish_details empty', () => { + (importTaxonomies as any).envUidMapper = { x: 'y' }; + const apiOptions: any = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { uid: 't', locale: 'en-us', publish_details: [] }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + expect((importTaxonomies as any).serializePublishTaxonomies(apiOptions).apiData).to.be.undefined; + }); + + it('returns undefined when no env mapping resolves', () => { + (importTaxonomies as any).envUidMapper = {}; + const apiOptions: any = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { + uid: 'tax1', + locale: 'en-us', + publish_details: [{ environment: 'missing' }], + }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + expect((importTaxonomies as any).serializePublishTaxonomies(apiOptions).apiData).to.be.undefined; + }); + + it('returns undefined when taxonomy.locale missing', () => { + (importTaxonomies as any).envUidMapper = { e: 'd' }; + const apiOptions: any = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { + uid: 'tax1', + publish_details: [{ environment: 'e' }], + }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + expect((importTaxonomies as any).serializePublishTaxonomies(apiOptions).apiData).to.be.undefined; + }); + }); + describe('createSuccessAndFailedFile', () => { it('should write all four files when data exists', () => { (importTaxonomies as any).createSuccessAndFailedFile.restore(); @@ -1123,10 +1224,8 @@ describe('ImportTaxonomies', () => { sandbox.stub(importTaxonomies as any, 'withLoadingSpinner').callsFake(async (msg: string, fn: () => Promise) => { return await fn(); }); - sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([1]); - sandbox.stub(importTaxonomies as any, 'createSimpleProgress').returns({ - updateStatus: sinon.stub() - }); + sandbox.stub(importTaxonomies as any, 'analyzeTaxonomies').resolves([1, 0]); + sandbox.stub(importTaxonomies as any, 'createNestedProgress').returns(nestedProgressMock(sandbox)); // Make prepareMapperDirectories reject with the error sandbox.stub(importTaxonomies as any, 'prepareMapperDirectories').rejects(new Error('Directory creation failed')); From 20a3e9c9f3a5bf3cb35eebd7b9feb3178bb820f9 Mon Sep 17 00:00:00 2001 From: raj pandey Date: Mon, 11 May 2026 14:48:18 +0530 Subject: [PATCH 04/61] refactored taxonomy publish import --- .../src/import/modules/taxonomies.ts | 91 +++----------- .../contentstack-import/src/utils/index.ts | 5 + .../src/utils/taxonomy-publish-utils.ts | 78 ++++++++++++ .../unit/import/modules/taxonomies.test.ts | 97 --------------- .../unit/utils/taxonomy-publish-utils.test.ts | 113 ++++++++++++++++++ 5 files changed, 211 insertions(+), 173 deletions(-) create mode 100644 packages/contentstack-import/src/utils/taxonomy-publish-utils.ts create mode 100644 packages/contentstack-import/test/unit/utils/taxonomy-publish-utils.test.ts diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index c57462a6c..402905647 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -5,7 +5,17 @@ import { log, handleAndLogError, CLIProgressManager } from '@contentstack/cli-ut import { PATH_CONSTANTS } from '../../constants'; import BaseClass, { ApiOptions } from './base-class'; -import { fsUtil, fileHelper, MODULE_CONTEXTS, MODULE_NAMES, PROCESS_STATUS, PROCESS_NAMES } from '../../utils'; +import { + fsUtil, + fileHelper, + MODULE_CONTEXTS, + MODULE_NAMES, + PROCESS_STATUS, + PROCESS_NAMES, + readEnvUidMapperSync, + warnIfEnvMapperEmpty, + serializePublishTaxonomies, +} from '../../utils'; import { ModuleClassParams, TaxonomiesConfig } from '../../types'; export default class ImportTaxonomies extends BaseClass { @@ -20,7 +30,6 @@ export default class ImportTaxonomies extends BaseClass { private termsFailsPath: string; private localesFilePath: string; private envUidMapperPath: string; - private envUidMapper: Record = {}; private isLocaleBasedStructure: boolean = false; public createdTaxonomies: Record = {}; public failedTaxonomies: Record = {}; @@ -387,30 +396,6 @@ export default class ImportTaxonomies extends BaseClass { // --- Publish --- - /** - * Reads source env UID → destination stack env UID map produced during environments import. - */ - private readEnvUidMapperSync(): Record { - if (!fileHelper.fileExistsSync(this.envUidMapperPath)) { - log.debug(`Environment UID mapper not found at ${this.envUidMapperPath}`, this.importConfig.context); - return {}; - } - - try { - const raw = fsUtil.readFile(this.envUidMapperPath, true) as Record; - const out: Record = {}; - for (const [k, v] of Object.entries(raw || {})) { - if (v !== undefined && v !== null && String(v).trim() !== '') { - out[k] = String(v); - } - } - return out; - } catch { - log.debug('Failed to read environment UID mapper', this.importConfig.context); - return {}; - } - } - private countPublishEligibleTaxonomies(envMapper: Record): number { let count = 0; for (const key of Object.keys(this.taxonomies || {})) { @@ -454,18 +439,9 @@ export default class ImportTaxonomies extends BaseClass { return jobs; } - private loadEnvUidMapper(): void { - this.envUidMapper = this.readEnvUidMapperSync(); - if (isEmpty(this.envUidMapper)) { - log.warn( - 'Environment UID mapper is empty; taxonomy publishing is skipped. Import environments first or ensure mapper/environments/uid-mapping.json exists.', - this.importConfig.context, - ); - } - } - async processTaxonomyPublishing(): Promise { - this.loadEnvUidMapper(); + const envUidMapper = readEnvUidMapperSync(this.envUidMapperPath, this.importConfig.context); + warnIfEnvMapperEmpty(envUidMapper, this.importConfig.context); const jobs = this.collectTaxonomyPublishJobs(); if (jobs.length === 0) { @@ -506,7 +482,7 @@ export default class ImportTaxonomies extends BaseClass { apiContent: jobs as unknown as Record[], processName: 'publish taxonomies', apiParams: { - serializeData: this.serializePublishTaxonomies.bind(this), + serializeData: (opts: ApiOptions) => serializePublishTaxonomies(opts, envUidMapper), reject: onReject, resolve: onSuccess, entity: 'publish-taxonomies', @@ -519,43 +495,6 @@ export default class ImportTaxonomies extends BaseClass { ); } - /** - * Builds taxonomy publish payload: destination env UIDs from mapper, locales from taxonomy.locale, items: [{ uid }]. - */ - serializePublishTaxonomies(apiOptions: ApiOptions): ApiOptions { - const job = apiOptions.apiData as { taxonomy?: Record }; - const taxonomy = job?.taxonomy; - - if (!taxonomy?.publish_details?.length || !taxonomy?.locale) { - apiOptions.apiData = undefined; - return apiOptions; - } - - const environments: string[] = []; - for (const pub of taxonomy.publish_details as any[]) { - const sourceEnvUid = pub?.environment; - if (!sourceEnvUid) continue; - const destUid = this.envUidMapper[String(sourceEnvUid)]; - if (destUid && !environments.includes(destUid)) { - environments.push(destUid); - } - } - - if (environments.length === 0) { - apiOptions.apiData = undefined; - return apiOptions; - } - - const locales = [String(taxonomy.locale)]; - apiOptions.apiData = { - environments, - locales, - items: [{ uid: taxonomy.uid }], - }; - - return apiOptions; - } - // --- Mapper output --- /** @@ -631,7 +570,7 @@ export default class ImportTaxonomies extends BaseClass { this.isLocaleBasedStructure = this.detectAndScanLocaleStructure(); const taxonomyCount = Object.keys(this.taxonomies || {}).length; - const envMapper = this.readEnvUidMapperSync(); + const envMapper = readEnvUidMapperSync(this.envUidMapperPath, this.importConfig.context); const publishJobCount = this.countPublishEligibleTaxonomies(envMapper); log.debug( diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index 5d64eb515..d32128232 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -31,4 +31,9 @@ export { } from './entries-helper'; export * from './common-helper'; export { lookUpTaxonomy, lookUpTerms } from './taxonomies-helper'; +export { + readEnvUidMapperSync, + warnIfEnvMapperEmpty, + serializePublishTaxonomies, +} from './taxonomy-publish-utils'; export { MODULE_CONTEXTS, MODULE_NAMES, PROCESS_NAMES, PROCESS_STATUS } from './constants'; diff --git a/packages/contentstack-import/src/utils/taxonomy-publish-utils.ts b/packages/contentstack-import/src/utils/taxonomy-publish-utils.ts new file mode 100644 index 000000000..4aa48ddca --- /dev/null +++ b/packages/contentstack-import/src/utils/taxonomy-publish-utils.ts @@ -0,0 +1,78 @@ +import isEmpty from 'lodash/isEmpty'; +import { log } from '@contentstack/cli-utilities'; +import { ApiOptions } from '../import/modules/base-class'; +import type { Context } from '../types'; +import { fsUtil, fileExistsSync } from './file-helper'; + +/** + * Reads source env UID → destination stack env UID map produced during environments import. + */ +export function readEnvUidMapperSync(envUidMapperPath: string, context: Context): Record { + if (!fileExistsSync(envUidMapperPath)) { + log.debug(`Environment UID mapper not found at ${envUidMapperPath}`, context); + return {}; + } + + try { + const raw = fsUtil.readFile(envUidMapperPath, true) as Record; + const out: Record = {}; + for (const [k, v] of Object.entries(raw || {})) { + if (v !== undefined && v !== null && String(v).trim() !== '') { + out[k] = String(v); + } + } + return out; + } catch { + log.debug('Failed to read environment UID mapper', context); + return {}; + } +} + +export function warnIfEnvMapperEmpty(envUidMapper: Record, context: Context): void { + if (isEmpty(envUidMapper)) { + log.warn( + 'Environment UID mapper is empty; taxonomy publishing is skipped. Import environments first or ensure mapper/environments/uid-mapping.json exists.', + context, + ); + } +} + +/** + * Builds taxonomy publish payload: destination env UIDs from mapper, locales from taxonomy.locale, items: [{ uid }]. + */ +export function serializePublishTaxonomies( + apiOptions: ApiOptions, + envUidMapper: Record, +): ApiOptions { + const job = apiOptions.apiData as { taxonomy?: Record }; + const taxonomy = job?.taxonomy; + + if (!taxonomy?.publish_details?.length || !taxonomy?.locale) { + apiOptions.apiData = undefined; + return apiOptions; + } + + const environments: string[] = []; + for (const pub of taxonomy.publish_details as any[]) { + const sourceEnvUid = pub?.environment; + if (!sourceEnvUid) continue; + const destUid = envUidMapper[String(sourceEnvUid)]; + if (destUid && !environments.includes(destUid)) { + environments.push(destUid); + } + } + + if (environments.length === 0) { + apiOptions.apiData = undefined; + return apiOptions; + } + + const locales = [String(taxonomy.locale)]; + apiOptions.apiData = { + environments, + locales, + items: [{ uid: taxonomy.uid }], + }; + + return apiOptions; +} diff --git a/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts b/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts index 6744a4f85..5dd22cae4 100644 --- a/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts +++ b/packages/contentstack-import/test/unit/import/modules/taxonomies.test.ts @@ -396,103 +396,6 @@ describe('ImportTaxonomies', () => { }); }); - describe('serializePublishTaxonomies', () => { - it('maps source environment UIDs to destination UIDs and sets items to uid only', () => { - (importTaxonomies as any).envUidMapper = { bltSrc: 'bltDest' }; - const apiOptions: any = { - entity: 'publish-taxonomies', - apiData: { - taxonomy: { - uid: 'tax1', - locale: 'en-us', - publish_details: [{ environment: 'bltSrc', time: '', user: '' }], - }, - }, - resolve: sandbox.stub(), - reject: sandbox.stub(), - }; - - const result = (importTaxonomies as any).serializePublishTaxonomies(apiOptions); - - expect(result.apiData).to.deep.equal({ - environments: ['bltDest'], - locales: ['en-us'], - items: [{ uid: 'tax1' }], - }); - }); - - it('dedupes multiple publish_details environments', () => { - (importTaxonomies as any).envUidMapper = { e1: 'd1', e2: 'd2' }; - const apiOptions: any = { - entity: 'publish-taxonomies', - apiData: { - taxonomy: { - uid: 'tax2', - locale: 'fr-fr', - publish_details: [{ environment: 'e1' }, { environment: 'e2' }, { environment: 'e1' }], - }, - }, - resolve: sandbox.stub(), - reject: sandbox.stub(), - }; - - const result = (importTaxonomies as any).serializePublishTaxonomies(apiOptions); - - expect(result.apiData.environments).to.deep.equal(['d1', 'd2']); - expect(result.apiData.locales).to.deep.equal(['fr-fr']); - expect(result.apiData.items).to.deep.equal([{ uid: 'tax2' }]); - }); - - it('returns undefined when publish_details empty', () => { - (importTaxonomies as any).envUidMapper = { x: 'y' }; - const apiOptions: any = { - entity: 'publish-taxonomies', - apiData: { - taxonomy: { uid: 't', locale: 'en-us', publish_details: [] }, - }, - resolve: sandbox.stub(), - reject: sandbox.stub(), - }; - - expect((importTaxonomies as any).serializePublishTaxonomies(apiOptions).apiData).to.be.undefined; - }); - - it('returns undefined when no env mapping resolves', () => { - (importTaxonomies as any).envUidMapper = {}; - const apiOptions: any = { - entity: 'publish-taxonomies', - apiData: { - taxonomy: { - uid: 'tax1', - locale: 'en-us', - publish_details: [{ environment: 'missing' }], - }, - }, - resolve: sandbox.stub(), - reject: sandbox.stub(), - }; - - expect((importTaxonomies as any).serializePublishTaxonomies(apiOptions).apiData).to.be.undefined; - }); - - it('returns undefined when taxonomy.locale missing', () => { - (importTaxonomies as any).envUidMapper = { e: 'd' }; - const apiOptions: any = { - entity: 'publish-taxonomies', - apiData: { - taxonomy: { - uid: 'tax1', - publish_details: [{ environment: 'e' }], - }, - }, - resolve: sandbox.stub(), - reject: sandbox.stub(), - }; - - expect((importTaxonomies as any).serializePublishTaxonomies(apiOptions).apiData).to.be.undefined; - }); - }); - describe('createSuccessAndFailedFile', () => { it('should write all four files when data exists', () => { (importTaxonomies as any).createSuccessAndFailedFile.restore(); diff --git a/packages/contentstack-import/test/unit/utils/taxonomy-publish-utils.test.ts b/packages/contentstack-import/test/unit/utils/taxonomy-publish-utils.test.ts new file mode 100644 index 000000000..de834c865 --- /dev/null +++ b/packages/contentstack-import/test/unit/utils/taxonomy-publish-utils.test.ts @@ -0,0 +1,113 @@ +import { expect } from 'chai'; +import * as sinon from 'sinon'; +import { serializePublishTaxonomies } from '../../../src/utils/taxonomy-publish-utils'; +import type { ApiOptions } from '../../../src/import/modules/base-class'; + +describe('taxonomy-publish-utils', () => { + let sandbox: sinon.SinonSandbox; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('serializePublishTaxonomies', () => { + it('maps source environment UIDs to destination UIDs and sets items to uid only', () => { + const envUidMapper = { bltSrc: 'bltDest' }; + const apiOptions: ApiOptions = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { + uid: 'tax1', + locale: 'en-us', + publish_details: [{ environment: 'bltSrc', time: '', user: '' }], + }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + const result = serializePublishTaxonomies(apiOptions, envUidMapper); + + expect(result.apiData).to.deep.equal({ + environments: ['bltDest'], + locales: ['en-us'], + items: [{ uid: 'tax1' }], + }); + }); + + it('dedupes multiple publish_details environments', () => { + const envUidMapper = { e1: 'd1', e2: 'd2' }; + const apiOptions: ApiOptions = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { + uid: 'tax2', + locale: 'fr-fr', + publish_details: [{ environment: 'e1' }, { environment: 'e2' }, { environment: 'e1' }], + }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + const result = serializePublishTaxonomies(apiOptions, envUidMapper); + + expect((result.apiData as any).environments).to.deep.equal(['d1', 'd2']); + expect((result.apiData as any).locales).to.deep.equal(['fr-fr']); + expect((result.apiData as any).items).to.deep.equal([{ uid: 'tax2' }]); + }); + + it('returns undefined when publish_details empty', () => { + const envUidMapper = { x: 'y' }; + const apiOptions: ApiOptions = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { uid: 't', locale: 'en-us', publish_details: [] }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + expect(serializePublishTaxonomies(apiOptions, envUidMapper).apiData).to.be.undefined; + }); + + it('returns undefined when no env mapping resolves', () => { + const envUidMapper = {}; + const apiOptions: ApiOptions = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { + uid: 'tax1', + locale: 'en-us', + publish_details: [{ environment: 'missing' }], + }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + expect(serializePublishTaxonomies(apiOptions, envUidMapper).apiData).to.be.undefined; + }); + + it('returns undefined when taxonomy.locale missing', () => { + const envUidMapper = { e: 'd' }; + const apiOptions: ApiOptions = { + entity: 'publish-taxonomies', + apiData: { + taxonomy: { + uid: 'tax1', + publish_details: [{ environment: 'e' }], + }, + }, + resolve: sandbox.stub(), + reject: sandbox.stub(), + }; + + expect(serializePublishTaxonomies(apiOptions, envUidMapper).apiData).to.be.undefined; + }); + }); +}); From 4e39de1364ab3c1d3535d207ea5eb9e18e2ccbd5 Mon Sep 17 00:00:00 2001 From: shafeeqd959 Date: Mon, 18 May 2026 12:28:55 +0530 Subject: [PATCH 05/61] added assets support in export-query --- .talismanrc | 4 + .../src/index.ts | 1 + .../query-export/am-asset-query-exporter.ts | 242 ++++++++++++++++++ .../src/query-export/index.ts | 1 + .../src/types/cs-assets-api.ts | 40 +++ .../src/utils/cs-assets-api-adapter.ts | 54 ++++ .../am-asset-query-exporter.test.ts | 152 +++++++++++ .../contentstack-query-export/package.json | 1 + .../src/commands/cm/stacks/export-query.ts | 4 +- .../src/core/query-executor.ts | 137 ++++++++-- .../src/types/index.ts | 13 + .../src/utils/config-handler.ts | 11 +- .../test/unit/query-executor.test.ts | 42 ++- 13 files changed, 676 insertions(+), 26 deletions(-) create mode 100644 packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts create mode 100644 packages/contentstack-asset-management/src/query-export/index.ts create mode 100644 packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts diff --git a/.talismanrc b/.talismanrc index ed36e71da..3fcbc91dd 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,8 @@ fileignoreconfig: - filename: pnpm-lock.yaml checksum: 2f6edbc19377e3a857884f00e31c498b660cc4f64b46892aee8eecf2f1ca9978 + - filename: packages/contentstack-query-export/src/core/query-executor.ts + checksum: a8d3688a519eb6a941bcdb22f41347df87539e7ab2413665b3193614dc40622d + - filename: packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts + checksum: 19c19d237e1dbe339d024c92049e24493d30b133422d2a10173fe3718413370c version: '1.0' diff --git a/packages/contentstack-asset-management/src/index.ts b/packages/contentstack-asset-management/src/index.ts index c66c638d0..c062f899b 100644 --- a/packages/contentstack-asset-management/src/index.ts +++ b/packages/contentstack-asset-management/src/index.ts @@ -3,3 +3,4 @@ export * from './types'; export * from './utils'; export * from './export'; export * from './import'; +export * from './query-export'; diff --git a/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts b/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts new file mode 100644 index 000000000..77b110553 --- /dev/null +++ b/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts @@ -0,0 +1,242 @@ +import { resolve as pResolve } from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { Readable } from 'node:stream'; +import { log, handleAndLogError, configHandler } from '@contentstack/cli-utilities'; + +import type { AmAssetQueryExportOptions, CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; +import type { ExportContext } from '../types/export-types'; +import ExportAssetTypes from '../export/asset-types'; +import ExportFields from '../export/fields'; +import { CSAssetsExportAdapter } from '../export/base'; +import { getAssetItems, writeStreamToFile } from '../utils/export-helpers'; +import { runInBatches } from '../utils/concurrent-batch'; + +const DEFAULT_ASSET_BATCH_SIZE = 100; +const SEARCH_PAGE_LIMIT = 50; + +/** + * Query-based AM 2.0 asset exporter. + * Exports only referenced asset UIDs from entries into the `spaces/` directory layout. + */ +export class AmAssetQueryExporter { + private readonly options: AmAssetQueryExportOptions; + + constructor(options: AmAssetQueryExportOptions) { + this.options = options; + } + + async export(assetUIDs: string[]): Promise { + const { linkedWorkspaces, exportDir, context } = this.options; + + if (!assetUIDs.length) { + log.info('No asset UIDs to export for AM 2.0 query export', context); + return; + } + + if (!linkedWorkspaces.length) { + log.warn('No linked workspaces configured for AM 2.0 asset query export', context); + return; + } + + log.info( + `Starting AM 2.0 query asset export (${assetUIDs.length} UID(s), ${linkedWorkspaces.length} space(s))`, + context, + ); + + const spacesRootPath = pResolve(exportDir, 'spaces'); + await mkdir(spacesRootPath, { recursive: true }); + + const apiConfig: CSAssetsAPIConfig = { + baseURL: this.options.csAssetsUrl, + headers: { organization_uid: this.options.org_uid }, + context, + }; + + const exportContext: ExportContext = { + spacesRootPath, + context, + securedAssets: this.options.securedAssets, + chunkFileSizeMb: this.options.chunkFileSizeMb, + apiConcurrency: this.options.apiConcurrency, + downloadAssetsConcurrency: this.options.downloadAssetsConcurrency, + }; + + const batchSize = this.options.assetBatchSize ?? DEFAULT_ASSET_BATCH_SIZE; + + try { + await this.bootstrapSharedModules(apiConfig, exportContext, linkedWorkspaces[0].space_uid); + + for (const workspace of linkedWorkspaces) { + try { + await this.exportWorkspaceAssets(apiConfig, exportContext, workspace, assetUIDs, batchSize); + } catch (err) { + handleAndLogError( + err, + { ...(context as Record), spaceUid: workspace.space_uid }, + `Failed AM 2.0 query export for space ${workspace.space_uid}`, + ); + } + } + + log.success('AM 2.0 query asset export completed', context); + } catch (err) { + handleAndLogError(err, context as Record, 'AM 2.0 query asset export failed'); + throw err; + } + } + + private async bootstrapSharedModules( + apiConfig: CSAssetsAPIConfig, + exportContext: ExportContext, + firstSpaceUid: string, + ): Promise { + const sharedFieldsDir = pResolve(exportContext.spacesRootPath, 'fields'); + const sharedAssetTypesDir = pResolve(exportContext.spacesRootPath, 'asset_types'); + await mkdir(sharedFieldsDir, { recursive: true }); + await mkdir(sharedAssetTypesDir, { recursive: true }); + + const exportAssetTypes = new ExportAssetTypes(apiConfig, exportContext); + const exportFields = new ExportFields(apiConfig, exportContext); + await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]); + } + + private async exportWorkspaceAssets( + apiConfig: CSAssetsAPIConfig, + exportContext: ExportContext, + workspace: LinkedWorkspace, + assetUIDs: string[], + batchSize: number, + ): Promise { + const { branchName, context } = this.options; + const workspaceExporter = new QueryExportWorkspaceAdapter(apiConfig, exportContext); + await workspaceExporter.start(workspace, assetUIDs, branchName || 'main', batchSize); + log.debug(`AM 2.0 query export finished for space ${workspace.space_uid}`, context); + } +} + +/** + * Per-space export: search by UID, write metadata/files, download binaries. + */ +class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter { + async start( + workspace: LinkedWorkspace, + assetUIDs: string[], + branchName: string, + uidBatchSize: number, + ): Promise { + await this.init(); + + const spaceDir = pResolve(this.exportContext.spacesRootPath, workspace.space_uid); + await mkdir(spaceDir, { recursive: true }); + + const spaceResponse = await this.getSpace(workspace.space_uid); + const space = spaceResponse.space; + const metadata = { + ...space, + workspace_uid: workspace.uid, + is_default: workspace.is_default, + branch: branchName, + }; + await writeFile(pResolve(spaceDir, 'metadata.json'), JSON.stringify(metadata, null, 2)); + + const assetsDir = pResolve(spaceDir, 'assets'); + await mkdir(assetsDir, { recursive: true }); + + const spaceRef = { space_uid: workspace.space_uid, workspace: workspace.uid }; + const assetItems = await this.searchAllAssets(assetUIDs, spaceRef, uidBatchSize); + + const folders = assetItems.filter((item) => (item as { is_dir?: boolean }).is_dir === true); + const files = assetItems.filter((item) => (item as { is_dir?: boolean }).is_dir !== true); + + await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); + + await this.writeItemsToChunkedJson( + assetsDir, + 'assets.json', + 'assets', + ['uid', 'url', 'filename', 'file_name', 'parent_uid'], + files, + ); + + await this.downloadAssets(files, assetsDir, workspace.space_uid); + } + + private async searchAllAssets( + assetUIDs: string[], + spaceRef: { space_uid: string; workspace: string }, + uidBatchSize: number, + ): Promise>> { + const seen = new Set(); + const results: Array> = []; + + for (let i = 0; i < assetUIDs.length; i += uidBatchSize) { + const uidBatch = assetUIDs.slice(i, i + uidBatchSize); + let skip = 0; + let pageItems: unknown[]; + + do { + const response = await this.searchAssets({ + assetUIDs: uidBatch, + spaces: [spaceRef], + skip, + limit: SEARCH_PAGE_LIMIT, + }); + pageItems = getAssetItems(response); + if (pageItems.length === 0 && Array.isArray((response as { assets?: unknown[] }).assets)) { + pageItems = (response as { assets: unknown[] }).assets; + } + + for (const item of pageItems) { + const record = item as Record; + const key = String(record.uid ?? record.asset_id ?? record._uid ?? ''); + if (key && !seen.has(key)) { + seen.add(key); + results.push(record); + } + } + + skip += pageItems.length; + } while (pageItems.length === SEARCH_PAGE_LIMIT); + } + + return results; + } + + private async downloadAssets( + items: Array>, + assetsDir: string, + spaceUid: string, + ): Promise { + const downloadable = items.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))); + if (downloadable.length === 0) { + log.debug(`No downloadable assets for space ${spaceUid}`, this.exportContext.context); + return; + } + + const filesDir = pResolve(assetsDir, 'files'); + await mkdir(filesDir, { recursive: true }); + + const securedAssets = this.exportContext.securedAssets ?? false; + const authtoken = securedAssets ? configHandler.get('authtoken') : null; + + await runInBatches(downloadable, this.downloadAssetsBatchConcurrency, async (asset) => { + const uid = String(asset.uid ?? asset._uid); + const url = String(asset.url); + const filename = String(asset.filename ?? asset.file_name ?? 'asset'); + try { + const separator = url.includes('?') ? '&' : '?'; + const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; + const response = await fetch(downloadUrl); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const body = response.body; + if (!body) throw new Error('No response body'); + const nodeStream = Readable.fromWeb(body as Parameters[0]); + const assetFolderPath = pResolve(filesDir, uid); + await mkdir(assetFolderPath, { recursive: true }); + await writeStreamToFile(nodeStream, pResolve(assetFolderPath, filename)); + } catch (e) { + log.debug(`Failed to download asset ${uid} in space ${spaceUid}: ${e}`, this.exportContext.context); + } + }); + } +} diff --git a/packages/contentstack-asset-management/src/query-export/index.ts b/packages/contentstack-asset-management/src/query-export/index.ts new file mode 100644 index 000000000..181aca119 --- /dev/null +++ b/packages/contentstack-asset-management/src/query-export/index.ts @@ -0,0 +1 @@ +export { AmAssetQueryExporter } from './am-asset-query-exporter'; diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index 6c8c379e5..79f313686 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -119,6 +119,28 @@ export type CSAssetsAPIConfig = { * Adapter interface for Contentstack Assets API calls. * Used by export and (future) import. */ +/** Space + workspace pair for AM search API. */ +export type SearchSpaceRef = { + space_uid: string; + workspace: string; +}; + +/** Parameters for POST /api/search (asset query export). */ +export type SearchAssetsParams = { + assetUIDs: string[]; + spaces: SearchSpaceRef[]; + skip?: number; + limit?: number; +}; + +/** Response shape from POST /api/search for assets. */ +export type SearchAssetsResponse = { + count?: number; + assets?: unknown[]; + items?: unknown[]; + folders?: unknown[]; +}; + export interface ICSAssetsAdapter { init(): Promise; listSpaces(): Promise; @@ -127,8 +149,26 @@ export interface ICSAssetsAdapter { getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; getWorkspaceAssetTypes(spaceUid: string): Promise; + searchAssets(params: SearchAssetsParams): Promise; } +/** Options for query-based AM asset export (referenced assets from entries). */ +export type AmAssetQueryExportOptions = { + linkedWorkspaces: LinkedWorkspace[]; + exportDir: string; + branchName: string; + csAssetsUrl: string; + org_uid: string; + apiKey?: string; + context?: Record; + securedAssets?: boolean; + chunkFileSizeMb?: number; + apiConcurrency?: number; + downloadAssetsConcurrency?: number; + /** Max UIDs per search request ($in batch). */ + assetBatchSize?: number; +}; + /** * Options for exporting space structure (used by export app after fetching linked workspaces). */ diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index dcdf26a11..2ee4c2230 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -12,11 +12,41 @@ import type { CreateSpacePayload, FieldsResponse, ICSAssetsAdapter, + SearchAssetsParams, + SearchAssetsResponse, Space, SpaceResponse, SpacesListResponse, } from '../types/cs-assets-api'; +/** Default fields requested from POST /api/search for asset export. */ +export const DEFAULT_SEARCH_ASSET_FIELDS = [ + 'asset_id', + 'uid', + 'title', + 'file_name', + 'description', + 'parent_uid', + 'is_dir', + 'dimensions', + 'file_size', + 'content_type', + 'asset_type', + 'url', + 'tags', + 'created_at', + 'updated_at', + 'created_by', + 'updated_by', + 'path', + 'locale', + 'space_uid', + 'version', + 'publish_details', + 'ACL', + '_asset_scan_status', +] as const; + export class CSAssetsAdapter implements ICSAssetsAdapter { private readonly config: CSAssetsAPIConfig; private readonly apiClient: HttpClient; @@ -223,6 +253,30 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { return result; } + /** + * POST /api/search — query assets by UID within linked spaces (AM 2.0 query export). + */ + async searchAssets(params: SearchAssetsParams): Promise { + await this.init(); + const { assetUIDs, spaces, skip = 0, limit = 50 } = params; + if (!assetUIDs.length) { + return { count: 0, assets: [] }; + } + const body = { + query: { uid: { $in: assetUIDs } }, + skip, + limit, + object_type: 'asset', + fields: [...DEFAULT_SEARCH_ASSET_FIELDS], + spaces, + }; + log.debug( + `Searching assets (skip=${skip}, limit=${limit}, uids=${assetUIDs.length}, spaces=${spaces.length})`, + this.config.context, + ); + return this.postJson('/api/search', body); + } + // --------------------------------------------------------------------------- // POST helpers // --------------------------------------------------------------------------- diff --git a/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts b/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts new file mode 100644 index 000000000..af3d6cae5 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts @@ -0,0 +1,152 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as fs from 'node:fs/promises'; +import { resolve as pResolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { HttpClient, authenticationHandler } from '@contentstack/cli-utilities'; + +import { AmAssetQueryExporter } from '../../../src/query-export/am-asset-query-exporter'; +import ExportAssetTypes from '../../../src/export/asset-types'; +import ExportFields from '../../../src/export/fields'; +import { CSAssetsExportAdapter } from '../../../src/export/base'; +import { CSAssetsAdapter } from '../../../src/utils/cs-assets-api-adapter'; +import * as concurrentBatch from '../../../src/utils/concurrent-batch'; + +import type { AmAssetQueryExportOptions } from '../../../src/types/cs-assets-api'; + +describe('AmAssetQueryExporter', () => { + let exportDir: string; + let searchAssetsStub: sinon.SinonStub; + const baseOptions: AmAssetQueryExportOptions = { + linkedWorkspaces: [{ uid: 'main', space_uid: 'space-1', is_default: true }], + exportDir: '', + branchName: 'main', + csAssetsUrl: 'https://am.example.com', + org_uid: 'org-1', + context: { command: 'export-query' }, + assetBatchSize: 2, + }; + + beforeEach(async () => { + exportDir = await fs.mkdtemp(pResolve(tmpdir(), 'am-query-export-')); + baseOptions.exportDir = exportDir; + + sinon.stub(ExportFields.prototype, 'start').resolves(); + sinon.stub(ExportAssetTypes.prototype, 'start').resolves(); + sinon.stub(CSAssetsExportAdapter.prototype, 'init').resolves(); + sinon.stub(CSAssetsExportAdapter.prototype, 'getSpace').resolves({ + space: { uid: 'space-1', title: 'Test Space' }, + }); + searchAssetsStub = sinon.stub(CSAssetsExportAdapter.prototype, 'searchAssets').resolves({ + assets: [ + { uid: 'asset-1', url: 'https://cdn.example.com/a1.png', file_name: 'a1.png', is_dir: false }, + { uid: 'asset-2', url: 'https://cdn.example.com/a2.png', file_name: 'a2.png', is_dir: false }, + ], + }); + sinon.stub(CSAssetsExportAdapter.prototype as any, 'writeItemsToChunkedJson').resolves(); + sinon.stub(concurrentBatch, 'runInBatches').callsFake(async (items, _concurrency, handler) => { + for (let i = 0; i < items.length; i++) { + await handler(items[i], i); + } + }); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should return early when no asset UIDs are provided', async () => { + const exporter = new AmAssetQueryExporter(baseOptions); + await exporter.export([]); + + expect((ExportFields.prototype.start as sinon.SinonStub).called).to.be.false; + }); + + it('should bootstrap shared fields and asset types', async () => { + const exporter = new AmAssetQueryExporter(baseOptions); + await exporter.export(['asset-1']); + + expect((ExportFields.prototype.start as sinon.SinonStub).calledOnceWith('space-1')).to.be.true; + expect((ExportAssetTypes.prototype.start as sinon.SinonStub).calledOnceWith('space-1')).to.be.true; + }); + + it('should call searchAssets with batched UIDs and space reference', async () => { + const exporter = new AmAssetQueryExporter(baseOptions); + await exporter.export(['asset-1', 'asset-2', 'asset-3']); + + expect(searchAssetsStub.called).to.be.true; + const firstCall = searchAssetsStub.getCall(0).args[0]; + expect(firstCall.spaces).to.deep.equal([{ space_uid: 'space-1', workspace: 'main' }]); + expect(firstCall.assetUIDs).to.deep.equal(['asset-1', 'asset-2']); + }); + + it('should write space metadata and asset files under spaces/', async () => { + const exporter = new AmAssetQueryExporter(baseOptions); + await exporter.export(['asset-1']); + + const metadataPath = pResolve(exportDir, 'spaces', 'space-1', 'metadata.json'); + const metadata = JSON.parse(await fs.readFile(metadataPath, 'utf-8')); + expect(metadata.uid).to.equal('space-1'); + expect(metadata.workspace_uid).to.equal('main'); + + const foldersPath = pResolve(exportDir, 'spaces', 'space-1', 'assets', 'folders.json'); + const folders = JSON.parse(await fs.readFile(foldersPath, 'utf-8')); + expect(folders).to.be.an('array').that.is.empty; + }); +}); + +describe('CSAssetsAdapter.searchAssets', () => { + const baseConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + + let fetchStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(HttpClient.prototype, 'headers').returnsThis(); + sinon.stub(HttpClient.prototype, 'baseUrl').returnsThis(); + sinon.stub(authenticationHandler, 'getAuthDetails').resolves(); + sinon.stub(authenticationHandler, 'isOauthEnabled').get(() => false); + sinon.stub(authenticationHandler, 'accessToken').get(() => 'test-token'); + + fetchStub = sinon.stub(global, 'fetch').resolves({ + ok: true, + json: async () => ({ count: 1, assets: [{ uid: 'a1' }] }), + } as Response); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should POST to /api/search with uid $in query', async () => { + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.searchAssets({ + assetUIDs: ['uid-1', 'uid-2'], + spaces: [{ space_uid: 'space-1', workspace: 'main' }], + skip: 0, + limit: 50, + }); + + expect(fetchStub.calledOnce).to.be.true; + const [url, init] = fetchStub.firstCall.args; + expect(url).to.equal('https://am.example.com/api/search'); + expect(init.method).to.equal('POST'); + const body = JSON.parse(init.body); + expect(body.query).to.deep.equal({ uid: { $in: ['uid-1', 'uid-2'] } }); + expect(body.object_type).to.equal('asset'); + expect(body.spaces).to.deep.equal([{ space_uid: 'space-1', workspace: 'main' }]); + }); + + it('should return empty result when assetUIDs is empty', async () => { + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.searchAssets({ + assetUIDs: [], + spaces: [{ space_uid: 'space-1', workspace: 'main' }], + }); + + expect(fetchStub.called).to.be.false; + expect(result).to.deep.equal({ count: 0, assets: [] }); + }); +}); diff --git a/packages/contentstack-query-export/package.json b/packages/contentstack-query-export/package.json index 1bb79f2a1..e2a32317a 100644 --- a/packages/contentstack-query-export/package.json +++ b/packages/contentstack-query-export/package.json @@ -5,6 +5,7 @@ "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { + "@contentstack/cli-asset-management": "~1.0.0-beta.1", "@contentstack/cli-cm-export": "~2.0.0-beta.17", "@contentstack/cli-command": "~2.0.0-beta.7", "@contentstack/cli-utilities": "~2.0.0-beta.8", diff --git a/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts b/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts index e70e6c2de..240d363ff 100644 --- a/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts +++ b/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts @@ -12,7 +12,7 @@ import { } from '@contentstack/cli-utilities'; import { QueryExporter } from '../../../core/query-executor'; import { QueryExportConfig } from '../../../types'; -import { setupQueryExportConfig, setupBranches, createLogContext } from '../../../utils'; +import { setupQueryExportConfig, setupBranches, createLogContext, applyRegionToQueryExportConfig } from '../../../utils'; export default class ExportQueryCommand extends Command { static description = 'Export content from a stack using query-based filtering'; @@ -80,7 +80,7 @@ export default class ExportQueryCommand extends Command { // Setup export configuration const exportQueryConfig = await setupQueryExportConfig(flags); exportQueryConfig.host = this.cmaHost; - exportQueryConfig.region = this.region; + applyRegionToQueryExportConfig(exportQueryConfig, this.region); if (this.developerHubUrl) { exportQueryConfig.developerHubBaseUrl = this.developerHubUrl; diff --git a/packages/contentstack-query-export/src/core/query-executor.ts b/packages/contentstack-query-export/src/core/query-executor.ts index dd48183df..0d9ca5f57 100644 --- a/packages/contentstack-query-export/src/core/query-executor.ts +++ b/packages/contentstack-query-export/src/core/query-executor.ts @@ -4,7 +4,9 @@ import { log, handleAndLogError, readContentTypeSchemas, + managementSDKClient, } from '@contentstack/cli-utilities'; +import { AmAssetQueryExporter } from '@contentstack/cli-asset-management'; import * as fs from 'fs'; import * as path from 'path'; import { QueryExportConfig, Modules } from '../types'; @@ -44,6 +46,9 @@ export class QueryExporter { // Step 2: Always export general modules await this.exportGeneralModules(); + // Step 3: Resolve AM 2.0 linked workspaces from branch settings + await this.fetchLinkedWorkspaces(); + // Step 4: Export queried modules await this.exportQueriedModule(parsedQuery); @@ -59,6 +64,56 @@ export class QueryExporter { log.success('Query-based export completed successfully!', this.exportQueryConfig.context); } + /** + * Fetch linked workspaces (am_v2) from branch settings for AM 2.0 asset routing. + */ + private async fetchLinkedWorkspaces(): Promise { + const branchName = this.exportQueryConfig.branchName || 'main'; + try { + const branch = await this.stackAPIClient + .branch(branchName) + .fetch({ include_settings: true } as Record); + const linked = (branch as { settings?: { am_v2?: { linked_workspaces?: QueryExportConfig['linkedWorkspaces'] } } }) + ?.settings?.am_v2?.linked_workspaces; + this.exportQueryConfig.linkedWorkspaces = Array.isArray(linked) ? linked : []; + log.debug( + `Linked workspaces for AM 2.0: ${this.exportQueryConfig.linkedWorkspaces?.length ?? 0}`, + this.exportQueryConfig.context, + ); + } catch (error) { + log.warn( + `Could not fetch linked workspaces for branch ${branchName}, using legacy asset export`, + this.exportQueryConfig.context, + ); + this.exportQueryConfig.linkedWorkspaces = []; + } + } + + private isAM2AssetExport(): boolean { + return ( + (this.exportQueryConfig.linkedWorkspaces?.length ?? 0) > 0 && + Boolean(this.exportQueryConfig.csAssetsUrl) + ); + } + + /** + * Resolve organization UID for AM 2.0 API calls. + */ + private async resolveOrgUid(): Promise { + if (this.exportQueryConfig.org_uid) { + return this.exportQueryConfig.org_uid; + } + try { + const tempAPIClient = await managementSDKClient({ host: this.exportQueryConfig.host }); + const stackData = await tempAPIClient.stack({ api_key: this.exportQueryConfig.stackApiKey }).fetch(); + this.exportQueryConfig.org_uid = (stackData as { org_uid?: string })?.org_uid ?? ''; + return this.exportQueryConfig.org_uid; + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Failed to resolve organization UID'); + return ''; + } + } + // export general modules private async exportGeneralModules(): Promise { log.info('Exporting general modules...', this.exportQueryConfig.context); @@ -266,28 +321,75 @@ export class QueryExporter { log.info('Starting export of referenced assets...', this.exportQueryConfig.context); try { - const assetsDir = path.join( - sanitizePath(this.exportQueryConfig.exportDir), - sanitizePath(this.exportQueryConfig.branchName || ''), - 'assets', - ); - - const metadataFilePath = path.join(assetsDir, 'metadata.json'); - const assetFilePath = path.join(assetsDir, 'assets.json'); - - // Define temp file paths - const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json'); - const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json'); - const assetHandler = new AssetReferenceHandler(this.exportQueryConfig); // Extract referenced asset UIDs from all entries log.debug('Extracting referenced assets from entries', this.exportQueryConfig.context); const assetUIDs = assetHandler.extractReferencedAssets(); - if (assetUIDs.length > 0) { - log.info(`Found ${assetUIDs.length} referenced assets to export`, this.exportQueryConfig.context); + if (assetUIDs.length === 0) { + log.info('No referenced assets found in entries', this.exportQueryConfig.context); + return; + } + + log.info(`Found ${assetUIDs.length} referenced assets to export`, this.exportQueryConfig.context); + + if (this.isAM2AssetExport()) { + await this.exportReferencedAssetsAM2(assetUIDs); + return; + } + + await this.exportReferencedAssetsLegacy(assetUIDs); + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting referenced assets'); + throw error; + } + } + + /** + * AM 2.0: export referenced assets into spaces/ via Contentstack Assets search API. + */ + private async exportReferencedAssetsAM2(assetUIDs: string[]): Promise { + log.info('Using AM 2.0 asset export (spaces/)', this.exportQueryConfig.context); + const org_uid = await this.resolveOrgUid(); + if (!org_uid) { + throw new Error('Organization UID is required for AM 2.0 asset export'); + } + const amExporter = new AmAssetQueryExporter({ + linkedWorkspaces: this.exportQueryConfig.linkedWorkspaces ?? [], + exportDir: this.exportQueryConfig.exportDir, + branchName: this.exportQueryConfig.branchName || 'main', + csAssetsUrl: this.exportQueryConfig.csAssetsUrl!, + org_uid, + apiKey: this.exportQueryConfig.stackApiKey, + context: this.exportQueryConfig.context as unknown as Record, + securedAssets: this.exportQueryConfig.securedAssets, + assetBatchSize: this.exportQueryConfig.assetBatchSize, + }); + + await amExporter.export(assetUIDs); + log.success('Referenced assets exported successfully (AM 2.0)', this.exportQueryConfig.context); + } + + /** + * AM 1.0: export referenced assets into legacy assets/ via CMA export module. + */ + private async exportReferencedAssetsLegacy(assetUIDs: string[]): Promise { + const assetsDir = path.join( + sanitizePath(this.exportQueryConfig.exportDir), + sanitizePath(this.exportQueryConfig.branchName || ''), + 'assets', + ); + + const metadataFilePath = path.join(assetsDir, 'metadata.json'); + const assetFilePath = path.join(assetsDir, 'assets.json'); + + // Define temp file paths + const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json'); + const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json'); + + try { fs.mkdirSync(assetsDir, { recursive: true }); // Define batch size - can be configurable through exportQueryConfig @@ -383,11 +485,8 @@ export class QueryExporter { log.info(`Temporary files cleaned up`, this.exportQueryConfig.context); log.success('Referenced assets exported successfully', this.exportQueryConfig.context); - } else { - log.info('No referenced assets found in entries', this.exportQueryConfig.context); - } } catch (error) { - handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting referenced assets'); + handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting legacy referenced assets'); throw error; } } diff --git a/packages/contentstack-query-export/src/types/index.ts b/packages/contentstack-query-export/src/types/index.ts index 1a036ea24..6c96ce564 100644 --- a/packages/contentstack-query-export/src/types/index.ts +++ b/packages/contentstack-query-export/src/types/index.ts @@ -31,8 +31,15 @@ export interface Region { cma: string; cda: string; uiHost: string; + csAssetsUrl?: string; } +export type LinkedWorkspace = { + uid: string; + space_uid: string; + is_default: boolean; +}; + export type Modules = | 'stack' | 'locales' @@ -203,6 +210,12 @@ export interface QueryExportConfig extends DefaultConfig { batchDelayMs?: number; assetBatchSize?: number; assetBatchDelayMs?: number; + /** AM 2.0 linked workspaces from branch settings (am_v2.linked_workspaces). */ + linkedWorkspaces?: LinkedWorkspace[]; + /** Contentstack Assets API base URL for AM 2.0 export. */ + csAssetsUrl?: string; + /** Organization UID for AM 2.0 API headers. */ + org_uid?: string; context?: LogContext; // Log context for centralized logging } diff --git a/packages/contentstack-query-export/src/utils/config-handler.ts b/packages/contentstack-query-export/src/utils/config-handler.ts index 94eccfc65..1a3ba02a4 100644 --- a/packages/contentstack-query-export/src/utils/config-handler.ts +++ b/packages/contentstack-query-export/src/utils/config-handler.ts @@ -1,9 +1,18 @@ import * as path from 'path'; -import { QueryExportConfig } from '../types'; +import { QueryExportConfig, Region } from '../types'; import { sanitizePath, pathValidator, configHandler, isAuthenticated } from '@contentstack/cli-utilities'; import config from '../config'; import { askAPIKey } from './common-helper'; +/** + * Apply region-derived AM 2.0 settings after the command resolves region. + */ +export function applyRegionToQueryExportConfig(exportQueryConfig: QueryExportConfig, region?: Region): void { + if (!region) return; + exportQueryConfig.region = region; + exportQueryConfig.csAssetsUrl = region.csAssetsUrl; +} + export async function setupQueryExportConfig(flags: any): Promise { const exportDir = sanitizePath(flags['data-dir'] || pathValidator('export')); diff --git a/packages/contentstack-query-export/test/unit/query-executor.test.ts b/packages/contentstack-query-export/test/unit/query-executor.test.ts index a3cfd4bc6..a7d3b3c7c 100644 --- a/packages/contentstack-query-export/test/unit/query-executor.test.ts +++ b/packages/contentstack-query-export/test/unit/query-executor.test.ts @@ -3,6 +3,7 @@ import * as sinon from 'sinon'; import { QueryExporter } from '../../src/core/query-executor'; import { QueryParser } from '../../src/utils/query-parser'; import { ModuleExporter } from '../../src/core/module-exporter'; +import { AmAssetQueryExporter } from '@contentstack/cli-asset-management'; import * as logger from '../../src/utils/logger'; import { ReferencedContentTypesHandler, @@ -23,7 +24,11 @@ describe('QueryExporter', () => { // Mock management client mockManagementClient = { - stack: sandbox.stub().returns({}), + stack: sandbox.stub().returns({ + branch: sandbox.stub().returns({ + fetch: sandbox.stub().resolves({ settings: { am_v2: { linked_workspaces: [] } } }), + }), + }), }; // Mock export configuration @@ -74,6 +79,7 @@ describe('QueryExporter', () => { describe('execute', () => { let queryParserStub: sinon.SinonStub; let exportGeneralModulesStub: sinon.SinonStub; + let fetchLinkedWorkspacesStub: sinon.SinonStub; let exportQueriedModuleStub: sinon.SinonStub; let expandSchemaClosureStub: sinon.SinonStub; let exportContentModulesStub: sinon.SinonStub; @@ -83,6 +89,7 @@ describe('QueryExporter', () => { modules: { entries: { content_type_uid: 'test_page' } }, }); exportGeneralModulesStub = sandbox.stub(queryExporter as any, 'exportGeneralModules').resolves(); + fetchLinkedWorkspacesStub = sandbox.stub(queryExporter as any, 'fetchLinkedWorkspaces').resolves(); exportQueriedModuleStub = sandbox.stub(queryExporter as any, 'exportQueriedModule').resolves(); expandSchemaClosureStub = sandbox.stub(queryExporter as any, 'expandSchemaClosure').resolves(); exportContentModulesStub = sandbox.stub(queryExporter as any, 'exportContentModules').resolves(); @@ -104,6 +111,7 @@ describe('QueryExporter', () => { sinon.assert.callOrder( queryParserStub, exportGeneralModulesStub, + fetchLinkedWorkspacesStub, exportQueriedModuleStub, expandSchemaClosureStub, exportContentModulesStub, @@ -569,9 +577,11 @@ describe('QueryExporter', () => { describe('exportReferencedAssets', () => { let moduleExporterStub: sinon.SinonStub; let assetHandlerStub: any; + let amExporterStub: sinon.SinonStub; beforeEach(() => { moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); + amExporterStub = sandbox.stub(AmAssetQueryExporter.prototype, 'export').resolves(); // Mock AssetReferenceHandler assetHandlerStub = { @@ -582,15 +592,39 @@ describe('QueryExporter', () => { .callsFake(assetHandlerStub.extractReferencedAssets); }); - it('should export referenced assets when found', async () => { + it('should export referenced assets when found (AM 1.0 legacy path)', async () => { + mockConfig.linkedWorkspaces = []; + mockConfig.csAssetsUrl = undefined; + queryExporter = new QueryExporter(mockManagementClient, mockConfig); + const legacyModuleExporterStub = sandbox + .stub((queryExporter as any).moduleExporter, 'exportModule') + .resolves(); + await (queryExporter as any).exportReferencedAssets(); - expect(moduleExporterStub.calledOnce).to.be.true; - const exportCall = moduleExporterStub.getCall(0); + expect(legacyModuleExporterStub.calledOnce).to.be.true; + expect(amExporterStub.called).to.be.false; + const exportCall = legacyModuleExporterStub.getCall(0); expect(exportCall.args[0]).to.equal('assets'); expect(exportCall.args[1].query.modules.assets.uid.$in).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); }); + it('should use AmAssetQueryExporter when AM 2.0 linked workspaces and csAssetsUrl are set', async () => { + mockConfig.linkedWorkspaces = [{ uid: 'main', space_uid: 'space-1', is_default: true }]; + mockConfig.csAssetsUrl = 'https://am.example.com'; + mockConfig.org_uid = 'org-1'; + queryExporter = new QueryExporter(mockManagementClient, mockConfig); + const am2ModuleExporterStub = sandbox + .stub((queryExporter as any).moduleExporter, 'exportModule') + .resolves(); + + await (queryExporter as any).exportReferencedAssets(); + + expect(amExporterStub.calledOnce).to.be.true; + expect(amExporterStub.firstCall.args[0]).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); + expect(am2ModuleExporterStub.called).to.be.false; + }); + it('should skip export when no assets found', async () => { assetHandlerStub.extractReferencedAssets.returns([]); From ab3991a2e4c0c91654514fb43d6cd2de1ba86a30 Mon Sep 17 00:00:00 2001 From: shafeeqd959 Date: Mon, 18 May 2026 12:32:35 +0530 Subject: [PATCH 06/61] updated unit test pipeline --- .github/workflows/unit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 62c048f9b..7e7119677 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -25,7 +25,7 @@ jobs: - name: Prune pnpm store run: pnpm store prune - name: Install Dependencies - run: pnpm install --frozen-lockfile + run: pnpm install --no-frozen-lockfile - name: Build all plugins run: | From b007415f17e7aa0c37bc9ac7693d770495c4768d Mon Sep 17 00:00:00 2001 From: shafeeqd959 Date: Mon, 18 May 2026 13:15:08 +0530 Subject: [PATCH 07/61] updated naming --- ...xporter.ts => cs-assets-query-exporter.ts} | 24 +++++------ .../src/query-export/index.ts | 2 +- .../src/types/cs-assets-api.ts | 6 +-- .../src/utils/cs-assets-api-adapter.ts | 2 +- ...st.ts => cs-assets-query-exporter.test.ts} | 18 ++++----- .../src/core/query-executor.ts | 40 +++++++++---------- .../src/types/index.ts | 6 +-- .../src/utils/config-handler.ts | 2 +- .../test/unit/query-executor.test.ts | 20 +++++----- 9 files changed, 60 insertions(+), 60 deletions(-) rename packages/contentstack-asset-management/src/query-export/{am-asset-query-exporter.ts => cs-assets-query-exporter.ts} (89%) rename packages/contentstack-asset-management/test/unit/query-export/{am-asset-query-exporter.test.ts => cs-assets-query-exporter.test.ts} (90%) diff --git a/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts similarity index 89% rename from packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts rename to packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts index 77b110553..f66487bb7 100644 --- a/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts +++ b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts @@ -3,7 +3,7 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { Readable } from 'node:stream'; import { log, handleAndLogError, configHandler } from '@contentstack/cli-utilities'; -import type { AmAssetQueryExportOptions, CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; +import type { CsAssetsQueryExportOptions, CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; import type { ExportContext } from '../types/export-types'; import ExportAssetTypes from '../export/asset-types'; import ExportFields from '../export/fields'; @@ -15,13 +15,13 @@ const DEFAULT_ASSET_BATCH_SIZE = 100; const SEARCH_PAGE_LIMIT = 50; /** - * Query-based AM 2.0 asset exporter. + * Query-based Contentstack Assets exporter. * Exports only referenced asset UIDs from entries into the `spaces/` directory layout. */ -export class AmAssetQueryExporter { - private readonly options: AmAssetQueryExportOptions; +export class CsAssetsQueryExporter { + private readonly options: CsAssetsQueryExportOptions; - constructor(options: AmAssetQueryExportOptions) { + constructor(options: CsAssetsQueryExportOptions) { this.options = options; } @@ -29,17 +29,17 @@ export class AmAssetQueryExporter { const { linkedWorkspaces, exportDir, context } = this.options; if (!assetUIDs.length) { - log.info('No asset UIDs to export for AM 2.0 query export', context); + log.info('No asset UIDs to export for Contentstack Assets query export', context); return; } if (!linkedWorkspaces.length) { - log.warn('No linked workspaces configured for AM 2.0 asset query export', context); + log.warn('No linked workspaces configured for Contentstack Assets query export', context); return; } log.info( - `Starting AM 2.0 query asset export (${assetUIDs.length} UID(s), ${linkedWorkspaces.length} space(s))`, + `Starting Contentstack Assets query export (${assetUIDs.length} UID(s), ${linkedWorkspaces.length} space(s))`, context, ); @@ -73,14 +73,14 @@ export class AmAssetQueryExporter { handleAndLogError( err, { ...(context as Record), spaceUid: workspace.space_uid }, - `Failed AM 2.0 query export for space ${workspace.space_uid}`, + `Failed Contentstack Assets query export for space ${workspace.space_uid}`, ); } } - log.success('AM 2.0 query asset export completed', context); + log.success('Contentstack Assets query export completed', context); } catch (err) { - handleAndLogError(err, context as Record, 'AM 2.0 query asset export failed'); + handleAndLogError(err, context as Record, 'Contentstack Assets query export failed'); throw err; } } @@ -110,7 +110,7 @@ export class AmAssetQueryExporter { const { branchName, context } = this.options; const workspaceExporter = new QueryExportWorkspaceAdapter(apiConfig, exportContext); await workspaceExporter.start(workspace, assetUIDs, branchName || 'main', batchSize); - log.debug(`AM 2.0 query export finished for space ${workspace.space_uid}`, context); + log.debug(`Contentstack Assets query export finished for space ${workspace.space_uid}`, context); } } diff --git a/packages/contentstack-asset-management/src/query-export/index.ts b/packages/contentstack-asset-management/src/query-export/index.ts index 181aca119..a46638b88 100644 --- a/packages/contentstack-asset-management/src/query-export/index.ts +++ b/packages/contentstack-asset-management/src/query-export/index.ts @@ -1 +1 @@ -export { AmAssetQueryExporter } from './am-asset-query-exporter'; +export { CsAssetsQueryExporter } from './cs-assets-query-exporter'; diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index 79f313686..cab8389d9 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -119,7 +119,7 @@ export type CSAssetsAPIConfig = { * Adapter interface for Contentstack Assets API calls. * Used by export and (future) import. */ -/** Space + workspace pair for AM search API. */ +/** Space + workspace pair for Contentstack Assets search API. */ export type SearchSpaceRef = { space_uid: string; workspace: string; @@ -152,8 +152,8 @@ export interface ICSAssetsAdapter { searchAssets(params: SearchAssetsParams): Promise; } -/** Options for query-based AM asset export (referenced assets from entries). */ -export type AmAssetQueryExportOptions = { +/** Options for query-based Contentstack Assets export (referenced assets from entries). */ +export type CsAssetsQueryExportOptions = { linkedWorkspaces: LinkedWorkspace[]; exportDir: string; branchName: string; diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index 2ee4c2230..aa33703a8 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -254,7 +254,7 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { } /** - * POST /api/search — query assets by UID within linked spaces (AM 2.0 query export). + * POST /api/search — query assets by UID within linked spaces (Contentstack Assets query export). */ async searchAssets(params: SearchAssetsParams): Promise { await this.init(); diff --git a/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts similarity index 90% rename from packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts rename to packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts index af3d6cae5..5c8ddd4ba 100644 --- a/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts +++ b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts @@ -5,19 +5,19 @@ import { resolve as pResolve } from 'node:path'; import { tmpdir } from 'node:os'; import { HttpClient, authenticationHandler } from '@contentstack/cli-utilities'; -import { AmAssetQueryExporter } from '../../../src/query-export/am-asset-query-exporter'; +import { CsAssetsQueryExporter } from '../../../src/query-export/cs-assets-query-exporter'; import ExportAssetTypes from '../../../src/export/asset-types'; import ExportFields from '../../../src/export/fields'; import { CSAssetsExportAdapter } from '../../../src/export/base'; import { CSAssetsAdapter } from '../../../src/utils/cs-assets-api-adapter'; import * as concurrentBatch from '../../../src/utils/concurrent-batch'; -import type { AmAssetQueryExportOptions } from '../../../src/types/cs-assets-api'; +import type { CsAssetsQueryExportOptions } from '../../../src/types/cs-assets-api'; -describe('AmAssetQueryExporter', () => { +describe('CsAssetsQueryExporter', () => { let exportDir: string; let searchAssetsStub: sinon.SinonStub; - const baseOptions: AmAssetQueryExportOptions = { + const baseOptions: CsAssetsQueryExportOptions = { linkedWorkspaces: [{ uid: 'main', space_uid: 'space-1', is_default: true }], exportDir: '', branchName: 'main', @@ -28,7 +28,7 @@ describe('AmAssetQueryExporter', () => { }; beforeEach(async () => { - exportDir = await fs.mkdtemp(pResolve(tmpdir(), 'am-query-export-')); + exportDir = await fs.mkdtemp(pResolve(tmpdir(), 'cs-assets-query-export-')); baseOptions.exportDir = exportDir; sinon.stub(ExportFields.prototype, 'start').resolves(); @@ -56,14 +56,14 @@ describe('AmAssetQueryExporter', () => { }); it('should return early when no asset UIDs are provided', async () => { - const exporter = new AmAssetQueryExporter(baseOptions); + const exporter = new CsAssetsQueryExporter(baseOptions); await exporter.export([]); expect((ExportFields.prototype.start as sinon.SinonStub).called).to.be.false; }); it('should bootstrap shared fields and asset types', async () => { - const exporter = new AmAssetQueryExporter(baseOptions); + const exporter = new CsAssetsQueryExporter(baseOptions); await exporter.export(['asset-1']); expect((ExportFields.prototype.start as sinon.SinonStub).calledOnceWith('space-1')).to.be.true; @@ -71,7 +71,7 @@ describe('AmAssetQueryExporter', () => { }); it('should call searchAssets with batched UIDs and space reference', async () => { - const exporter = new AmAssetQueryExporter(baseOptions); + const exporter = new CsAssetsQueryExporter(baseOptions); await exporter.export(['asset-1', 'asset-2', 'asset-3']); expect(searchAssetsStub.called).to.be.true; @@ -81,7 +81,7 @@ describe('AmAssetQueryExporter', () => { }); it('should write space metadata and asset files under spaces/', async () => { - const exporter = new AmAssetQueryExporter(baseOptions); + const exporter = new CsAssetsQueryExporter(baseOptions); await exporter.export(['asset-1']); const metadataPath = pResolve(exportDir, 'spaces', 'space-1', 'metadata.json'); diff --git a/packages/contentstack-query-export/src/core/query-executor.ts b/packages/contentstack-query-export/src/core/query-executor.ts index 0d9ca5f57..bc0042149 100644 --- a/packages/contentstack-query-export/src/core/query-executor.ts +++ b/packages/contentstack-query-export/src/core/query-executor.ts @@ -6,7 +6,7 @@ import { readContentTypeSchemas, managementSDKClient, } from '@contentstack/cli-utilities'; -import { AmAssetQueryExporter } from '@contentstack/cli-asset-management'; +import { CsAssetsQueryExporter } from '@contentstack/cli-asset-management'; import * as fs from 'fs'; import * as path from 'path'; import { QueryExportConfig, Modules } from '../types'; @@ -46,7 +46,7 @@ export class QueryExporter { // Step 2: Always export general modules await this.exportGeneralModules(); - // Step 3: Resolve AM 2.0 linked workspaces from branch settings + // Step 3: Resolve linked workspaces from branch settings (Contentstack Assets) await this.fetchLinkedWorkspaces(); // Step 4: Export queried modules @@ -65,7 +65,7 @@ export class QueryExporter { } /** - * Fetch linked workspaces (am_v2) from branch settings for AM 2.0 asset routing. + * Fetch linked workspaces from branch settings for Contentstack Assets export routing. */ private async fetchLinkedWorkspaces(): Promise { const branchName = this.exportQueryConfig.branchName || 'main'; @@ -77,19 +77,19 @@ export class QueryExporter { ?.settings?.am_v2?.linked_workspaces; this.exportQueryConfig.linkedWorkspaces = Array.isArray(linked) ? linked : []; log.debug( - `Linked workspaces for AM 2.0: ${this.exportQueryConfig.linkedWorkspaces?.length ?? 0}`, + `Linked workspaces for Contentstack Assets: ${this.exportQueryConfig.linkedWorkspaces?.length ?? 0}`, this.exportQueryConfig.context, ); } catch (error) { log.warn( - `Could not fetch linked workspaces for branch ${branchName}, using legacy asset export`, + `Could not fetch linked workspaces for branch ${branchName}, using stack assets export`, this.exportQueryConfig.context, ); this.exportQueryConfig.linkedWorkspaces = []; } } - private isAM2AssetExport(): boolean { + private isCsAssetsExport(): boolean { return ( (this.exportQueryConfig.linkedWorkspaces?.length ?? 0) > 0 && Boolean(this.exportQueryConfig.csAssetsUrl) @@ -97,7 +97,7 @@ export class QueryExporter { } /** - * Resolve organization UID for AM 2.0 API calls. + * Resolve organization UID for Contentstack Assets API calls. */ private async resolveOrgUid(): Promise { if (this.exportQueryConfig.org_uid) { @@ -334,12 +334,12 @@ export class QueryExporter { log.info(`Found ${assetUIDs.length} referenced assets to export`, this.exportQueryConfig.context); - if (this.isAM2AssetExport()) { - await this.exportReferencedAssetsAM2(assetUIDs); + if (this.isCsAssetsExport()) { + await this.exportReferencedCsAssets(assetUIDs); return; } - await this.exportReferencedAssetsLegacy(assetUIDs); + await this.exportReferencedStackAssets(assetUIDs); } catch (error) { handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting referenced assets'); throw error; @@ -347,16 +347,16 @@ export class QueryExporter { } /** - * AM 2.0: export referenced assets into spaces/ via Contentstack Assets search API. + * Export referenced assets into spaces/ via Contentstack Assets search API. */ - private async exportReferencedAssetsAM2(assetUIDs: string[]): Promise { - log.info('Using AM 2.0 asset export (spaces/)', this.exportQueryConfig.context); + private async exportReferencedCsAssets(assetUIDs: string[]): Promise { + log.info('Using Contentstack Assets export (spaces/)', this.exportQueryConfig.context); const org_uid = await this.resolveOrgUid(); if (!org_uid) { - throw new Error('Organization UID is required for AM 2.0 asset export'); + throw new Error('Organization UID is required for Contentstack Assets export'); } - const amExporter = new AmAssetQueryExporter({ + const csAssetsExporter = new CsAssetsQueryExporter({ linkedWorkspaces: this.exportQueryConfig.linkedWorkspaces ?? [], exportDir: this.exportQueryConfig.exportDir, branchName: this.exportQueryConfig.branchName || 'main', @@ -368,14 +368,14 @@ export class QueryExporter { assetBatchSize: this.exportQueryConfig.assetBatchSize, }); - await amExporter.export(assetUIDs); - log.success('Referenced assets exported successfully (AM 2.0)', this.exportQueryConfig.context); + await csAssetsExporter.export(assetUIDs); + log.success('Referenced assets exported successfully (Contentstack Assets)', this.exportQueryConfig.context); } /** - * AM 1.0: export referenced assets into legacy assets/ via CMA export module. + * Export referenced assets into stack assets/ via CMA export module. */ - private async exportReferencedAssetsLegacy(assetUIDs: string[]): Promise { + private async exportReferencedStackAssets(assetUIDs: string[]): Promise { const assetsDir = path.join( sanitizePath(this.exportQueryConfig.exportDir), sanitizePath(this.exportQueryConfig.branchName || ''), @@ -486,7 +486,7 @@ export class QueryExporter { log.info(`Temporary files cleaned up`, this.exportQueryConfig.context); log.success('Referenced assets exported successfully', this.exportQueryConfig.context); } catch (error) { - handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting legacy referenced assets'); + handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting stack referenced assets'); throw error; } } diff --git a/packages/contentstack-query-export/src/types/index.ts b/packages/contentstack-query-export/src/types/index.ts index 6c96ce564..8791069e9 100644 --- a/packages/contentstack-query-export/src/types/index.ts +++ b/packages/contentstack-query-export/src/types/index.ts @@ -210,11 +210,11 @@ export interface QueryExportConfig extends DefaultConfig { batchDelayMs?: number; assetBatchSize?: number; assetBatchDelayMs?: number; - /** AM 2.0 linked workspaces from branch settings (am_v2.linked_workspaces). */ + /** Linked workspaces from branch settings (Contentstack Assets). */ linkedWorkspaces?: LinkedWorkspace[]; - /** Contentstack Assets API base URL for AM 2.0 export. */ + /** Contentstack Assets API base URL. */ csAssetsUrl?: string; - /** Organization UID for AM 2.0 API headers. */ + /** Organization UID for Contentstack Assets API headers. */ org_uid?: string; context?: LogContext; // Log context for centralized logging } diff --git a/packages/contentstack-query-export/src/utils/config-handler.ts b/packages/contentstack-query-export/src/utils/config-handler.ts index 1a3ba02a4..c281c07b0 100644 --- a/packages/contentstack-query-export/src/utils/config-handler.ts +++ b/packages/contentstack-query-export/src/utils/config-handler.ts @@ -5,7 +5,7 @@ import config from '../config'; import { askAPIKey } from './common-helper'; /** - * Apply region-derived AM 2.0 settings after the command resolves region. + * Apply region-derived Contentstack Assets settings after the command resolves region. */ export function applyRegionToQueryExportConfig(exportQueryConfig: QueryExportConfig, region?: Region): void { if (!region) return; diff --git a/packages/contentstack-query-export/test/unit/query-executor.test.ts b/packages/contentstack-query-export/test/unit/query-executor.test.ts index a7d3b3c7c..ecf9670d9 100644 --- a/packages/contentstack-query-export/test/unit/query-executor.test.ts +++ b/packages/contentstack-query-export/test/unit/query-executor.test.ts @@ -3,7 +3,7 @@ import * as sinon from 'sinon'; import { QueryExporter } from '../../src/core/query-executor'; import { QueryParser } from '../../src/utils/query-parser'; import { ModuleExporter } from '../../src/core/module-exporter'; -import { AmAssetQueryExporter } from '@contentstack/cli-asset-management'; +import { CsAssetsQueryExporter } from '@contentstack/cli-asset-management'; import * as logger from '../../src/utils/logger'; import { ReferencedContentTypesHandler, @@ -577,11 +577,11 @@ describe('QueryExporter', () => { describe('exportReferencedAssets', () => { let moduleExporterStub: sinon.SinonStub; let assetHandlerStub: any; - let amExporterStub: sinon.SinonStub; + let csAssetsExporterStub: sinon.SinonStub; beforeEach(() => { moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); - amExporterStub = sandbox.stub(AmAssetQueryExporter.prototype, 'export').resolves(); + csAssetsExporterStub = sandbox.stub(CsAssetsQueryExporter.prototype, 'export').resolves(); // Mock AssetReferenceHandler assetHandlerStub = { @@ -592,7 +592,7 @@ describe('QueryExporter', () => { .callsFake(assetHandlerStub.extractReferencedAssets); }); - it('should export referenced assets when found (AM 1.0 legacy path)', async () => { + it('should export referenced assets when found (stack assets path)', async () => { mockConfig.linkedWorkspaces = []; mockConfig.csAssetsUrl = undefined; queryExporter = new QueryExporter(mockManagementClient, mockConfig); @@ -603,26 +603,26 @@ describe('QueryExporter', () => { await (queryExporter as any).exportReferencedAssets(); expect(legacyModuleExporterStub.calledOnce).to.be.true; - expect(amExporterStub.called).to.be.false; + expect(csAssetsExporterStub.called).to.be.false; const exportCall = legacyModuleExporterStub.getCall(0); expect(exportCall.args[0]).to.equal('assets'); expect(exportCall.args[1].query.modules.assets.uid.$in).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); }); - it('should use AmAssetQueryExporter when AM 2.0 linked workspaces and csAssetsUrl are set', async () => { + it('should use CsAssetsQueryExporter when linked workspaces and csAssetsUrl are set', async () => { mockConfig.linkedWorkspaces = [{ uid: 'main', space_uid: 'space-1', is_default: true }]; mockConfig.csAssetsUrl = 'https://am.example.com'; mockConfig.org_uid = 'org-1'; queryExporter = new QueryExporter(mockManagementClient, mockConfig); - const am2ModuleExporterStub = sandbox + const csAssetsModuleExporterStub = sandbox .stub((queryExporter as any).moduleExporter, 'exportModule') .resolves(); await (queryExporter as any).exportReferencedAssets(); - expect(amExporterStub.calledOnce).to.be.true; - expect(amExporterStub.firstCall.args[0]).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); - expect(am2ModuleExporterStub.called).to.be.false; + expect(csAssetsExporterStub.calledOnce).to.be.true; + expect(csAssetsExporterStub.firstCall.args[0]).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); + expect(csAssetsModuleExporterStub.called).to.be.false; }); it('should skip export when no assets found', async () => { From 6333e8f78f7046015d33a4e1975de5951a55692b Mon Sep 17 00:00:00 2001 From: harshitha-cstk Date: Thu, 21 May 2026 13:05:29 +0530 Subject: [PATCH 08/61] feat: migrate @contentstack/apps-cli to cli-plugins monorepo and add initial implementation --- .cursor/rules/README.md | 3 +- .github/config/release.json | 3 + .github/workflows/unit-test.yml | 4 + APPS-CLI-MIGRATION.md | 45 ++ README.md | 1 + packages/contentstack-apps-cli/.eslintignore | 1 + packages/contentstack-apps-cli/.eslintrc | 42 ++ packages/contentstack-apps-cli/.gitignore | 21 + packages/contentstack-apps-cli/.mocharc.json | 12 + packages/contentstack-apps-cli/LICENSE | 21 + packages/contentstack-apps-cli/README.md | 313 ++++++++++++ packages/contentstack-apps-cli/bin/dev | 17 + packages/contentstack-apps-cli/bin/dev.cmd | 3 + packages/contentstack-apps-cli/bin/run | 5 + packages/contentstack-apps-cli/bin/run.cmd | 3 + .../examples/create-launch-project.json | 14 + packages/contentstack-apps-cli/package.json | 114 +++++ .../src/app-cli-base-command.ts | 33 ++ .../contentstack-apps-cli/src/base-command.ts | 199 ++++++++ .../src/commands/app/create.ts | 463 +++++++++++++++++ .../src/commands/app/delete.ts | 86 ++++ .../src/commands/app/deploy.ts | 306 +++++++++++ .../src/commands/app/get.ts | 68 +++ .../src/commands/app/index.ts | 39 ++ .../src/commands/app/install.ts | 155 ++++++ .../src/commands/app/reinstall.ts | 137 +++++ .../src/commands/app/uninstall.ts | 82 +++ .../src/commands/app/update.ts | 219 ++++++++ .../contentstack-apps-cli/src/config/index.ts | 13 + .../src/config/manifest.json | 105 ++++ .../src/factories/uninstall-app-factory.ts | 15 + .../src/graphql/queries.ts | 29 ++ packages/contentstack-apps-cli/src/index.ts | 1 + .../src/interfaces/uninstall-app.ts | 18 + .../src/messages/index.ts | 173 +++++++ .../src/strategies/uninstall-all.ts | 61 +++ .../src/strategies/uninstall-selected.ts | 63 +++ .../contentstack-apps-cli/src/types/app.ts | 151 ++++++ .../contentstack-apps-cli/src/types/index.ts | 2 + .../contentstack-apps-cli/src/types/utils.ts | 29 ++ .../src/util/api-request-handler.ts | 61 +++ .../src/util/common-utils.ts | 430 ++++++++++++++++ .../src/util/error-helper.ts | 7 + packages/contentstack-apps-cli/src/util/fs.ts | 34 ++ .../contentstack-apps-cli/src/util/index.ts | 4 + .../src/util/inquirer.ts | 471 +++++++++++++++++ .../contentstack-apps-cli/src/util/log.ts | 188 +++++++ .../test/helpers/init.js | 7 + .../contentstack-apps-cli/test/tsconfig.json | 10 + .../test/unit/commands/app/create.test.ts | 473 ++++++++++++++++++ .../test/unit/commands/app/delete.test.ts | 114 +++++ .../test/unit/commands/app/deploy.test.ts | 405 +++++++++++++++ .../test/unit/commands/app/get.test.ts | 206 ++++++++ .../test/unit/commands/app/install.test.ts | 166 ++++++ .../test/unit/commands/app/reinstall.test.ts | 225 +++++++++ .../test/unit/commands/app/uninstall.test.ts | 353 +++++++++++++ .../test/unit/commands/app/update.test.ts | 290 +++++++++++ .../test/unit/config/manifest.json | 28 ++ .../test/unit/config/org_manifest.json | 29 ++ .../test/unit/helpers/auth-stub-helper.ts | 36 ++ .../test/unit/mock/boilerplate.zip | Bin 0 -> 328 bytes .../test/unit/mock/common.mock.json | 154 ++++++ .../test/unit/mock/config.json | 3 + .../test/unit/util/common-utils.test.ts | 144 ++++++ .../test/unit/util/inquirer.test.ts | 177 +++++++ packages/contentstack-apps-cli/tsconfig.json | 24 + 66 files changed, 7107 insertions(+), 1 deletion(-) create mode 100644 APPS-CLI-MIGRATION.md create mode 100644 packages/contentstack-apps-cli/.eslintignore create mode 100644 packages/contentstack-apps-cli/.eslintrc create mode 100644 packages/contentstack-apps-cli/.gitignore create mode 100644 packages/contentstack-apps-cli/.mocharc.json create mode 100644 packages/contentstack-apps-cli/LICENSE create mode 100644 packages/contentstack-apps-cli/README.md create mode 100755 packages/contentstack-apps-cli/bin/dev create mode 100644 packages/contentstack-apps-cli/bin/dev.cmd create mode 100755 packages/contentstack-apps-cli/bin/run create mode 100644 packages/contentstack-apps-cli/bin/run.cmd create mode 100644 packages/contentstack-apps-cli/examples/create-launch-project.json create mode 100644 packages/contentstack-apps-cli/package.json create mode 100644 packages/contentstack-apps-cli/src/app-cli-base-command.ts create mode 100644 packages/contentstack-apps-cli/src/base-command.ts create mode 100644 packages/contentstack-apps-cli/src/commands/app/create.ts create mode 100644 packages/contentstack-apps-cli/src/commands/app/delete.ts create mode 100644 packages/contentstack-apps-cli/src/commands/app/deploy.ts create mode 100644 packages/contentstack-apps-cli/src/commands/app/get.ts create mode 100644 packages/contentstack-apps-cli/src/commands/app/index.ts create mode 100644 packages/contentstack-apps-cli/src/commands/app/install.ts create mode 100644 packages/contentstack-apps-cli/src/commands/app/reinstall.ts create mode 100644 packages/contentstack-apps-cli/src/commands/app/uninstall.ts create mode 100644 packages/contentstack-apps-cli/src/commands/app/update.ts create mode 100644 packages/contentstack-apps-cli/src/config/index.ts create mode 100644 packages/contentstack-apps-cli/src/config/manifest.json create mode 100644 packages/contentstack-apps-cli/src/factories/uninstall-app-factory.ts create mode 100644 packages/contentstack-apps-cli/src/graphql/queries.ts create mode 100644 packages/contentstack-apps-cli/src/index.ts create mode 100644 packages/contentstack-apps-cli/src/interfaces/uninstall-app.ts create mode 100644 packages/contentstack-apps-cli/src/messages/index.ts create mode 100644 packages/contentstack-apps-cli/src/strategies/uninstall-all.ts create mode 100644 packages/contentstack-apps-cli/src/strategies/uninstall-selected.ts create mode 100644 packages/contentstack-apps-cli/src/types/app.ts create mode 100644 packages/contentstack-apps-cli/src/types/index.ts create mode 100644 packages/contentstack-apps-cli/src/types/utils.ts create mode 100644 packages/contentstack-apps-cli/src/util/api-request-handler.ts create mode 100644 packages/contentstack-apps-cli/src/util/common-utils.ts create mode 100644 packages/contentstack-apps-cli/src/util/error-helper.ts create mode 100644 packages/contentstack-apps-cli/src/util/fs.ts create mode 100644 packages/contentstack-apps-cli/src/util/index.ts create mode 100644 packages/contentstack-apps-cli/src/util/inquirer.ts create mode 100755 packages/contentstack-apps-cli/src/util/log.ts create mode 100755 packages/contentstack-apps-cli/test/helpers/init.js create mode 100755 packages/contentstack-apps-cli/test/tsconfig.json create mode 100644 packages/contentstack-apps-cli/test/unit/commands/app/create.test.ts create mode 100644 packages/contentstack-apps-cli/test/unit/commands/app/delete.test.ts create mode 100644 packages/contentstack-apps-cli/test/unit/commands/app/deploy.test.ts create mode 100644 packages/contentstack-apps-cli/test/unit/commands/app/get.test.ts create mode 100644 packages/contentstack-apps-cli/test/unit/commands/app/install.test.ts create mode 100644 packages/contentstack-apps-cli/test/unit/commands/app/reinstall.test.ts create mode 100644 packages/contentstack-apps-cli/test/unit/commands/app/uninstall.test.ts create mode 100644 packages/contentstack-apps-cli/test/unit/commands/app/update.test.ts create mode 100644 packages/contentstack-apps-cli/test/unit/config/manifest.json create mode 100644 packages/contentstack-apps-cli/test/unit/config/org_manifest.json create mode 100644 packages/contentstack-apps-cli/test/unit/helpers/auth-stub-helper.ts create mode 100644 packages/contentstack-apps-cli/test/unit/mock/boilerplate.zip create mode 100644 packages/contentstack-apps-cli/test/unit/mock/common.mock.json create mode 100644 packages/contentstack-apps-cli/test/unit/mock/config.json create mode 100644 packages/contentstack-apps-cli/test/unit/util/common-utils.test.ts create mode 100644 packages/contentstack-apps-cli/test/unit/util/inquirer.test.ts create mode 100644 packages/contentstack-apps-cli/tsconfig.json diff --git a/.cursor/rules/README.md b/.cursor/rules/README.md index cbb51b61b..325fa7b53 100644 --- a/.cursor/rules/README.md +++ b/.cursor/rules/README.md @@ -37,7 +37,7 @@ Context-aware rules that load automatically based on the files you're editing, o ### Monorepo Structure -This is a **CLI plugins** monorepo with 12 plugin packages under `packages/`: +This is a **CLI plugins** monorepo with plugin packages under `packages/`, including: - `contentstack-audit` - Stack audit and fix operations - `contentstack-bootstrap` - Seed/bootstrap stacks with content - `contentstack-branches` - Git-based branch management for stacks @@ -50,6 +50,7 @@ This is a **CLI plugins** monorepo with 12 plugin packages under `packages/`: - `contentstack-migration` - Content migration workflows - `contentstack-seed` - Seed stacks with generated data - `contentstack-variants` - Manage content variants +- `contentstack-apps-cli` - Developer Hub apps (`app:*` commands; npm `@contentstack/apps-cli`) All plugins depend on: - `@contentstack/cli-command` - Base Command class diff --git a/.github/config/release.json b/.github/config/release.json index 078a8824b..1ea83997c 100755 --- a/.github/config/release.json +++ b/.github/config/release.json @@ -1,6 +1,8 @@ { "releaseAll": true, "plugins": { + "variants": false, + "query-export": false, "utilities": false, "command": false, "config": false, @@ -17,6 +19,7 @@ "dev-dependencies": false, "launch": false, "branches": false, + "apps-cli": false, "core": false } } diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index eecc68d16..7c306cae7 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -65,3 +65,7 @@ jobs: - name: Run tests for Contentstack Query Export working-directory: ./packages/contentstack-query-export run: npm run test:unit + + - name: Run tests for Contentstack Apps CLI + working-directory: ./packages/contentstack-apps-cli + run: npm run test:unit:report:json diff --git a/APPS-CLI-MIGRATION.md b/APPS-CLI-MIGRATION.md new file mode 100644 index 000000000..d3b9af325 --- /dev/null +++ b/APPS-CLI-MIGRATION.md @@ -0,0 +1,45 @@ +# Apps CLI migration: standalone repo → cli-plugins monorepo + +## Summary + +The **@contentstack/apps-cli** plugin has moved from [contentstack/contentstack-apps-cli](https://github.com/contentstack/contentstack-apps-cli) into [contentstack/cli-plugins](https://github.com/contentstack/cli-plugins) at **`packages/contentstack-apps-cli`**. + +The **npm package name is unchanged**: `@contentstack/apps-cli`. + +## Repository and issues + +| Before | After | +| --- | --- | +| `github.com/contentstack/contentstack-apps-cli` | `github.com/contentstack/cli-plugins` → `packages/contentstack-apps-cli` | +| Issues on standalone repo | [cli-plugins issues](https://github.com/contentstack/cli-plugins/issues) | + +## Version lines (1.x vs 2.x) + +| CLI line | cli-plugins branch | Apps plugin | +| --- | --- | --- | +| **1.x** | `v1-dev` / `v1-beta` | Version **1.7.x**; `@contentstack/cli-command` ~1.8.2, `@contentstack/cli-utilities` ~1.18.x; chalk v4 | +| **2.x beta** | `v2-dev` / `v2-beta` | Version **2.0.0-beta.x**; 2.x beta core packages; chalk v5 | + +Develop and release each line on its branch. + +## Install (unchanged) + +```bash +csdx plugins:install @contentstack/apps-cli +``` + +## Local development (cli-dev-workspace) + +```bash +cd cli-dev-workspace +pnpm install +pnpm --filter @contentstack/apps-cli run build +pnpm -C cli/packages/contentstack run build +``` + +Core CLI must list `@contentstack/apps-cli` as `workspace:*` and register it in `oclif.plugins` — see [cli](https://github.com/contentstack/cli) `packages/contentstack/package.json`. + +## Contributor docs + +- [AGENTS.md](./AGENTS.md) +- [skills/contentstack-apps/SKILL.md](./skills/contentstack-apps/SKILL.md) diff --git a/README.md b/README.md index 60eca7d5e..b3660d6ba 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ CLI supports content management scripts through which you can perform the follow - Migrate HTML RTE to JSON RTE content - Change Master Locale - Use Bootstrap plugin +- Manage Developer Hub apps (`app:*` via `@contentstack/apps-cli`) - Use Tsgen plugin diff --git a/packages/contentstack-apps-cli/.eslintignore b/packages/contentstack-apps-cli/.eslintignore new file mode 100644 index 000000000..9b1c8b133 --- /dev/null +++ b/packages/contentstack-apps-cli/.eslintignore @@ -0,0 +1 @@ +/dist diff --git a/packages/contentstack-apps-cli/.eslintrc b/packages/contentstack-apps-cli/.eslintrc new file mode 100644 index 000000000..aa58bce72 --- /dev/null +++ b/packages/contentstack-apps-cli/.eslintrc @@ -0,0 +1,42 @@ +{ + "env": { + "node": true + }, + "parser": "@typescript-eslint/parser", + "parserOptions": { + "project": "tsconfig.json", + "sourceType": "module" + }, + "plugins": [ + "@typescript-eslint" + ], + "extends": [ + "plugin:@typescript-eslint/recommended" + ], + "ignorePatterns": [ + "lib/**/*", + "test/**/*" + ], + "rules": { + "@typescript-eslint/no-unused-vars": [ + "error", + { + "args": "none" + } + ], + "@typescript-eslint/prefer-namespace-keyword": "error", + "quotes": "off", + "semi": "off", + "@typescript-eslint/no-redeclare": "off", + "eqeqeq": [ + "error", + "smart" + ], + "id-match": "error", + "no-eval": "error", + "no-var": "error", + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-require-imports": "off", + "prefer-const": "error" + } +} \ No newline at end of file diff --git a/packages/contentstack-apps-cli/.gitignore b/packages/contentstack-apps-cli/.gitignore new file mode 100644 index 000000000..976172982 --- /dev/null +++ b/packages/contentstack-apps-cli/.gitignore @@ -0,0 +1,21 @@ +*-debug.log +*-error.log +/.nyc_output +/dist +/lib +/package-lock.json +/tmp +/yarn.lock +node_modules +oclif.manifest.json +.env +*.log +tsconfig.tsbuildinfo +dependabot.yml +.vscode +*.todo +/bkp +.editorconfig +oclif.manifest.json +*.env +.vscode/ diff --git a/packages/contentstack-apps-cli/.mocharc.json b/packages/contentstack-apps-cli/.mocharc.json new file mode 100644 index 000000000..4a09d1446 --- /dev/null +++ b/packages/contentstack-apps-cli/.mocharc.json @@ -0,0 +1,12 @@ +{ + "require": [ + "test/helpers/init.js", + "ts-node/register" + ], + "watch-extensions": [ + "ts" + ], + "recursive": true, + "reporter": "spec", + "timeout": 60000 +} diff --git a/packages/contentstack-apps-cli/LICENSE b/packages/contentstack-apps-cli/LICENSE new file mode 100644 index 000000000..aff1142ee --- /dev/null +++ b/packages/contentstack-apps-cli/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Contentstack + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/contentstack-apps-cli/README.md b/packages/contentstack-apps-cli/README.md new file mode 100644 index 000000000..06e597559 --- /dev/null +++ b/packages/contentstack-apps-cli/README.md @@ -0,0 +1,313 @@ +> **Source of truth:** [cli-plugins](https://github.com/contentstack/cli-plugins) — `packages/contentstack-apps-cli` (v1 line: `v1-dev` / `v1-beta`) +> Migrated from [contentstack-apps-cli](https://github.com/contentstack/contentstack-apps-cli). See [APPS-CLI-MIGRATION.md](../../APPS-CLI-MIGRATION.md). + + + + +# @contentstack/apps-cli + +Contentstack lets you develop apps in your organization using the Developer Hub portal. With the Apps CLI plugin, Contentstack CLI allows you to perform the CRUD operations on your app in Developer Hub and then use the app in your organization or stack by installing or uninstalling your app as required. + +## How to install this plugin + +```shell +$ csdx plugins:install @contentstack/apps-cli +``` + +## How to use this plugin + +This plugin requires you to be authenticated using [csdx auth:login](https://www.contentstack.com/docs/developers/cli/authenticate-with-the-cli/). + + +```sh-session +$ npm install -g @contentstack/apps-cli +$ csdx COMMAND +running command... +$ csdx (--version|-v) +@contentstack/apps-cli/1.6.1 darwin-arm64 node-v18.20.2 +$ csdx --help [COMMAND] +USAGE + $ csdx COMMAND +... +``` + + +# Commands + + +* [`csdx app`](#csdx-app) +* [`csdx app:create`](#csdx-appcreate) +* [`csdx app:delete`](#csdx-appdelete) +* [`csdx app:deploy`](#csdx-appdeploy) +* [`csdx app:get`](#csdx-appget) +* [`csdx app:install`](#csdx-appinstall) +* [`csdx app:reinstall`](#csdx-appreinstall) +* [`csdx app:uninstall`](#csdx-appuninstall) +* [`csdx app:update`](#csdx-appupdate) + +## `csdx app` + +Apps CLI plugin + +``` +USAGE + $ csdx app + +DESCRIPTION + Apps CLI plugin + +EXAMPLES + $ csdx app:create + + $ csdx app:delete + + $ csdx app:deploy + + $ csdx app:get + + $ csdx app:install + + $ csdx app:reinstall + + $ csdx app:uninstall + + $ csdx app:update +``` + +_See code: [src/commands/app/index.ts](https://github.com/contentstack/apps-cli/blob/v1.6.1/src/commands/app/index.ts)_ + +## `csdx app:create` + +Create a new app in Developer Hub and optionally clone a boilerplate locally. + +``` +USAGE + $ csdx app:create [--org ] [-n ] [--app-type stack|organization] [-c ] [-d ] + [--boilerplate ] + +FLAGS + -c, --config= Path of the external config + -d, --data-dir= Current working directory. + -n, --name= Name of the app to be created + --app-type=