feat: add pagination and test cases for AM package

Author: naman-contentstack

.talismanrc

@@ -1,12 +1,14 @@
 fileignoreconfig:
-  - filename: pnpm-lock.yaml
-    checksum: 2b0f2461ea1bb240a9210b9cf99dc403a756199712b7270f9792a590480451bd
-  - filename: packages/contentstack-import/test/unit/import/modules/base-class.test.ts
-    checksum: fe372852d5f2f3f57ef62c603406c30ccecdb444c17133ac0b21dda399b962c0
-  - filename: packages/contentstack-bulk-operations/test/unit/commands/bulk-am-assets.test.ts
-    checksum: f8d21db7db0ca2eebe7cc40af0a59f02e74e1689efb6d50a1072dc5ca3e03e9b
-  - filename: packages/contentstack-export/src/export/modules/taxonomies.ts
-    checksum: b6d077118280bc88385405f504f921468a9fd490ac37a4a21f741be729fd1ca3
-  - filename: packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts
-    checksum: cab2ad4d897d23f04f988c1f018a9583ab7f0ee1815994d7bc9fce23dea70073
+- filename: packages/contentstack-asset-management/test/unit/import/asset-types.test.ts
+  checksum: fa4c08a47a52b8bd27353b9ba712e6128674315610d3372b53abe6af361ba0b7
+- filename: packages/contentstack-asset-management/test/unit/import/fields.test.ts
+  checksum: c2830111af2bf72c5a661fe2251a3520a865c42c3980289195a08d713703e4b1
+- filename: packages/contentstack-asset-management/test/unit/import/spaces.test.ts
+  checksum: f95e4b7ce04d4beb70c7f70c762f03541588c211a8c15ba68d426c98215a0769
+- filename: packages/contentstack-asset-management/test/unit/import/assets.test.ts
+  checksum: 37613b7f2b5812282eedf87e0b9f87efc8eebb5e3cf75afc5f0eaa3605be4b5d
+- filename: packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts
+  checksum: 3d9d252ffa66a28380a015857fe7358fe7e2ba2efe90adaf715f9ff2a408450f
+- filename: packages/contentstack-asset-management/test/unit/import/base.test.ts
+  checksum: c6ed81639052b5905f481e90d6c17e19b099b30916d4cf6bf7eabfe33fd15530
 version: '1.0'

packages/contentstack-asset-management/src/constants/index.ts

@@ -4,6 +4,8 @@ export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1;
 export const FALLBACK_AM_API_CONCURRENCY = 5;
 /** @deprecated Use FALLBACK_AM_API_CONCURRENCY */
 export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY;
+export const FALLBACK_AM_API_PAGE_SIZE = 100;
+export const FALLBACK_AM_API_FETCH_CONCURRENCY = 5;
 
 /** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */
 export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [

packages/contentstack-asset-management/src/export/assets.ts

@@ -28,8 +28,8 @@ export default class ExportAssets extends CSAssetsExportAdapter {
     log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context);
 
     const [folders, assetsData] = await Promise.all([
-      this.getWorkspaceFolders(workspace.space_uid, workspace.uid),
-      this.getWorkspaceAssets(workspace.space_uid, workspace.uid),
+      this.getWorkspaceFolders(workspace.space_uid, workspace.uid, this.apiPageSize, this.apiFetchConcurrency),
+      this.getWorkspaceAssets(workspace.space_uid, workspace.uid, this.apiPageSize, this.apiFetchConcurrency),
     ]);
 
     const assetItems = getAssetItems(assetsData);

packages/contentstack-asset-management/src/export/base.ts

@@ -5,7 +5,7 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack
 import type { CSAssetsAPIConfig } from '../types/cs-assets-api';
 import type { ExportContext } from '../types/export-types';
 import { CSAssetsAdapter } from '../utils/cs-assets-api-adapter';
-import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index';
+import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index';
 
 export type { ExportContext };
 
@@ -82,6 +82,14 @@ export class CSAssetsExportAdapter extends CSAssetsAdapter {
     return this.exportContext.downloadAssetsConcurrency ?? this.apiConcurrency;
   }
 
+  protected get apiPageSize(): number {
+    return this.exportContext.pageSize ?? FALLBACK_AM_API_PAGE_SIZE;
+  }
+
+  protected get apiFetchConcurrency(): number {
+    return this.exportContext.fetchConcurrency ?? FALLBACK_AM_API_FETCH_CONCURRENCY;
+  }
+
   protected getAssetTypesDir(): string {
     return pResolve(this.exportContext.spacesRootPath, 'asset_types');
   }

packages/contentstack-asset-management/src/export/spaces.ts

@@ -79,6 +79,8 @@ export class ExportSpaces {
       chunkFileSizeMb,
       apiConcurrency: this.options.apiConcurrency,
       downloadAssetsConcurrency: this.options.downloadAssetsConcurrency,
+      pageSize: this.options.pageSize,
+      fetchConcurrency: this.options.fetchConcurrency,
     };
 
     const sharedFieldsDir = pResolve(spacesRootPath, 'fields');

packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts

@@ -4,7 +4,7 @@ import { join, resolve } from 'node:path';
 
 import { formatError, log } from '@contentstack/cli-utilities';
 
-import { IMPORT_ASSETS_MAPPER_FILES, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
+import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE, IMPORT_ASSETS_MAPPER_FILES, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
 import type { CSAssetsAPIConfig, ImportContext } from '../types/cs-assets-api';
 import type { AssetMapperImportSetupResult, RunAssetMapperImportSetupParams } from '../types/import-setup-asset-mapper';
 import ImportAssets from '../import/assets';
@@ -25,7 +25,7 @@ export default class ImportSetupAssetMappers extends AssetManagementImportSetupA
   private async fetchExistingSpaceUidsInOrg(apiConfig: CSAssetsAPIConfig): Promise<Set<string>> {
     const adapter = new CSAssetsAdapter(apiConfig);
     await adapter.init();
-    const { spaces } = await adapter.listSpaces();
+    const { spaces } = await adapter.listSpaces(FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY);
     const uids = new Set<string>();
     for (const s of spaces) {
       if (s.uid) {

packages/contentstack-asset-management/src/import/spaces.ts

@@ -10,7 +10,7 @@ import type {
   ImportSpacesOptions,
   SpaceMapping,
 } from '../types/cs-assets-api';
-import { CS_ASSETS_MAIN_PROCESS_NAME, PROCESS_NAMES, getSpaceProcessName } from '../constants/index';
+import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY, PROCESS_NAMES, getSpaceProcessName } from '../constants/index';
 import { CSAssetsAdapter } from '../utils/cs-assets-api-adapter';
 import ImportAssetTypes from './asset-types';
 import ImportFields from './fields';
@@ -112,7 +112,7 @@ export class ImportSpaces {
     try {
       const adapterForList = new CSAssetsAdapter(apiConfig);
       await adapterForList.init();
-      const { spaces } = await adapterForList.listSpaces();
+      const { spaces } = await adapterForList.listSpaces(FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY);
       for (const s of spaces) {
         if (s.uid) existingSpaceUids.add(s.uid);
       }

packages/contentstack-asset-management/src/types/cs-assets-api.ts

@@ -167,11 +167,11 @@ export type SearchAssetsResponse = {
 
 export interface ICSAssetsAdapter {
   init(): Promise<void>;
-  listSpaces(): Promise<SpacesListResponse>;
+  listSpaces(pageSize?: number, fetchConcurrency?: number): Promise<SpacesListResponse>;
   getSpace(spaceUid: string): Promise<SpaceResponse>;
   getWorkspaceFields(spaceUid: string): Promise<FieldsResponse>;
-  getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise<unknown>;
-  getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise<unknown>;
+  getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise<unknown>;
+  getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise<unknown>;
   getWorkspaceAssetTypes(spaceUid: string): Promise<AssetTypesResponse>;
   searchAssets(params: SearchAssetsParams): Promise<SearchAssetsResponse>;
   bulkDeleteAssets(
@@ -233,6 +233,8 @@ export type AssetManagementExportOptions = {
    * Max parallel asset file downloads per workspace.
    */
   downloadAssetsConcurrency?: number;
+  pageSize?: number;
+  fetchConcurrency?: number;
 };
 
 // ---------------------------------------------------------------------------

packages/contentstack-asset-management/src/types/export-types.ts

@@ -5,6 +5,8 @@ export type ExportContext = {
   chunkFileSizeMb?: number;
   apiConcurrency?: number;
   downloadAssetsConcurrency?: number;
+  pageSize?: number;
+  fetchConcurrency?: number;
 };
 
 /**

packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts

@@ -2,6 +2,9 @@ import { readFileSync } from 'node:fs';
 import { basename } from 'node:path';
 import { HttpClient, log, authenticationHandler, handleAndLogError } from '@contentstack/cli-utilities';
 
+import { chunkArray } from './concurrent-batch';
+import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE } from '../constants/index';
+
 import type {
   CSAssetsAPIConfig,
   AssetTypesResponse,
@@ -189,11 +192,17 @@ export class CSAssetsAdapter implements ICSAssetsAdapter {
     }
   }
 
-  async listSpaces(): Promise<SpacesListResponse> {
+  async listSpaces(pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise<SpacesListResponse> {
     log.debug('Fetching all spaces in org', this.config.context);
-    const result = await this.getSpaceLevel<SpacesListResponse>('', '/api/spaces', {});
-    log.debug(`Fetched ${result?.count ?? result?.spaces?.length ?? '?'} space(s)`, this.config.context);
-    return result;
+    const items = await this.fetchAllPages(
+      '',
+      '/api/spaces',
+      'spaces',
+      pageSize,
+      fetchConcurrency,
+    );
+    log.debug(`Fetched ${items.length} space(s)`, this.config.context);
+    return { spaces: items as Space[], count: items.length };
   }
 
   async getSpace(spaceUid: string): Promise<SpaceResponse> {
@@ -230,22 +239,73 @@ export class CSAssetsAdapter implements ICSAssetsAdapter {
     return result;
   }
 
-  async getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise<unknown> {
-    return this.getWorkspaceCollection(
+
+  /**
+   * Fetch all pages of a paginated collection by issuing the first request to determine
+   * the total count, then issuing remaining page requests with controlled concurrency.
+   */
+  private async fetchAllPages(
+    spaceUid: string,
+    path: string,
+    itemsKey: string,
+    pageSize: number,
+    concurrency: number,
+    baseParams: Record<string, unknown> = {},
+  ): Promise<unknown[]> {
+    const first = await this.getSpaceLevel<Record<string, unknown>>(spaceUid, path, {
+      ...baseParams, limit: String(pageSize), skip: '0',
+    });
+
+    const total: number = Number(first?.count ?? 0);
+    const firstItems: unknown[] = Array.isArray(first?.[itemsKey]) ? (first[itemsKey] as unknown[]) : [];
+    if (firstItems.length >= total) return firstItems;
+
+    const skips = Array.from(
+      { length: Math.ceil(total / pageSize) - 1 },
+      (_, i) => (i + 1) * pageSize,
+    );
+
+    const skipBatches = chunkArray(skips, concurrency);
+    const rest: unknown[] = [];
+
+    for (const batch of skipBatches) {
+      const pages = await Promise.all(
+        batch.map((skip) =>
+          this.getSpaceLevel<Record<string, unknown>>(spaceUid, path, {
+            ...baseParams, limit: String(pageSize), skip: String(skip),
+          }).then((r) => (Array.isArray(r?.[itemsKey]) ? (r[itemsKey] as unknown[]) : [])),
+        ),
+      );
+      rest.push(...pages.flat());
+    }
+
+    return [...firstItems, ...rest];
+  }
+
+  async getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise<unknown> {
+    const baseParams: Record<string, unknown> = workspaceUid ? { workspace: workspaceUid } : {};
+    const items = await this.fetchAllPages(
       spaceUid,
       `/api/spaces/${encodeURIComponent(spaceUid)}/assets`,
       'assets',
-      workspaceUid ? { workspace: workspaceUid } : {},
+      pageSize,
+      fetchConcurrency,
+      baseParams,
     );
+    return { assets: items, count: items.length };
   }
 
-  async getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise<unknown> {
-    return this.getWorkspaceCollection(
+  async getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise<unknown> {
+    const baseParams: Record<string, unknown> = workspaceUid ? { workspace: workspaceUid } : {};
+    const items = await this.fetchAllPages(
       spaceUid,
       `/api/spaces/${encodeURIComponent(spaceUid)}/folders`,
       'folders',
-      workspaceUid ? { workspace: workspaceUid } : {},
+      pageSize,
+      fetchConcurrency,
+      baseParams,
     );
+    return { folders: items, count: items.length };
   }
 
   async getWorkspaceAssetTypes(spaceUid: string): Promise<AssetTypesResponse> {