AWS-SDK Migration

index.ts

@@ -153,9 +153,8 @@ export const storage = {
             AwsClient: require('./lib/storage/data/external/AwsClient'),
             AzureClient: require('./lib/storage/data/external/AzureClient'),
             GcpClient: require('./lib/storage/data/external/GcpClient'),
-            GCP: require('./lib/storage/data/external/GCP/GcpService'),
+            GCP: require('./lib/storage/data/external/GCP'),
             GcpUtils: require('./lib/storage/data/external/GCP/GcpUtils'),
-            GcpSigner: require('./lib/storage/data/external/GCP/GcpSigner'),
             PfsClient: require('./lib/storage/data/external/PfsClient'),
             backendUtils: require('./lib/storage/data/external/utils'),
         },

lib/auth/v2/getCanonicalizedAmzHeaders.ts

@@ -9,7 +9,16 @@ export default function getCanonicalizedAmzHeaders(headers: Record<string, strin
         (val: string) => val.startsWith('x-amz-');
     const amzHeaders = Object.keys(headers)
         .filter(filterFn)
-        .map(val => [val.trim(), headers[val].trim()]);
+        .map(val => {
+            const headerValue = headers[val];
+            // AWS SDK v3 can pass header values as arrays (for multiple values),
+            // strings, or other types. We need to normalize them before calling .trim()
+            // Per HTTP spec and AWS Signature v2, multiple values are joined with commas
+            const stringValue = Array.isArray(headerValue) 
+                ? headerValue.join(',') 
+                : String(headerValue);
+            return [val.trim(), stringValue.trim()];
+        });
     /*
     AWS docs state that duplicate headers should be combined
     in the same header with values concatenated with

lib/auth/v4/createCanonicalRequest.ts

@@ -73,7 +73,14 @@ export default function createCanonicalRequest(
     // canonical headers
     const canonicalHeadersList = signedHeadersList.map((signedHeader: any) => {
         if (pHeaders[signedHeader] !== undefined) {
-            const trimmedHeader = pHeaders[signedHeader]
+            const headerValue = pHeaders[signedHeader];
+            // AWS SDK v3 can pass header values as arrays (for multiple values),
+            // strings, or other types. We need to normalize them before calling .trim()
+            // Per HTTP spec and AWS Signature v4, multiple values are joined with commas
+            const stringValue = Array.isArray(headerValue) 
+                ? headerValue.join(',') 
+                : String(headerValue);
+            const trimmedHeader = stringValue
                 .trim().replace(/\s+/g, ' ');
             return `${signedHeader}:${trimmedHeader}\n`;
         }

lib/auth/v4/streamingV4/constructChunkStringToSign.ts

@@ -24,9 +24,7 @@ export default function constructChunkStringToSign(
         currentChunkHash = constants.emptyStringHash;
     } else {
         const hash = crypto.createHash('sha256');
-        const temp = justDataChunk instanceof Buffer
-            ? hash.update(justDataChunk)
-            : hash.update(justDataChunk, 'binary');
+        const temp = hash.update(justDataChunk);
         currentChunkHash = temp.digest('hex');
     }
     return `AWS4-HMAC-SHA256-PAYLOAD\n${timestamp}\n` +

lib/network/kmsAWS/Client.ts

@@ -3,7 +3,16 @@
 import { arsenalErrorAWSKMS } from '../utils';
 import { Agent as HttpAgent } from 'http';
 import { Agent as HttpsAgent } from 'https';
-import { KMS, AWSError } from 'aws-sdk';
+import { KMSClient, 
+    CreateKeyCommand, 
+    ScheduleKeyDeletionCommand,
+    GenerateDataKeyCommand, 
+    EncryptCommand, 
+    DecryptCommand, 
+    ListKeysCommand, 
+    NotFoundException, 
+    KMSInvalidStateException } from '@aws-sdk/client-kms';
+const { NodeHttpHandler } = require('@smithy/node-http-handler');
 import * as werelogs from 'werelogs';
 import assert from 'assert';
 import { KMSInterface, KmsBackend, getKeyIdFromArn, KmsProtocol, KmsType, makeBackend } from '../KMSInterface';
@@ -45,42 +54,39 @@ interface ClientOptions {
 
 export default class Client implements KMSInterface {
     private _supportsDefaultKeyPerAccount: boolean;
-    private client: KMS;
+    private client: KMSClient;
     public readonly backend: KmsBackend<KmsType.external>;
     public readonly noAwsArn?: boolean;
 
     constructor(options: ClientOptions) {
         this._supportsDefaultKeyPerAccount = true;
-        const { providerName, tls, ak, sk, region, endpoint, noAwsArn } = options.kmsAWS;
+        const { providerName,tls, ak, sk, region, endpoint, noAwsArn } = options.kmsAWS;
 
-        const httpOptions = tls ? {
-            agent: new HttpsAgent({
+        const requestHandler = new NodeHttpHandler({
+            httpAgent: !tls ? new HttpAgent({
+                keepAlive: true,
+            }) : undefined,
+            httpsAgent: tls ? new HttpsAgent({
                 keepAlive: true,
                 rejectUnauthorized: tls.rejectUnauthorized,
                 ca: tls.ca,
                 cert: tls.cert,
                 minVersion: tls.minVersion,
                 maxVersion: tls.maxVersion,
                 key: tls.key,
-            }),
-        } : {
-            agent: new HttpAgent({
-                keepAlive: true,
-            }),
-        };
+            }) : undefined,
+        });
 
         const credentials = (ak && sk) ? {
-            credentials: {
-                accessKeyId: ak,
-                secretAccessKey: sk,
-            },
+            accessKeyId: ak,
+            secretAccessKey: sk,
         } : undefined;
 
-        this.client = new KMS({
+        this.client = new KMSClient({
             region,
             endpoint,
-            httpOptions,
-            ...credentials,
+            credentials,
+            requestHandler,
         });
         this.backend = makeBackend(KmsType.external, KmsProtocol.aws_kms, providerName);
         this.noAwsArn = noAwsArn;
@@ -118,31 +124,22 @@ export default class Client implements KMSInterface {
 
     createMasterKey(logger: werelogs.Logger, cb: (err: Error | null, keyId?: string, keyArn?: string) => void): void {
         logger.debug('AWS KMS: creating master encryption key');
-        this.client.createKey({}, (err: AWSError, data) => {
-            if (err) {
-                const error = arsenalErrorAWSKMS(err);
-                logger.error('AWS KMS: failed to create master encryption key', { err });
-                cb(error);
-                return;
-            }
+        this.client.send(new CreateKeyCommand({})).then(data => {
             const keyMetadata = data?.KeyMetadata;
             logger.debug("AWS KMS: master encryption key created", { KeyMetadata: keyMetadata });
             let keyId: string;
             if (this.noAwsArn) {
-                // Use KeyId when ARN is not wanted
-                // eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain
-                keyId = keyMetadata?.KeyId!;
+                keyId = keyMetadata?.KeyId || '';
             } else {
                 // Prefer ARN, but fall back to KeyId if ARN is missing
-                // eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain
-                keyId = keyMetadata?.Arn ?? keyMetadata?.KeyId!;
+                keyId = keyMetadata?.Arn ?? (keyMetadata?.KeyId || '');
             }
-            // May produce double arn prefix: scality arn + aws arn
-            // arn:scality:kms:external:aws_kms:custom:key/arn:aws:kms:region:accountId:key/cbd69d33-ba8e-4b56-8cfe
-            // If this is a problem, a config flag should be used to hide the scality arn when returning the KMS KeyId
-            // or aws arn when creating the KMS Key
             const arn = `${this.backend.arnPrefix}${keyId}`;
             cb(null, keyId, arn);
+        }).catch(err => {
+            const error = arsenalErrorAWSKMS(err);
+            logger.error('AWS KMS: failed to create master encryption key', { err });
+            cb(error);
         });
     }
 
@@ -163,30 +160,24 @@ export default class Client implements KMSInterface {
             KeyId: masterKeyId,
             PendingWindowInDays: 7,
         };
-        this.client.scheduleKeyDeletion(params, (err: AWSError, data) => {
-            if (err) {
-                if (err.code === 'NotFoundException' || err.code === 'KMSInvalidStateException') {
-                    // master key does not exist or is already pending deletion
-                    logger.info('AWS KMS: key does not exist or is already pending deletion',
-                        { masterKeyId, error: err });
-                    cb(null);
-                    return;
-                }
-
-                const error = arsenalErrorAWSKMS(err);
-                logger.error('AWS KMS: failed to delete master encryption key', { err });
-                cb(error);
-                return;
-            }
+        const command = new ScheduleKeyDeletionCommand(params);
 
+        this.client.send(command).then(data => {
             if (data?.KeyState && data.KeyState !== 'PendingDeletion') {
                 const error = arsenalErrorAWSKMS('key is not in PendingDeletion state');
-                logger.error('AWS KMS: failed to delete master encryption key', { err, data });
+                logger.error('AWS KMS: failed to delete master encryption key', { data });
                 cb(error);
                 return;
             }
-
             cb(null);
+        }).catch(err => {
+            if (err instanceof NotFoundException || err instanceof KMSInvalidStateException) {
+                logger.info('AWS KMS: key does not exist or is already pending deletion', { masterKeyId, error: err });
+                return cb(null);
+            }
+            const error = arsenalErrorAWSKMS(err);
+            logger.error('AWS KMS: failed to delete master encryption key', { err });
+            return cb(error);
         });
     }
 
@@ -199,31 +190,24 @@ export default class Client implements KMSInterface {
         const masterKeyId = getKeyIdFromArn(masterKeyIdOrArn);
         logger.debug("AWS KMS: generating data key", { cryptoScheme, masterKeyId, masterKeyIdOrArn });
         assert.strictEqual(cryptoScheme, 1);
-
         const params = {
             KeyId: masterKeyId,
-            KeySpec: 'AES_256',
+            KeySpec: 'AES_256' as const,
         };
-
-        this.client.generateDataKey(params, (err: AWSError, data) => {
-            if (err) {
-                const error = arsenalErrorAWSKMS(err);
-                logger.error('AWS KMS: failed to generate data key', { err });
-                cb(error);
-                return;
-            }
-
+        this.client.send(new GenerateDataKeyCommand(params)).then(data => {
             if (!data) {
                 const error = arsenalErrorAWSKMS("failed to generate data key: empty response");
                 logger.error("AWS KMS: failed to generate data key: empty response");
                 cb(error);
                 return;
             }
-
             const isolatedPlaintext = this.safePlaintext(data.Plaintext as Buffer);
-
             logger.debug('AWS KMS: data key generated');
             cb(null, isolatedPlaintext, Buffer.from(data.CiphertextBlob as Uint8Array));
+        }).catch(err => {
+            const error = arsenalErrorAWSKMS(err);
+            logger.error('AWS KMS: failed to generate data key', { err });
+            cb(error);
         });
     }
 
@@ -244,14 +228,7 @@ export default class Client implements KMSInterface {
             Plaintext: plainTextDataKey,
         };
 
-        this.client.encrypt(params, (err: AWSError, data) => {
-            if (err) {
-                const error = arsenalErrorAWSKMS(err);
-                logger.error('AWS KMS: failed to cipher data key', { err });
-                cb(error);
-                return;
-            }
-
+        this.client.send(new EncryptCommand(params)).then(data => {
             if (!data) {
                 const error = arsenalErrorAWSKMS("failed to cipher data key: empty response");
                 logger.error("AWS KMS: failed to cipher data key: empty response");
@@ -262,6 +239,10 @@ export default class Client implements KMSInterface {
             logger.debug('AWS KMS: data key ciphered');
             cb(null, Buffer.from(data.CiphertextBlob as Uint8Array));
             return;
+        }).catch(err => {
+            const error = arsenalErrorAWSKMS(err);
+            logger.error('AWS KMS: failed to cipher data key', { err });
+            cb(error);
         });
     }
 
@@ -281,14 +262,7 @@ export default class Client implements KMSInterface {
             CiphertextBlob: cipheredDataKey,
         };
 
-        this.client.decrypt(params, (err: AWSError, data) => {
-            if (err) {
-                const error = arsenalErrorAWSKMS(err);
-                logger.error('AWS KMS: failed to decipher data key', { err });
-                cb(error);
-                return;
-            }
-
+        this.client.send(new DecryptCommand(params)).then(data => {
             if (!data) {
                 const error = arsenalErrorAWSKMS("failed to decipher data key: empty response");
                 logger.error("AWS KMS: failed to decipher data key: empty response");
@@ -300,41 +274,30 @@ export default class Client implements KMSInterface {
 
             logger.debug('AWS KMS: data key deciphered');
             cb(null, isolatedPlaintext);
+        }).catch(err => {
+            const error = arsenalErrorAWSKMS(err);
+            logger.error('AWS KMS: failed to decipher data key', { err });
+            cb(error);
         });
     }
 
     /**
-     * NOTE1: S3C-4833 KMS healthcheck is disabled in CloudServer
-     * NOTE2: The best approach for implementing the AWS KMS health check is still under consideration.
-     * In the meantime, this method is commented out to prevent potential issues related to costs or permissions.
-     *
-     * Reasons for commenting out:
-     * - frequent API calls can lead to increased expenses.
-     * - access key secret key used must have `kms:ListKeys` permissions
-     *
-     * Future potential actions:
-     * - implement caching mechanisms to reduce the number of API calls.
-     * - differentiate between error types (e.g., 500 vs. 403) for more effective error handling.
+     * Healthcheck function to verify KMS connectivity
      */
-    /*
     healthcheck(logger: werelogs.Logger, cb: (err: Error | null) => void): void {
         logger.debug("AWS KMS: performing healthcheck");
 
-        const params = {
+        const command = new ListKeysCommand({
             Limit: 1,
-        };
-
-        this.client.listKeys(params, (err, data) => {
-            if (err) {
-                const error = arsenalErrorAWSKMS(err);
-                logger.error("AWS KMS healthcheck: failed to list keys", { err });
-                cb(error);
-                return;
-            }
+        });
 
+        this.client.send(command).then(() => {
             logger.debug("AWS KMS healthcheck: list keys succeeded");
             cb(null);
+        }).catch(err => {
+            const error = arsenalErrorAWSKMS(err);
+            logger.error("AWS KMS healthcheck: failed to list keys", { err });
+            cb(error);
         });
     }
-    */
 }

lib/network/utils.ts

@@ -1,4 +1,3 @@
-import type { AWSError } from 'aws-sdk';
 import { ArsenalError, errorInstances } from '../errors';
 import { allowedKmsErrors } from '../errors/kmsErrors';
 
@@ -32,20 +31,37 @@ export function arsenalErrorKMIP(err: string | Error) {
 
 const allowedKmsErrorCodes = Object.keys(allowedKmsErrors) as unknown as (keyof typeof allowedKmsErrors)[];
 
+// Local AWSError type for compatibility with v3 error handling
+export type AWSError = Error & {
+    name?: string;
+    $fault?: 'client' | 'server';
+    $metadata?: {
+        httpStatusCode?: number;
+        requestId?: string;
+        attempts?: number;
+        totalRetryDelay?: number;
+    };
+    $retryable?: {
+        throttling?: boolean;
+    };
+    message?: string;
+};
+
 function isAWSError(err: string | Error | AWSError): err is AWSError {
-    return (err as AWSError).code !== undefined
-        && (err as AWSError).retryable !== undefined;
+    return (err as AWSError).name !== undefined
+        && (err as AWSError).$metadata !== undefined;
 }
 
 export function arsenalErrorAWSKMS(err: string | Error | AWSError) {
     if (isAWSError(err)) {
-        if (allowedKmsErrorCodes.includes(err.code as keyof typeof allowedKmsErrors)) {
-            return errorInstances[`KMS.${err.code}`].customizeDescription(err.message);
+        const errorCode = err.name;
+        if (allowedKmsErrorCodes.includes(errorCode as keyof typeof allowedKmsErrors)) {
+            return errorInstances[`KMS.${errorCode}`].customizeDescription(err.message);
         } else {
             // Encapsulate into a generic ArsenalError but keep the aws error code
             return ArsenalError.unflatten({
                 is_arsenal_error: true,
-                type: `KMS.${err.code}`, // aws s3 prefix kms errors with KMS.
+                type: `KMS.${errorCode}`, // aws s3 prefix kms errors with KMS.
                 code: 500,
                 description: `unexpected AWS_KMS error`,
                 stack: err.stack,