fix upload

Author: endink

package.json

@@ -7,15 +7,15 @@
     "start": "webpack-dev-server --open development",
     "pub": "local-package-publisher -u&&local-package-publisher -p",
     "unpub": "local-package-publisher -u",
-    "build": "tsc",
+    "build": "tsc --excludeDirectories test",
     "format": "prettier --write \"src/**/*.ts\" \"src/**/*.js\"",
     "lint": "tslint -p tsconfig.json",
     "lint:fix": "tslint --fix -p tsconfig.json",
     "test": "jest",
     "test-c": "jest --coverage",
     "debug": "node --inspect-brk node_modules/.bin/jest --runInBand --no-cache --no-watchman",
     "debug-win": "node --inspect-brk ./node_modules/jest/bin/jest.js",
-    "prettier": "prettier -c --write \"src/**\""
+    "pretty": "prettier -c --write \"src/**\""
   },
   "jest": {
     "testEnvironment": "node"
@@ -40,15 +40,13 @@
   "homepage": "https://github.com/endink/infra-typescript-sdk",
   "dependencies": {
     "ali-oss": "^6.15.2",
-    "minio": "^7.0.18",
     "umi-request": "^1.3.5"
   },
   "devDependencies": {
     "@types/ali-oss": "^6.0.8",
     "@types/enzyme": "^3.10.8",
     "@types/html-webpack-plugin": "^3.2.5",
     "@types/jest": "^26.0.23",
-    "@types/minio": "^7.0.7",
     "@types/node": "^15.0.2",
     "@types/webpack": "^5.28.0",
     "css-loader": "^5.2.4",

src/minio/minio.ts

@@ -1,15 +1,15 @@
 import { isNullOrEmptyString } from "../utils";
-import { RequestResponse, ResponseError } from "umi-request";
+import { RequestResponse } from "umi-request";
 import { AssumedCredentials, MinioConfig } from ".";
 import { ExtendedRequestMethod, ExtendedRequestOptionsInit } from "../request";
 import { ApplicationError, BucketPolicy } from "../core";
-import * as Minio from "minio";
-import { PresignedResult, UploadResult } from "./types";
+import { PresignedUrl } from "./types";
 
 export interface MinioOptions {
     configURL: string;
     stsTokenURL: string;
-    genUrl: string;
+    presignReadUrl: string;
+    presignUploadURL: string;
 }
 
 interface MinioContext {
@@ -20,10 +20,6 @@ interface MinioContext {
 
 const minioContext: MinioContext = { tokenTime: Date.now().valueOf() };
 
-type SimpleStringValue = {
-    value: string;
-};
-
 export class MinioUtils {
     private constructor(private request: ExtendedRequestMethod, public options: MinioOptions) {}
 
@@ -33,7 +29,8 @@ export class MinioUtils {
         const options: MinioOptions = {
             configURL: `${base}/objects/cnf`,
             stsTokenURL: `${base}/objects/assume-role`,
-            genUrl: `${base}/objects/pre-sign-url`
+            presignReadUrl: `${base}/objects/pre-sign-url`,
+            presignUploadURL: `${base}/objects/pre-sign-upload`
         };
 
         return new MinioUtils(request, options);
@@ -91,7 +88,7 @@ export class MinioUtils {
         return { data: minioContext, response: { ok: true } } as any;
     }
 
-    public generateObjectUrl(key: string): string | undefined {
+    public generateObjectUrl(key: string): string {
         const config = minioContext.config;
         if (config && !isNullOrEmptyString(config.publicBucket)) {
             const file = key.startsWith("/") ? key.substr(1, key.length - 1) : key;
@@ -101,71 +98,92 @@ export class MinioUtils {
                 return `${config.schema}://${config.host}/${config.publicBucket}/${file}`;
             }
         }
+        return "";
     }
 
     public async presignedObjectUrl(
-        filePath: string,
+        key: string,
         resOptions?: Omit<ExtendedRequestOptionsInit, "method" | "data">
-    ): Promise<RequestResponse<PresignedResult & ApplicationError>> {
-        const file = encodeURI(filePath);
+    ): Promise<RequestResponse<PresignedUrl & ApplicationError>> {
+        const file = encodeURI(key);
         const requestOptions = { ...(resOptions || {}), method: "POST" };
-        const { response, data } = await this.request<SimpleStringValue & ApplicationError>(
-            `${this.options.genUrl}?key=${file}`,
+        return await this.request<PresignedUrl & ApplicationError>(
+            `${this.options.presignReadUrl}?key=${file}`,
             requestOptions
         );
-        return { data: { url: (data?.value || "") }, response }
     }
 
-
-    private putObjectAsync(
-        client: Minio.Client,
-        bucketName: string,
-        objectName: string,
-        stream: any
-    ): Promise<RequestResponse<UploadResult>> {
-        return new Promise((resolve, reject) => {
-            client.putObject(bucketName, objectName, stream, (err, etag) => {
-                if (err === undefined || err === null) {
-                    resolve({ data: { etag }, response: { ok: true, status: 200, type: "default" } as any });
-                } else {
-                    const data: UploadResult = { etag };
-                    if (typeof err === "string") {
-                        data.error = err as any;
-                        data.error_description = err as any;
-                    } else {
-                        data.error = err?.name;
-                        data.error_description = err?.message;
-                    }
-
-                    reject({ data, response: { ok: false, status: 500 } as any });
-                }
-            });
-        });
+    public async presignedUploadUrl(
+        key: string,
+        bucket: BucketPolicy = BucketPolicy.Public,
+        resOptions?: Omit<ExtendedRequestOptionsInit, "method" | "data">
+    ): Promise<RequestResponse<PresignedUrl & ApplicationError>> {
+        const r = await this.getMinioContext();
+        if (!r.response.ok) {
+            return r as any;
+        }
+        const file = encodeURI(key);
+        const requestOptions = { ...(resOptions || {}), method: "POST" };
+        return await this.request<PresignedUrl & ApplicationError>(
+            `${this.options.presignUploadURL}?key=${file}&b=${bucket}`,
+            requestOptions
+        );
     }
 
-    public async upload(bucketPolicy: BucketPolicy, key: string, stream: any): Promise<RequestResponse<UploadResult>> {
-        const r = await this.getMinioContext();
+    private getFileName(key: string): string {
+        const correctKey = key.replace("\\", "/");
+        const index = correctKey.lastIndexOf("/");
+        return correctKey.substr(index);
+    }
 
-        const { response, data } = r;
-        if (!r.response.ok) {
-            return { data: data as any, response };
+    public async upload(
+        key: string,
+        blobOrString: Blob | string,
+        bucketPolicy: BucketPolicy = BucketPolicy.Public
+    ): Promise<RequestResponse<PresignedUrl & ApplicationError>> {
+        const fileAPI = window.File && window.FileReader && window.FileList && window.Blob;
+        if (!fileAPI) {
+            return {
+                response: { ok: false, status: 0, statusText: "file api unsupported" } as any,
+                data: {
+                    error: "file_api_unsupported",
+                    error_description: "file api unsupported by current broswser."
+                } as any
+            };
         }
 
-        const { stsToken, config } = r.data;
-        const cnf = config as MinioConfig;
+        const r = await this.getMinioContext();
+        const fileName = this.getFileName(key);
+        const length = typeof blobOrString === "string" ? blobOrString.length : blobOrString.size;
+        const blob = typeof blobOrString === "string" ? new Blob([blobOrString], { type: "plain/text" }) : blobOrString;
 
-        const bucketName = bucketPolicy === BucketPolicy.Private ? cnf.privateBucket : cnf.publicBucket;
+        const presignedResult = await this.presignedUploadUrl(key, bucketPolicy);
 
-        const minioClient = new Minio.Client({
-            endPoint: cnf.host,
-            port: cnf.port,
-            region: cnf.region,
-            useSSL: cnf.schema === "https",
-            accessKey: stsToken?.accessKey || "",
-            secretKey: stsToken?.secretKey || "",
-            sessionToken:stsToken?.sessionToken || "",
-        });
+        if (presignedResult.response.ok) {
+            const formData = new FormData();
+            formData.append("file", blob, fileName);
 
-        return await this.putObjectAsync(minioClient, bucketName, key, stream);
+            const { url, contentType } = presignedResult.data;
+            const { response, data } = await this.request.put(url, {
+                data: formData,
+                headers: { "Content-Length": `${length}`, "Content-Type":"multipart/form-data" }
+            });
+
+            if (response.ok) {
+                if (bucketPolicy === BucketPolicy.Private) {
+                    const presigned = await this.presignedObjectUrl(key);
+                    if (presigned.response.ok) {
+                        return presigned;
+                    }
+                }
+                return {
+                    response: { ok: true, status: 200 },
+                    data: { url: this.generateObjectUrl(key), expireInSeconds: Number.MAX_VALUE, contentType }
+                } as any;
+            }
+            return { response, data } as any;
+        } else {
+            return presignedResult as any;
+        }
     }
 }

src/minio/types.ts

@@ -20,9 +20,11 @@ export interface AssumedCredentials {
 }
 
 export interface UploadResult extends ApplicationError {
-    etag: string;
+    url: string;
 }
 
-export interface PresignedResult {
+export interface PresignedUrl {
     url: string;
+    expireInSeconds: number;
+    contentType: string;
 }

src/request/index.ts

@@ -117,14 +117,14 @@ const requestContext: Pick<RequestOptions, "accessTokenUrl" | "checkTokenUrl"> &
     checkTokenUrl: ""
 };
 
-export function initRequestNoneOAuth2() : ExtendedRequestMethod {
+export function initRequestNoneOAuth2(): ExtendedRequestMethod {
     return initRequest({
         clientId: "",
-        clientSecret:"",
+        clientSecret: "",
         noneOAuth2: true,
-        accessTokenUrl:"",
-        checkTokenUrl:""
-    })
+        accessTokenUrl: "",
+        checkTokenUrl: ""
+    });
 }
 
 export function initRequest(options: RequestOptions, session?: OAuth2Session): ExtendedRequestMethod {

src/request/types.ts

@@ -20,5 +20,5 @@ export interface RequestOptions {
     errorDescriber?: Record<string, string>;
     httpCodeDescriber?: Record<string, string>;
     errorHandlers?: CustomErrorHandler[];
-    noneOAuth2?: boolean
+    noneOAuth2?: boolean;
 }

src/typings.d.ts

@@ -0,0 +1,4 @@
+declare module "*.png" {
+    const content: any;
+    export default content;
+}

test/minio.test.ts

@@ -1,26 +1,42 @@
-import { generateUUID } from "../src/utils";
-import { BucketPolicy } from "../src/core";
 import { MinioUtils } from "../src/minio";
 import { initRequestNoneOAuth2 } from "../src/request";
 
 const request = initRequestNoneOAuth2();
 
 const minioUtils = MinioUtils.create(request, "http://localhost:9898");
 
-test("testAssumeRole", async ()=>{
+test("testFetchConfig", async () => {
    const resp = await minioUtils.fetchConfig(false, true);
-   const{ response, data }= resp;
+   const { response, data } = resp;
    expect(response.ok).toBeTruthy();
 });
 
-test("testAssumeRoleUpload", async ()=>{
-    const resp = await minioUtils.fetchConfig(false, true);
-    expect(resp.response.ok).toBeTruthy();
-
-    const file = generateUUID()
-    const uploadResp = await minioUtils.upload(BucketPolicy.Public, `aaa/${file}.txt`, "XXXXXX");
-    const { response, data } = uploadResp;
-    expect(response.ok).toBeTruthy();
-    expect(response.status).toEqual(200);
-    expect(data.etag.length >0).toBeTruthy();
- });
+test("testPresignedUploadUrl", async () => {
+   const resp = await minioUtils.fetchConfig(false, true);
+   expect(resp.response.ok).toBeTruthy();
+
+   const { response, data } = await minioUtils.presignedUploadUrl("kkkk/aaa.jpg");
+   expect(response.ok).toBeTruthy();
+
+   expect(data.contentType?.length).toBeDefined()
+   expect(data.error).toBeUndefined();
+   expect(data.url).toBeDefined();
+   expect(data.expireInSeconds > 0).toBeTruthy();
+
+   console.log(data);
+});
+
+
+test("testUpload", async () => {
+   const blob = new Blob(["xxxxx"], { type: "text/plain" });
+
+   const { response, data } = await minioUtils.upload("test/aaa.txt", blob);
+   expect(response.ok).toBeTruthy();
+
+   expect(data.contentType?.length).toBeDefined()
+   expect(data.error).toBeUndefined();
+   expect(data.url).toBeDefined();
+   expect(data.expireInSeconds > 0).toBeTruthy();
+
+   console.log(data);
+});

tsconfig.json

@@ -24,5 +24,5 @@
     // "suppressImplicitAnyIndexErrors": true, //阻止对缺少索引签名的索引对象报错
   },
   "exclude": ["node_modules"],
-  "include": ["src/**/*", "/src/typings.d.ts"]
+  "include": ["src/**/*", "src/typings.d.ts"]
 }

webpack.config.js

@@ -32,7 +32,7 @@ module.exports = {
                 use: {
                     loader: "url-loader",
                     options: {
-                        limit: 20,
+                        limit: 100 * 1024,
                     },
                 },
             },