feat: migrate apps' and dataset's avatar to minio

Author: xqvvu

packages/service/common/s3/buckets/base.ts

@@ -1,13 +1,19 @@
-import { Client } from 'minio';
+import { Client, type RemoveOptions, type CopyConditions, type LifecycleConfig } from 'minio';
 import {
   defaultS3Options,
+  type CreatePostPresignedUrlOptions,
   type CreatePostPresignedUrlParams,
   type CreatePostPresignedUrlResult,
   type S3BucketName,
   type S3Options
 } from '../types';
 import type { IBucketBasicOperations } from '../interface';
-import { createObjectKey, createPresignedUrlExpires, inferContentType } from '../helpers';
+import {
+  createObjectKey,
+  createPresignedUrlExpires,
+  createTempObjectKey,
+  inferContentType
+} from '../helpers';
 
 export class S3BaseBucket implements IBucketBasicOperations {
   public client: Client;
@@ -16,11 +22,11 @@ export class S3BaseBucket implements IBucketBasicOperations {
    *
    * @param _bucket the bucket you want to operate
    * @param options the options for the s3 client
-   * @param afterInit the function to be called after instantiating the s3 service
+   * @param afterInits the function to be called after instantiating the s3 service
    */
   constructor(
     private readonly _bucket: S3BucketName,
-    private readonly afterInit?: () => Promise<void> | void,
+    private readonly afterInits?: (() => Promise<void> | void)[],
     public options: Partial<S3Options> = defaultS3Options
   ) {
     options = { ...defaultS3Options, ...options };
@@ -31,45 +37,54 @@ export class S3BaseBucket implements IBucketBasicOperations {
       if (!(await this.exist())) {
         await this.client.makeBucket(this._bucket);
       }
-      await this.afterInit?.();
+      await Promise.all(this.afterInits?.map((afterInit) => afterInit()) ?? []);
     };
     init();
   }
 
-  async exist(): Promise<boolean> {
-    return await this.client.bucketExists(this._bucket);
-  }
-
   get name(): string {
     return this._bucket;
   }
 
-  upload(): Promise<void> {
-    throw new Error('Method not implemented.');
+  async move(src: string, dst: string, options?: CopyConditions): Promise<void> {
+    const bucket = this.name;
+    await this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options);
+    return this.client.removeObject(bucket, src);
   }
 
-  download(): Promise<void> {
-    throw new Error('Method not implemented.');
+  copy(src: string, dst: string, options?: CopyConditions): ReturnType<Client['copyObject']> {
+    return this.client.copyObject(this.name, src, dst, options);
   }
 
-  delete(objectKey: string): Promise<void> {
-    return this.client.removeObject(this._bucket, objectKey);
+  exist(): Promise<boolean> {
+    return this.client.bucketExists(this.name);
+  }
+
+  delete(objectKey: string, options?: RemoveOptions): Promise<void> {
+    return this.client.removeObject(this.name, objectKey, options);
   }
 
   get(): Promise<void> {
     throw new Error('Method not implemented.');
   }
 
+  lifecycle(): Promise<LifecycleConfig | null> {
+    return this.client.getBucketLifecycle(this.name);
+  }
+
   async createPostPresignedUrl(
-    params: CreatePostPresignedUrlParams
+    params: CreatePostPresignedUrlParams,
+    options: CreatePostPresignedUrlOptions = {}
   ): Promise<CreatePostPresignedUrlResult> {
-    const maxFileSize = this.options.maxFileSize as number;
+    const { temporay } = options;
     const contentType = inferContentType(params.filename);
+    const maxFileSize = this.options.maxFileSize as number;
+    const key = temporay ? createTempObjectKey(params) : createObjectKey(params);
 
     const policy = this.client.newPostPolicy();
-    policy.setBucket(this._bucket);
+    policy.setKey(key);
+    policy.setBucket(this.name);
     policy.setContentType(contentType);
-    policy.setKey(createObjectKey(params));
     policy.setContentLengthRange(1, maxFileSize);
     policy.setExpires(createPresignedUrlExpires(10));
     policy.setUserMetaData({

packages/service/common/s3/buckets/public.ts

@@ -2,26 +2,36 @@ import { S3BaseBucket } from './base';
 import { createBucketPolicy } from '../helpers';
 import {
   S3Buckets,
+  type CreatePostPresignedUrlOptions,
   type CreatePostPresignedUrlParams,
   type CreatePostPresignedUrlResult,
   type S3Options
 } from '../types';
 import type { IPublicBucketOperations } from '../interface';
+import { lifecycleOfTemporaryAvatars } from '../lifecycle';
 
 export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperations {
   constructor(options?: Partial<S3Options>) {
     super(
       S3Buckets.public,
-      async () => {
-        const bucket = this.name;
-        const policy = createBucketPolicy(bucket);
-        try {
-          await this.client.setBucketPolicy(bucket, policy);
-        } catch (error) {
-          // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error,
-          // maybe we can ignore the error, or we have other plan to handle this.
+      [
+        // set bucket policy
+        async () => {
+          const bucket = this.name;
+          const policy = createBucketPolicy(bucket);
+          try {
+            await this.client.setBucketPolicy(bucket, policy);
+          } catch (error) {
+            // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error,
+            // maybe we can ignore the error, or we have other plan to handle this.
+          }
+        },
+        // set bucket lifecycle
+        async () => {
+          const bucket = this.name;
+          await this.client.setBucketLifecycle(bucket, lifecycleOfTemporaryAvatars);
         }
-      },
+      ],
       options
     );
   }
@@ -36,8 +46,9 @@ export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperati
   }
 
   override createPostPresignedUrl(
-    params: Omit<CreatePostPresignedUrlParams, 'visibility'>
+    params: Omit<CreatePostPresignedUrlParams, 'visibility'>,
+    options: CreatePostPresignedUrlOptions = {}
   ): Promise<CreatePostPresignedUrlResult> {
-    return super.createPostPresignedUrl({ ...params, visibility: 'public' });
+    return super.createPostPresignedUrl({ ...params, visibility: 'public' }, options);
   }
 }

packages/service/common/s3/helpers.ts

@@ -69,3 +69,11 @@ export function createObjectKey({ source, teamId, filename }: CreateObjectKeyPar
   const id = crypto.randomBytes(16).toString('hex');
   return `${source}/${teamId}/${date}/${id}_${filename}`;
 }
+
+/**
+ * create temporary s3 object key by source, team ID and filename
+ */
+export function createTempObjectKey(params: CreateObjectKeyParams): string {
+  const origin = createObjectKey(params);
+  return `temp/${origin}`;
+}

packages/service/common/s3/interface.ts

@@ -1,14 +1,17 @@
-import type { CreateObjectKeyParams } from './types';
+import { type LifecycleConfig, type Client, type CopyConditions, type RemoveOptions } from 'minio';
+import type { CreateObjectKeyParams, CreatePostPresignedUrlOptions } from './types';
 
 export interface IBucketBasicOperations {
   get name(): string;
-  exist(): Promise<boolean>;
-  upload(): Promise<void>;
-  download(): Promise<void>;
-  delete(objectKey: string): Promise<void>;
   get(): Promise<void>;
+  exist(): Promise<boolean>;
+  delete(objectKey: string, options?: RemoveOptions): Promise<void>;
+  move(src: string, dst: string, options?: CopyConditions): Promise<void>;
+  copy(src: string, dst: string, options?: CopyConditions): ReturnType<Client['copyObject']>;
+  lifecycle(): Promise<LifecycleConfig | null>;
   createPostPresignedUrl(
-    params: CreateObjectKeyParams
+    params: CreateObjectKeyParams,
+    options?: CreatePostPresignedUrlOptions
   ): Promise<{ url: string; fields: Record<string, string> }>;
 }
 

packages/service/common/s3/lifecycle.ts

@@ -0,0 +1,23 @@
+import { type LifecycleRule, type LifecycleConfig } from 'minio';
+
+export function createLifeCycleConfig(rule: LifecycleRule): LifecycleConfig {
+  return {
+    Rule: [rule]
+  };
+}
+
+export function assembleLifeCycleConfigs(...configs: LifecycleConfig[]): LifecycleConfig {
+  if (configs.length === 0) return { Rule: [] };
+  return {
+    Rule: configs.flatMap((config) => config.Rule)
+  };
+}
+
+export const lifecycleOfTemporaryAvatars = createLifeCycleConfig({
+  ID: 'Temporary Avatars Rule',
+  Prefix: 'temp/avatar/',
+  Status: 'Enabled',
+  Expiration: {
+    Days: 1
+  }
+});

packages/service/common/s3/sources/avatar.ts

@@ -1,6 +1,8 @@
 import { S3BaseSource } from './base';
 import {
   S3Sources,
+  S3APIPrefix,
+  type CreatePostPresignedUrlOptions,
   type CreatePostPresignedUrlParams,
   type CreatePostPresignedUrlResult,
   type S3Options
@@ -17,20 +19,36 @@ class S3AvatarSource extends S3BaseSource<S3PublicBucket> {
   }
 
   override createPostPresignedUrl(
-    params: Omit<CreatePostPresignedUrlParams, 'source' | 'visibility'>
+    params: Omit<CreatePostPresignedUrlParams, 'source' | 'visibility'>,
+    options: CreatePostPresignedUrlOptions = {}
   ): Promise<CreatePostPresignedUrlResult> {
-    return this.bucket.createPostPresignedUrl({
-      ...params,
-      source: S3Sources.avatar
-    });
+    return this.bucket.createPostPresignedUrl(
+      {
+        ...params,
+        source: S3Sources.avatar
+      },
+      options
+    );
   }
 
   createPublicUrl(objectKey: string): string {
     return this.bucket.createPublicUrl(objectKey);
   }
 
-  removeAvatar(objectKey: string): Promise<void> {
-    return this.bucket.delete(objectKey);
+  createAvatarObjectKey(avatarWithPrefix: string): string {
+    return avatarWithPrefix.replace(S3APIPrefix.avatar, '');
+  }
+
+  removeAvatar(avatarWithPrefix: string): Promise<void> {
+    const avatarObjectKey = this.createAvatarObjectKey(avatarWithPrefix);
+    return this.bucket.delete(avatarObjectKey);
+  }
+
+  async moveAvatarFromTemp(tempAvatarWithPrefix: string): Promise<string> {
+    const tempAvatarObjectKey = this.createAvatarObjectKey(tempAvatarWithPrefix);
+    const avatarObjectKey = tempAvatarObjectKey.replace(`${S3Sources.temp}/`, '');
+    await this.bucket.move(tempAvatarObjectKey, avatarObjectKey);
+    return S3APIPrefix.avatar + avatarObjectKey;
   }
 }
 

packages/service/common/s3/types.ts

@@ -69,7 +69,8 @@ export const S3SourcesSchema = z.enum([
   'dataset',
   'dataset-image',
   'invoice',
-  'rawtext'
+  'rawtext',
+  'temp'
 ]);
 export const S3Sources = S3SourcesSchema.enum;
 export type S3SourceType = z.infer<typeof S3SourcesSchema>;
@@ -87,8 +88,18 @@ export const CreatePostPresignedUrlParamsSchema = z.object({
 });
 export type CreatePostPresignedUrlParams = z.infer<typeof CreatePostPresignedUrlParamsSchema>;
 
+export const CreatePostPresignedUrlOptionsSchema = z.object({
+  temporay: z.boolean().optional()
+});
+export type CreatePostPresignedUrlOptions = z.infer<typeof CreatePostPresignedUrlOptionsSchema>;
+
 export const CreatePostPresignedUrlResultSchema = z.object({
   url: z.string().min(1),
   fields: z.record(z.string(), z.string())
 });
 export type CreatePostPresignedUrlResult = z.infer<typeof CreatePostPresignedUrlResultSchema>;
+
+export const S3APIPrefix = {
+  avatar: '/api/system/img/'
+} as const;
+export type S3APIPrefixType = (typeof S3APIPrefix)[keyof typeof S3APIPrefix];