feat(filestore): full S3 compatibility

packages/database/src/implementations/mongodb/MongoDb.ts

@@ -132,7 +132,7 @@ export default class MongoDB implements Driver
 
         if (entry === null)
         {
-            throw new RecordNotFound(`Record ${type} found: ${id}`);
+            throw new RecordNotFound(`Record ${type} not found: ${id}`);
         }
 
         return this.#buildRecordData(entry as Document, fields);

packages/filestore/README.md

@@ -11,29 +11,39 @@ npm install @theshelf/filestore
 
 ## Implementations
 
-Currently, there are two implementations:
+Currently, there are three implementations:
 
+* **AWS** - persistent AWS S3 object storage.
 * **Memory** - non-persistent in memory storage (suited for testing).
-* **Minio** - persistent S3 compatible object storage.
+* **S3** - persistent S3 compatible object storage.
 
 ## Configuration
 
 The used implementation needs to be configured in the `.env` file.
 
 ```env
-FILE_STORE_IMPLEMENTATION="minio" # (memory | minio)
+FILE_STORE_IMPLEMENTATION="minio" # (aws | memory | s3)
 ```
 
-In case of Minio, additional configuration is required.
+In case of an S3 compatible storage, additional configuration is required.
 
 ```env
-MINIO_END_POINT="address"
-MINIO_PORT_NUMBER=9000
-MINIO_USE_SSL=true
-MINIO_ACCESS_KEY="development"
-MINIO_SECRET_KEY="secret"
+S3_END_POINT="http://objectstore.com"
+S3_ROOT_USER="development"
+S3_ROOT_PASSWORD="secret"
+S3_BUCKET_NAME="your-bucket-name"
+S3_REGION="local"
 ```
 
+In case of AWS, additional configuration is required.
+
+```env
+AWS_REGION="eu-central-1"
+AWS_BUCKET_NAME="your-bucket-name"
+```
+
+The AWS S3 client uses the **Default Credential Provider Chain** for authentication. More information can be found in the [AWS  documentation](https://docs.aws.amazon.com/sdkref/latest/guide/standardized-credentials.html).
+
 ## How to use
 
 An instance of the configured file storage implementation can be imported for performing file operations.

packages/filestore/package.json

@@ -19,6 +19,6 @@
   "types": "dist/index.d.ts",
   "exports": "./dist/index.js",
   "dependencies": {
-    "minio": "8.0.6"
+    "@aws-sdk/client-s3": "^3.948.0"
   }
 }

packages/filestore/src/implementation.ts

@@ -2,11 +2,13 @@
 import type { FileStore } from './definitions/interfaces.js';
 import UnknownImplementation from './errors/UnknownImplementation.js';
 import createMemoryFS from './implementations/memory/create.js';
-import createMinioFS from './implementations/minio/create.js';
+import createS3FS from './implementations/s3/create.js';
+import createS3AwsFS from './implementations/s3/createAws.js';
 
 const implementations = new Map<string, () => FileStore>([
+    ['aws', createS3AwsFS],
     ['memory', createMemoryFS],
-    ['minio', createMinioFS],
+    ['s3', createS3FS],
 ]);
 
 const DEFAULT_FILE_STORE_IMPLEMENTATION = 'memory';

packages/filestore/src/implementations/minio/MinioFS.ts → packages/filestore/src/implementations/s3/S3.ts

@@ -1,36 +1,39 @@
 
-import type { ClientOptions } from 'minio';
-import { Client } from 'minio';
+import type { Readable } from 'node:stream';
+import type { S3ClientConfig } from '@aws-sdk/client-s3';
+import { NotFound, S3Client, HeadObjectCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, ListBucketsCommand, CreateBucketCommand } from '@aws-sdk/client-s3';
 
 import type { FileStore } from '../../definitions/interfaces.js';
 import FileNotFound from '../../errors/FileNotFound.js';
 import NotConnected from '../../errors/NotConnected.js';
 
-const BUCKET_NAME = 'comify';
-
-export default class MinioFS implements FileStore
+export default class S3 implements FileStore
 {
-    readonly #configuration: ClientOptions;
-    #client?: Client;
+    readonly #configuration: S3ClientConfig;
+    readonly #bucketName: string;
+    #client?: S3Client;
 
-    constructor(configuration: ClientOptions)
+    constructor(configuration: S3ClientConfig, bucketName: string)
     {
         this.#configuration = configuration;
+        this.#bucketName = bucketName;
     }
-
+    
     get connected()
     {
         return this.#client !== undefined;
     }
 
     async connect(): Promise<void>
     {
-        this.#client = new Client(this.#configuration);
-
-        if (await this.#client.bucketExists(BUCKET_NAME) === false)
-        {
-            await this.#client.makeBucket(BUCKET_NAME);
-        }
+        this.#client = new S3Client(this.#configuration);
+
+        const buckets = await this.#client.send(new ListBucketsCommand({}));
+        const bucketExists = buckets.Buckets?.some(bucket => bucket.Name === this.#bucketName);
+
+        if (bucketExists === true) return;
+
+        await this.#client.send(new CreateBucketCommand({ Bucket: this.#bucketName }));
     }
 
     async disconnect(): Promise<void>
@@ -40,6 +43,7 @@ export default class MinioFS implements FileStore
             throw new NotConnected();
         }
 
+        this.#client.destroy();
         this.#client = undefined;
     }
 
@@ -49,7 +53,7 @@ export default class MinioFS implements FileStore
 
         try
         {
-            await client.statObject(BUCKET_NAME, path);
+            await client.send(new HeadObjectCommand({ Bucket: this.#bucketName, Key: path }));
 
             return true;
         }
@@ -72,7 +76,7 @@ export default class MinioFS implements FileStore
 
         try
         {
-            await client.putObject(BUCKET_NAME, path, data);
+            await client.send(new PutObjectCommand({Bucket: this.#bucketName, Key: path, Body: data}));
         }
         catch (error)
         {
@@ -86,8 +90,21 @@ export default class MinioFS implements FileStore
 
         try
         {
-            const stream = await client.getObject(BUCKET_NAME, path);
-            const chunks = await stream.toArray();
+            const response = await client.send(new GetObjectCommand({ Bucket: this.#bucketName, Key: path }));
+            const body = response.Body;
+
+            if (body === undefined)
+            {
+                throw new FileNotFound(path);
+            }
+
+            const stream = body as Readable;
+            const chunks: Uint8Array[] = [];
+
+            for await (const chunk of stream)
+            {
+                chunks.push(chunk);
+            }
 
             return Buffer.concat(chunks);
         }
@@ -103,7 +120,7 @@ export default class MinioFS implements FileStore
 
         try
         {
-            await client.removeObject(BUCKET_NAME, path);
+            await client.send(new DeleteObjectCommand({ Bucket: this.#bucketName, Key: path }));
         }
         catch (error)
         {
@@ -113,10 +130,10 @@ export default class MinioFS implements FileStore
 
     async clear(): Promise<void>
     {
-        return;  // Deliberately not implemented
+        return; // Deliberately not implemented
     }
 
-    #getClient(): Client
+    #getClient(): S3Client
     {
         if (this.#client === undefined)
         {
@@ -128,17 +145,11 @@ export default class MinioFS implements FileStore
 
     #handleError(error: unknown, path: string): unknown
     {
-        if (error instanceof Error && this.#isNotFoundError(error))
+        if (error instanceof NotFound)
         {
             return new FileNotFound(path);
         }
 
         return error;
     }
-
-    #isNotFoundError(error: Error): boolean
-    {
-        return error.message.startsWith('The specified key does not exist')
-            || error.message.startsWith('Not Found');
-    }
 }

packages/filestore/src/implementations/s3/create.ts

@@ -0,0 +1,23 @@
+
+import S3 from './S3.js';
+
+export default function create(): S3
+{
+    const endpoint = process.env.S3_END_POINT ?? 'undefined';
+    const accessKey = process.env.S3_ROOT_USER ?? 'undefined';
+    const secretKey = process.env.S3_ROOT_PASSWORD ?? 'undefined';
+    const bucket = process.env.S3_BUCKET_NAME ?? 'undefined';
+    const region = process.env.S3_REGION ?? 'local';
+
+    const clientConfig = { 
+        region: region,
+        credentials: {
+            accessKeyId: accessKey,
+            secretAccessKey: secretKey
+        },
+        forcePathStyle: true,
+        endpoint: endpoint
+    };
+
+    return new S3(clientConfig, bucket);
+}

packages/filestore/src/implementations/s3/createAws.ts

@@ -0,0 +1,10 @@
+
+import S3 from './S3.js';
+
+export default function create(): S3
+{
+    const region = process.env.AWS_REGION ?? 'eu-central-1';
+    const bucket = process.env.AWS_BUCKET_NAME ?? 'undefined';
+
+    return new S3({ region: region }, bucket);
+}