refactor(core): Alllow using local minio for s3 object storage

packages/@n8n/config/src/index.ts

@@ -7,7 +7,6 @@ import { EndpointsConfig } from './configs/endpoints.config';
 import { EventBusConfig } from './configs/event-bus.config';
 import { ExecutionsConfig } from './configs/executions.config';
 import { ExternalSecretsConfig } from './configs/external-secrets.config';
-import { ExternalStorageConfig } from './configs/external-storage.config';
 import { GenericConfig } from './configs/generic.config';
 import { LicenseConfig } from './configs/license.config';
 import { LoggingConfig } from './configs/logging.config';
@@ -62,9 +61,6 @@ export class GlobalConfig {
 	@Nested
 	nodes: NodesConfig;
 
-	@Nested
-	externalStorage: ExternalStorageConfig;
-
 	@Nested
 	workflows: WorkflowsConfig;
 

packages/@n8n/config/test/config.test.ts

@@ -135,19 +135,6 @@ describe('GlobalConfig', () => {
 			endpoint: 'https://api.n8n.io/api/versions/',
 			infoUrl: 'https://docs.n8n.io/hosting/installation/updating/',
 		},
-		externalStorage: {
-			s3: {
-				host: '',
-				bucket: {
-					name: '',
-					region: '',
-				},
-				credentials: {
-					accessKey: '',
-					accessSecret: '',
-				},
-			},
-		},
 		workflows: {
 			defaultName: 'My workflow',
 			callerPolicyDefaultOption: 'workflowsFromSameOwner',

packages/cli/src/commands/base-command.ts

@@ -189,42 +189,10 @@ export abstract class BaseCommand extends Command {
 	private async _initObjectStoreService(options = { isReadOnly: false }) {
 		const objectStoreService = Container.get(ObjectStoreService);
 
-		const { host, bucket, credentials } = this.globalConfig.externalStorage.s3;
-
-		if (host === '') {
-			throw new ApplicationError(
-				'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.',
-			);
-		}
-
-		if (bucket.name === '') {
-			throw new ApplicationError(
-				'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.',
-			);
-		}
-
-		if (bucket.region === '') {
-			throw new ApplicationError(
-				'External storage bucket region not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_REGION`.',
-			);
-		}
-
-		if (credentials.accessKey === '') {
-			throw new ApplicationError(
-				'External storage access key not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_KEY`.',
-			);
-		}
-
-		if (credentials.accessSecret === '') {
-			throw new ApplicationError(
-				'External storage access secret not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_SECRET`.',
-			);
-		}
-
 		this.logger.debug('Initializing object store service');
 
 		try {
-			await objectStoreService.init(host, bucket, credentials);
+			await objectStoreService.init();
 			objectStoreService.setReadonly(options.isReadOnly);
 
 			this.logger.debug('Object store init completed');

packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts

@@ -3,6 +3,7 @@ import { mock } from 'jest-mock-extended';
 import { Readable } from 'stream';
 
 import { ObjectStoreService } from '@/binary-data/object-store/object-store.service.ee';
+import type { S3Config } from '@/binary-data/object-store/s3.config';
 import { writeBlockedMessage } from '@/binary-data/object-store/utils';
 
 jest.mock('axios');
@@ -18,6 +19,12 @@ const mockError = new Error('Something went wrong!');
 const fileId =
 	'workflows/ObogjVbqpNOQpiyV/executions/999/binary_data/71f6209b-5d48-41a2-a224-80d529d8bb32';
 const mockBuffer = Buffer.from('Test data');
+const s3Config = mock<S3Config>({
+	host: mockHost,
+	bucket: mockBucket,
+	credentials: mockCredentials,
+	protocol: 'https',
+});
 
 const toDeletionXml = (filename: string) => `<Delete>
 <Object><Key>${filename}</Key></Object>
@@ -26,9 +33,9 @@ const toDeletionXml = (filename: string) => `<Delete>
 let objectStoreService: ObjectStoreService;
 
 beforeEach(async () => {
-	objectStoreService = new ObjectStoreService(mock());
+	objectStoreService = new ObjectStoreService(mock(), s3Config);
 	mockAxios.request.mockResolvedValueOnce({ status: 200 }); // for checkConnection
-	await objectStoreService.init(mockHost, mockBucket, mockCredentials);
+	await objectStoreService.init();
 	jest.restoreAllMocks();
 });
 

packages/core/src/binary-data/object-store/object-store.service.ee.ts

@@ -1,6 +1,6 @@
 import { Service } from '@n8n/di';
 import { sign } from 'aws4';
-import type { Request as Aws4Options, Credentials as Aws4Credentials } from 'aws4';
+import type { Request as Aws4Options } from 'aws4';
 import axios from 'axios';
 import type { AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig, Method } from 'axios';
 import { ApplicationError } from 'n8n-workflow';
@@ -9,43 +9,42 @@ import type { Readable } from 'stream';
 
 import { Logger } from '@/logging/logger';
 
-import type {
-	Bucket,
-	ConfigSchemaCredentials,
-	ListPage,
-	MetadataResponseHeaders,
-	RawListPage,
-	RequestOptions,
-} from './types';
+import { S3Config } from './s3.config';
+import type { ListPage, MetadataResponseHeaders, RawListPage, RequestOptions } from './types';
 import { isStream, parseXml, writeBlockedMessage } from './utils';
 import type { BinaryData } from '../types';
 
 @Service()
 export class ObjectStoreService {
-	private host = '';
-
-	private bucket: Bucket = { region: '', name: '' };
-
-	private credentials: Aws4Credentials = { accessKeyId: '', secretAccessKey: '' };
+	private baseUrl: string;
 
 	private isReady = false;
 
 	private isReadOnly = false;
 
-	constructor(private readonly logger: Logger) {}
+	constructor(
+		private readonly logger: Logger,
+		private readonly s3Config: S3Config,
+	) {
+		const { host, bucket, protocol } = s3Config;
 
-	async init(host: string, bucket: Bucket, credentials: ConfigSchemaCredentials) {
-		this.host = host;
-		this.bucket.name = bucket.name;
-		this.bucket.region = bucket.region;
+		if (host === '') {
+			throw new ApplicationError(
+				'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.',
+			);
+		}
 
-		this.credentials = {
-			accessKeyId: credentials.accessKey,
-			secretAccessKey: credentials.accessSecret,
-		};
+		if (bucket.name === '') {
+			throw new ApplicationError(
+				'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.',
+			);
+		}
 
-		await this.checkConnection();
+		this.baseUrl = `${protocol}://${host}/${bucket.name}`;
+	}
 
+	async init() {
+		await this.checkConnection();
 		this.setReady(true);
 	}
 
@@ -65,7 +64,7 @@ export class ObjectStoreService {
 	async checkConnection() {
 		if (this.isReady) return;
 
-		return await this.request('HEAD', this.host, this.bucket.name);
+		return await this.request('HEAD', '');
 	}
 
 	/**
@@ -84,9 +83,7 @@ export class ObjectStoreService {
 		if (metadata.fileName) headers['x-amz-meta-filename'] = metadata.fileName;
 		if (metadata.mimeType) headers['Content-Type'] = metadata.mimeType;
 
-		const path = `/${this.bucket.name}/${filename}`;
-
-		return await this.request('PUT', this.host, path, { headers, body: buffer });
+		return await this.request('PUT', filename, { headers, body: buffer });
 	}
 
 	/**
@@ -97,9 +94,7 @@ export class ObjectStoreService {
 	async get(fileId: string, { mode }: { mode: 'buffer' }): Promise<Buffer>;
 	async get(fileId: string, { mode }: { mode: 'stream' }): Promise<Readable>;
 	async get(fileId: string, { mode }: { mode: 'stream' | 'buffer' }) {
-		const path = `${this.bucket.name}/${fileId}`;
-
-		const { data } = await this.request('GET', this.host, path, {
+		const { data } = await this.request('GET', fileId, {
 			responseType: mode === 'buffer' ? 'arraybuffer' : 'stream',
 		});
 
@@ -116,9 +111,7 @@ export class ObjectStoreService {
 	 * @doc https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html
 	 */
 	async getMetadata(fileId: string) {
-		const path = `${this.bucket.name}/${fileId}`;
-
-		const response = await this.request('HEAD', this.host, path);
+		const response = await this.request('HEAD', fileId);
 
 		return response.headers as MetadataResponseHeaders;
 	}
@@ -129,9 +122,7 @@ export class ObjectStoreService {
 	 * @doc https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html
 	 */
 	async deleteOne(fileId: string) {
-		const path = `${this.bucket.name}/${fileId}`;
-
-		return await this.request('DELETE', this.host, path);
+		return await this.request('DELETE', fileId);
 	}
 
 	/**
@@ -154,9 +145,7 @@ export class ObjectStoreService {
 			'Content-MD5': createHash('md5').update(body).digest('base64'),
 		};
 
-		const path = `${this.bucket.name}/?delete`;
-
-		return await this.request('POST', this.host, path, { headers, body });
+		return await this.request('POST', '?delete', { headers, body });
 	}
 
 	/**
@@ -192,7 +181,7 @@ export class ObjectStoreService {
 
 		if (nextPageToken) qs['continuation-token'] = nextPageToken;
 
-		const { data } = await this.request('GET', this.host, this.bucket.name, { qs });
+		const { data } = await this.request('GET', '', { qs });
 
 		if (typeof data !== 'string') {
 			throw new TypeError(`Expected XML string but received ${typeof data}`);
@@ -243,28 +232,46 @@ export class ObjectStoreService {
 
 	private async request<T>(
 		method: Method,
-		host: string,
 		rawPath = '',
 		{ qs, headers, body, responseType }: RequestOptions = {},
 	) {
-		const path = this.toPath(rawPath, qs);
+		const {
+			host,
+			bucket: { region },
+		} = this.s3Config;
+		let url = this.baseUrl;
+		if (rawPath && rawPath !== '/') {
+			url = `${url}/${rawPath}`;
+		}
+		if (qs) {
+			url +=
+				'?' +
+				Object.entries(qs)
+					.map(([key, value]) => `${key}=${value}`)
+					.join('&');
+		}
+		const path = new URL(url).pathname;
 
 		const optionsToSign: Aws4Options = {
 			method,
 			service: 's3',
-			region: this.bucket.region,
+			region,
 			host,
 			path,
 		};
 
 		if (headers) optionsToSign.headers = headers;
 		if (body) optionsToSign.body = body;
 
-		const signedOptions = sign(optionsToSign, this.credentials);
+		const { accessKey, accessSecret } = this.s3Config.credentials;
+		const signedOptions = sign(optionsToSign, {
+			accessKeyId: accessKey,
+			secretAccessKey: accessSecret,
+		});
 
 		const config: AxiosRequestConfig = {
 			method,
-			url: `https://${host}${path}`,
+			url,
 			headers: signedOptions.headers,
 		};
 

packages/@n8n/config/src/configs/external-storage.config.ts → packages/core/src/binary-data/object-store/s3.config.ts

@@ -1,4 +1,4 @@
-import { Config, Env, Nested } from '../decorators';
+import { Config, Env, Nested } from '@n8n/config';
 
 @Config
 class S3BucketConfig {
@@ -23,20 +23,17 @@ class S3CredentialsConfig {
 }
 
 @Config
-class S3Config {
+export class S3Config {
 	/** Host of the n8n bucket in S3-compatible external storage @example "s3.us-east-1.amazonaws.com" */
 	@Env('N8N_EXTERNAL_STORAGE_S3_HOST')
 	host: string = '';
 
-	@Nested
-	bucket: S3BucketConfig;
+	@Env('N8N_EXTERNAL_STORAGE_S3_PROTOCOL')
+	protocol: 'http' | 'https' = 'https';
 
 	@Nested
-	credentials: S3CredentialsConfig;
-}
+	bucket: S3BucketConfig = new S3BucketConfig();
 
-@Config
-export class ExternalStorageConfig {
 	@Nested
-	s3: S3Config;
+	credentials: S3CredentialsConfig = new S3CredentialsConfig();
 }

packages/core/src/binary-data/object-store/types.ts

@@ -24,8 +24,6 @@ type Item = {
 
 export type ListPage = Omit<RawListPage['listBucketResult'], 'contents'> & { contents: Item[] };
 
-export type Bucket = { region: string; name: string };
-
 export type RequestOptions = {
 	qs?: Record<string, string | number>;
 	headers?: Record<string, string | number>;
@@ -38,5 +36,3 @@ export type MetadataResponseHeaders = AxiosResponseHeaders & {
 	'content-type'?: string;
 	'x-amz-meta-filename'?: string;
 } & BinaryData.PreWriteMetadata;
-
-export type ConfigSchemaCredentials = { accessKey: string; accessSecret: string };