diff --git a/packages/@n8n/config/src/index.ts b/packages/@n8n/config/src/index.ts index f462ef94248cf4..2437210a5d0d0c 100644 --- a/packages/@n8n/config/src/index.ts +++ b/packages/@n8n/config/src/index.ts @@ -7,7 +7,6 @@ import { EndpointsConfig } from './configs/endpoints.config'; import { EventBusConfig } from './configs/event-bus.config'; import { ExecutionsConfig } from './configs/executions.config'; import { ExternalSecretsConfig } from './configs/external-secrets.config'; -import { ExternalStorageConfig } from './configs/external-storage.config'; import { GenericConfig } from './configs/generic.config'; import { LicenseConfig } from './configs/license.config'; import { LoggingConfig } from './configs/logging.config'; @@ -62,9 +61,6 @@ export class GlobalConfig { @Nested nodes: NodesConfig; - @Nested - externalStorage: ExternalStorageConfig; - @Nested workflows: WorkflowsConfig; diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index d9499d784952ff..4fcf8676446255 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -135,19 +135,6 @@ describe('GlobalConfig', () => { endpoint: 'https://api.n8n.io/api/versions/', infoUrl: 'https://docs.n8n.io/hosting/installation/updating/', }, - externalStorage: { - s3: { - host: '', - bucket: { - name: '', - region: '', - }, - credentials: { - accessKey: '', - accessSecret: '', - }, - }, - }, workflows: { defaultName: 'My workflow', callerPolicyDefaultOption: 'workflowsFromSameOwner', diff --git a/packages/cli/src/commands/base-command.ts b/packages/cli/src/commands/base-command.ts index 9bfd992b3d58cd..5e07b8e350b0a5 100644 --- a/packages/cli/src/commands/base-command.ts +++ b/packages/cli/src/commands/base-command.ts @@ -189,42 +189,10 @@ export abstract class BaseCommand extends Command { private async _initObjectStoreService(options = { isReadOnly: false }) { const objectStoreService = Container.get(ObjectStoreService); - const { host, bucket, credentials } = this.globalConfig.externalStorage.s3; - - if (host === '') { - throw new ApplicationError( - 'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.', - ); - } - - if (bucket.name === '') { - throw new ApplicationError( - 'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.', - ); - } - - if (bucket.region === '') { - throw new ApplicationError( - 'External storage bucket region not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_REGION`.', - ); - } - - if (credentials.accessKey === '') { - throw new ApplicationError( - 'External storage access key not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_KEY`.', - ); - } - - if (credentials.accessSecret === '') { - throw new ApplicationError( - 'External storage access secret not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_SECRET`.', - ); - } - this.logger.debug('Initializing object store service'); try { - await objectStoreService.init(host, bucket, credentials); + await objectStoreService.init(); objectStoreService.setReadonly(options.isReadOnly); this.logger.debug('Object store init completed'); diff --git a/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts b/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts index f5d2924eb55f87..1b53a6317d4b93 100644 --- a/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts +++ b/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts @@ -3,6 +3,7 @@ import { mock } from 'jest-mock-extended'; import { Readable } from 'stream'; import { ObjectStoreService } from '@/binary-data/object-store/object-store.service.ee'; +import type { S3Config } from '@/binary-data/object-store/s3.config'; import { writeBlockedMessage } from '@/binary-data/object-store/utils'; jest.mock('axios'); @@ -18,6 +19,12 @@ const mockError = new Error('Something went wrong!'); const fileId = 'workflows/ObogjVbqpNOQpiyV/executions/999/binary_data/71f6209b-5d48-41a2-a224-80d529d8bb32'; const mockBuffer = Buffer.from('Test data'); +const s3Config = mock({ + host: mockHost, + bucket: mockBucket, + credentials: mockCredentials, + protocol: 'https', +}); const toDeletionXml = (filename: string) => ` ${filename} @@ -26,9 +33,9 @@ const toDeletionXml = (filename: string) => ` let objectStoreService: ObjectStoreService; beforeEach(async () => { - objectStoreService = new ObjectStoreService(mock()); + objectStoreService = new ObjectStoreService(mock(), s3Config); mockAxios.request.mockResolvedValueOnce({ status: 200 }); // for checkConnection - await objectStoreService.init(mockHost, mockBucket, mockCredentials); + await objectStoreService.init(); jest.restoreAllMocks(); }); diff --git a/packages/core/src/binary-data/object-store/object-store.service.ee.ts b/packages/core/src/binary-data/object-store/object-store.service.ee.ts index 508477d50e6f9e..5e505f0e9a1bab 100644 --- a/packages/core/src/binary-data/object-store/object-store.service.ee.ts +++ b/packages/core/src/binary-data/object-store/object-store.service.ee.ts @@ -1,6 +1,6 @@ import { Service } from '@n8n/di'; import { sign } from 'aws4'; -import type { Request as Aws4Options, Credentials as Aws4Credentials } from 'aws4'; +import type { Request as Aws4Options } from 'aws4'; import axios from 'axios'; import type { AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig, Method } from 'axios'; import { ApplicationError } from 'n8n-workflow'; @@ -9,43 +9,42 @@ import type { Readable } from 'stream'; import { Logger } from '@/logging/logger'; -import type { - Bucket, - ConfigSchemaCredentials, - ListPage, - MetadataResponseHeaders, - RawListPage, - RequestOptions, -} from './types'; +import { S3Config } from './s3.config'; +import type { ListPage, MetadataResponseHeaders, RawListPage, RequestOptions } from './types'; import { isStream, parseXml, writeBlockedMessage } from './utils'; import type { BinaryData } from '../types'; @Service() export class ObjectStoreService { - private host = ''; - - private bucket: Bucket = { region: '', name: '' }; - - private credentials: Aws4Credentials = { accessKeyId: '', secretAccessKey: '' }; + private baseUrl: string; private isReady = false; private isReadOnly = false; - constructor(private readonly logger: Logger) {} + constructor( + private readonly logger: Logger, + private readonly s3Config: S3Config, + ) { + const { host, bucket, protocol } = s3Config; - async init(host: string, bucket: Bucket, credentials: ConfigSchemaCredentials) { - this.host = host; - this.bucket.name = bucket.name; - this.bucket.region = bucket.region; + if (host === '') { + throw new ApplicationError( + 'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.', + ); + } - this.credentials = { - accessKeyId: credentials.accessKey, - secretAccessKey: credentials.accessSecret, - }; + if (bucket.name === '') { + throw new ApplicationError( + 'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.', + ); + } - await this.checkConnection(); + this.baseUrl = `${protocol}://${host}/${bucket.name}`; + } + async init() { + await this.checkConnection(); this.setReady(true); } @@ -65,7 +64,7 @@ export class ObjectStoreService { async checkConnection() { if (this.isReady) return; - return await this.request('HEAD', this.host, this.bucket.name); + return await this.request('HEAD', ''); } /** @@ -84,9 +83,7 @@ export class ObjectStoreService { if (metadata.fileName) headers['x-amz-meta-filename'] = metadata.fileName; if (metadata.mimeType) headers['Content-Type'] = metadata.mimeType; - const path = `/${this.bucket.name}/${filename}`; - - return await this.request('PUT', this.host, path, { headers, body: buffer }); + return await this.request('PUT', filename, { headers, body: buffer }); } /** @@ -97,9 +94,7 @@ export class ObjectStoreService { async get(fileId: string, { mode }: { mode: 'buffer' }): Promise; async get(fileId: string, { mode }: { mode: 'stream' }): Promise; async get(fileId: string, { mode }: { mode: 'stream' | 'buffer' }) { - const path = `${this.bucket.name}/${fileId}`; - - const { data } = await this.request('GET', this.host, path, { + const { data } = await this.request('GET', fileId, { responseType: mode === 'buffer' ? 'arraybuffer' : 'stream', }); @@ -116,9 +111,7 @@ export class ObjectStoreService { * @doc https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html */ async getMetadata(fileId: string) { - const path = `${this.bucket.name}/${fileId}`; - - const response = await this.request('HEAD', this.host, path); + const response = await this.request('HEAD', fileId); return response.headers as MetadataResponseHeaders; } @@ -129,9 +122,7 @@ export class ObjectStoreService { * @doc https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html */ async deleteOne(fileId: string) { - const path = `${this.bucket.name}/${fileId}`; - - return await this.request('DELETE', this.host, path); + return await this.request('DELETE', fileId); } /** @@ -154,9 +145,7 @@ export class ObjectStoreService { 'Content-MD5': createHash('md5').update(body).digest('base64'), }; - const path = `${this.bucket.name}/?delete`; - - return await this.request('POST', this.host, path, { headers, body }); + return await this.request('POST', '?delete', { headers, body }); } /** @@ -192,7 +181,7 @@ export class ObjectStoreService { if (nextPageToken) qs['continuation-token'] = nextPageToken; - const { data } = await this.request('GET', this.host, this.bucket.name, { qs }); + const { data } = await this.request('GET', '', { qs }); if (typeof data !== 'string') { throw new TypeError(`Expected XML string but received ${typeof data}`); @@ -243,16 +232,30 @@ export class ObjectStoreService { private async request( method: Method, - host: string, rawPath = '', { qs, headers, body, responseType }: RequestOptions = {}, ) { - const path = this.toPath(rawPath, qs); + const { + host, + bucket: { region }, + } = this.s3Config; + let url = this.baseUrl; + if (rawPath && rawPath !== '/') { + url = `${url}/${rawPath}`; + } + if (qs) { + url += + '?' + + Object.entries(qs) + .map(([key, value]) => `${key}=${value}`) + .join('&'); + } + const path = new URL(url).pathname; const optionsToSign: Aws4Options = { method, service: 's3', - region: this.bucket.region, + region, host, path, }; @@ -260,11 +263,15 @@ export class ObjectStoreService { if (headers) optionsToSign.headers = headers; if (body) optionsToSign.body = body; - const signedOptions = sign(optionsToSign, this.credentials); + const { accessKey, accessSecret } = this.s3Config.credentials; + const signedOptions = sign(optionsToSign, { + accessKeyId: accessKey, + secretAccessKey: accessSecret, + }); const config: AxiosRequestConfig = { method, - url: `https://${host}${path}`, + url, headers: signedOptions.headers, }; diff --git a/packages/@n8n/config/src/configs/external-storage.config.ts b/packages/core/src/binary-data/object-store/s3.config.ts similarity index 75% rename from packages/@n8n/config/src/configs/external-storage.config.ts rename to packages/core/src/binary-data/object-store/s3.config.ts index 6e5fbd64d89ad6..41dc92b588f330 100644 --- a/packages/@n8n/config/src/configs/external-storage.config.ts +++ b/packages/core/src/binary-data/object-store/s3.config.ts @@ -1,4 +1,4 @@ -import { Config, Env, Nested } from '../decorators'; +import { Config, Env, Nested } from '@n8n/config'; @Config class S3BucketConfig { @@ -23,20 +23,17 @@ class S3CredentialsConfig { } @Config -class S3Config { +export class S3Config { /** Host of the n8n bucket in S3-compatible external storage @example "s3.us-east-1.amazonaws.com" */ @Env('N8N_EXTERNAL_STORAGE_S3_HOST') host: string = ''; - @Nested - bucket: S3BucketConfig; + @Env('N8N_EXTERNAL_STORAGE_S3_PROTOCOL') + protocol: 'http' | 'https' = 'https'; @Nested - credentials: S3CredentialsConfig; -} + bucket: S3BucketConfig = new S3BucketConfig(); -@Config -export class ExternalStorageConfig { @Nested - s3: S3Config; + credentials: S3CredentialsConfig = new S3CredentialsConfig(); } diff --git a/packages/core/src/binary-data/object-store/types.ts b/packages/core/src/binary-data/object-store/types.ts index 49726f5c43c0d7..20390cf243ca81 100644 --- a/packages/core/src/binary-data/object-store/types.ts +++ b/packages/core/src/binary-data/object-store/types.ts @@ -24,8 +24,6 @@ type Item = { export type ListPage = Omit & { contents: Item[] }; -export type Bucket = { region: string; name: string }; - export type RequestOptions = { qs?: Record; headers?: Record; @@ -38,5 +36,3 @@ export type MetadataResponseHeaders = AxiosResponseHeaders & { 'content-type'?: string; 'x-amz-meta-filename'?: string; } & BinaryData.PreWriteMetadata; - -export type ConfigSchemaCredentials = { accessKey: string; accessSecret: string };