From b4ea7a7ab3d33aa40dd218036e782374ba121ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20B=C3=A4ttig?= Date: Fri, 26 Jul 2024 22:28:53 +0200 Subject: [PATCH 1/3] feat: add Cloudflare R2 provider --- src/config.ts | 13 ++ src/providers/cloudflare-r2/provider.ts | 230 +++++++++++++++++++++ src/providers/cloudflare-r2/transformer.ts | 16 ++ src/providers/providers.ts | 1 + src/providers/transformers.ts | 1 + src/utils/r2.ts | 45 ++++ 6 files changed, 306 insertions(+) create mode 100644 src/providers/cloudflare-r2/provider.ts create mode 100644 src/providers/cloudflare-r2/transformer.ts create mode 100644 src/utils/r2.ts diff --git a/src/config.ts b/src/config.ts index 180ffe9..b5481e6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -53,6 +53,19 @@ export type ProviderConfig = { /* An optional function to generate the bucket asset key. */ generateAssetKey?: (filePathOrURL: string, folder: string) => string; }; + + 'cloudflare-r2'?: { + endpoint: string; + accountId: string; + bucket?: string; + bucketUrlPublic?: string; + accessKeyId?: string; + secretAccessKey?: string; + apiToken?: string; + jurisdiction?: string; + /* An optional function to generate the bucket asset key. */ + generateAssetKey?: (filePathOrURL: string, folder: string) => string; + }; }; export type VideoConfig = Partial; diff --git a/src/providers/cloudflare-r2/provider.ts b/src/providers/cloudflare-r2/provider.ts new file mode 100644 index 0000000..1d95ba3 --- /dev/null +++ b/src/providers/cloudflare-r2/provider.ts @@ -0,0 +1,230 @@ +import { ReadStream, createReadStream } from 'node:fs'; +import { Readable } from 'node:stream'; +import fs from 'node:fs/promises'; +import { env } from 'node:process'; +import { fetch as uFetch } from 'undici'; +import chalk from 'chalk'; +import cuid2 from '@paralleldrive/cuid2'; +import { S3Client } from '@aws-sdk/client-s3'; + +import { updateAsset, Asset } from '../../assets.js'; +import { getVideoConfig } from '../../config.js'; +import { findBucket, createBucket, putBucketCors, putObject } from '../../utils/s3.js'; +import { createAssetKey } from '../../utils/provider.js'; +import { isRemote } from '../../utils/utils.js'; +import log from '../../utils/logger.js'; +import { publicAccessR2Bucket } from '../../utils/r2.js'; + +export type CloudflareR2Metadata = { + bucket?: string; + endpoint?: string; + key?: string; +}; + +// Why 11? +// - Reasonable id length visually in the src URL +// - Familiarity with the length of YouTube IDs +// - It would take more than 300 million buckets to have a 50% chance of a collision. +// - "These go to eleven" https://www.youtube.com/watch?v=F7IZZXQ89Oc +const createId = cuid2.init({ length: 11 }); + +let s3: S3Client; +let bucketName: string; +let bucketUrlPublic: string; +let accountId: string; +let endpoint: string; + +async function initR2() { + const { providerConfig } = await getVideoConfig(); + const CloudflareR2Config = providerConfig['cloudflare-r2']; + + accountId = CloudflareR2Config?.accountId ?? env.R2_ACCOUNT_ID ?? ''; + bucketName = CloudflareR2Config?.bucket ?? ''; + bucketUrlPublic = CloudflareR2Config?.bucketUrlPublic ?? ''; + + const jurisdiction = CloudflareR2Config?.jurisdiction ?? ''; + const baseEndpoint = jurisdiction === '' ? 'r2.cloudflarestorage.com' : `${jurisdiction}.r2.cloudflarestorage.com`; + endpoint = `https://${accountId}.${baseEndpoint}`; + + s3 ??= new S3Client({ + endpoint, + // region does not have any impact on Cloudflare R2 + region: 'auto', + credentials: { + accessKeyId: CloudflareR2Config?.accessKeyId ?? env.R2_ACCESS_KEY_ID ?? '', + secretAccessKey: CloudflareR2Config?.secretAccessKey ?? env.R2_SECRET_ACCESS_KEY ?? '', + }, + }); + + if (!bucketName) { + try { + const bucket = await findBucket(s3, (bucket) => bucket.Name?.startsWith('next-videos-')); + + if (bucket) { + bucketName = bucket.Name!; + log.info(log.label('Using existing Cloudflare R2 bucket:'), bucketName); + } + } catch (err) { + log.error('Error listing Cloudflare R2 buckets'); + console.error(err); + } + } + + if (!bucketName) { + bucketName = `next-videos-${createId()}`; + log.info(log.label('Creating Cloudflare R2 bucket:'), bucketName); + + try { + await createBucket(s3, bucketName, {}); + await putBucketCors(s3, bucketName); + } catch (err) { + log.error('Error creating Cloudflare R2 bucket'); + console.error(err); + } + } + + if (!bucketUrlPublic && bucketName) { + const cloudflareApiToken = CloudflareR2Config?.apiToken ?? env.R2_CF_API_TOKEN ?? ''; + let bucketPublicId: string; + if (cloudflareApiToken) { + try { + bucketPublicId = (await publicAccessR2Bucket(accountId, bucketName, cloudflareApiToken)) ?? ''; + bucketUrlPublic = `https://${bucketPublicId}`; + } catch (e) { + log.error(`Error setting Public access for Cloudflare R2 bucket: ${bucketName}`); + console.error(e); + return; + } + } + } +} + +export async function uploadLocalFile(asset: Asset) { + const filePath = asset.originalFilePath; + + if (!filePath) { + log.error('No filePath provided for asset.'); + console.error(asset); + return; + } + + // Handle imported remote videos. + if (isRemote(filePath)) { + return uploadRequestedFile(asset); + } + + if (asset.status === 'ready') { + return; + } else if (asset.status === 'uploading') { + // Right now this re-starts the upload from the beginning. + // We should probably do something smarter here. + log.info(log.label('Resuming upload:'), filePath); + } + + await updateAsset(filePath, { + status: 'uploading', + }); + + await initR2(); + + if (!bucketUrlPublic) { + log.error( + `Public access configuration missing: + Neither the Cloudflare API Key nor the bucketUrlPublic URL is specified for the bucket "${bucketName}". + + To enable public access, you must ensure one of the following: + 1. **Configure the Bucket for Public Access:** + - Make sure the bucket "${bucketName}" is configured for public access + and specify the public URL in the provider configuration under the key 'bucketUrlPublic'. + - For detailed instructions, refer to the Cloudflare documentation: + https://developers.cloudflare.com/r2/buckets/public-buckets/ + + 2. **Provide a Cloudflare API Key:** + - You can specify a Cloudflare API Key with R2 Admin read & write permissions using the environment variable: R2_CF_API_TOKEN. + - This API Key will allow us to enable public access for the bucket and retrieve the public URL using the Cloudflare API. + - To create an API Token, visit: + https://dash.cloudflare.com/?to=/:account/r2/api-tokens` + ); + return; + } + + const fileStats = await fs.stat(filePath); + const stream = createReadStream(filePath); + + return putAsset(filePath, fileStats.size, stream); +} + +export async function uploadRequestedFile(asset: Asset) { + const filePath = asset.originalFilePath; + + if (!filePath) { + log.error('No URL provided for asset.'); + console.error(asset); + return; + } + + if (asset.status === 'ready') { + return; + } + + await updateAsset(filePath, { + status: 'uploading', + }); + + await initR2(); + + const response = await uFetch(filePath); + const size = Number(response.headers.get('content-length')); + const stream = response.body; + + if (!stream) { + log.error('Error fetching the requested file:', filePath); + return; + } + + return putAsset(filePath, size, Readable.fromWeb(stream)); +} + +async function putAsset(filePath: string, size: number, stream: ReadStream | Readable) { + log.info(log.label('Uploading file:'), `${filePath} (${size} bytes)`); + + let key; + try { + key = await createAssetKey(filePath, 'cloudflare-r2'); + + await putObject(s3, { + ACL: 'public-read', + Bucket: bucketName, + Key: key, + Body: stream, + ContentLength: size, + }); + + if (stream instanceof ReadStream) { + stream.close(); + } + } catch (e) { + log.error('Error uploading to Cloudflare R2'); + console.error(e); + return; + } + + log.success(log.label('File uploaded:'), `${filePath} (${size} bytes)`); + + const updatedAsset = await updateAsset(filePath, { + status: 'ready', + providerMetadata: { + 'cloudflare-r2': { + endpoint, + bucketUrlPublic, + bucket: bucketName, + key, + } as CloudflareR2Metadata, + }, + }); + + const url = updatedAsset.sources?.[0].src; + log.space(chalk.gray('>'), log.label('URL:'), url); + + return updatedAsset; +} diff --git a/src/providers/cloudflare-r2/transformer.ts b/src/providers/cloudflare-r2/transformer.ts new file mode 100644 index 0000000..d67de01 --- /dev/null +++ b/src/providers/cloudflare-r2/transformer.ts @@ -0,0 +1,16 @@ +import type { Asset, AssetSource } from '../../assets.js'; + +export function transform(asset: Asset) { + const providerMetadata = asset.providerMetadata?.['cloudflare-r2']; + if (!providerMetadata) return asset; + + const src = new URL(providerMetadata.bucketUrlPublic); + src.pathname = providerMetadata.key; + + const source: AssetSource = { src: `${src}` }; + + return { + ...asset, + sources: [source], + }; +} diff --git a/src/providers/providers.ts b/src/providers/providers.ts index 49d53a2..bfa2f84 100644 --- a/src/providers/providers.ts +++ b/src/providers/providers.ts @@ -2,3 +2,4 @@ export * as mux from './mux/provider.js'; export * as vercelBlob from './vercel-blob/provider.js'; export * as backblaze from './backblaze/provider.js'; export * as amazonS3 from './amazon-s3/provider.js'; +export * as cloudflareR2 from './cloudflare-r2/provider.js' diff --git a/src/providers/transformers.ts b/src/providers/transformers.ts index 1559add..05b158d 100644 --- a/src/providers/transformers.ts +++ b/src/providers/transformers.ts @@ -2,3 +2,4 @@ export * as mux from './mux/transformer.js'; export * as vercelBlob from './vercel-blob/transformer.js'; export * as backblaze from './backblaze/transformer.js'; export * as amazonS3 from './amazon-s3/transformer.js'; +export * as cloudflareR2 from './cloudflare-r2/transformer.js'; diff --git a/src/utils/r2.ts b/src/utils/r2.ts new file mode 100644 index 0000000..0f4c1fd --- /dev/null +++ b/src/utils/r2.ts @@ -0,0 +1,45 @@ +import { request } from 'undici'; + +interface CloudflareR2PolicyResponse { + success: boolean; + errors: Array<{ code: number; message: string }>; + messages: Array<{ code: number; message: string }>; + result: { + publicId: string; + onlyViaCnames: string[]; + }; +} + +export async function publicAccessR2Bucket( + accountId: string, + bucketName: string, + apiToken: string +): Promise { + const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucketName}/policy?access=PublicUrlAndCnames`; + + try { + const { statusCode, body } = await request(url, { + method: 'PUT', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + }); + + const responseBody: CloudflareR2PolicyResponse = (await body.json()) as CloudflareR2PolicyResponse; + + if (statusCode !== 200 || !responseBody.success) { + throw new Error( + `Failed to set public access. Status code: ${statusCode}, Error details: ${JSON.stringify(responseBody.errors)}` + ); + } + + if (responseBody.result.onlyViaCnames.length > 0) { + return responseBody.result.onlyViaCnames[0]; + } else { + return `${responseBody.result.publicId}.r2.dev`; + } + } catch (error) { + throw new Error(`Error setting public access: ${(error as Error).message}`); + } +} From 60b7d0c04d10b67d8e704ff051b6da1c66485159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20B=C3=A4ttig?= Date: Sat, 27 Jul 2024 21:31:19 +0200 Subject: [PATCH 2/3] docs: add Cloudflare R2 --- README.md | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a5032d8..2a2fb9a 100644 --- a/README.md +++ b/README.md @@ -363,23 +363,24 @@ Supported providers with their required environment variables: | [`vercel-blob`](https://vercel.com/docs/storage/vercel-blob) | `BLOB_READ_WRITE_TOKEN` | | [Pricing](https://vercel.com/docs/storage/vercel-blob/usage-and-pricing) | | [`backblaze`](https://www.backblaze.com/cloud-storage) | `BACKBLAZE_ACCESS_KEY_ID`
`BACKBLAZE_SECRET_ACCESS_KEY` | `endpoint`
`bucket` (optional) | [Pricing](https://www.backblaze.com/cloud-storage/pricing) | | [`amazon-s3`](https://aws.amazon.com/s3) | `AWS_ACCESS_KEY_ID`
`AWS_SECRET_ACCESS_KEY` | `endpoint`
`bucket` (optional) | [Pricing](https://aws.amazon.com/s3/pricing/) | +| [`cloudflare-r2`](https://developers.cloudflare.com/r2/) | `R2_ACCESS_KEY_ID`
`R2_SECRET_ACCESS_KEY`
`R2_CF_API_TOKEN` (optional when `bucketUrlPublic` set) | `accountId`
`bucket` (optional)
`bucketUrlPublic` (optional when `R2_CF_API_TOKEN` set) | [Pricing](https://developers.cloudflare.com/r2/pricing/) | #### Provider feature set -| | Mux (default) | Vercel Blob | Backblaze | Amazon S3 | -| ---------------------------- | ------------- | ----------- | --------- | --------- | -| Off-repo storage | ✅ | ✅ | ✅ | ✅ | -| Delivery via CDN | ✅ | ✅ | - | - | -| BYO player | ✅ | ✅ | ✅ | ✅ | -| Compressed for streaming | ✅ | - | - | - | -| Adapt to slow networks (HLS) | ✅ | - | - | - | -| Automatic placeholder poster | ✅ | - | - | - | -| Timeline hover thumbnails | ✅ | - | - | - | -| Stream any source format | ✅ | * | * | * | -| AI captions & subtitles | ✅ | - | - | - | -| Video analytics | ✅ | - | - | - | -| Pricing | Minutes-based | GB-based | GB-based | GB-based | +| | Mux (default) | Vercel Blob | Backblaze | Amazon S3 | Cloudflare R2 | +| ---------------------------- | ------------- | ----------- | --------- | --------- | ------------- | +| Off-repo storage | ✅ | ✅ | ✅ | ✅ | ✅ | +| Delivery via CDN | ✅ | ✅ | - | - | ✅ | +| BYO player | ✅ | ✅ | ✅ | ✅ | ✅ | +| Compressed for streaming | ✅ | - | - | - | | +| Adapt to slow networks (HLS) | ✅ | - | - | - | | +| Automatic placeholder poster | ✅ | - | - | - | | +| Timeline hover thumbnails | ✅ | - | - | - | | +| Stream any source format | ✅ | * | * | * | * | +| AI captions & subtitles | ✅ | - | - | - | | +| Video analytics | ✅ | - | - | - | | +| Pricing | Minutes-based | GB-based | GB-based | GB-based | GB-based | *Web-compatible MP4 files required for hosting providers without video processing @@ -420,6 +421,25 @@ Supported providers with their required environment variables: ``` +## Cloudflare R2 Bucket Public Access + +
+By default, Cloudflare R2 Buckets are not publicly accessible. To enable public access, you must ensure one of the following: + +1. Configure the Bucket for Public Access: + - Provide a `bucket` Name in the provider configuration and ensure it is configured for public access + - Specify the public URL in the provider configuration under the `bucketUrlPublic` key + - For detailed instructions, refer to the Cloudflare documentation: + https://developers.cloudflare.com/r2/buckets/public-buckets/ + +2. Provide a Cloudflare API Key: + - You can specify a Cloudflare API Key with R2 Admin read & write permissions using the environment variable: `R2_CF_API_TOKEN` + - This API Key will allow the provider to enable public access for the bucket and retrieve the public URL using the Cloudflare API + - You don't need to create a bucket manually + - To create an API Token, visit: + https://dash.cloudflare.com/?to=/:account/r2/api-tokens +
+ ## Roadmap ### v0 From 80a9f5687dfe7c736a25057c9094a237d57f752a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20B=C3=A4ttig?= Date: Sun, 25 Aug 2024 20:37:02 +0200 Subject: [PATCH 3/3] remove juristication and acountId in favor of endpoint --- README.md | 2 +- src/config.ts | 2 -- src/providers/cloudflare-r2/provider.ts | 7 ++----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2a2fb9a..a7faad8 100644 --- a/README.md +++ b/README.md @@ -363,7 +363,7 @@ Supported providers with their required environment variables: | [`vercel-blob`](https://vercel.com/docs/storage/vercel-blob) | `BLOB_READ_WRITE_TOKEN` | | [Pricing](https://vercel.com/docs/storage/vercel-blob/usage-and-pricing) | | [`backblaze`](https://www.backblaze.com/cloud-storage) | `BACKBLAZE_ACCESS_KEY_ID`
`BACKBLAZE_SECRET_ACCESS_KEY` | `endpoint`
`bucket` (optional) | [Pricing](https://www.backblaze.com/cloud-storage/pricing) | | [`amazon-s3`](https://aws.amazon.com/s3) | `AWS_ACCESS_KEY_ID`
`AWS_SECRET_ACCESS_KEY` | `endpoint`
`bucket` (optional) | [Pricing](https://aws.amazon.com/s3/pricing/) | -| [`cloudflare-r2`](https://developers.cloudflare.com/r2/) | `R2_ACCESS_KEY_ID`
`R2_SECRET_ACCESS_KEY`
`R2_CF_API_TOKEN` (optional when `bucketUrlPublic` set) | `accountId`
`bucket` (optional)
`bucketUrlPublic` (optional when `R2_CF_API_TOKEN` set) | [Pricing](https://developers.cloudflare.com/r2/pricing/) | +| [`cloudflare-r2`](https://developers.cloudflare.com/r2/) | `R2_ACCESS_KEY_ID`
`R2_SECRET_ACCESS_KEY`
`R2_CF_API_TOKEN` (optional when `bucketUrlPublic` set) | `bucket` (optional)
`bucketUrlPublic` (optional when `R2_CF_API_TOKEN` set) | [Pricing](https://developers.cloudflare.com/r2/pricing/) | #### Provider feature set diff --git a/src/config.ts b/src/config.ts index b5481e6..e6b541d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -56,13 +56,11 @@ export type ProviderConfig = { 'cloudflare-r2'?: { endpoint: string; - accountId: string; bucket?: string; bucketUrlPublic?: string; accessKeyId?: string; secretAccessKey?: string; apiToken?: string; - jurisdiction?: string; /* An optional function to generate the bucket asset key. */ generateAssetKey?: (filePathOrURL: string, folder: string) => string; }; diff --git a/src/providers/cloudflare-r2/provider.ts b/src/providers/cloudflare-r2/provider.ts index 1d95ba3..8f6614f 100644 --- a/src/providers/cloudflare-r2/provider.ts +++ b/src/providers/cloudflare-r2/provider.ts @@ -38,13 +38,10 @@ async function initR2() { const { providerConfig } = await getVideoConfig(); const CloudflareR2Config = providerConfig['cloudflare-r2']; - accountId = CloudflareR2Config?.accountId ?? env.R2_ACCOUNT_ID ?? ''; bucketName = CloudflareR2Config?.bucket ?? ''; bucketUrlPublic = CloudflareR2Config?.bucketUrlPublic ?? ''; - - const jurisdiction = CloudflareR2Config?.jurisdiction ?? ''; - const baseEndpoint = jurisdiction === '' ? 'r2.cloudflarestorage.com' : `${jurisdiction}.r2.cloudflarestorage.com`; - endpoint = `https://${accountId}.${baseEndpoint}`; + endpoint = CloudflareR2Config?.endpoint ?? ''; + accountId = endpoint.split('.')[0].replace(/^https?:\/\//, ''); s3 ??= new S3Client({ endpoint,