-
Notifications
You must be signed in to change notification settings - Fork 36
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: Cloudflare R2 provider #277
Merged
Merged
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,227 @@ | ||
| import { ReadStream, createReadStream } from 'node:fs'; | ||
| import { Readable } from 'node:stream'; | ||
| import fs from 'node:fs/promises'; | ||
| import { env } from 'node:process'; | ||
| import { fetch as uFetch } from 'undici'; | ||
| import chalk from 'chalk'; | ||
| import cuid2 from '@paralleldrive/cuid2'; | ||
| import { S3Client } from '@aws-sdk/client-s3'; | ||
|
|
||
| import { updateAsset, Asset } from '../../assets.js'; | ||
| import { getVideoConfig } from '../../config.js'; | ||
| import { findBucket, createBucket, putBucketCors, putObject } from '../../utils/s3.js'; | ||
| import { createAssetKey } from '../../utils/provider.js'; | ||
| import { isRemote } from '../../utils/utils.js'; | ||
| import log from '../../utils/logger.js'; | ||
| import { publicAccessR2Bucket } from '../../utils/r2.js'; | ||
|
|
||
| export type CloudflareR2Metadata = { | ||
| bucket?: string; | ||
| endpoint?: string; | ||
| key?: string; | ||
| }; | ||
|
|
||
| // Why 11? | ||
| // - Reasonable id length visually in the src URL | ||
| // - Familiarity with the length of YouTube IDs | ||
| // - It would take more than 300 million buckets to have a 50% chance of a collision. | ||
| // - "These go to eleven" https://www.youtube.com/watch?v=F7IZZXQ89Oc | ||
| const createId = cuid2.init({ length: 11 }); | ||
|
|
||
| let s3: S3Client; | ||
| let bucketName: string; | ||
| let bucketUrlPublic: string; | ||
| let accountId: string; | ||
| let endpoint: string; | ||
|
|
||
| async function initR2() { | ||
| const { providerConfig } = await getVideoConfig(); | ||
| const CloudflareR2Config = providerConfig['cloudflare-r2']; | ||
|
|
||
| bucketName = CloudflareR2Config?.bucket ?? ''; | ||
| bucketUrlPublic = CloudflareR2Config?.bucketUrlPublic ?? ''; | ||
| endpoint = CloudflareR2Config?.endpoint ?? ''; | ||
| accountId = endpoint.split('.')[0].replace(/^https?:\/\//, ''); | ||
|
|
||
| s3 ??= new S3Client({ | ||
| endpoint, | ||
| // region does not have any impact on Cloudflare R2 | ||
| region: 'auto', | ||
| credentials: { | ||
| accessKeyId: CloudflareR2Config?.accessKeyId ?? env.R2_ACCESS_KEY_ID ?? '', | ||
| secretAccessKey: CloudflareR2Config?.secretAccessKey ?? env.R2_SECRET_ACCESS_KEY ?? '', | ||
| }, | ||
| }); | ||
|
|
||
| if (!bucketName) { | ||
| try { | ||
| const bucket = await findBucket(s3, (bucket) => bucket.Name?.startsWith('next-videos-')); | ||
|
|
||
| if (bucket) { | ||
| bucketName = bucket.Name!; | ||
| log.info(log.label('Using existing Cloudflare R2 bucket:'), bucketName); | ||
| } | ||
| } catch (err) { | ||
| log.error('Error listing Cloudflare R2 buckets'); | ||
| console.error(err); | ||
| } | ||
| } | ||
|
|
||
| if (!bucketName) { | ||
| bucketName = `next-videos-${createId()}`; | ||
| log.info(log.label('Creating Cloudflare R2 bucket:'), bucketName); | ||
|
|
||
| try { | ||
| await createBucket(s3, bucketName, {}); | ||
| await putBucketCors(s3, bucketName); | ||
| } catch (err) { | ||
| log.error('Error creating Cloudflare R2 bucket'); | ||
| console.error(err); | ||
| } | ||
| } | ||
|
|
||
| if (!bucketUrlPublic && bucketName) { | ||
| const cloudflareApiToken = CloudflareR2Config?.apiToken ?? env.R2_CF_API_TOKEN ?? ''; | ||
| let bucketPublicId: string; | ||
| if (cloudflareApiToken) { | ||
| try { | ||
| bucketPublicId = (await publicAccessR2Bucket(accountId, bucketName, cloudflareApiToken)) ?? ''; | ||
| bucketUrlPublic = `https://${bucketPublicId}`; | ||
| } catch (e) { | ||
| log.error(`Error setting Public access for Cloudflare R2 bucket: ${bucketName}`); | ||
| console.error(e); | ||
| return; | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| export async function uploadLocalFile(asset: Asset) { | ||
| const filePath = asset.originalFilePath; | ||
|
|
||
| if (!filePath) { | ||
| log.error('No filePath provided for asset.'); | ||
| console.error(asset); | ||
| return; | ||
| } | ||
|
|
||
| // Handle imported remote videos. | ||
| if (isRemote(filePath)) { | ||
| return uploadRequestedFile(asset); | ||
| } | ||
|
|
||
| if (asset.status === 'ready') { | ||
| return; | ||
| } else if (asset.status === 'uploading') { | ||
| // Right now this re-starts the upload from the beginning. | ||
| // We should probably do something smarter here. | ||
| log.info(log.label('Resuming upload:'), filePath); | ||
| } | ||
|
|
||
| await updateAsset(filePath, { | ||
| status: 'uploading', | ||
| }); | ||
|
|
||
| await initR2(); | ||
|
|
||
| if (!bucketUrlPublic) { | ||
| log.error( | ||
| `Public access configuration missing: | ||
| Neither the Cloudflare API Key nor the bucketUrlPublic URL is specified for the bucket "${bucketName}". | ||
|
|
||
| To enable public access, you must ensure one of the following: | ||
| 1. **Configure the Bucket for Public Access:** | ||
| - Make sure the bucket "${bucketName}" is configured for public access | ||
| and specify the public URL in the provider configuration under the key 'bucketUrlPublic'. | ||
| - For detailed instructions, refer to the Cloudflare documentation: | ||
| https://developers.cloudflare.com/r2/buckets/public-buckets/ | ||
|
|
||
| 2. **Provide a Cloudflare API Key:** | ||
| - You can specify a Cloudflare API Key with R2 Admin read & write permissions using the environment variable: R2_CF_API_TOKEN. | ||
| - This API Key will allow us to enable public access for the bucket and retrieve the public URL using the Cloudflare API. | ||
| - To create an API Token, visit: | ||
| https://dash.cloudflare.com/?to=/:account/r2/api-tokens` | ||
| ); | ||
| return; | ||
| } | ||
|
|
||
| const fileStats = await fs.stat(filePath); | ||
| const stream = createReadStream(filePath); | ||
|
|
||
| return putAsset(filePath, fileStats.size, stream); | ||
| } | ||
|
|
||
| export async function uploadRequestedFile(asset: Asset) { | ||
| const filePath = asset.originalFilePath; | ||
|
|
||
| if (!filePath) { | ||
| log.error('No URL provided for asset.'); | ||
| console.error(asset); | ||
| return; | ||
| } | ||
|
|
||
| if (asset.status === 'ready') { | ||
| return; | ||
| } | ||
|
|
||
| await updateAsset(filePath, { | ||
| status: 'uploading', | ||
| }); | ||
|
|
||
| await initR2(); | ||
|
|
||
| const response = await uFetch(filePath); | ||
| const size = Number(response.headers.get('content-length')); | ||
| const stream = response.body; | ||
|
|
||
| if (!stream) { | ||
| log.error('Error fetching the requested file:', filePath); | ||
| return; | ||
| } | ||
|
|
||
| return putAsset(filePath, size, Readable.fromWeb(stream)); | ||
| } | ||
|
|
||
| async function putAsset(filePath: string, size: number, stream: ReadStream | Readable) { | ||
| log.info(log.label('Uploading file:'), `${filePath} (${size} bytes)`); | ||
|
|
||
| let key; | ||
| try { | ||
| key = await createAssetKey(filePath, 'cloudflare-r2'); | ||
|
|
||
| await putObject(s3, { | ||
| ACL: 'public-read', | ||
| Bucket: bucketName, | ||
| Key: key, | ||
| Body: stream, | ||
| ContentLength: size, | ||
| }); | ||
|
|
||
| if (stream instanceof ReadStream) { | ||
| stream.close(); | ||
| } | ||
| } catch (e) { | ||
| log.error('Error uploading to Cloudflare R2'); | ||
| console.error(e); | ||
| return; | ||
| } | ||
|
|
||
| log.success(log.label('File uploaded:'), `${filePath} (${size} bytes)`); | ||
|
|
||
| const updatedAsset = await updateAsset(filePath, { | ||
| status: 'ready', | ||
| providerMetadata: { | ||
| 'cloudflare-r2': { | ||
| endpoint, | ||
| bucketUrlPublic, | ||
| bucket: bucketName, | ||
| key, | ||
| } as CloudflareR2Metadata, | ||
| }, | ||
| }); | ||
|
|
||
| const url = updatedAsset.sources?.[0].src; | ||
| log.space(chalk.gray('>'), log.label('URL:'), url); | ||
|
|
||
| return updatedAsset; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| import type { Asset, AssetSource } from '../../assets.js'; | ||
|
|
||
| export function transform(asset: Asset) { | ||
| const providerMetadata = asset.providerMetadata?.['cloudflare-r2']; | ||
| if (!providerMetadata) return asset; | ||
|
|
||
| const src = new URL(providerMetadata.bucketUrlPublic); | ||
| src.pathname = providerMetadata.key; | ||
|
|
||
| const source: AssetSource = { src: `${src}` }; | ||
|
|
||
| return { | ||
| ...asset, | ||
| sources: [source], | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| import { request } from 'undici'; | ||
|
|
||
| interface CloudflareR2PolicyResponse { | ||
| success: boolean; | ||
| errors: Array<{ code: number; message: string }>; | ||
| messages: Array<{ code: number; message: string }>; | ||
| result: { | ||
| publicId: string; | ||
| onlyViaCnames: string[]; | ||
| }; | ||
| } | ||
|
|
||
| export async function publicAccessR2Bucket( | ||
| accountId: string, | ||
| bucketName: string, | ||
| apiToken: string | ||
| ): Promise<string> { | ||
| const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucketName}/policy?access=PublicUrlAndCnames`; | ||
|
|
||
| try { | ||
| const { statusCode, body } = await request(url, { | ||
| method: 'PUT', | ||
| headers: { | ||
| Authorization: `Bearer ${apiToken}`, | ||
| 'Content-Type': 'application/json', | ||
| }, | ||
| }); | ||
|
|
||
| const responseBody: CloudflareR2PolicyResponse = (await body.json()) as CloudflareR2PolicyResponse; | ||
|
|
||
| if (statusCode !== 200 || !responseBody.success) { | ||
| throw new Error( | ||
| `Failed to set public access. Status code: ${statusCode}, Error details: ${JSON.stringify(responseBody.errors)}` | ||
| ); | ||
| } | ||
|
|
||
| if (responseBody.result.onlyViaCnames.length > 0) { | ||
| return responseBody.result.onlyViaCnames[0]; | ||
| } else { | ||
| return `${responseBody.result.publicId}.r2.dev`; | ||
| } | ||
| } catch (error) { | ||
| throw new Error(`Error setting public access: ${(error as Error).message}`); | ||
| } | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is there any way
bucketUrlPubliccan be constructed from endpoint, accountId and bucket?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Without a Cloudflare API Token (R2_CF_API_TOKEN) to my knowledge no.
There are two ways for public access:
Both must be enabled first on a bucket level.
If a Cloudflare API Token is given to the provider the r2.dev subdomain will be enabled when creating the bucket (if none is given). Making it usable instantly without manual interaction.