From bf28a4bb48c6330ce9892039a56158ef126a9133 Mon Sep 17 00:00:00 2001 From: Wesley Luyten Date: Wed, 7 Feb 2024 13:04:56 -0600 Subject: [PATCH] feat: custom local remote source folder & custom asset bucket key (#161) * feat: maintain local folder structure in bucket This could be a breaking change in your app if you use S3, Vercel or Backblaze!!! related issue #149 * feat: add remoteSourceAssetPath & generateAssetKey * fix: make it possible to store in root of bucket --- src/assets.ts | 101 +++++++++++++++---------- src/cli/sync.ts | 26 ++++++- src/config.ts | 77 +++++++++++-------- src/providers/amazon-s3/provider.ts | 14 ++-- src/providers/amazon-s3/transformer.ts | 4 +- src/providers/backblaze/provider.ts | 16 ++-- src/providers/backblaze/transformer.ts | 4 +- src/providers/vercel-blob/provider.ts | 9 ++- src/request-handler.ts | 6 +- src/utils/provider.ts | 26 +++++++ src/utils/utils.ts | 10 +++ src/webpack-loader.ts | 5 +- src/with-next-video.ts | 4 + tests/components/video.test.tsx | 10 ++- 14 files changed, 213 insertions(+), 99 deletions(-) create mode 100644 src/utils/provider.ts diff --git a/src/assets.ts b/src/assets.ts index a6db1d4..0d208ab 100644 --- a/src/assets.ts +++ b/src/assets.ts @@ -1,16 +1,26 @@ -import { relative } from 'node:path'; +import * as path from 'node:path'; import { cwd } from 'node:process'; -import { stat, readFile, writeFile } from 'node:fs/promises'; +import { stat, readFile, writeFile, mkdir } from 'node:fs/promises'; import { getVideoConfig } from './config.js'; -import { deepMerge, camelCase } from './utils/utils.js'; +import { deepMerge, camelCase, isRemote, toSafePath } from './utils/utils.js'; import * as transformers from './providers/transformers.js'; export interface Asset { - status: 'sourced' | 'pending' | 'uploading' | 'processing' | 'ready' | 'error'; + status: + | 'sourced' + | 'pending' + | 'uploading' + | 'processing' + | 'ready' + | 'error'; originalFilePath: string; + // TODO: should we add a `filePath` field which would store the file path + // without the configurable folder? This would allow us to change the folder + // without having to update the file paths in the assets. + // filePath?: string; provider: string; providerMetadata?: { - [provider: string]: { [key: string]: any } + [provider: string]: { [key: string]: any }; }; poster?: string; sources?: AssetSource[]; @@ -32,19 +42,46 @@ export interface AssetSource { } export async function getAsset(filePath: string): Promise { - const assetPath = await getAssetConfigPath(filePath); - const file = await readFile(assetPath); + const assetConfigPath = await getAssetConfigPath(filePath); + const file = await readFile(assetConfigPath); const asset = JSON.parse(file.toString()); return asset; } -export async function createAsset(filePath: string, assetDetails?: Partial) { +export async function getAssetConfigPath(filePath: string) { + return `${await getAssetPath(filePath)}.json`; +} + +async function getAssetPath(filePath: string) { + if (!isRemote(filePath)) return filePath; + + const { folder, remoteSourceAssetPath = defaultRemoteSourceAssetPath } = + await getVideoConfig(); + + if (!folder) throw new Error('Missing video `folder` config.'); + + // Add the asset directory and make remote url a safe file path. + return path.join(folder, remoteSourceAssetPath(filePath)); +} + +function defaultRemoteSourceAssetPath(url: string) { + const urlObj = new URL(url); + // Strip the https from the asset path. + // Strip the search params from the file path so in most cases it'll + // have a video file extension and not a query string in the end. + return toSafePath(decodeURIComponent(`${urlObj.hostname}${urlObj.pathname}`)); +} + +export async function createAsset( + filePath: string, + assetDetails?: Partial +) { const videoConfig = await getVideoConfig(); - const assetPath = await getAssetConfigPath(filePath); + const assetConfigPath = await getAssetConfigPath(filePath); let originalFilePath = filePath; - if (!isRemote(filePath)) { - originalFilePath = relative(cwd(), filePath); + if (!isRemote(filePath)) { + originalFilePath = path.relative(cwd(), filePath); } const newAssetDetails: Asset = { @@ -66,7 +103,10 @@ export async function createAsset(filePath: string, assetDetails?: Partial) { - const assetPath = await getAssetConfigPath(filePath); +export async function updateAsset( + filePath: string, + assetDetails: Partial +) { + const assetConfigPath = await getAssetConfigPath(filePath); const currentAsset = await getAsset(filePath); if (!currentAsset) { @@ -92,35 +135,17 @@ export async function updateAsset(filePath: string, assetDetails: Partial newAssetDetails = transformAsset(transformers, newAssetDetails); - await writeFile(assetPath, JSON.stringify(newAssetDetails)); + await writeFile(assetConfigPath, JSON.stringify(newAssetDetails)); return newAssetDetails; } -export async function getAssetConfigPath(filePath: string) { - if (isRemote(filePath)) { - const VIDEOS_DIR = (await getVideoConfig()).folder; - if (!VIDEOS_DIR) throw new Error('Missing video `folder` config.'); - - // Add the asset directory and make remote url a safe file path. - return `${VIDEOS_DIR}/${toSafePath(filePath)}.json`; +type TransformerRecord = Record< + string, + { + transform: (asset: Asset, props?: any) => Asset; } - return `${filePath}.json` -} - -function isRemote(filePath: string) { - return /^https?:\/\//.test(filePath); -} - -function toSafePath(str: string) { - return str - .replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, '') - .replace(/[^a-zA-Z0-9._-]+/g, '_'); -} - -type TransformerRecord = Record Asset; -}>; +>; function transformAsset(transformers: TransformerRecord, asset: Asset) { const provider = asset.provider; diff --git a/src/cli/sync.ts b/src/cli/sync.ts index 13bd3c5..724a3f0 100644 --- a/src/cli/sync.ts +++ b/src/cli/sync.ts @@ -54,10 +54,22 @@ export async function handler(argv: Arguments) { const directoryPath = path.join(cwd(), argv.dir as string); try { - const files = await readdir(directoryPath); + const dirents = await readdir(directoryPath, { + recursive: true, + withFileTypes: true, + }); + + // Filter out directories and get relative file paths. + const files = dirents + .filter((dirent) => dirent.isFile()) + .map((dirent) => + path.join(path.relative(directoryPath, dirent.path), dirent.name) + ); const jsonFiles = files.filter((file) => file.endsWith('.json')); - const otherFiles = files.filter((file) => !file.match(/(^|[\/\\])\..*|\.json$/)); + const otherFiles = files.filter( + (file) => !file.match(/(^|[\/\\])\..*|\.json$/) + ); if (argv.watch) { const version = await getNextVideoVersion(); @@ -89,7 +101,11 @@ export async function handler(argv: Arguments) { // If the existing asset is 'pending', 'uploading', or 'processing', run // it back through the local video handler. const assetStatus = existingAsset?.status; - if (assetStatus && ['sourced', 'pending', 'uploading', 'processing'].includes(assetStatus)) { + + if ( + assetStatus && + ['sourced', 'pending', 'uploading', 'processing'].includes(assetStatus) + ) { const videoConfig = await getVideoConfig(); return callHandler('local.video.added', existingAsset, videoConfig); } @@ -116,7 +132,9 @@ export async function handler(argv: Arguments) { if (processed.length > 0) { const s = processed.length === 1 ? '' : 's'; - log.success(`Processed (or resumed processing) ${processed.length} video${s}`); + log.success( + `Processed (or resumed processing) ${processed.length} video${s}` + ); } } catch (err: any) { if (err.code === 'ENOENT' && err.path === directoryPath) { diff --git a/src/config.ts b/src/config.ts index 05d33dc..7c915dc 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,7 +6,6 @@ import { pathToFileURL } from 'node:url'; * Video configurations */ export type VideoConfigComplete = { - /** The folder in your project where you will put all video source files. */ folder: string; @@ -14,22 +13,43 @@ export type VideoConfigComplete = { path: string; /* The default provider that will deliver your video. */ - provider: string; + provider: keyof ProviderConfig; /* Config by provider. */ - providerConfig: { - backblaze?: { - endpoint: string; - bucket?: string; - }, - 'amazon-s3'?: { - endpoint: string; - bucket?: string; - accessKeyId?: string; - secretAccessKey?: string; - }, - } -} + providerConfig: ProviderConfig; + + /* An optional function to generate the local asset path for remote sources. */ + remoteSourceAssetPath?: (url: string) => string; +}; + +export type ProviderConfig = { + mux?: { + generateAssetKey: undefined; + }; + + 'vercel-blob'?: { + /* An optional function to generate the bucket asset key. */ + generateAssetKey?: (filePathOrURL: string, folder: string) => string; + }; + + backblaze?: { + endpoint: string; + bucket?: string; + accessKeyId?: string; + secretAccessKey?: string; + /* An optional function to generate the bucket asset key. */ + generateAssetKey?: (filePathOrURL: string, folder: string) => string; + }; + + 'amazon-s3'?: { + endpoint: string; + bucket?: string; + accessKeyId?: string; + secretAccessKey?: string; + /* An optional function to generate the bucket asset key. */ + generateAssetKey?: (filePathOrURL: string, folder: string) => string; + }; +}; export type VideoConfig = Partial; @@ -38,33 +58,30 @@ export const videoConfigDefault: VideoConfigComplete = { path: '/api/video', provider: 'mux', providerConfig: {}, -} +}; /** * The video config is set in `next.config.js` and passed to the `withNextVideo` function. * The video config is then stored as an environment variable __NEXT_VIDEO_OPTS. */ export async function getVideoConfig(): Promise { - if (!env['__NEXT_VIDEO_OPTS']) { - // Import the app's next.config.(m)js file so the env variable - // __NEXT_VIDEO_OPTS set in with-next-video.ts can be used. - try { - await importConfig('next.config.js'); - } catch (err) { - console.error(err); + let nextConfig; - try { - await importConfig('next.config.mjs'); - } catch { - console.error('Failed to load next.config.js or next.config.mjs'); - } + try { + nextConfig = await importConfig('next.config.js'); + } catch (err) { + try { + nextConfig = await importConfig('next.config.mjs'); + } catch { + console.error('Failed to load next.config.js or next.config.mjs'); } } - return JSON.parse(env['__NEXT_VIDEO_OPTS'] ?? '{}'); + + return nextConfig?.serverRuntimeConfig?.nextVideo; } async function importConfig(file: string) { const absFilePath = path.resolve(cwd(), file); const fileUrl = pathToFileURL(absFilePath).href; - return import(/* webpackIgnore: true */ fileUrl); + return (await import(/* webpackIgnore: true */ fileUrl))?.default; } diff --git a/src/providers/amazon-s3/provider.ts b/src/providers/amazon-s3/provider.ts index d3a9c50..b7315e5 100644 --- a/src/providers/amazon-s3/provider.ts +++ b/src/providers/amazon-s3/provider.ts @@ -1,7 +1,6 @@ import { ReadStream, createReadStream } from 'node:fs'; import { Readable } from 'node:stream'; import fs from 'node:fs/promises'; -import path from 'node:path'; import { env } from 'node:process'; import { fetch as uFetch } from 'undici'; import chalk from 'chalk'; @@ -11,13 +10,14 @@ import { S3Client } from '@aws-sdk/client-s3'; import { updateAsset, Asset } from '../../assets.js'; import { getVideoConfig } from '../../config.js'; import { findBucket, createBucket, putBucketCors, putObject, putBucketAcl } from '../../utils/s3.js'; +import { createAssetKey } from '../../utils/provider.js'; +import { isRemote } from '../../utils/utils.js'; import log from '../../utils/logger.js'; export type AmazonS3Metadata = { bucket?: string; endpoint?: string; - accessKeyId?: string; - secretAccessKey?: string; + key?: string; } // Why 11? @@ -98,7 +98,7 @@ export async function uploadLocalFile(asset: Asset) { } // Handle imported remote videos. - if (filePath && /^https?:\/\//.test(filePath)) { + if (isRemote(filePath)) { return uploadRequestedFile(asset); } @@ -156,11 +156,14 @@ export async function uploadRequestedFile(asset: Asset) { async function putAsset(filePath: string, size: number, stream: ReadStream | Readable) { log.info(log.label('Uploading file:'), `${filePath} (${size} bytes)`); + let key; try { + key = await createAssetKey(filePath, 'amazon-s3'); + await putObject(s3, { ACL: 'public-read', Bucket: bucketName, - Key: path.basename(filePath), + Key: key, Body: stream, ContentLength: size, }); @@ -182,6 +185,7 @@ async function putAsset(filePath: string, size: number, stream: ReadStream | Rea 'amazon-s3': { endpoint, bucket: bucketName, + key, } as AmazonS3Metadata }, }); diff --git a/src/providers/amazon-s3/transformer.ts b/src/providers/amazon-s3/transformer.ts index 7f8d1af..7b018ac 100644 --- a/src/providers/amazon-s3/transformer.ts +++ b/src/providers/amazon-s3/transformer.ts @@ -6,9 +6,7 @@ export function transform(asset: Asset) { const src = new URL(providerMetadata.endpoint); src.hostname = `${providerMetadata.bucket}.${src.hostname}`; - - const basename = asset.originalFilePath.split('/').pop(); - if (basename) src.pathname = basename + src.pathname = providerMetadata.key; const source: AssetSource = { src: `${src}` }; diff --git a/src/providers/backblaze/provider.ts b/src/providers/backblaze/provider.ts index 58bf1c0..3cfd232 100644 --- a/src/providers/backblaze/provider.ts +++ b/src/providers/backblaze/provider.ts @@ -1,7 +1,6 @@ import { ReadStream, createReadStream } from 'node:fs'; import { Readable } from 'node:stream'; import fs from 'node:fs/promises'; -import path from 'node:path'; import { env } from 'node:process'; import { fetch as uFetch } from 'undici'; import chalk from 'chalk'; @@ -11,11 +10,14 @@ import { S3Client } from '@aws-sdk/client-s3'; import { updateAsset, Asset } from '../../assets.js'; import { getVideoConfig } from '../../config.js'; import { findBucket, createBucket, putBucketCors, putObject } from '../../utils/s3.js'; +import { createAssetKey } from '../../utils/provider.js'; +import { isRemote } from '../../utils/utils.js'; import log from '../../utils/logger.js'; export type BackblazeMetadata = { bucket?: string; endpoint?: string; + key?: string; } // Why 11? @@ -42,8 +44,8 @@ async function initS3() { endpoint, region, credentials: { - accessKeyId: env.BACKBLAZE_ACCESS_KEY_ID ?? '', - secretAccessKey: env.BACKBLAZE_SECRET_ACCESS_KEY ?? '', + accessKeyId: backblazeConfig?.accessKeyId ?? env.BACKBLAZE_ACCESS_KEY_ID ?? '', + secretAccessKey: backblazeConfig?.secretAccessKey ?? env.BACKBLAZE_SECRET_ACCESS_KEY ?? '', } }); @@ -87,7 +89,7 @@ export async function uploadLocalFile(asset: Asset) { } // Handle imported remote videos. - if (filePath && /^https?:\/\//.test(filePath)) { + if (isRemote(filePath)) { return uploadRequestedFile(asset); } @@ -145,10 +147,13 @@ export async function uploadRequestedFile(asset: Asset) { async function putAsset(filePath: string, size: number, stream: ReadStream | Readable) { log.info(log.label('Uploading file:'), `${filePath} (${size} bytes)`); + let key; try { + key = await createAssetKey(filePath, 'backblaze'); + await putObject(s3, { Bucket: bucketName, - Key: path.basename(filePath), + Key: key, Body: stream, ContentLength: size, }); @@ -170,6 +175,7 @@ async function putAsset(filePath: string, size: number, stream: ReadStream | Rea backblaze: { endpoint, bucket: bucketName, + key, } as BackblazeMetadata }, }); diff --git a/src/providers/backblaze/transformer.ts b/src/providers/backblaze/transformer.ts index 3a06d14..9400ea3 100644 --- a/src/providers/backblaze/transformer.ts +++ b/src/providers/backblaze/transformer.ts @@ -6,9 +6,7 @@ export function transform(asset: Asset) { const src = new URL(providerMetadata.endpoint); src.hostname = `${providerMetadata.bucket}.${src.hostname}`; - - const basename = asset.originalFilePath.split('/').pop(); - if (basename) src.pathname = basename + src.pathname = providerMetadata.key; const source: AssetSource = { src: `${src}` }; diff --git a/src/providers/vercel-blob/provider.ts b/src/providers/vercel-blob/provider.ts index 6c405ff..03562c6 100644 --- a/src/providers/vercel-blob/provider.ts +++ b/src/providers/vercel-blob/provider.ts @@ -5,6 +5,8 @@ import { put } from '@vercel/blob'; import chalk from 'chalk'; import { updateAsset, Asset } from '../../assets.js'; +import { createAssetKey } from '../../utils/provider.js'; +import { isRemote } from '../../utils/utils.js'; import log from '../../utils/logger.js'; export const config = { @@ -26,7 +28,7 @@ export async function uploadLocalFile(asset: Asset) { } // Handle imported remote videos. - if (filePath && /^https?:\/\//.test(filePath)) { + if (isRemote(filePath)) { return uploadRequestedFile(asset); } @@ -80,9 +82,11 @@ export async function uploadRequestedFile(asset: Asset) { async function putAsset(filePath: string, size: number, stream: ReadStream | ReadableStream) { log.info(log.label('Uploading file:'), `${filePath} (${size} bytes)`); + let key; let blob; try { - blob = await put(filePath, stream, { access: 'public' }); + key = await createAssetKey(filePath, 'vercel-blob'); + blob = await put(key, stream, { access: 'public' }); if (stream instanceof ReadStream) { stream.close(); @@ -100,6 +104,7 @@ async function putAsset(filePath: string, size: number, stream: ReadStream | Rea status: 'ready', providerMetadata: { 'vercel-blob': { + key, url: blob.url, contentType: blob.contentType, } as VercelBlobMetadata diff --git a/src/request-handler.ts b/src/request-handler.ts index 41c8a9f..94391ca 100644 --- a/src/request-handler.ts +++ b/src/request-handler.ts @@ -2,6 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { callHandler } from './process.js'; import { createAsset, getAsset } from './assets.js'; import { getVideoConfig } from './config.js'; +import { isRemote } from './utils/utils.js'; // App Router export async function GET(request: Request) { @@ -26,10 +27,7 @@ async function handleRequest(url?: string | null) { }; } - const remoteRegex = /^https?:\/\//; - const isRemote = remoteRegex.test(url); - - if (!isRemote) { + if (!isRemote(url)) { // todo: handle local files via string src return { status: 400, diff --git a/src/utils/provider.ts b/src/utils/provider.ts new file mode 100644 index 0000000..849f622 --- /dev/null +++ b/src/utils/provider.ts @@ -0,0 +1,26 @@ +import * as path from 'node:path'; + +import { getVideoConfig } from '../config.js'; +import { isRemote } from './utils.js'; + +import type { ProviderConfig } from '../config.js'; + +export async function createAssetKey(filePathOrURL: string, provider: keyof ProviderConfig) { + const { folder, providerConfig } = await getVideoConfig(); + const config = providerConfig[provider]; + const { generateAssetKey = defaultGenerateAssetKey } = config ?? {}; + return generateAssetKey(filePathOrURL, folder); +} + +function defaultGenerateAssetKey(filePathOrURL: string, folder: string) { + // By default local imports keep the same local folder structure. + if (!isRemote(filePathOrURL)) return filePathOrURL; + + const url = new URL(filePathOrURL); + + // Remote imports are stored in the configured videos folder with just the file name. + // This could easily lead to collisions if the same file name is used in different + // remote sources. There are many ways to generate unique asset keys from remote sources, + // so we leave this up to the user to configure if needed. + return path.posix.join(folder, path.basename(decodeURIComponent(url.pathname))); +} diff --git a/src/utils/utils.ts b/src/utils/utils.ts index e4b61d7..d9bfbec 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -6,6 +6,16 @@ export function camelCase(name: string) { return name.replace(/[-_]([a-z])/g, ($0, $1) => $1.toUpperCase()); } +export function isRemote(filePath: string) { + return /^https?:\/\//.test(filePath); +} + +export function toSafePath(str: string) { + return str + .replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, '') + .replace(/[^a-zA-Z0-9._-]+/g, '_'); +} + /** * Performs a deep merge of objects and returns a new object. * Does not modify objects (immutable) and merges arrays via concatenation. diff --git a/src/webpack-loader.ts b/src/webpack-loader.ts index 402fc37..768e529 100644 --- a/src/webpack-loader.ts +++ b/src/webpack-loader.ts @@ -6,7 +6,8 @@ import { createAsset, getAssetConfigPath } from './assets.js'; export const raw = true; export default async function loader(this: any, source: Buffer) { - const assetPath = path.resolve(await getAssetConfigPath(this.resourcePath)); + const importPath = `${this.resourcePath}${this.resourceQuery ?? ''}`; + const assetPath = path.resolve(await getAssetConfigPath(importPath)); this.addDependency(assetPath); @@ -14,7 +15,7 @@ export default async function loader(this: any, source: Buffer) { try { asset = await readFile(assetPath, 'utf-8'); } catch { - asset = JSON.stringify(await createAsset(this.resourcePath, { + asset = JSON.stringify(await createAsset(importPath, { status: 'sourced' })); } diff --git a/src/with-next-video.ts b/src/with-next-video.ts index 5b641cc..230584b 100644 --- a/src/with-next-video.ts +++ b/src/with-next-video.ts @@ -39,6 +39,10 @@ export async function withNextVideo(nextConfig: any, videoConfig?: VideoConfig) } return Object.assign({}, nextConfig, { + serverRuntimeConfig: { + ...nextConfig.serverRuntimeConfig, + nextVideo: videoConfigComplete, + }, webpack(config: any, options: any) { if (!options.defaultLoaders) { throw new Error( diff --git a/tests/components/video.test.tsx b/tests/components/video.test.tsx index 7718bce..b52cb8d 100644 --- a/tests/components/video.test.tsx +++ b/tests/components/video.test.tsx @@ -1,31 +1,35 @@ import assert from 'node:assert'; import { test } from 'node:test'; import { setTimeout } from 'node:timers/promises'; -import { create, act } from 'react-test-renderer'; +import { create } from 'react-test-renderer'; import React from 'react'; import asset from '../factories/BBB-720p-1min.mp4.json' assert { type: "json" }; import Video from '../../src/components/video.js'; -test('renders a video container', () => { +test('renders a video container', async () => { const wrapper = create(