feat: custom local remote source folder & custom asset bucket key (#161)

* feat: maintain local folder structure in bucket This could be a breaking change in your app if you use S3, Vercel or Backblaze!!! related issue #149 * feat: add remoteSourceAssetPath & generateAssetKey * fix: make it possible to store in root of bucket
muxinc · Feb 7, 2024 · bf28a4b · bf28a4b
1 parent 02b15fc
commit bf28a4b
Show file tree

Hide file tree

Showing 14 changed files with 213 additions and 99 deletions.
diff --git a/src/assets.ts b/src/assets.ts
@@ -1,16 +1,26 @@
-import { relative } from 'node:path';
+import * as path from 'node:path';
 import { cwd } from 'node:process';
-import { stat, readFile, writeFile } from 'node:fs/promises';
+import { stat, readFile, writeFile, mkdir } from 'node:fs/promises';
 import { getVideoConfig } from './config.js';
-import { deepMerge, camelCase } from './utils/utils.js';
+import { deepMerge, camelCase, isRemote, toSafePath } from './utils/utils.js';
 import * as transformers from './providers/transformers.js';
 
 export interface Asset {
-  status: 'sourced' | 'pending' | 'uploading' | 'processing' | 'ready' | 'error';
+  status:
+    | 'sourced'
+    | 'pending'
+    | 'uploading'
+    | 'processing'
+    | 'ready'
+    | 'error';
   originalFilePath: string;
+  // TODO: should we add a `filePath` field which would store the file path
+  // without the configurable folder? This would allow us to change the folder
+  // without having to update the file paths in the assets.
+  // filePath?: string;
   provider: string;
   providerMetadata?: {
-    [provider: string]: { [key: string]: any }
+    [provider: string]: { [key: string]: any };
   };
   poster?: string;
   sources?: AssetSource[];
@@ -32,19 +42,46 @@ export interface AssetSource {
 }
 
 export async function getAsset(filePath: string): Promise<Asset | undefined> {
-  const assetPath = await getAssetConfigPath(filePath);
-  const file = await readFile(assetPath);
+  const assetConfigPath = await getAssetConfigPath(filePath);
+  const file = await readFile(assetConfigPath);
   const asset = JSON.parse(file.toString());
   return asset;
 }
 
-export async function createAsset(filePath: string, assetDetails?: Partial<Asset>) {
+export async function getAssetConfigPath(filePath: string) {
+  return `${await getAssetPath(filePath)}.json`;
+}
+
+async function getAssetPath(filePath: string) {
+  if (!isRemote(filePath)) return filePath;
+
+  const { folder, remoteSourceAssetPath = defaultRemoteSourceAssetPath } =
+    await getVideoConfig();
+
+  if (!folder) throw new Error('Missing video `folder` config.');
+
+  // Add the asset directory and make remote url a safe file path.
+  return path.join(folder, remoteSourceAssetPath(filePath));
+}
+
+function defaultRemoteSourceAssetPath(url: string) {
+  const urlObj = new URL(url);
+  // Strip the https from the asset path.
+  // Strip the search params from the file path so in most cases it'll
+  // have a video file extension and not a query string in the end.
+  return toSafePath(decodeURIComponent(`${urlObj.hostname}${urlObj.pathname}`));
+}
+
+export async function createAsset(
+  filePath: string,
+  assetDetails?: Partial<Asset>
+) {
   const videoConfig = await getVideoConfig();
-  const assetPath = await getAssetConfigPath(filePath);
+  const assetConfigPath = await getAssetConfigPath(filePath);
 
   let originalFilePath = filePath;
-  if (!isRemote(filePath))  {
-    originalFilePath = relative(cwd(), filePath);
+  if (!isRemote(filePath)) {
+    originalFilePath = path.relative(cwd(), filePath);
   }
 
   const newAssetDetails: Asset = {
@@ -66,7 +103,10 @@ export async function createAsset(filePath: string, assetDetails?: Partial<Asset
   }
 
   try {
-    await writeFile(assetPath, JSON.stringify(newAssetDetails), { flag: 'wx' });
+    await mkdir(path.dirname(assetConfigPath), { recursive: true });
+    await writeFile(assetConfigPath, JSON.stringify(newAssetDetails), {
+      flag: 'wx',
+    });
   } catch (err: any) {
     if (err.code === 'EEXIST') {
       // The file already exists, and that's ok in this case. Ignore the error.
@@ -78,8 +118,11 @@ export async function createAsset(filePath: string, assetDetails?: Partial<Asset
   return newAssetDetails;
 }
 
-export async function updateAsset(filePath: string, assetDetails: Partial<Asset>) {
-  const assetPath = await getAssetConfigPath(filePath);
+export async function updateAsset(
+  filePath: string,
+  assetDetails: Partial<Asset>
+) {
+  const assetConfigPath = await getAssetConfigPath(filePath);
   const currentAsset = await getAsset(filePath);
 
   if (!currentAsset) {
@@ -92,35 +135,17 @@ export async function updateAsset(filePath: string, assetDetails: Partial<Asset>
 
   newAssetDetails = transformAsset(transformers, newAssetDetails);
 
-  await writeFile(assetPath, JSON.stringify(newAssetDetails));
+  await writeFile(assetConfigPath, JSON.stringify(newAssetDetails));
 
   return newAssetDetails;
 }
 
-export async function getAssetConfigPath(filePath: string) {
-  if (isRemote(filePath)) {
-    const VIDEOS_DIR = (await getVideoConfig()).folder;
-    if (!VIDEOS_DIR) throw new Error('Missing video `folder` config.');
-
-    // Add the asset directory and make remote url a safe file path.
-    return `${VIDEOS_DIR}/${toSafePath(filePath)}.json`;
+type TransformerRecord = Record<
+  string,
+  {
+    transform: (asset: Asset, props?: any) => Asset;
   }
-  return `${filePath}.json`
-}
-
-function isRemote(filePath: string) {
-  return /^https?:\/\//.test(filePath);
-}
-
-function toSafePath(str: string) {
-  return str
-    .replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, '')
-    .replace(/[^a-zA-Z0-9._-]+/g, '_');
-}
-
-type TransformerRecord = Record<string, {
-  transform: (asset: Asset, props?: any) => Asset;
-}>;
+>;
 
 function transformAsset(transformers: TransformerRecord, asset: Asset) {
   const provider = asset.provider;

diff --git a/src/cli/sync.ts b/src/cli/sync.ts
@@ -54,10 +54,22 @@ export async function handler(argv: Arguments) {
   const directoryPath = path.join(cwd(), argv.dir as string);
 
   try {
-    const files = await readdir(directoryPath);
+    const dirents = await readdir(directoryPath, {
+      recursive: true,
+      withFileTypes: true,
+    });
+
+    // Filter out directories and get relative file paths.
+    const files = dirents
+      .filter((dirent) => dirent.isFile())
+      .map((dirent) =>
+        path.join(path.relative(directoryPath, dirent.path), dirent.name)
+      );
 
     const jsonFiles = files.filter((file) => file.endsWith('.json'));
-    const otherFiles = files.filter((file) => !file.match(/(^|[\/\\])\..*|\.json$/));
+    const otherFiles = files.filter(
+      (file) => !file.match(/(^|[\/\\])\..*|\.json$/)
+    );
 
     if (argv.watch) {
       const version = await getNextVideoVersion();
@@ -89,7 +101,11 @@ export async function handler(argv: Arguments) {
       // If the existing asset is 'pending', 'uploading', or 'processing', run
       // it back through the local video handler.
       const assetStatus = existingAsset?.status;
-      if (assetStatus && ['sourced', 'pending', 'uploading', 'processing'].includes(assetStatus)) {
+
+      if (
+        assetStatus &&
+        ['sourced', 'pending', 'uploading', 'processing'].includes(assetStatus)
+      ) {
         const videoConfig = await getVideoConfig();
         return callHandler('local.video.added', existingAsset, videoConfig);
       }
@@ -116,7 +132,9 @@ export async function handler(argv: Arguments) {
 
     if (processed.length > 0) {
       const s = processed.length === 1 ? '' : 's';
-      log.success(`Processed (or resumed processing) ${processed.length} video${s}`);
+      log.success(
+        `Processed (or resumed processing) ${processed.length} video${s}`
+      );
     }
   } catch (err: any) {
     if (err.code === 'ENOENT' && err.path === directoryPath) {

diff --git a/src/config.ts b/src/config.ts
@@ -6,30 +6,50 @@ import { pathToFileURL } from 'node:url';
  * Video configurations
  */
 export type VideoConfigComplete = {
-
   /** The folder in your project where you will put all video source files. */
   folder: string;
 
   /** The route of the video API request for string video source URLs. */
   path: string;
 
   /* The default provider that will deliver your video. */
-  provider: string;
+  provider: keyof ProviderConfig;
 
   /* Config by provider. */
-  providerConfig: {
-    backblaze?: {
-      endpoint: string;
-      bucket?: string;
-    },
-    'amazon-s3'?: {
-      endpoint: string;
-      bucket?: string;
-      accessKeyId?: string;
-      secretAccessKey?: string;
-    },
-  }
-}
+  providerConfig: ProviderConfig;
+
+  /* An optional function to generate the local asset path for remote sources. */
+  remoteSourceAssetPath?: (url: string) => string;
+};
+
+export type ProviderConfig = {
+  mux?: {
+    generateAssetKey: undefined;
+  };
+
+  'vercel-blob'?: {
+    /* An optional function to generate the bucket asset key. */
+    generateAssetKey?: (filePathOrURL: string, folder: string) => string;
+  };
+
+  backblaze?: {
+    endpoint: string;
+    bucket?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    /* An optional function to generate the bucket asset key. */
+    generateAssetKey?: (filePathOrURL: string, folder: string) => string;
+  };
+
+  'amazon-s3'?: {
+    endpoint: string;
+    bucket?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    /* An optional function to generate the bucket asset key. */
+    generateAssetKey?: (filePathOrURL: string, folder: string) => string;
+  };
+};
 
 export type VideoConfig = Partial<VideoConfigComplete>;
 
@@ -38,33 +58,30 @@ export const videoConfigDefault: VideoConfigComplete = {
   path: '/api/video',
   provider: 'mux',
   providerConfig: {},
-}
+};
 
 /**
  * The video config is set in `next.config.js` and passed to the `withNextVideo` function.
  * The video config is then stored as an environment variable __NEXT_VIDEO_OPTS.
  */
 export async function getVideoConfig(): Promise<VideoConfigComplete> {
-  if (!env['__NEXT_VIDEO_OPTS']) {
-    // Import the app's next.config.(m)js file so the env variable
-    // __NEXT_VIDEO_OPTS set in with-next-video.ts can be used.
-    try {
-      await importConfig('next.config.js');
-    } catch (err) {
-      console.error(err);
+  let nextConfig;
 
-      try {
-        await importConfig('next.config.mjs');
-      } catch {
-        console.error('Failed to load next.config.js or next.config.mjs');
-      }
+  try {
+    nextConfig = await importConfig('next.config.js');
+  } catch (err) {
+    try {
+      nextConfig = await importConfig('next.config.mjs');
+    } catch {
+      console.error('Failed to load next.config.js or next.config.mjs');
     }
   }
-  return JSON.parse(env['__NEXT_VIDEO_OPTS'] ?? '{}');
+
+  return nextConfig?.serverRuntimeConfig?.nextVideo;
 }
 
 async function importConfig(file: string) {
   const absFilePath = path.resolve(cwd(), file);
   const fileUrl = pathToFileURL(absFilePath).href;
-  return import(/* webpackIgnore: true */ fileUrl);
+  return (await import(/* webpackIgnore: true */ fileUrl))?.default;
 }
diff --git a/src/providers/amazon-s3/provider.ts b/src/providers/amazon-s3/provider.ts
@@ -1,7 +1,6 @@
 import { ReadStream, createReadStream } from 'node:fs';
 import { Readable } from 'node:stream';
 import fs from 'node:fs/promises';
-import path from 'node:path';
 import { env } from 'node:process';
 import { fetch as uFetch } from 'undici';
 import chalk from 'chalk';
@@ -11,13 +10,14 @@ import { S3Client } from '@aws-sdk/client-s3';
 import { updateAsset, Asset } from '../../assets.js';
 import { getVideoConfig } from '../../config.js';
 import { findBucket, createBucket, putBucketCors, putObject, putBucketAcl } from '../../utils/s3.js';
+import { createAssetKey } from '../../utils/provider.js';
+import { isRemote } from '../../utils/utils.js';
 import log from '../../utils/logger.js';
 
 export type AmazonS3Metadata = {
   bucket?: string;
   endpoint?: string;
-  accessKeyId?: string;
-  secretAccessKey?: string;
+  key?: string;
 }
 
 // Why 11?
@@ -98,7 +98,7 @@ export async function uploadLocalFile(asset: Asset) {
   }
 
   // Handle imported remote videos.
-  if (filePath && /^https?:\/\//.test(filePath)) {
+  if (isRemote(filePath)) {
     return uploadRequestedFile(asset);
   }
 
@@ -156,11 +156,14 @@ export async function uploadRequestedFile(asset: Asset) {
 async function putAsset(filePath: string, size: number, stream: ReadStream | Readable) {
   log.info(log.label('Uploading file:'), `${filePath} (${size} bytes)`);
 
+  let key;
   try {
+    key = await createAssetKey(filePath, 'amazon-s3');
+
     await putObject(s3, {
       ACL: 'public-read',
       Bucket: bucketName,
-      Key: path.basename(filePath),
+      Key: key,
       Body: stream,
       ContentLength: size,
     });
@@ -182,6 +185,7 @@ async function putAsset(filePath: string, size: number, stream: ReadStream | Rea
       'amazon-s3': {
         endpoint,
         bucket: bucketName,
+        key,
       } as AmazonS3Metadata
     },
   });

diff --git a/src/providers/amazon-s3/transformer.ts b/src/providers/amazon-s3/transformer.ts
@@ -6,9 +6,7 @@ export function transform(asset: Asset) {
 
   const src = new URL(providerMetadata.endpoint);
   src.hostname = `${providerMetadata.bucket}.${src.hostname}`;
-
-  const basename = asset.originalFilePath.split('/').pop();
-  if (basename) src.pathname = basename
+  src.pathname = providerMetadata.key;
 
   const source: AssetSource = { src: `${src}` };