fix: fix HLS concat on new FFmpeg pipeline

Includes some other fixes around audio channels and padding on VAAPI

Fixes #974

server/src/ffmpeg/FFmpegFactory.ts

@@ -0,0 +1,18 @@
+import { Channel } from '@/db/schema/Channel.ts';
+import { FfmpegSettings } from '@tunarr/types';
+import { FfmpegStreamFactory } from './FfmpegStreamFactory.ts';
+import { FFMPEG } from './ffmpeg.ts';
+import { IFFMPEG } from './ffmpegBase.ts';
+
+export class FFmpegFactory {
+  static getFFmpegPipelineBuilder(
+    settings: FfmpegSettings,
+    channel: Channel,
+  ): IFFMPEG {
+    if (settings.useNewFfmpegPipeline) {
+      return new FfmpegStreamFactory(settings, channel);
+    } else {
+      return new FFMPEG(settings, channel);
+    }
+  }
+}

server/src/ffmpeg/FfmpegStreamFactory.ts

@@ -39,7 +39,11 @@ import { FfmpegState } from './builder/state/FfmpegState.ts';
 import { FrameState } from './builder/state/FrameState.ts';
 import { FrameSize } from './builder/types.ts';
 import { ConcatOptions, StreamSessionOptions } from './ffmpeg.ts';
-import { IFFMPEG } from './ffmpegBase.ts';
+import {
+  ConcatStreamModeToChildMode,
+  HlsWrapperOptions,
+  IFFMPEG,
+} from './ffmpegBase.ts';
 import { FFMPEGInfo } from './ffmpegInfo.ts';
 
 export class FfmpegStreamFactory extends IFFMPEG {
@@ -57,7 +61,7 @@ export class FfmpegStreamFactory extends IFFMPEG {
 
   async createConcatSession(
     streamUrl: string,
-    opts: DeepReadonly<Partial<ConcatOptions>>,
+    opts: DeepReadonly<ConcatOptions>,
   ): Promise<FfmpegTranscodeSession> {
     const concatInput = new ConcatInputSource(
       new HttpStreamSource(streamUrl),
@@ -73,24 +77,25 @@ export class FfmpegStreamFactory extends IFFMPEG {
 
     const calculator = new FfmpegPlaybackParamsCalculator(this.ffmpegSettings);
 
-    const pipeline = pipelineBuilder.build(
+    const pipeline = pipelineBuilder.concat(
+      concatInput,
       FfmpegState.create({
         version: await this.ffmpegInfo.getVersion(),
         outputFormat: opts.outputFormat ?? MpegTsOutputFormat,
         metadataServiceProvider: 'Tunarr',
         metadataServiceName: this.channel.name,
         ptsOffset: 0,
       }),
-      new FrameState({
-        ...calculator.calculateForHlsConcat(),
-        scaledSize: FrameSize.fromResolution(
-          this.ffmpegSettings.targetResolution,
-        ),
-        paddedSize: FrameSize.fromResolution(
-          this.ffmpegSettings.targetResolution,
-        ),
-        isAnamorphic: false,
-      }),
+      // new FrameState({
+      //   ...calculator.calculateForHlsConcat(),
+      //   scaledSize: FrameSize.fromResolution(
+      //     this.ffmpegSettings.targetResolution,
+      //   ),
+      //   paddedSize: FrameSize.fromResolution(
+      //     this.ffmpegSettings.targetResolution,
+      //   ),
+      //   isAnamorphic: false,
+      // }),
     );
 
     return new FfmpegTranscodeSession(
@@ -105,8 +110,9 @@ export class FfmpegStreamFactory extends IFFMPEG {
     );
   }
 
-  async createWrapperConcatSession(
+  async createHlsWrapperSession(
     streamUrl: string,
+    opts: HlsWrapperOptions,
   ): Promise<FfmpegTranscodeSession> {
     const concatInput = new ConcatInputSource(
       new HttpStreamSource(streamUrl),
@@ -116,12 +122,21 @@ export class FfmpegStreamFactory extends IFFMPEG {
       }),
     );
 
+    switch (ConcatStreamModeToChildMode[opts.mode]) {
+      case 'hls':
+        return this.createHlsConcatSession(concatInput);
+      case 'hls_slower':
+        return this.createHlsSlowerConcatSession(concatInput);
+    }
+  }
+
+  private async createHlsConcatSession(concatInput: ConcatInputSource) {
     const pipelineBuilder = await this.pipelineBuilderFactory
       .builder(this.ffmpegSettings)
       .setConcatInputSource(concatInput)
       .build();
 
-    const pipeline = pipelineBuilder.hlsConcat(
+    const pipeline = pipelineBuilder.hlsWrap(
       concatInput,
       FfmpegState.forConcat(
         await this.ffmpegInfo.getVersion(),
@@ -141,6 +156,12 @@ export class FfmpegStreamFactory extends IFFMPEG {
     );
   }
 
+  private async createHlsSlowerConcatSession(
+    concatInput: ConcatInputSource,
+  ): Promise<FfmpegTranscodeSession> {
+    throw new Error('not yet implemented');
+  }
+
   async createStreamSession({
     // TODO Fix these dumb params
     streamSource,
@@ -234,7 +255,7 @@ export class FfmpegStreamFactory extends IFFMPEG {
           AudioStream.create({
             index: isNaN(audioStreamIndex) ? 1 : audioStreamIndex,
             codec: audioStream.codec ?? 'unknown',
-            channels: playbackParams.audioChannels ?? -2,
+            channels: audioStream.channels ?? -2,
           }),
         ],
         audioState,

server/src/ffmpeg/builder/filter/vaapi/ScaleVaapiFilter.ts

@@ -10,9 +10,12 @@ export class ScaleVaapiFilter extends FilterOption {
     private paddedSize: FrameSize,
   ) {
     super();
+    this.filter = this.genFilter();
   }
 
-  public get filter(): string {
+  public readonly filter: string;
+
+  private genFilter(): string {
     let scale = '';
 
     if (this.currentState.scaledSize.equals(this.scaledSize)) {
@@ -23,6 +26,7 @@ export class ScaleVaapiFilter extends FilterOption {
       let aspectRatio = '';
       if (!this.scaledSize.equals(this.paddedSize)) {
         // Set cropped size
+        aspectRatio = ':force_original_aspect_ratio=decrease';
       }
 
       let squareScale = '';

server/src/ffmpeg/builder/options/AudioOutputOptions.ts

@@ -6,7 +6,6 @@ export const AudioChannelsOutputOption = (
   desiredChannels: number,
 ) => {
   const opts: string[] = [];
-  // TODO Audio format constants
   if (
     sourceChannels !== desiredChannels ||
     (audioFormat === 'aac' && desiredChannels > 2)

server/src/ffmpeg/builder/options/OutputOption.ts

@@ -57,6 +57,9 @@ export const MetadataServiceNameOutputOption = (serviceName: string) =>
 export const DoNotMapMetadataOutputOption = () =>
   makeConstantOutputOption(['-map_metadata', '-1']);
 
+export const MapAllStreamsOutputOption = () =>
+  makeConstantOutputOption(['-map', '0']);
+
 export const NoSceneDetectOutputOption = (
   value: number,
 ): ConstantOutputOption =>
@@ -90,12 +93,13 @@ export const FrameRateOutputOption = (
 export const VideoTrackTimescaleOutputOption = (scale: number) =>
   makeConstantOutputOption(['-video_track_timescale', scale.toString()]);
 
-export const MpegTsOutputFormatOption = () =>
+export const MpegTsOutputFormatOption = (initialDiscontinuity?: boolean) =>
   makeConstantOutputOption([
     '-f',
     'mpegts',
-    '-mpegts_flags',
-    '+initial_discontinuity',
+    ...(initialDiscontinuity
+      ? ['-mpegts_flags', '+initial_discontinuity']
+      : []),
   ]);
 
 export const Mp4OutputFormatOption = () =>

server/src/ffmpeg/builder/pipeline/BasePipelineBuilder.ts

@@ -81,6 +81,7 @@ import {
   DoNotMapMetadataOutputOption,
   FastStartOutputOption,
   FrameRateOutputOption,
+  MapAllStreamsOutputOption,
   MatroskaOutputFormatOption,
   MetadataServiceNameOutputOption,
   MetadataServiceProviderOutputOption,
@@ -169,18 +170,23 @@ export abstract class BasePipelineBuilder implements PipelineBuilder {
   protected context: PipelineBuilderContext;
 
   constructor(
-    protected videoInputSource: VideoInputSource,
+    protected nullableVideoInputSource: Nullable<VideoInputSource>,
     private audioInputSource: Nullable<AudioInputSource>,
     protected watermarkInputSource: Nullable<WatermarkInputSource>,
     protected concatInputSource: Nullable<ConcatInputSource>,
     protected ffmpegCapabilities: FfmpegCapabilities,
   ) {}
 
+  get videoInputSource(): VideoInputSource {
+    // Only use this on video pipelines!!!
+    return this.nullableVideoInputSource!;
+  }
+
   validate(): Nullable<Error> {
     return null;
   }
 
-  hlsConcat(input: ConcatInputSource, state: FfmpegState) {
+  concat(input: ConcatInputSource, state: FfmpegState) {
     const pipelineSteps: PipelineStep[] = [
       new NoStdInOption(),
       new HideBannerOption(),
@@ -203,10 +209,8 @@ export abstract class BasePipelineBuilder implements PipelineBuilder {
       input.addOption(new ConcatHttpReconnectOptions());
     }
 
-    if (this.ffmpegState.threadCount) {
-      pipelineSteps.unshift(
-        new ThreadCountOption(this.ffmpegState.threadCount),
-      );
+    if (state.threadCount) {
+      pipelineSteps.unshift(new ThreadCountOption(state.threadCount));
     }
 
     pipelineSteps.push(NoSceneDetectOutputOption(0), new CopyAllEncoder());
@@ -231,6 +235,46 @@ export abstract class BasePipelineBuilder implements PipelineBuilder {
     });
   }
 
+  hlsWrap(input: ConcatInputSource, state: FfmpegState) {
+    const pipelineSteps: PipelineStep[] = [
+      new NoStdInOption(),
+      new HideBannerOption(),
+      new ThreadCountOption(1),
+      new LogLevelOption(state.logLevel),
+      new NoStatsOption(),
+      new StandardFormatFlags(),
+      MapAllStreamsOutputOption(),
+      new CopyAllEncoder(),
+    ];
+
+    if (input.protocol === 'http') {
+      input.addOption(new ConcatHttpReconnectOptions());
+    }
+
+    input.addOption(new ReadrateInputOption(this.ffmpegCapabilities, 0));
+    if (state.metadataServiceName) {
+      pipelineSteps.push(
+        MetadataServiceNameOutputOption(state.metadataServiceName),
+      );
+    }
+    if (state.metadataServiceProvider) {
+      pipelineSteps.push(
+        MetadataServiceProviderOutputOption(state.metadataServiceProvider),
+      );
+    }
+    pipelineSteps.push(
+      MpegTsOutputFormatOption(false),
+      PipeProtocolOutputOption(),
+    );
+
+    return new Pipeline(pipelineSteps, {
+      videoInput: null,
+      audioInput: null,
+      concatInput: input,
+      watermarkInput: null,
+    });
+  }
+
   build(ffmpegState: FfmpegState, desiredState: FrameState): Pipeline {
     this.context = {
       videoStream: first(this.videoInputSource.streams),
@@ -284,6 +328,10 @@ export abstract class BasePipelineBuilder implements PipelineBuilder {
       this.pipelineSteps.push(TimeLimitOutputOption(this.ffmpegState.duration));
     }
 
+    if (isNull(this.nullableVideoInputSource)) {
+      throw new Error('FFmpeg pipeline currently requires a video input');
+    }
+
     if (
       this.videoInputSource.protocol === 'http' &&
       this.videoInputSource.continuity === 'discrete'
@@ -585,7 +633,6 @@ export abstract class BasePipelineBuilder implements PipelineBuilder {
           isNonEmptyString(this.ffmpegState.hlsSegmentTemplate) &&
           isNonEmptyString(this.ffmpegState.hlsBaseStreamUrl)
         ) {
-          console.log(this.ffmpegState);
           this.pipelineSteps.push(
             new HlsOutputFormat(
               this.desiredState,

server/src/ffmpeg/builder/pipeline/PipelineBuilder.ts

@@ -6,6 +6,28 @@ import { Pipeline } from './Pipeline.ts';
 
 export interface PipelineBuilder {
   validate(): Nullable<Error>;
-  hlsConcat(input: ConcatInputSource, state: FfmpegState): Pipeline;
+
+  /**
+   * Takes input source in ffconcat format and returns the concatenated output stream
+   * in the given output format. Simply copies input streams.
+   * @param input
+   * @param state
+   */
+  concat(input: ConcatInputSource, state: FfmpegState): Pipeline;
+
+  /**
+   * Takes m3u8 HLS playlist as input and returns a continuous output stream
+   * in the given output format. Simple copies input streams.
+   * @param input
+   * @param state
+   */
+  hlsWrap(input: ConcatInputSource, state: FfmpegState): Pipeline;
+
+  /**
+   * Calculates an ffmpeg pipeline using the inputted ffmpeg state and desired
+   * output state.
+   * @param currentState
+   * @param desiredState
+   */
   build(currentState: FfmpegState, desiredState: FrameState): Pipeline;
 }

server/src/ffmpeg/builder/pipeline/PipelineBuilderFactory.ts

@@ -7,7 +7,7 @@ import { HardwareAccelerationMode } from '@/ffmpeg/builder/types.ts';
 import { FFMPEGInfo } from '@/ffmpeg/ffmpegInfo.ts';
 import { Nullable } from '@/types/util.ts';
 import { FfmpegSettings } from '@tunarr/types';
-import { isNull, isUndefined } from 'lodash-es';
+import { isUndefined } from 'lodash-es';
 import { DeepReadonly } from 'ts-essentials';
 import { PipelineBuilder } from './PipelineBuilder.js';
 import { NvidiaPipelineBuilder } from './hardware/NvidiaPipelineBuilder.ts';
@@ -78,12 +78,6 @@ class PipelineBuilderFactory$Builder {
     );
     const binaryCapabilities = await info.getCapabilities();
 
-    if (isNull(this.videoInputSource)) {
-      // Audio-only pipeline builder??
-      throw new Error('Not yet implemented');
-      // return new SoftwarePipelineBuilder()
-    }
-
     switch (this.hardwareAccelerationMode) {
       case 'cuda':
         return new NvidiaPipelineBuilder(

server/src/ffmpeg/builder/pipeline/hardware/NvidiaPipelineBuilder.ts

@@ -49,7 +49,7 @@ export class NvidiaPipelineBuilder extends SoftwarePipelineBuilder {
   constructor(
     private hardwareCapabilities: BaseFfmpegHardwareCapabilities,
     binaryCapabilities: FfmpegCapabilities,
-    videoInputFile: VideoInputSource,
+    videoInputFile: Nullable<VideoInputSource>,
     audioInputFile: Nullable<AudioInputSource>,
     concatInputSource: Nullable<ConcatInputSource>,
     watermarkInputSource: Nullable<WatermarkInputSource>,
@@ -358,7 +358,6 @@ export class NvidiaPipelineBuilder extends SoftwarePipelineBuilder {
         this.watermarkInputSource &&
         currentState.frameDataLocation === FrameDataLocation.Hardware
       ) {
-        this.logger.debug('Using software encoder');
         const hwDownloadFilter = new HardwareDownloadCudaFilter(
           currentState.pixelFormat,
           null,
@@ -367,7 +366,6 @@ export class NvidiaPipelineBuilder extends SoftwarePipelineBuilder {
         steps.push(hwDownloadFilter);
       }
 
-      console.debug('pixelFormat', currentState, this.ffmpegState);
       if (
         currentState.frameDataLocation === FrameDataLocation.Hardware &&
         this.ffmpegState.encoderHwAccelMode === HardwareAccelerationMode.None

server/src/ffmpeg/builder/pipeline/hardware/VaapiPipelineBuilder.ts

@@ -58,7 +58,7 @@ export class VaapiPipelineBuilder extends SoftwarePipelineBuilder {
   constructor(
     private hardwareCapabilities: BaseFfmpegHardwareCapabilities,
     binaryCapabilities: FfmpegCapabilities,
-    videoInputFile: VideoInputSource,
+    videoInputFile: Nullable<VideoInputSource>,
     audioInputFile: Nullable<AudioInputSource>,
     watermarkInputSource: Nullable<WatermarkInputSource>,
     concatInputSource: Nullable<ConcatInputSource>,

server/src/ffmpeg/builder/pipeline/hardware/VideoToolboxPipelineBuilder.ts

@@ -25,7 +25,7 @@ export class VideoToolboxPipelineBuilder extends SoftwarePipelineBuilder {
   constructor(
     private hardwareCapabilities: BaseFfmpegHardwareCapabilities,
     binaryCapabilities: FfmpegCapabilities,
-    videoInputFile: VideoInputSource,
+    videoInputFile: Nullable<VideoInputSource>,
     audioInputFile: Nullable<AudioInputSource>,
     concatInputSource: Nullable<ConcatInputSource>,
     watermarkInputSource: Nullable<WatermarkInputSource>,