Custom camera + HaishinKit (v2.0.0-rc.2): VideoSize and Audio problems.

[abhinandan-chakraborty]

Hi, I always like using Haishinkit's camera directly but in my project, there is a sudden need to use AVCaptureDataOutputSynchronizerDelegate to manipulate CMSampleBuffer from dual wide-angle camera stream synced at the same time. First let me post the sample I'm working on:

private class MultiCamModel: NSObject, ObservableObject, AVCaptureDataOutputSynchronizerDelegate,
  AVCaptureAudioDataOutputSampleBufferDelegate
{

  //variables
  private let sessionQueue = DispatchQueue(label: "com.demo.sessionQueue")
  var dualVideoSession = AVCaptureMultiCamSession()
  var synchronizer: AVCaptureDataOutputSynchronizer?
  let wideDataOutput = AVCaptureVideoDataOutput()
  let ultrawideDataOutput = AVCaptureVideoDataOutput()
  let audioDataOutput = AVCaptureAudioDataOutput()

  //haishinkit
  private var stream: RTMPStream?
  private let mixer = MediaMixer()
  private let connection = RTMPConnection()

  //configure camera
  private func configureCaptureSession() {
    sessionQueue.async { [weak self] in
      guard let self else { return }
      do {

        LBLogger.with(kHaishinKitIdentifier).level = .trace

        dualVideoSession.beginConfiguration()

        stream = RTMPStream(connection: connection)  //<-- RTMPStream

        Task {
          let settings = VideoCodecSettings(
            videoSize: .init(width: 3840, height: 1080), bitRate: 4 * 1000 * 1000)  //<-- ERROR to append stream when 'videoSize' is used. Works well without 'videoSize' but the video is cropped within the default frame of 854x480
          await self.stream?.setVideoSettings(settings)
        }

        guard
          let cameraDevice = AVCaptureDevice.default(
            .builtInDualWideCamera, for: .video, position: .back),
          let microphoneDevice = AVCaptureDevice.default(for: .audio)
        else {
          print("no camera or microphone device")
          return
        }

        let cameraInput = try AVCaptureDeviceInput(device: cameraDevice)
        let microphoneInput = try AVCaptureDeviceInput(device: microphoneDevice)

        guard
          let widePort = cameraInput.ports(
            for: .video, sourceDeviceType: .builtInWideAngleCamera, sourceDevicePosition: .back
          ).first,
          let ultrawidePort = cameraInput.ports(
            for: .video, sourceDeviceType: .builtInUltraWideCamera, sourceDevicePosition: .back
          ).first,
          let audioPort = microphoneInput.ports(
            for: .audio, sourceDeviceType: microphoneDevice.deviceType,
            sourceDevicePosition: microphoneDevice.position
          ).first
        else {
          print("Could not obtain input ports")
          return
        }

        //start of connection hell
        dualVideoSession.addInputWithNoConnections(cameraInput)
        dualVideoSession.addInputWithNoConnections(microphoneInput)

        dualVideoSession.addOutputWithNoConnections(wideDataOutput)
        dualVideoSession.addOutputWithNoConnections(ultrawideDataOutput)
        dualVideoSession.addOutputWithNoConnections(audioDataOutput)

        let wideAngleCameraConnection = AVCaptureConnection(
          inputPorts: [widePort], output: wideDataOutput)
        let ultrawideAngleCameraConnection = AVCaptureConnection(
          inputPorts: [ultrawidePort], output: ultrawideDataOutput)
        let microphoneConnection = AVCaptureConnection(
          inputPorts: [audioPort], output: audioDataOutput)

        dualVideoSession.addConnection(wideAngleCameraConnection)
        dualVideoSession.addConnection(ultrawideAngleCameraConnection)
        dualVideoSession.addConnection(microphoneConnection)
        //end of connection hell

        synchronizer = AVCaptureDataOutputSynchronizer(dataOutputs: [
          wideDataOutput, ultrawideDataOutput,
        ])

        synchronizer!.setDelegate(self, queue: sessionQueue)  //listen for video data
        audioDataOutput.setSampleBufferDelegate(self, queue: sessionQueue)  //listen for audio data

        dualVideoSession.commitConfiguration()
        dualVideoSession.startRunning()

        //success in setup
      } catch {
        //err
      }
    }
  }

  //listen to callback for Video Data
  func dataOutputSynchronizer(
    _ synchronizer: AVCaptureDataOutputSynchronizer,
    didOutput synchronizedDataCollection: AVCaptureSynchronizedDataCollection
  ) {
    //maipulate multiple video frames and once the work is done then append the new sample buffer into RTMPStream
    let imageBuffer = CMSampleBufferGetImageBuffer(newSampleBuffer!)!
    Task {
      await stream?.append(newSampleBuffer!)  //<-- Append to Stream
    }
  }

  //listen for Audio Data
  func captureOutput(
    _ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    from connection: AVCaptureConnection
  ) {
    //<-- HOW to Append SampleBuffer to RTMPStream?
  }
}

So now I have few questions:
1. How to set videoSize in VideoCodecSettings without causing this error ?

2024-27-10 16:46:51.393 [Error] [com.haishinkit.HaishinKit] [VideoCodec.swift:81] append(_:) > failedToPrepare(status: -12902)

2. How to append Audio in the stream? I tried this code

  //listen for Audio Data
  func captureOutput(
    _ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,
    from connection: AVCaptureConnection
  ) {
    //COPIED FROM STACKOVERFLOW
    guard let description: CMFormatDescription = CMSampleBufferGetFormatDescription(sampleBuffer),
      let sampleRate: Float64 = description.audioStreamBasicDescription?.mSampleRate,
      let channelsPerFrame: UInt32 = description.audioStreamBasicDescription?.mChannelsPerFrame
    else { return }
    guard let blockBuffer: CMBlockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else {
      return
    }

    let samplesCount = CMSampleBufferGetNumSamples(sampleBuffer)
    let audioFormat = AVAudioFormat(
      commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: AVAudioChannelCount(1),
      interleaved: false)
    let buffer = AVAudioPCMBuffer(
      pcmFormat: audioFormat!, frameCapacity: AVAudioFrameCount(samplesCount))!
    buffer.frameLength = buffer.frameCapacity
    var dataPointer: UnsafeMutablePointer<Int8>?
    CMBlockBufferGetDataPointer(
      blockBuffer, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: nil,
      dataPointerOut: &dataPointer)

    guard var channel: UnsafeMutablePointer<Float> = buffer.floatChannelData?[0],
      let data = dataPointer
    else { return }

    var data16 = UnsafeRawPointer(data).assumingMemoryBound(to: Int16.self)

    for _ in 0...samplesCount - 1 {
      channel.pointee = Float32(data16.pointee) / Float32(Int16.max)
      channel += 1
      for _ in 0...channelsPerFrame - 1 {
        data16 += 1
      }
    }
    let audioTime = AVAudioTime(sampleTime: .zero, atRate: sampleRate)
    await stream?.append(buffer, when: audioTime) //<-- Append audio to RTMPStream
  }

but it causes this error:

2024-27-10 17:28:29.184 [Warn] [com.haishinkit.HaishinKit] [RTMPStream.swift:790] append(_:when:) > invalidSequence