Magewell Examples and main Muxer and Demuxer

build.gradle.kts

@@ -49,6 +49,11 @@ mainDependencies {
    api("com.hierynomus:sshj:0.31.0")
    api("us.ihmc:scs2-definition:17-0.26.0")
 
+   api("org.bytedeco:javacv:1.5.10")
+   api("org.bytedeco:javacpp:1.5.10")
+   api("org.bytedeco:javacv-platform:1.5.10")
+   api("org.freedesktop.gstreamer:gst1-java-core:1.4.0")
+
    var javaFXVersion = "17.0.2"
    api(ihmc.javaFXModule("base", javaFXVersion))
    api(ihmc.javaFXModule("controls", javaFXVersion))

src/main/java/us/ihmc/robotDataLogger/logger/MagewellDemuxer.java

@@ -0,0 +1,83 @@
+package us.ihmc.robotDataLogger.logger;
+
+import org.bytedeco.javacv.FFmpegFrameGrabber;
+import org.bytedeco.javacv.Frame;
+import org.bytedeco.javacv.FrameGrabber;
+
+import java.io.File;
+
+/**
+ * This class takes a video file and returns given information about its frames when requested
+ */
+public class MagewellDemuxer
+{
+    private static final String MAGEWELL_DEMUXER = "MageWell Demuxer";
+    private final FFmpegFrameGrabber grabber;
+
+    public MagewellDemuxer(File videoFile)
+    {
+        try
+        {
+            grabber = new FFmpegFrameGrabber(videoFile);
+            grabber.start();
+        }
+        catch (FrameGrabber.Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public String getName()
+    {
+        return MAGEWELL_DEMUXER;
+    }
+
+    public int getImageHeight()
+    {
+        return grabber.getImageHeight();
+    }
+
+    public int getImageWidth()
+    {
+        return grabber.getImageWidth();
+    }
+
+    public long getCurrentPTS()
+    {
+        return grabber.getTimestamp();
+    }
+
+    public void seekToPTS(long videoTimestamp)
+    {
+        try
+        {
+            grabber.setTimestamp(videoTimestamp);
+        }
+        catch (FFmpegFrameGrabber.Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public int getFrameNumber()
+    {
+        return grabber.getFrameNumber();
+    }
+
+    public Frame getNextFrame()
+    {
+        try
+        {
+            return grabber.grabFrame();
+        }
+        catch (FrameGrabber.Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public double getFrameRate()
+    {
+        return grabber.getVideoFrameRate();
+    }
+}

src/main/java/us/ihmc/robotDataLogger/logger/MagewellMuxer.java

@@ -0,0 +1,92 @@
+package us.ihmc.robotDataLogger.logger;
+
+import org.bytedeco.ffmpeg.global.avcodec;
+import org.bytedeco.ffmpeg.global.avutil;
+import org.bytedeco.javacv.FFmpegFrameRecorder;
+import org.bytedeco.javacv.FFmpegFrameRecorder.Exception;
+import org.bytedeco.javacv.Frame;
+
+import java.io.File;
+
+public class MagewellMuxer
+{
+   private final FFmpegFrameRecorder recorder;
+
+   public MagewellMuxer(File videoCaptureFile, int captureWidth, int captureHeight)
+   {
+      recorder = new FFmpegFrameRecorder(videoCaptureFile, captureWidth, captureHeight);
+
+      recorder.setVideoOption("tune", "zerolatency"); // https://trac.ffmpeg.org/wiki/StreamingGuide
+      recorder.setFormat("mov");
+
+      // For information about these settings visit https://trac.ffmpeg.org/wiki/Encode/H.264
+      recorder.setVideoOption("preset", "ultrafast");
+      recorder.setVideoOption("crf", "27");
+      recorder.setVideoBitrate(60000000); // 6000 kb/s
+
+      // This video codec is deprecated, so in order to use it without errors we have to set the pixel format and strictly allow FFMPEG to use it
+      recorder.setVideoCodec(avcodec.AV_CODEC_ID_MJPEG);
+      recorder.setPixelFormat(avutil.AV_PIX_FMT_YUV420P);
+      recorder.setVideoOption("strict", "-2");
+      // Frame rate of video recordings
+      recorder.setFrameRate(60);
+   }
+
+   public void start() throws Exception
+   {
+      recorder.start();
+   }
+
+   /**
+    * This method only works if {@link MagewellMuxer#start()} has been called first
+    *
+    * @param capturedFrame  the frame we want to save to the video
+    * @param videoTimestamp is the timestamp in which to set the frame at
+    */
+   public void recordFrame(Frame capturedFrame, long videoTimestamp)
+   {
+      // Ensure the video timestamp is ahead of the record's current timestamp
+      if (videoTimestamp > recorder.getTimestamp())
+      {
+         // We tell the recorder to write this frame at this timestamp
+         recorder.setTimestamp(videoTimestamp);
+      }
+
+      // This is where a frame is record, and we then need to store the timestamps, so they are synced
+      try
+      {
+         if (!recorder.isCloseOutputStream())
+         {
+            recorder.record(capturedFrame);
+         }
+      }
+      catch (Exception e)
+      {
+         throw new RuntimeException(e);
+      }
+   }
+
+   public boolean isCloseOutputStream()
+   {
+      return recorder.isCloseOutputStream();
+   }
+
+   public long getTimeStamp()
+   {
+      return recorder.getTimestamp();
+   }
+
+   public void close()
+   {
+      recorder.setCloseOutputStream(true);
+      try
+      {
+         recorder.flush();
+         recorder.stop();
+      }
+      catch (Exception e)
+      {
+         throw new RuntimeException(e);
+      }
+   }
+}

src/test/java/us/ihmc/robotDataLogger/captureVideo/ExampleBytedecoJavaCVWindowsScreenRecord.java

@@ -0,0 +1,108 @@
+package us.ihmc.robotDataLogger.captureVideo;
+
+import org.bytedeco.ffmpeg.global.avcodec;
+import org.bytedeco.javacv.*;
+import us.ihmc.log.LogTools;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+/**
+ * This example class records the screen and saves it to file, this can be played back with Bytedeco as well
+ */
+public class ExampleBytedecoJavaCVWindowsScreenRecord
+{
+   private static long startTime = 0;
+
+   public static String videoPath = "ihmc-robot-data-logger/out/windowsBytedecoScreenRecordingVideo.mov";
+   public static final String timestampPath = "ihmc-robot-data-logger/out/windowsBytedecoScreenRecordingTimestamps.dat";
+   private static FileWriter timestampWriter;
+
+   public static File videoFile = new File(videoPath);
+   public static File timestampFile = new File(timestampPath);
+   public static void main(String[] args) throws IOException
+   {
+
+      try (FrameGrabber grabber = new FFmpegFrameGrabber("desktop"))
+      {
+         grabber.setFormat("gdigrab");
+         grabber.setFrameRate(60);
+         grabber.start();
+
+         setupTimestampWriter();
+
+         try(FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(videoFile, 1280, 720))
+         {
+            // Trying these settings for now (H264 is a bad setting because of slicing)
+            recorder.setVideoOption("tune", "zerolatency");
+            recorder.setVideoCodec(avcodec.AV_CODEC_ID_MPEG4);
+            recorder.setFormat("mov");
+            recorder.setFrameRate(60);
+
+            recorder.start();
+
+            int timer = 0;
+            Frame capturedFrame;
+
+            while (timer < 50 && ((capturedFrame = grabber.grab()) != null))
+            {
+               // Keeps track of time for the recorder because often times it gets offset
+               if (startTime == 0)
+                  startTime = System.currentTimeMillis();
+
+               long videoTS = 1000 * (System.currentTimeMillis() - startTime);
+
+               if (videoTS > recorder.getTimestamp())
+               {
+                  if (timer % 5 == 0)
+                     System.out.println("Lip-flap correction: " + videoTS + " : " + recorder.getTimestamp() + " -> " + (videoTS - recorder.getTimestamp()));
+
+                  // We tell the recorder to write this frame at this timestamp
+                  recorder.setTimestamp(videoTS);
+               }
+
+               recorder.record(capturedFrame);
+               // System.nanoTime() represents the controllerTimestamp in this example since its fake
+               writeTimestampToFile(System.nanoTime(), recorder.getTimestamp());
+
+               timer++;
+            }
+
+            recorder.stop();
+            grabber.stop();
+            timestampWriter.close();
+         }
+      }
+   }
+
+   public static void setupTimestampWriter()
+   {
+      try
+      {
+         timestampWriter = new FileWriter(timestampFile);
+         timestampWriter.write(1 + "\n");
+         timestampWriter.write(60000 + "\n");
+      }
+      catch (IOException e)
+      {
+         LogTools.info("Didn't setup the timestamp file correctly");
+      }
+   }
+
+   /**
+    * Write the timestamp sent from the controller, and then we get the timestamp of the camera, and write both
+    * of those to a file.
+    */
+   public static void writeTimestampToFile(long controllerTimestamp, long pts)
+   {
+      try
+      {
+         timestampWriter.write(controllerTimestamp + " " + pts + "\n");
+      }
+      catch (IOException e)
+      {
+         e.printStackTrace();
+      }
+   }
+}