diff --git a/scs2-session-logger/build.gradle.kts b/scs2-session-logger/build.gradle.kts index 7d803ee9e..412e75bf8 100644 --- a/scs2-session-logger/build.gradle.kts +++ b/scs2-session-logger/build.gradle.kts @@ -14,7 +14,7 @@ mainDependencies { api("us.ihmc:scs2-session:source") api("us.ihmc:scs2-simulation:source") // TODO Need to fix this, it needs the Robot. - api("us.ihmc:ihmc-robot-data-logger:0.29.3") + api("us.ihmc:ihmc-robot-data-logger:0.29.4") api("com.github.luben:zstd-jni:1.5.5-10") api("org.antlr:antlr4-runtime:4.13.1") //api("org.lz4:lz4-java:1.8.0") diff --git a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/BlackMagicVideoDataReader.java b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/BlackMagicVideoDataReader.java new file mode 100644 index 000000000..b38ad2d98 --- /dev/null +++ b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/BlackMagicVideoDataReader.java @@ -0,0 +1,144 @@ +package us.ihmc.scs2.sessionVisualizer.jfx.session.log; + +import us.ihmc.codecs.demuxer.MP4VideoDemuxer; +import us.ihmc.codecs.generated.YUVPicture; +import us.ihmc.concurrent.ConcurrentCopier; +import us.ihmc.robotDataLogger.Camera; +import us.ihmc.scs2.session.log.ProgressConsumer; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; + +public class BlackMagicVideoDataReader implements VideoDataReader +{ + private final TimestampScrubber timestampScrubber; + private final String name; + + private final MP4VideoDemuxer demuxer; + private final JavaFXPictureConverter converter = new JavaFXPictureConverter(); + + private final File videoFile; + private final Camera camera; + private final ConcurrentCopier imageBuffer = new ConcurrentCopier<>(FrameData::new); + + public BlackMagicVideoDataReader(Camera camera, File dataDirectory, boolean hasTimeBase) throws IOException + { + this.camera = camera; + name = camera.getNameAsString(); + boolean interlaced = camera.getInterlaced(); + + if (!hasTimeBase) + { + System.err.println("Video data is using timestamps instead of frame numbers. Falling back to seeking based on timestamp."); + } + + videoFile = new File(dataDirectory, camera.getVideoFileAsString()); + + if (!videoFile.exists()) + { + throw new IOException("Cannot find video: " + videoFile); + } + + demuxer = new MP4VideoDemuxer(videoFile); + + File timestampFile = new File(dataDirectory, camera.getTimestampFileAsString()); + this.timestampScrubber = new TimestampScrubber(timestampFile, hasTimeBase, interlaced); + } + + @Override + public int getImageHeight() + { + return 0; + } + + @Override + public int getImageWidth() + { + return 0; + } + + public void readVideoFrame(long queryRobotTimestamp) + { + long videoTimestamp = timestampScrubber.getVideoTimestampFromRobotTimestamp(queryRobotTimestamp); + long currentRobotTimestamp = timestampScrubber.getCurrentRobotTimestamp(); + + try + { + demuxer.seekToPTS(videoTimestamp); + FrameData copyForWriting = imageBuffer.getCopyForWriting(); + copyForWriting.queryRobotTimestamp = queryRobotTimestamp; + copyForWriting.currentRobotTimestamp = currentRobotTimestamp; + copyForWriting.currentVideoTimestamp = videoTimestamp; + copyForWriting.currentDemuxerTimestamp = demuxer.getCurrentPTS(); + YUVPicture nextFrame = demuxer.getNextFrame(); // Increment frame index after getting frame. + copyForWriting.frame = converter.toFXImage(nextFrame, copyForWriting.frame); + + imageBuffer.commit(); + } + catch (IOException e) + { + e.printStackTrace(); + } + } + + public void cropVideo(File outputFile, File timestampFile, long startTimestamp, long endTimestamp, ProgressConsumer monitor) throws IOException + { + long startVideoTimestamp = timestampScrubber.getVideoTimestampFromRobotTimestamp(startTimestamp); + long endVideoTimestamp = timestampScrubber.getVideoTimestampFromRobotTimestamp(endTimestamp); + + int framerate = VideoConverter.cropBlackMagicVideo(videoFile, outputFile, startVideoTimestamp, endVideoTimestamp, monitor); + + PrintWriter timestampWriter = new PrintWriter(timestampFile); + timestampWriter.println(1); + timestampWriter.println(framerate); + + long pts = 0; + /* + * PTS gets reordered to be monotonically increasing starting from 0 + */ + for (int i = 0; i < timestampScrubber.getRobotTimestampsLength(); i++) + { + long robotTimestamp = timestampScrubber.getRobotTimestampAtIndex(i); + + if (robotTimestamp >= startTimestamp && robotTimestamp <= endTimestamp) + { + timestampWriter.print(robotTimestamp); + timestampWriter.print(" "); + timestampWriter.println(pts); + pts++; + } + else if (robotTimestamp > endTimestamp) + { + break; + } + } + + timestampWriter.close(); + } + + public String getName() + { + return name; + } + + public Camera getCamera() + { + return camera; + } + + public FrameData pollCurrentFrame() + { + return imageBuffer.getCopyForReading(); + } + + public int getCurrentIndex() + { + return timestampScrubber.getCurrentIndex(); + } + + public boolean replacedRobotTimestampsContainsIndex(int index) + { + return timestampScrubber.getReplacedRobotTimestampIndex(index); + } +} diff --git a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/FrameData.java b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/FrameData.java new file mode 100644 index 000000000..12cc579e8 --- /dev/null +++ b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/FrameData.java @@ -0,0 +1,16 @@ +package us.ihmc.scs2.sessionVisualizer.jfx.session.log; + +import javafx.scene.image.WritableImage; + +/** + * This class is used for debugging timestamp delays in the videos when viewing them in SCS2. In order to see the timestamp debugging information change the + * boolean in {@link VideoViewer#LOGGER_VIDEO_DEBUG} to true. This can also be set as an environmental variable. + */ +public class FrameData +{ + public WritableImage frame; + public long queryRobotTimestamp; + public long currentRobotTimestamp; + public long currentVideoTimestamp; + public long currentDemuxerTimestamp; +} diff --git a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/MagewellVideoDataReader.java b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/MagewellVideoDataReader.java new file mode 100644 index 000000000..949471a10 --- /dev/null +++ b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/MagewellVideoDataReader.java @@ -0,0 +1,205 @@ +package us.ihmc.scs2.sessionVisualizer.jfx.session.log; + +import javafx.scene.image.Image; +import javafx.scene.image.PixelReader; +import javafx.scene.image.PixelWriter; +import javafx.scene.image.WritableImage; +import org.bytedeco.javacv.Frame; +import org.bytedeco.javacv.JavaFXFrameConverter; +import us.ihmc.robotDataLogger.Camera; +import us.ihmc.robotDataLogger.logger.MagewellDemuxer; +import us.ihmc.robotDataLogger.logger.MagewellMuxer; +import us.ihmc.scs2.session.log.ProgressConsumer; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; + +public class MagewellVideoDataReader implements VideoDataReader +{ + private final TimestampScrubber timestampScrubber; + private final String name; + + private final MagewellDemuxer magewellDemuxer; + + private final Camera camera; + private final FrameData frameData = new FrameData(); + + public MagewellVideoDataReader(Camera camera, File dataDirectory, boolean hasTimeBase) throws IOException + { + this.camera = camera; + name = camera.getNameAsString(); + boolean interlaced = camera.getInterlaced(); + + if (!hasTimeBase) + { + System.err.println("Video data is using timestamps instead of frame numbers. Falling back to seeking based on timestamp."); + } + + File videoFile = new File(dataDirectory, camera.getVideoFileAsString()); + + if (!videoFile.exists()) + { + throw new IOException("Cannot find video: " + videoFile); + } + + magewellDemuxer = new MagewellDemuxer(videoFile); + + File timestampFile = new File(dataDirectory, camera.getTimestampFileAsString()); + this.timestampScrubber = new TimestampScrubber(timestampFile, hasTimeBase, interlaced); + } + + public int getImageHeight() + { + return magewellDemuxer.getImageHeight(); + } + + public int getImageWidth() + { + return magewellDemuxer.getImageWidth(); + } + + public void readVideoFrame(long queryRobotTimestamp) + { + long currentVideoTimestamps = timestampScrubber.getVideoTimestampFromRobotTimestamp(queryRobotTimestamp); + long currentRobotTimestamp = timestampScrubber.getCurrentRobotTimestamp(); + + magewellDemuxer.seekToPTS(currentVideoTimestamps); + + // This is a copy that can be shown in the video view to debug timestamp issues + { + FrameData copyForWriting = frameData; + copyForWriting.queryRobotTimestamp = queryRobotTimestamp; + copyForWriting.currentRobotTimestamp = currentRobotTimestamp; + copyForWriting.currentVideoTimestamp = currentVideoTimestamps; + copyForWriting.currentDemuxerTimestamp = magewellDemuxer.getCurrentPTS(); + } + + Frame nextFrame = magewellDemuxer.getNextFrame(); + frameData.frame = convertFrameToWritableImage(nextFrame); + } + + /** + * This class converts a {@link Frame} to a {@link WritableImage} in order to be displayed correctly in JavaFX. + * + * @param frameToConvert is the next frame we want to visualize so we convert it to be compatible with JavaFX + * @return {@link WritableImage} + */ + public WritableImage convertFrameToWritableImage(Frame frameToConvert) + { + Image currentImage; + + if (frameToConvert == null) + { + return null; + } + + try (JavaFXFrameConverter frameConverter = new JavaFXFrameConverter()) + { + currentImage = frameConverter.convert(frameToConvert); + } + WritableImage writableImage = new WritableImage((int) currentImage.getWidth(), (int) currentImage.getHeight()); + PixelReader pixelReader = currentImage.getPixelReader(); + PixelWriter pixelWriter = writableImage.getPixelWriter(); + + for (int y = 0; y < currentImage.getHeight(); y++) + { + for (int x = 0; x < currentImage.getWidth(); x++) + { + pixelWriter.setArgb(x, y, pixelReader.getArgb(x, y)); + } + } + + return writableImage; + } + + public Frame getFrame() + { + return magewellDemuxer.getNextFrame(); + } + + public void cropVideo(File outputFile, File timestampFile, long startTimestamp, long endTimestamp, ProgressConsumer progressConsumer) throws IOException + { + long startVideoTimestamp = timestampScrubber.getVideoTimestampFromRobotTimestamp(startTimestamp); + long endVideoTimestamp = timestampScrubber.getVideoTimestampFromRobotTimestamp(endTimestamp); + + long[] robotTimestampsForCroppedLog = timestampScrubber.getCroppedRobotTimestamps(startTimestamp, endTimestamp); + long[] videoTimestampsForCroppedLog = new long[robotTimestampsForCroppedLog.length]; + int i = 0; + + // This stuff is used to print to SCS2 so the user knows how the cropped log is going, progress wise + long startFrame = getFrameAtTimestamp(startVideoTimestamp, magewellDemuxer); // This also moves the stream to the startFrame + long endFrame = getFrameAtTimestamp(endVideoTimestamp, magewellDemuxer); + long numberOfFrames = endFrame - startFrame; + int frameRate = (int) magewellDemuxer.getFrameRate(); + + magewellDemuxer.seekToPTS(startVideoTimestamp); + + PrintWriter timestampWriter = new PrintWriter(timestampFile); + timestampWriter.println(1 + "\n" + frameRate); + + long startTime = System.currentTimeMillis(); + + MagewellMuxer magewellMuxer = new MagewellMuxer(outputFile, magewellDemuxer.getImageWidth(), magewellDemuxer.getImageHeight()); + magewellMuxer.start(); + + Frame frame; + while ((frame = magewellDemuxer.getNextFrame()) != null && magewellDemuxer.getFrameNumber() <= endFrame) + { + // We want to write all the frames at once to get equal timestamps between frames. When recording from the camera we have a fixed rate at which we + // receive frames, so we don't need to worry about it, here however, we don't have that so we cna grab the next frame as fast as possible. However if the + // timestamps between frames aren't large enough, things won't work. (maybe :)) + long videoTimestamp = 1000 * (System.currentTimeMillis() - startTime); + magewellMuxer.recordFrame(frame, videoTimestamp); + videoTimestampsForCroppedLog[i] = magewellMuxer.getTimeStamp(); + i++; + + if (progressConsumer != null) + { + progressConsumer.info("frame %d/%d".formatted(magewellDemuxer.getFrameNumber() - startFrame, numberOfFrames)); + progressConsumer.progress((double) (magewellDemuxer.getFrameNumber() - startFrame) / (double) numberOfFrames); + } + } + + for (i = 0; i < videoTimestampsForCroppedLog.length; i++) + { + timestampWriter.print(robotTimestampsForCroppedLog[i]); + timestampWriter.print(" "); + timestampWriter.println(videoTimestampsForCroppedLog[i]); + } + + magewellMuxer.close(); + timestampWriter.close(); + } + + private static long getFrameAtTimestamp(long endCameraTimestamp, MagewellDemuxer magewellDemuxer) + { + magewellDemuxer.seekToPTS(endCameraTimestamp); + return magewellDemuxer.getFrameNumber(); + } + + public String getName() + { + return name; + } + + public Camera getCamera() + { + return camera; + } + + public FrameData pollCurrentFrame() + { + return frameData; + } + + public int getCurrentIndex() + { + return timestampScrubber.getCurrentIndex(); + } + + public boolean replacedRobotTimestampsContainsIndex(int index) + { + return timestampScrubber.getReplacedRobotTimestampIndex(index); + } +} diff --git a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/MultiVideoDataReader.java b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/MultiVideoDataReader.java index 0601a031c..1fc0bd968 100644 --- a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/MultiVideoDataReader.java +++ b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/MultiVideoDataReader.java @@ -7,6 +7,7 @@ import java.util.concurrent.Future; import us.ihmc.robotDataLogger.Camera; +import us.ihmc.robotDataLogger.CameraType; import us.ihmc.robotDataLogger.LogProperties; import us.ihmc.scs2.session.log.ProgressConsumer; import us.ihmc.scs2.sessionVisualizer.jfx.managers.BackgroundExecutorManager; @@ -27,7 +28,20 @@ public MultiVideoDataReader(File dataDirectory, LogProperties logProperties, Bac Camera camera = cameras.get(i); try { - VideoDataReader reader = new VideoDataReader(camera, dataDirectory, logProperties.getVideo().getHasTimebase()); + VideoDataReader reader; + if (camera.getType().toString().equals(CameraType.CAPTURE_CARD_MAGEWELL.toString())) + { + reader = new MagewellVideoDataReader(camera, dataDirectory, logProperties.getVideo().getHasTimebase()); + } + else if (camera.getType().toString().equals(CameraType.CAPTURE_CARD.toString())) + { + reader = new BlackMagicVideoDataReader(camera, dataDirectory, logProperties.getVideo().getHasTimebase()); + } + else + { // Older logs won't have the camera type set correctly, if there isn't a type set this as the only option + reader = new BlackMagicVideoDataReader(camera, dataDirectory, logProperties.getVideo().getHasTimebase()); + } + readers.add(reader); } catch (IOException e) @@ -62,7 +76,9 @@ public void crop(File selectedDirectory, long startTimestamp, long endTimestamp, progressConsumer.info("Cropping video (%s)".formatted(camera.getVideoFileAsString())); double progressPercentage = (double) i / (double) readers.size(); progressConsumer.progress(progressPercentage); - subProgressConsumer = progressConsumer.subProgress("Cropping video (%s): ".formatted(camera.getVideoFileAsString()), progressPercentage, (i + 1.0) / readers.size()); + subProgressConsumer = progressConsumer.subProgress("Cropping video (%s): ".formatted(camera.getVideoFileAsString()), + progressPercentage, + (i + 1.0) / readers.size()); } File timestampFile = new File(selectedDirectory, camera.getTimestampFileAsString()); diff --git a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/TimestampScrubber.java b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/TimestampScrubber.java new file mode 100644 index 000000000..af2016cbf --- /dev/null +++ b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/TimestampScrubber.java @@ -0,0 +1,237 @@ +package us.ihmc.scs2.sessionVisualizer.jfx.session.log; + +import gnu.trove.list.array.TLongArrayList; +import us.ihmc.euclid.tools.EuclidCoreTools; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.Arrays; + +public class TimestampScrubber +{ + private final boolean hasTimebase; + private final boolean interlaced; + private long[] robotTimestamps; + private long[] videoTimestamps; + + private int currentIndex = 0; + private long currentRobotTimestamp = 0; + private long videoTimestamp; + + private boolean[] replacedRobotTimestampIndex; + + public TimestampScrubber(File timestampFile, boolean hasTimebase, boolean interlaced) throws IOException + { + this.hasTimebase = hasTimebase; + this.interlaced = interlaced; + + parseTimestampData(timestampFile); + } + + private void parseTimestampData(File timestampFile) throws IOException + { + try (BufferedReader bufferedReader = new BufferedReader(new FileReader(timestampFile))) + { + String line; + if (hasTimebase) + { + if (bufferedReader.readLine() == null) + { + throw new IOException("Cannot read numerator"); + } + + if (bufferedReader.readLine() == null) + { + throw new IOException("Cannot read denumerator"); + } + } + + TLongArrayList robotTimestamps = new TLongArrayList(); + TLongArrayList videoTimestamps = new TLongArrayList(); + + while ((line = bufferedReader.readLine()) != null) + { + String[] stamps = line.split("\\s"); + long robotStamp = Long.parseLong(stamps[0]); + long videoStamp = Long.parseLong(stamps[1]); + + if (interlaced) + { + videoStamp /= 2; + } + + robotTimestamps.add(robotStamp); + videoTimestamps.add(videoStamp); + } + + this.robotTimestamps = robotTimestamps.toArray(); + this.videoTimestamps = videoTimestamps.toArray(); + } + catch (FileNotFoundException e) + { + throw new RuntimeException(e); + } + + checkAndReplaceDuplicates(); + } + + private void checkAndReplaceDuplicates() + { + replacedRobotTimestampIndex = new boolean[robotTimestamps.length]; + int duplicatesAtEndOfFile = getNumberOfDuplicatesAtEndOfFile(); + + for (int currentIndex = 0; currentIndex < robotTimestamps.length - duplicatesAtEndOfFile; ) + { + if (robotTimestamps[currentIndex] != robotTimestamps[currentIndex + 1]) + { + currentIndex++; + continue; + } + + // Keeps track of the duplicated index's so frames border can be adjusted + replacedRobotTimestampIndex[currentIndex + 1] = true; + + int nextNonDuplicateIndex = getNextNonDuplicateIndex(currentIndex); + for (int i = currentIndex; i < nextNonDuplicateIndex; i++) + { + long firstAdjustedTimestamp = (long) EuclidCoreTools.interpolate(robotTimestamps[i], + robotTimestamps[nextNonDuplicateIndex], + (double) 1 / (nextNonDuplicateIndex - i)); + robotTimestamps[i + 1] = firstAdjustedTimestamp; + } + + currentIndex = nextNonDuplicateIndex; + } + } + + private int getNumberOfDuplicatesAtEndOfFile() + { + int duplicatesAtEndOfFile = 1; + + for (int i = robotTimestamps.length - 1; i > 0; i--) + { + if (robotTimestamps[i] == robotTimestamps[i - 1]) + duplicatesAtEndOfFile++; + } + + return duplicatesAtEndOfFile; + } + + private int getNextNonDuplicateIndex(int index) + { + while (index < robotTimestamps.length - 1 && robotTimestamps[index] == robotTimestamps[index + 1]) + index++; + + return index + 1; + } + + /** + * Searches the list of robotTimestamps for the value closest to queryRobotTimestamp and returns that index. Then sets videoTimestamp to + * that index in oder to display the right frame. + * + * @param queryRobotTimestamp the value sent from the robot data in which we want to find the closest robotTimestamp in the instant file. + * @return the videoTimestamp that matches the index of the closest robotTimestamp in our instant file. + */ + public long getVideoTimestampFromRobotTimestamp(long queryRobotTimestamp) + { + currentIndex = searchRobotTimestampsForIndex(queryRobotTimestamp); + videoTimestamp = videoTimestamps[currentIndex]; + currentRobotTimestamp = robotTimestamps[currentIndex]; + + return videoTimestamp; + } + + private int searchRobotTimestampsForIndex(long queryRobotTimestamp) + { + if (queryRobotTimestamp <= robotTimestamps[0]) + return 0; + + if (queryRobotTimestamp >= robotTimestamps[robotTimestamps.length - 1]) + return robotTimestamps.length - 1; + + int index = Arrays.binarySearch(robotTimestamps, queryRobotTimestamp); + + if (index < 0) + { + int nextIndex = -index - 1; // insertionPoint + index = nextIndex; + } + + return index; + } + + public long[] getCroppedRobotTimestamps(long startRobotTimestamp, long endRobotTimestamp) + { + int startIndex = findIndexOfRobotTimestamps(startRobotTimestamp); + int endIndex = findIndexOfRobotTimestamps(endRobotTimestamp); + + return Arrays.copyOfRange(robotTimestamps, startIndex, endIndex + 1); + } + + private int findIndexOfRobotTimestamps(long value) + { + if (value <= robotTimestamps[0]) + return 0; + + if (value >= robotTimestamps[robotTimestamps.length - 1]) + return robotTimestamps.length - 1; + + int index = Arrays.binarySearch(robotTimestamps, value); + + if (index < 0) + { + int nextIndex = -index - 1; // insertionPoint + index = nextIndex; + } + + return index; + } + + public int getCurrentIndex() + { + return currentIndex; + } + + public long getCurrentRobotTimestamp() + { + return currentRobotTimestamp; + } + + public int getRobotTimestampsLength() + { + return robotTimestamps.length; + } + + public long getRobotTimestampAtIndex(int i) + { + return robotTimestamps[i]; + } + + public long getVideoTimestampAtIndex(int i) + { + return videoTimestamps[i]; + } + + public long[] getRobotTimestampsArray() + { + return robotTimestamps; + } + + public long[] getVideoTimestampsArray() + { + return videoTimestamps; + } + + public long getCurrentVideoTimestamp() + { + return videoTimestamp; + } + + public boolean getReplacedRobotTimestampIndex(int index) + { + return replacedRobotTimestampIndex[index]; + } +} diff --git a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoConverter.java b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoConverter.java index a01d7a4be..41fe4f7e5 100644 --- a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoConverter.java +++ b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoConverter.java @@ -5,27 +5,21 @@ import org.jcodec.containers.mp4.MP4Packet; -import us.ihmc.codecs.builder.H264Settings; -import us.ihmc.codecs.builder.MP4H264MovieBuilder; import us.ihmc.codecs.builder.MP4MJPEGMovieBuilder; import us.ihmc.codecs.demuxer.MP4VideoDemuxer; -import us.ihmc.codecs.generated.EProfileIdc; -import us.ihmc.codecs.generated.YUVPicture; import us.ihmc.scs2.session.log.ProgressConsumer; public class VideoConverter { - /** * @param source * @param target * @param startPTS * @param endPTS - * @param monitor * @return frame rate of the new video file * @throws IOException */ - public static int crop(File source, File target, long startPTS, long endPTS, ProgressConsumer progressConsumer) throws IOException + public static int cropBlackMagicVideo(File source, File target, long startPTS, long endPTS, ProgressConsumer progressConsumer) throws IOException { MP4MJPEGMovieBuilder builder = null; MP4VideoDemuxer demuxer = new MP4VideoDemuxer(source); @@ -58,54 +52,6 @@ public static int crop(File source, File target, long startPTS, long endPTS, Pro demuxer.delete(); return frameRate; - - } - - /** - * @param source - * @param target - * @param startPTS - * @param endPTS - * @param bitrate - * @param progressConsumer - * @return frame rate of the new video file - * @throws IOException - */ - public static void convert(File source, File target, long startPTS, long endPTS, ProgressConsumer progressConsumer) throws IOException - { - MP4H264MovieBuilder builder = null; - MP4VideoDemuxer demuxer = new MP4VideoDemuxer(source); - - H264Settings settings = new H264Settings(); - settings.setBitrate(10000); - settings.setProfileIdc(EProfileIdc.PRO_HIGH); - - int frameRate = getFrameRate(demuxer); - - long endFrame = getFrame(endPTS, demuxer); - long startFrame = getFrame(startPTS, demuxer); // This also moves the stream to the startFrame - long numberOfFrames = endFrame - startFrame; - - YUVPicture frame; - while ((frame = demuxer.getNextFrame()) != null && demuxer.getCurrentFrame() <= endFrame) - { - if (builder == null) - { - builder = new MP4H264MovieBuilder(target, frame.getWidth(), frame.getHeight(), frameRate, settings); - } - // frame.toYUV420(); - builder.encodeFrame(frame); - frame.delete(); - - if (progressConsumer != null) - { - progressConsumer.progress((int) ((double) (demuxer.getCurrentFrame() - startFrame) / (double) numberOfFrames)); - } - } - - builder.close(); - demuxer.delete(); - } private static int getFrameRate(MP4VideoDemuxer demuxer) throws IOException @@ -120,7 +66,6 @@ private static int getFrameRate(MP4VideoDemuxer demuxer) throws IOException System.out.println("Framerate is " + rate); return rate; - } private static long getFrame(long endPTS, MP4VideoDemuxer demuxer) throws IOException diff --git a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoDataReader.java b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoDataReader.java index 5d1099da6..818057285 100644 --- a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoDataReader.java +++ b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoDataReader.java @@ -1,351 +1,35 @@ package us.ihmc.scs2.sessionVisualizer.jfx.session.log; -import gnu.trove.list.array.TLongArrayList; -import javafx.scene.image.WritableImage; -import us.ihmc.codecs.demuxer.MP4VideoDemuxer; -import us.ihmc.codecs.generated.YUVPicture; -import us.ihmc.concurrent.ConcurrentCopier; -import us.ihmc.euclid.tools.EuclidCoreTools; import us.ihmc.robotDataLogger.Camera; import us.ihmc.scs2.session.log.ProgressConsumer; -import java.io.BufferedReader; import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; -import java.io.PrintWriter; -import java.util.Arrays; -public class VideoDataReader +/** + * This interface allows supporting different types of capture methods to be viewed back with SCS2. + */ +public interface VideoDataReader { - private final TimestampScrubber timestampScrubber; - private final String name; - private final MP4VideoDemuxer demuxer; - private final JavaFXPictureConverter converter = new JavaFXPictureConverter(); + int getImageHeight(); - private final File videoFile; - private final Camera camera; - private final ConcurrentCopier imageBuffer = new ConcurrentCopier<>(FrameData::new); + int getImageWidth(); - public VideoDataReader(Camera camera, File dataDirectory, boolean hasTimeBase) throws IOException - { - this.camera = camera; - name = camera.getNameAsString(); - boolean interlaced = camera.getInterlaced(); - - if (!hasTimeBase) - { - System.err.println("Video data is using timestamps instead of frame numbers. Falling back to seeking based on timestamp."); - } - - videoFile = new File(dataDirectory, camera.getVideoFileAsString()); - - if (!videoFile.exists()) - { - throw new IOException("Cannot find video: " + videoFile); - } - - demuxer = new MP4VideoDemuxer(videoFile); - - File timestampFile = new File(dataDirectory, camera.getTimestampFileAsString()); - this.timestampScrubber = new TimestampScrubber(timestampFile, hasTimeBase, interlaced); - } - - public void readVideoFrame(long queryRobotTimestamp) - { - long videoTimestamp = timestampScrubber.getVideoTimestamp(queryRobotTimestamp); - long currentRobotTimestamp = timestampScrubber.getCurrentRobotTimestamp(); + void readVideoFrame(long timestamp); - try - { - demuxer.seekToPTS(videoTimestamp); - FrameData copyForWriting = imageBuffer.getCopyForWriting(); - copyForWriting.queryRobotTimestamp = queryRobotTimestamp; - copyForWriting.robotTimestamp = currentRobotTimestamp; - copyForWriting.cameraCurrentPTS = videoTimestamp; - copyForWriting.demuxerCurrentPTS = demuxer.getCurrentPTS(); - YUVPicture nextFrame = demuxer.getNextFrame(); // Increment frame index after getting frame. - copyForWriting.frame = converter.toFXImage(nextFrame, copyForWriting.frame); + void cropVideo(File file, File file2, long test, long what, ProgressConsumer heck) throws IOException; - imageBuffer.commit(); - } - catch (IOException e) - { - e.printStackTrace(); - } - } + String getName(); - public void cropVideo(File outputFile, File timestampFile, long startTimestamp, long endTimestamp, ProgressConsumer monitor) throws IOException - { - long startVideoTimestamp = timestampScrubber.getVideoTimestamp(startTimestamp); - long endVideoTimestamp = timestampScrubber.getVideoTimestamp(endTimestamp); - - int framerate = VideoConverter.crop(videoFile, outputFile, startVideoTimestamp, endVideoTimestamp, monitor); - - PrintWriter timestampWriter = new PrintWriter(timestampFile); - timestampWriter.println(1); - timestampWriter.println(framerate); - - long pts = 0; - /* - * PTS gets reordered to be monotonically increasing starting from 0 - */ - for (int i = 0; i < timestampScrubber.getRobotTimestampsLength(); i++) - { - long robotTimestamp = timestampScrubber.getRobotTimestampAtIndex(i); + Camera getCamera(); - if (robotTimestamp >= startTimestamp && robotTimestamp <= endTimestamp) - { - timestampWriter.print(robotTimestamp); - timestampWriter.print(" "); - timestampWriter.println(pts); - pts++; - } - else if (robotTimestamp > endTimestamp) - { - break; - } - } - - timestampWriter.close(); - } - - public void exportVideo(File selectedFile, long startTimestamp, long endTimestamp, ProgressConsumer progreesConsumer) + default FrameData pollCurrentFrame() { - long startVideoTimestamp = timestampScrubber.getVideoTimestamp(startTimestamp); - long endVideoTimestamp = timestampScrubber.getVideoTimestamp(endTimestamp); - - try - { - VideoConverter.convert(videoFile, selectedFile, startVideoTimestamp, endVideoTimestamp, progreesConsumer); - } - catch (IOException e) - { - e.printStackTrace(); - } - } - - public String getName() - { - return name; - } - - public Camera getCamera() - { - return camera; - } - - public FrameData pollCurrentFrame() - { - return imageBuffer.getCopyForReading(); - } - - public int getCurrentIndex() - { - return timestampScrubber.getCurrentIndex(); - } - - public boolean replacedRobotTimestampsContainsIndex(int index) - { - return timestampScrubber.replacedRobotTimestampIndex[index]; - } - - public static class FrameData - { - public WritableImage frame; - public long queryRobotTimestamp; - public long robotTimestamp; - public long cameraCurrentPTS; - public long demuxerCurrentPTS; - } - - public static class TimestampScrubber - { - private final boolean hasTimebase; - private final boolean interlaced; - private long[] robotTimestamps; - private long[] videoTimestamps; - - private int currentIndex = 0; - private long currentRobotTimestamp = 0; - private long videoTimestamp; - private boolean[] replacedRobotTimestampIndex; - - public TimestampScrubber(File timestampFile, boolean hasTimebase, boolean interlaced) throws IOException - { - this.hasTimebase = hasTimebase; - this.interlaced = interlaced; - - parseTimestampData(timestampFile); - } - - private void parseTimestampData(File timestampFile) throws IOException - { - try (BufferedReader bufferedReader = new BufferedReader(new FileReader(timestampFile))) - { - String line; - if (hasTimebase) - { - if (bufferedReader.readLine() == null) - { - throw new IOException("Cannot read numerator"); - } - - if (bufferedReader.readLine() == null) - { - throw new IOException("Cannot read denumerator"); - } - } - - TLongArrayList robotTimestamps = new TLongArrayList(); - TLongArrayList videoTimestamps = new TLongArrayList(); - - while ((line = bufferedReader.readLine()) != null) - { - String[] stamps = line.split("\\s"); - long robotStamp = Long.parseLong(stamps[0]); - long videoStamp = Long.parseLong(stamps[1]); - - if (interlaced) - { - videoStamp /= 2; - } - - robotTimestamps.add(robotStamp); - videoTimestamps.add(videoStamp); - } - - this.robotTimestamps = robotTimestamps.toArray(); - this.videoTimestamps = videoTimestamps.toArray(); - } - catch (FileNotFoundException e) - { - throw new RuntimeException(e); - } - - checkAndReplaceDuplicates(); - } - - private void checkAndReplaceDuplicates() - { - replacedRobotTimestampIndex = new boolean[robotTimestamps.length]; - int duplicatesAtEndOfFile = getNumberOfDuplicatesAtEndOfFile(); - - for (int currentIndex = 0; currentIndex < robotTimestamps.length - duplicatesAtEndOfFile; ) - { - if (robotTimestamps[currentIndex] != robotTimestamps[currentIndex + 1]) - { - currentIndex++; - continue; - } - - // Keeps track of the duplicated index's so frames border can be adjusted - replacedRobotTimestampIndex[currentIndex + 1] = true; - - int nextNonDuplicateIndex = getNextNonDuplicateIndex(currentIndex); - for (int i = currentIndex; i < nextNonDuplicateIndex; i++) - { - long firstAdjustedTimestamp = (long) EuclidCoreTools.interpolate(robotTimestamps[i], - robotTimestamps[nextNonDuplicateIndex], - (double) 1 / (nextNonDuplicateIndex - i)); - robotTimestamps[i + 1] = firstAdjustedTimestamp; - } - - currentIndex = nextNonDuplicateIndex; - } - } - - private int getNumberOfDuplicatesAtEndOfFile() - { - int duplicatesAtEndOfFile = 1; - - for (int i = robotTimestamps.length - 1; i > 0; i--) - { - if (robotTimestamps[i] == robotTimestamps[i - 1]) - duplicatesAtEndOfFile++; - } - - return duplicatesAtEndOfFile; - } - - private int getNextNonDuplicateIndex(int index) - { - while (index < robotTimestamps.length - 1 && robotTimestamps[index] == robotTimestamps[index + 1]) - index++; - - return index + 1; - } - - /** - * Searches the list of robotTimestamps for the value closest to queryRobotTimestamp and returns that index. Then sets videoTimestamp to - * that index in oder to display the right frame. - * - * @param queryRobotTimestamp the value sent from the robot data in which we want to find the closest robotTimestamp in the instant file. - * @return the videoTimestamp that matches the index of the closest robotTimestamp in our instant file. - */ - public long getVideoTimestamp(long queryRobotTimestamp) - { - currentIndex = searchRobotTimestampsForIndex(queryRobotTimestamp); - videoTimestamp = videoTimestamps[currentIndex]; - currentRobotTimestamp = robotTimestamps[currentIndex]; - - return videoTimestamp; - } - - private int searchRobotTimestampsForIndex(long queryRobotTimestamp) - { - if (queryRobotTimestamp <= robotTimestamps[0]) - return 0; - - if (queryRobotTimestamp >= robotTimestamps[robotTimestamps.length - 1]) - return robotTimestamps.length - 1; - - int index = Arrays.binarySearch(robotTimestamps, queryRobotTimestamp); - - if (index < 0) - { - int nextIndex = -index - 1; // insertionPoint - index = nextIndex; - } - - return index; - } - - public int getCurrentIndex() - { - return currentIndex; - } - - public long getCurrentRobotTimestamp() - { - return currentRobotTimestamp; - } - - public int getRobotTimestampsLength() - { - return robotTimestamps.length; - } - - public long getRobotTimestampAtIndex(int i) - { - return robotTimestamps[i]; - } - - public long[] getRobotTimestampsArray() - { - return robotTimestamps; - } + return null; + }; - public long[] getVideoTimestampsArray() - { - return videoTimestamps; - } + int getCurrentIndex(); - public long getCurrentVideoTimestamp() - { - return videoTimestamp; - } - } + boolean replacedRobotTimestampsContainsIndex(int index); } diff --git a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoViewer.java b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoViewer.java index 3bac81a54..0b1d4d3ac 100644 --- a/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoViewer.java +++ b/scs2-session-visualizer-jfx/src/main/java/us/ihmc/scs2/sessionVisualizer/jfx/session/log/VideoViewer.java @@ -39,7 +39,6 @@ import javafx.util.Duration; import us.ihmc.scs2.session.SessionPropertiesHelper; import us.ihmc.scs2.sessionVisualizer.jfx.SessionVisualizerIOTools; -import us.ihmc.scs2.sessionVisualizer.jfx.session.log.VideoDataReader.FrameData; import us.ihmc.scs2.sessionVisualizer.jfx.tools.JavaFXMissingTools; public class VideoViewer @@ -54,9 +53,9 @@ public class VideoViewer private final StackPane thumbnailContainer = new StackPane(thumbnail); private final ImageView videoView = new ImageView(); private final Label queryRobotTimestampLabel = new Label(); - private final Label demuxerCurrentPTSLabel = new Label(); - private final Label cameraCurrentPTSLabel = new Label(); - private final Label robotTimestampLabel = new Label(); + private final Label currentDemuxerTimestampLabel = new Label(); + private final Label currentVideoTimestampLabel = new Label(); + private final Label currentRobotTimestampLabel = new Label(); private final BooleanProperty updateVideoView = new SimpleBooleanProperty(this, "updateVideoView", false); private final ObjectProperty videoWindowProperty = new SimpleObjectProperty<>(this, "videoWindow", null); @@ -178,14 +177,14 @@ private void setupVideoStatistics(AnchorPane anchorPane) videoStatisticBox.setBorder(generalBorder); VBox videoStatisticLabels = new VBox(new Label("queryRobotTimestamp"), - new Label("robotTimestamp"), - new Label("cameraCurrentPTS"), - new Label("demuxerCurrentPTS")); + new Label("currentRobotTimestamp"), + new Label("currentVideoTimestamp"), + new Label("currentDemuxerTimestamp")); videoStatisticLabels.setBackground(generalBackground); videoStatisticLabels.setBorder(noRightBorder); videoStatisticLabels.setPadding(textInsets); - VBox videoStatistics = new VBox(queryRobotTimestampLabel, robotTimestampLabel, cameraCurrentPTSLabel, demuxerCurrentPTSLabel); + VBox videoStatistics = new VBox(queryRobotTimestampLabel, currentRobotTimestampLabel, currentVideoTimestampLabel, currentDemuxerTimestampLabel); videoStatistics.setBackground(generalBackground); videoStatistics.setBorder(noLeftBorder); videoStatistics.setPadding(textInsets); @@ -230,7 +229,7 @@ public void update() { FrameData currentFrameData = reader.pollCurrentFrame(); - if (currentFrameData == null) + if (currentFrameData.frame == null) return; WritableImage currentFrame = currentFrameData.frame; @@ -244,9 +243,9 @@ public void update() { videoView.setImage(currentFrame); queryRobotTimestampLabel.setText(Long.toString(currentFrameData.queryRobotTimestamp)); - robotTimestampLabel.setText(Long.toString(currentFrameData.robotTimestamp)); - cameraCurrentPTSLabel.setText(Long.toString(currentFrameData.cameraCurrentPTS)); - demuxerCurrentPTSLabel.setText(Long.toString(currentFrameData.demuxerCurrentPTS)); + currentRobotTimestampLabel.setText(Long.toString(currentFrameData.currentRobotTimestamp)); + currentVideoTimestampLabel.setText(Long.toString(currentFrameData.currentVideoTimestamp)); + currentDemuxerTimestampLabel.setText(Long.toString(currentFrameData.currentDemuxerTimestamp)); if (imageViewRootPane.get() != null) { diff --git a/scs2-session-visualizer-jfx/src/test/java/us/ihmc/scs2/sessionVisualizer/jfx/session/TimestampScrubberTest.java b/scs2-session-visualizer-jfx/src/test/java/us/ihmc/scs2/sessionVisualizer/jfx/session/TimestampScrubberTest.java index e8397d822..ab79306d8 100644 --- a/scs2-session-visualizer-jfx/src/test/java/us/ihmc/scs2/sessionVisualizer/jfx/session/TimestampScrubberTest.java +++ b/scs2-session-visualizer-jfx/src/test/java/us/ihmc/scs2/sessionVisualizer/jfx/session/TimestampScrubberTest.java @@ -4,7 +4,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import us.ihmc.scs2.sessionVisualizer.jfx.session.log.VideoDataReader; +import us.ihmc.scs2.sessionVisualizer.jfx.session.log.TimestampScrubber; import java.io.File; import java.io.IOException; @@ -15,7 +15,7 @@ public class TimestampScrubberTest { - private static VideoDataReader.TimestampScrubber scrubber; + private static TimestampScrubber scrubber; private static long[] robotTimestamps; private static long[] videoTimestamps; @@ -29,7 +29,7 @@ public static void loadFileTimestamps() throws URISyntaxException, IOException // File timestampFile = new File("//10.7.4.48/LogData/Nadia/20230427_NadiaRunning/20230427_183903_NadiaRunningTallerCompleteFailRobotBreakMaybe/NadiaPoleNorth_Timestamps.dat"); File timestampFile = new File(Objects.requireNonNull(TimestampScrubberTest.class.getClassLoader().getResource("sessionLogs/Capture.dat")).toURI()); - scrubber = new VideoDataReader.TimestampScrubber(timestampFile, true, false); + scrubber = new TimestampScrubber(timestampFile, true, false); // Need to have one video in the log or this will fail robotTimestamps = scrubber.getRobotTimestampsArray(); @@ -60,7 +60,7 @@ public void testChronologicallyIncreasingRobotTimestamps() System.out.println(currentRobotTimestamp + " -- " + previousRobotTimestamp); Assertions.assertTrue(currentRobotTimestamp > previousRobotTimestamp, - "Cureent: " + currentRobotTimestamp + " and Previous: " + previousRobotTimestamp + " at Index: " + i); + "Current: " + currentRobotTimestamp + " and Previous: " + previousRobotTimestamp + " at Index: " + i); } } @@ -128,7 +128,7 @@ public void testGoingThroughRobotTimestampsInOrder() // Go through the robot timestamps in order and see if we get the desired video timestamp for (int i = 0; i < robotTimestamps.length - duplicatesAtEndOfFile; i++) { - long currentVideoTimestamp = scrubber.getVideoTimestamp(robotTimestamps[i]); + long currentVideoTimestamp = scrubber.getVideoTimestampFromRobotTimestamp(robotTimestamps[i]); assertEquals(currentVideoTimestamp, videoTimestamps[i]); } } @@ -138,7 +138,7 @@ public void testGoingThroughRobotTimestampsBackwards() { for (int i = robotTimestamps.length - duplicatesAtEndOfFile - 1; i > 0; i--) { - scrubber.getVideoTimestamp(robotTimestamps[i]); + scrubber.getVideoTimestampFromRobotTimestamp(robotTimestamps[i]); assertEquals(scrubber.getCurrentVideoTimestamp(), videoTimestamps[i]); } } @@ -149,7 +149,7 @@ public void testGoingThroughRobotTimestampsEveryOther() // Go through the robot timestamps by +=2, so we skip every other frame and see if we get the desired video timestamp for (int i = 0; i < robotTimestamps.length - duplicatesAtEndOfFile; i += 2) { - scrubber.getVideoTimestamp(robotTimestamps[i]); + scrubber.getVideoTimestampFromRobotTimestamp(robotTimestamps[i]); assertEquals(scrubber.getCurrentVideoTimestamp(), videoTimestamps[i], "For loop index: " + i); } } @@ -159,13 +159,13 @@ public void testGettingRandomTimestamp() { // Test grabbing random robot timestamps and checking to make sure we get the correct video timestamp // These robot timestamps need to be unique or the binary search will fail to get the correct video timestamp - scrubber.getVideoTimestamp(robotTimestamps[26]); + scrubber.getVideoTimestampFromRobotTimestamp(robotTimestamps[26]); assertEquals(scrubber.getCurrentVideoTimestamp(), videoTimestamps[26]); - scrubber.getVideoTimestamp(robotTimestamps[40]); + scrubber.getVideoTimestampFromRobotTimestamp(robotTimestamps[40]); assertEquals(scrubber.getCurrentVideoTimestamp(), videoTimestamps[40]); - scrubber.getVideoTimestamp(robotTimestamps[34]); + scrubber.getVideoTimestampFromRobotTimestamp(robotTimestamps[34]); assertEquals(scrubber.getCurrentVideoTimestamp(), videoTimestamps[34]); } @@ -173,7 +173,7 @@ public void testGettingRandomTimestamp() public void testSearchRobotTimestampsForIndex() { // Sets the currentIndex to the end of the videoTimestamps to test edge case - scrubber.getVideoTimestamp(Long.MAX_VALUE); + scrubber.getVideoTimestampFromRobotTimestamp(Long.MAX_VALUE); int endOfArray = scrubber.getCurrentIndex(); assertEquals(robotTimestamps.length - 1, endOfArray); @@ -184,7 +184,7 @@ public void testFileNotFoundException() { File badName = new File("This_is_a_bad_file_name_lol"); - Throwable thrown = assertThrows(RuntimeException.class, () -> new VideoDataReader.TimestampScrubber(badName, true, false)); + Throwable thrown = assertThrows(RuntimeException.class, () -> new TimestampScrubber(badName, true, false)); String messageException = thrown.getMessage().substring(0, 58); assertEquals("java.io.FileNotFoundException: " + badName, messageException); @@ -194,7 +194,7 @@ public void testFileNotFoundException() public void testInsertionPointWhenSearching() { // Trying to find a robotTimestamp that doesn't exist will cause an insertionPoint - scrubber.getVideoTimestamp(robotTimestamps[0] + 1); + scrubber.getVideoTimestampFromRobotTimestamp(robotTimestamps[0] + 1); int insertionPointIndex = scrubber.getCurrentIndex(); assertEquals(1, insertionPointIndex);