diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/SubtitleArguments.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/SubtitleArguments.kt index 84d6670f..2ccd43e0 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/SubtitleArguments.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/SubtitleArguments.kt @@ -2,6 +2,7 @@ package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.streams import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.SubtitleArgumentsDto import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.SubtitleStream +import kotlin.math.sqrt class SubtitleArguments(val subtitleStreams: List) { /** @@ -18,18 +19,27 @@ class SubtitleArguments(val subtitleStreams: List) { } private fun SubtitleStream.isCC(): Boolean { + if ((this.disposition?.captions ?: 0) > 0) { + return true + } val title = this.tags.title?.lowercase() ?: return false val keywords = listOf("cc", "closed caption") return keywords.any { title.contains(it) } } private fun SubtitleStream.isSHD(): Boolean { + if ((this.disposition?.hearing_impaired ?: 0) > 0) { + return true + } val title = this.tags.title?.lowercase() ?: return false val keywords = listOf("shd", "hh", "Hard-of-Hearing", "Hard of Hearing") return keywords.any { title.contains(it) } } private fun SubtitleStream.isSignOrSong(): Boolean { + if ((this.disposition?.lyrics ?: 0) > 0) { + return true + } val title = this.tags.title?.lowercase() ?: return false val keywords = listOf("song", "songs", "sign", "signs") return keywords.any { title.contains(it) } @@ -48,16 +58,19 @@ class SubtitleArguments(val subtitleStreams: List) { fun getSubtitleArguments(): List { val acceptable = subtitleStreams.filter { !it.isSignOrSong() } val codecFiltered = acceptable.filter { getFormatToCodec(it.codec_name) != null } - val mappedToType = - codecFiltered.map { getSubtitleType(it) to it }.filter { it.first in SubtitleType.entries } - .groupBy { it.second.tags.language ?: "eng" } - .mapValues { entry -> - val languageStreams = entry.value - val sortedStreams = languageStreams.sortedBy { SubtitleType.entries.indexOf(it.first) } - sortedStreams.firstOrNull()?.second - }.mapNotNull { it.value } - - return mappedToType.mapNotNull { stream -> + + val languageGrouped = codecFiltered.groupBy { it.tags.language ?: "eng" } + + val streamsToExtract = languageGrouped.mapNotNull { item -> + val types = item.value.map { getSubtitleType(it) } + if (types.none { t -> t == SubtitleType.DEFAULT } || types.count { t -> t == SubtitleType.DEFAULT} > 1) { + excludeLowFrameCount(item.value).sortedBy { it.tags.NUMBER_OF_FRAMES }.firstOrNull() + } else { + item.value.minByOrNull { s -> getSubtitleType(s) } + } + } + + return streamsToExtract.mapNotNull { stream -> getFormatToCodec(stream.codec_name)?.let { format -> SubtitleArgumentsDto( index = subtitleStreams.indexOf(stream), @@ -69,6 +82,22 @@ class SubtitleArguments(val subtitleStreams: List) { } + fun excludeLowFrameCount(streams: List): List { + val usable = streams.filter { (it.tags.NUMBER_OF_FRAMES ?: 0) > 0 } + val mean = usable.mapNotNull { it.tags.NUMBER_OF_FRAMES }.average() + val variance = usable.map { (it.tags.NUMBER_OF_FRAMES!! - mean) * (it.tags.NUMBER_OF_FRAMES!! - mean) }.average() + val standardDeviation = sqrt(variance) + + // Definer intervallet for "normale" rammer: mean ± 2 * standard deviation + val lowerBound = mean - 2 * standardDeviation + val upperBound = mean + 2 * standardDeviation + + return usable.filter { + val frameCount = it.tags.NUMBER_OF_FRAMES ?: 0 + frameCount.toDouble() in standardDeviation..upperBound + } + } + fun getFormatToCodec(codecName: String): String? { return when (codecName) { "ass" -> "ass" diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/SubtitleArgumentsTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/SubtitleArgumentsTest.kt new file mode 100644 index 00000000..2928e5ff --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/SubtitleArgumentsTest.kt @@ -0,0 +1,265 @@ +package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.streams + +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.SubtitleStream +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class SubtitleArgumentsTest { + val type = object : TypeToken>() {}.type + + @Test + fun validate1() { + val data = Gson().fromJson>(multipleSubtitleStreamsWithSameLanguage, type) + assertThat(data.all { it is SubtitleStream }).isTrue() + assertThat(data).isNotNull() + } + + @Test + fun validate2() { + val data = Gson().fromJson>(multipleSubtitleStreamsWithSameLanguage, type) + val args = SubtitleArguments(data) + val selectable = args.excludeLowFrameCount(data) + assertThat(selectable).hasSize(2) + assertThat(selectable.find { it.index == 4 }) + assertThat(selectable.find { it.index == 5 }) + } + + @Test + fun validate3() { + val data = Gson().fromJson>(multipleSubtitleStreamsWithSameLanguage, type) + val args = SubtitleArguments(data).getSubtitleArguments() + assertThat(args).hasSize(1) + assertThat(args.firstOrNull()?.index).isEqualTo(1) + } + + @Test + fun validate3_2() { + val data = Gson().fromJson>(multipleSubtitleStreamsWithSameLanguageWithDisposition, type) + val args = SubtitleArguments(data).getSubtitleArguments() + assertThat(args).hasSize(1) + assertThat(args.firstOrNull()?.index).isEqualTo(1) + } + + + val multipleSubtitleStreamsWithSameLanguage = """ + [{ + "index": 3, + "codec_name": "ass", + "codec_long_name": "ASS (Advanced SSA) subtitle", + "codec_type": "subtitle", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "r_frame_rate": "0/0", + "avg_frame_rate": "0/0", + "time_base": "1/1000", + "start_pts": 0, + "start_time": "0.000000", + "duration_ts": 1437083, + "duration": "1437.083000", + "extradata_size": 1967, + "tags": { + "language": "eng", + "title": "Forced", + "BPS": "5", + "DURATION": "00:21:42.640000000", + "NUMBER_OF_FRAMES": "14", + "NUMBER_OF_BYTES": "835", + "_STATISTICS_WRITING_APP": "mkvmerge v69.0.0 ('Day And Age') 64-bit", + "_STATISTICS_WRITING_DATE_UTC": "2024-10-04 08:12:59", + "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" + } + }, + { + "index": 4, + "codec_name": "ass", + "codec_long_name": "ASS (Advanced SSA) subtitle", + "codec_type": "subtitle", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "r_frame_rate": "0/0", + "avg_frame_rate": "0/0", + "time_base": "1/1000", + "start_pts": 0, + "start_time": "0.000000", + "duration_ts": 1437083, + "duration": "1437.083000", + "extradata_size": 1791, + "tags": { + "language": "eng", + "BPS": "129", + "DURATION": "00:22:26.550000000", + "NUMBER_OF_FRAMES": "356", + "NUMBER_OF_BYTES": "21787", + "_STATISTICS_WRITING_APP": "mkvmerge v69.0.0 ('Day And Age') 64-bit", + "_STATISTICS_WRITING_DATE_UTC": "2024-10-04 08:12:59", + "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" + } + }, + { + "index": 5, + "codec_name": "subrip", + "codec_long_name": "SubRip subtitle", + "codec_type": "subtitle", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "r_frame_rate": "0/0", + "avg_frame_rate": "0/0", + "time_base": "1/1000", + "start_pts": 790, + "start_time": "0.790000", + "tags": { + "language": "eng", + "title": "CC", + "BPS": "83", + "DURATION": "00:23:56.060000000", + "NUMBER_OF_FRAMES": "495", + "NUMBER_OF_BYTES": "14954", + "_STATISTICS_WRITING_APP": "mkvmerge v69.0.0 ('Day And Age') 64-bit", + "_STATISTICS_WRITING_DATE_UTC": "2024-10-04 08:12:59", + "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" + } + }] + """.trimIndent() + + val multipleSubtitleStreamsWithSameLanguageWithDisposition = """ + [{ + "index": 3, + "codec_name": "ass", + "codec_long_name": "ASS (Advanced SSA) subtitle", + "codec_type": "subtitle", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "r_frame_rate": "0/0", + "avg_frame_rate": "0/0", + "time_base": "1/1000", + "start_pts": 0, + "start_time": "0.000000", + "duration_ts": 1437083, + "duration": "1437.083000", + "extradata_size": 1967, + "disposition": { + "default": 1, + "dub": 0, + "original": 0, + "comment": 0, + "lyrics": 0, + "karaoke": 0, + "forced": 1, + "hearing_impaired": 0, + "visual_impaired": 0, + "clean_effects": 0, + "attached_pic": 0, + "timed_thumbnails": 0, + "non_diegetic": 0, + "captions": 0, + "descriptions": 0, + "metadata": 0, + "dependent": 0, + "still_image": 0 + }, + "tags": { + "language": "eng", + "title": "Forced", + "BPS": "5", + "DURATION": "00:21:42.640000000", + "NUMBER_OF_FRAMES": "14", + "NUMBER_OF_BYTES": "835", + "_STATISTICS_WRITING_APP": "mkvmerge v69.0.0 ('Day And Age') 64-bit", + "_STATISTICS_WRITING_DATE_UTC": "2024-10-04 08:12:59", + "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" + } + }, + { + "index": 4, + "codec_name": "ass", + "codec_long_name": "ASS (Advanced SSA) subtitle", + "codec_type": "subtitle", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "r_frame_rate": "0/0", + "avg_frame_rate": "0/0", + "time_base": "1/1000", + "start_pts": 0, + "start_time": "0.000000", + "duration_ts": 1437083, + "duration": "1437.083000", + "extradata_size": 1791, + "disposition": { + "default": 0, + "dub": 0, + "original": 0, + "comment": 0, + "lyrics": 0, + "karaoke": 0, + "forced": 0, + "hearing_impaired": 0, + "visual_impaired": 0, + "clean_effects": 0, + "attached_pic": 0, + "timed_thumbnails": 0, + "non_diegetic": 0, + "captions": 0, + "descriptions": 0, + "metadata": 0, + "dependent": 0, + "still_image": 0 + }, + "tags": { + "language": "eng", + "BPS": "129", + "DURATION": "00:22:26.550000000", + "NUMBER_OF_FRAMES": "356", + "NUMBER_OF_BYTES": "21787", + "_STATISTICS_WRITING_APP": "mkvmerge v69.0.0 ('Day And Age') 64-bit", + "_STATISTICS_WRITING_DATE_UTC": "2024-10-04 08:12:59", + "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" + } + }, + { + "index": 5, + "codec_name": "subrip", + "codec_long_name": "SubRip subtitle", + "codec_type": "subtitle", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "r_frame_rate": "0/0", + "avg_frame_rate": "0/0", + "time_base": "1/1000", + "start_pts": 790, + "start_time": "0.790000", + "disposition": { + "default": 0, + "dub": 0, + "original": 0, + "comment": 0, + "lyrics": 0, + "karaoke": 0, + "forced": 0, + "hearing_impaired": 0, + "visual_impaired": 0, + "clean_effects": 0, + "attached_pic": 0, + "timed_thumbnails": 0, + "non_diegetic": 0, + "captions": 0, + "descriptions": 0, + "metadata": 0, + "dependent": 0, + "still_image": 0 + }, + "tags": { + "language": "eng", + "title": "CC", + "BPS": "83", + "DURATION": "00:23:56.060000000", + "NUMBER_OF_FRAMES": "495", + "NUMBER_OF_BYTES": "14954", + "_STATISTICS_WRITING_APP": "mkvmerge v69.0.0 ('Day And Age') 64-bit", + "_STATISTICS_WRITING_DATE_UTC": "2024-10-04 08:12:59", + "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" + } + }] + """.trimIndent() +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/MediaStreams.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/MediaStreams.kt index 71970dcc..80ee082e 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/MediaStreams.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/MediaStreams.kt @@ -24,7 +24,7 @@ sealed class Stream( @Transient open val start_time: String, @Transient open val duration_ts: Long? = null, @Transient open val duration: String? = null, - @Transient open val disposition: Disposition, + @Transient open val disposition: Disposition? = null, @Transient open val tags: Tags ) @@ -133,7 +133,7 @@ data class SubtitleStream( override val start_time: String, override val duration: String?, override val duration_ts: Long?, - override val disposition: Disposition, + override val disposition: Disposition? = null, override val tags: Tags, val subtitle_tags: SubtitleTags ) : Stream( @@ -163,6 +163,7 @@ data class Disposition( val karaoke: Int, val forced: Int, val hearing_impaired: Int, + val captions: Int, val visual_impaired: Int, val clean_effects: Int, val attached_pic: Int, @@ -173,7 +174,7 @@ data class Tags( val title: String?, val BPS: String?, val DURATION: String?, - val NUMBER_OF_FRAMES: String?, + val NUMBER_OF_FRAMES: Int? = 0, val NUMBER_OF_BYTES: String?, val _STATISTICS_WRITING_APP: String?, val _STATISTICS_WRITING_DATE_UTC: String?,