From 50080af39cc9b076c3bbc7e7281544ec11351a10 Mon Sep 17 00:00:00 2001 From: Ben Henning Date: Mon, 27 May 2024 15:38:09 -0700 Subject: [PATCH] Add logic for compatibility-based downloads. This contains the structures and algorithms necessary for downloading complete Oppia topics based on specific app compatibility (and internal consistency of the lesson structures). --- .../android/scripts/gae/compat/BUILD.bazel | 30 + .../scripts/gae/compat/CompleteExploration.kt | 13 + .../scripts/gae/compat/CompleteTopicPack.kt | 17 + .../compat/StructureCompatibilityChecker.kt | 668 +++++++++++ .../gae/compat/SubtitledHtmlCollector.kt | 207 ++++ .../scripts/gae/compat/TopicPackRepository.kt | 1012 +++++++++++++++++ .../oppia/android/scripts/proto/BUILD.bazel | 11 + .../proto/download_list_versions.proto | 39 + 8 files changed, 1997 insertions(+) create mode 100644 scripts/src/java/org/oppia/android/scripts/gae/compat/BUILD.bazel create mode 100644 scripts/src/java/org/oppia/android/scripts/gae/compat/CompleteExploration.kt create mode 100644 scripts/src/java/org/oppia/android/scripts/gae/compat/CompleteTopicPack.kt create mode 100644 scripts/src/java/org/oppia/android/scripts/gae/compat/StructureCompatibilityChecker.kt create mode 100644 scripts/src/java/org/oppia/android/scripts/gae/compat/SubtitledHtmlCollector.kt create mode 100644 scripts/src/java/org/oppia/android/scripts/gae/compat/TopicPackRepository.kt create mode 100644 scripts/src/java/org/oppia/android/scripts/proto/download_list_versions.proto diff --git a/scripts/src/java/org/oppia/android/scripts/gae/compat/BUILD.bazel b/scripts/src/java/org/oppia/android/scripts/gae/compat/BUILD.bazel new file mode 100644 index 00000000000..6b40ada4f54 --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/gae/compat/BUILD.bazel @@ -0,0 +1,30 @@ +""" +Library for providing compatibility computations and support when determining a compatible closure +of valid sub-structures that compose a single topic convertible by the asset pipeline (and thus +playable by the app). +""" + +load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") + +kt_jvm_library( + name = "compat", + testonly = True, + srcs = [ + "CompleteExploration.kt", + "CompleteTopicPack.kt", + "StructureCompatibilityChecker.kt", + "SubtitledHtmlCollector.kt", + "TopicPackRepository.kt", + ], + visibility = [ + "//scripts/src/java/org/oppia/android/scripts/gae:__subpackages__", + ], + deps = [ + "//scripts/src/java/org/oppia/android/scripts/gae/json:api", + "//scripts/src/java/org/oppia/android/scripts/gae/json:model", + "//scripts/src/java/org/oppia/android/scripts/gae/proto:localization_tracker", + "//scripts/src/java/org/oppia/android/scripts/proto:download_list_versions_java_proto", + "//third_party:oppia_proto_api_java_protos", + "//third_party:org_jetbrains_kotlinx_kotlinx-coroutines-core", + ], +) diff --git a/scripts/src/java/org/oppia/android/scripts/gae/compat/CompleteExploration.kt b/scripts/src/java/org/oppia/android/scripts/gae/compat/CompleteExploration.kt new file mode 100644 index 00000000000..613531783ee --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/gae/compat/CompleteExploration.kt @@ -0,0 +1,13 @@ +package org.oppia.android.scripts.gae.compat + +import org.oppia.android.scripts.gae.json.GaeEntityTranslations +import org.oppia.android.scripts.gae.json.GaeExploration +import org.oppia.android.scripts.gae.json.VersionedStructure +import org.oppia.proto.v1.structure.LanguageType + +data class CompleteExploration( + val exploration: GaeExploration, + val translations: Map> +) { + val version: Int = exploration.version +} diff --git a/scripts/src/java/org/oppia/android/scripts/gae/compat/CompleteTopicPack.kt b/scripts/src/java/org/oppia/android/scripts/gae/compat/CompleteTopicPack.kt new file mode 100644 index 00000000000..dccf22111a7 --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/gae/compat/CompleteTopicPack.kt @@ -0,0 +1,17 @@ +package org.oppia.android.scripts.gae.compat + +import org.oppia.android.scripts.gae.json.GaeSkill +import org.oppia.android.scripts.gae.json.GaeStory +import org.oppia.android.scripts.gae.json.GaeSubtopicPage +import org.oppia.android.scripts.gae.json.GaeTopic +import org.oppia.proto.v1.structure.LanguageType +import org.oppia.proto.v1.structure.SubtopicPageIdDto + +data class CompleteTopicPack( + val topic: GaeTopic, + val subtopicPages: Map, + val stories: Map, + val explorations: Map, + val referencedSkills: Map, + val defaultLanguage: LanguageType +) diff --git a/scripts/src/java/org/oppia/android/scripts/gae/compat/StructureCompatibilityChecker.kt b/scripts/src/java/org/oppia/android/scripts/gae/compat/StructureCompatibilityChecker.kt new file mode 100644 index 00000000000..595f10ea51b --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/gae/compat/StructureCompatibilityChecker.kt @@ -0,0 +1,668 @@ +package org.oppia.android.scripts.gae.compat + +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.AudioVoiceoverHasInvalidAudioFormat +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.HtmlInTitleOrDescription +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.HtmlUnexpectedlyInUnicodeContent +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.MissingRequiredXlationLangForContentTranslation +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.MissingRequiredXlationLangForTitleOrDescFromWeb +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.StateHasInvalidInteractionId +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.StateSchemaVersionTooNew +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.TextHasInvalidTags +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.TextReferencesInvalidImageFormat +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.TextUsesImageTagWithMissingFilePath +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.ThumbnailHasInvalidColor +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.ThumbnailHasInvalidImageFormat +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.TopicHasNoKnownDependencies +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.TranslatedTextHasInvalidTags +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure.UnsupportedDefaultLanguageCode +import org.oppia.android.scripts.gae.compat.SubtitledHtmlCollector.SubtitledText +import org.oppia.android.scripts.gae.json.GaeAnswerGroup +import org.oppia.android.scripts.gae.json.GaeCustomizationArgValue +import org.oppia.android.scripts.gae.json.GaeEntityTranslations +import org.oppia.android.scripts.gae.json.GaeExploration +import org.oppia.android.scripts.gae.json.GaeHint +import org.oppia.android.scripts.gae.json.GaeInteractionCustomizationArgsMap +import org.oppia.android.scripts.gae.json.GaeInteractionInstance +import org.oppia.android.scripts.gae.json.GaeOutcome +import org.oppia.android.scripts.gae.json.GaeRecordedVoiceovers +import org.oppia.android.scripts.gae.json.GaeSkill +import org.oppia.android.scripts.gae.json.GaeSkillContents +import org.oppia.android.scripts.gae.json.GaeSolution +import org.oppia.android.scripts.gae.json.GaeState +import org.oppia.android.scripts.gae.json.GaeStory +import org.oppia.android.scripts.gae.json.GaeStoryNode +import org.oppia.android.scripts.gae.json.GaeSubtitledHtml +import org.oppia.android.scripts.gae.json.GaeSubtitledUnicode +import org.oppia.android.scripts.gae.json.GaeSubtopic +import org.oppia.android.scripts.gae.json.GaeSubtopicPage +import org.oppia.android.scripts.gae.json.GaeSubtopicPageContents +import org.oppia.android.scripts.gae.json.GaeTopic +import org.oppia.android.scripts.gae.json.GaeTranslatedContent +import org.oppia.android.scripts.gae.json.GaeWorkedExample +import org.oppia.android.scripts.gae.json.GaeWrittenTranslation +import org.oppia.android.scripts.gae.json.GaeWrittenTranslations +import org.oppia.android.scripts.gae.json.VersionedStructure +import org.oppia.android.scripts.gae.proto.LocalizationTracker +import org.oppia.android.scripts.gae.proto.LocalizationTracker.Companion.parseColorRgb +import org.oppia.android.scripts.gae.proto.LocalizationTracker.Companion.resolveLanguageCode +import org.oppia.android.scripts.gae.proto.LocalizationTracker.ContainerId +import org.oppia.android.scripts.gae.proto.LocalizationTracker.ContentContext.DESCRIPTION +import org.oppia.android.scripts.gae.proto.LocalizationTracker.ContentContext.TITLE +import org.oppia.android.scripts.proto.DownloadListVersions +import org.oppia.proto.v1.structure.LanguageType + +// TODO: Check SVG compatibility? +// TODO: Check image validity? +// TODO: Check audio validity? +// TODO: Check HTML parsability? +// TODO: Check math exp parsability? + +class StructureCompatibilityChecker( + private val constraints: CompatibilityConstraints, + private val localizationTracker: LocalizationTracker, + private val subtitledHtmlCollector: SubtitledHtmlCollector +) { + fun isTopicItselfCompatible(gaeTopic: GaeTopic): CompatibilityResult { + val containerId = ContainerId.createFrom(gaeTopic) + val defaultLanguage = gaeTopic.languageCode.resolveLanguageCode() + return CompatibilityResult.createFrom { + gaeTopic.id.checkIsValidTopicId(containerId) + + gaeTopic.name.checkTitleOrDescTextForHtml(containerId) + + gaeTopic.description.checkTitleOrDescTextForHtml(containerId) + + gaeTopic.thumbnailFilename.checkThumbnailFilename(containerId) + + gaeTopic.thumbnailBgColor.checkBackgroundHexColor(containerId) + + gaeTopic.languageCode.checkDefaultLanguageCode(containerId) + + checkHasRequiredWebTranslationsFor(containerId, defaultLanguage, TITLE, DESCRIPTION) + + gaeTopic.subtopics.flatMap { checkSubtopicCompatibility(gaeTopic.id, it, defaultLanguage) } + } + } + + private fun checkSubtopicCompatibility( + topicId: String, + gaeSubtopic: GaeSubtopic, + defaultLanguage: LanguageType + ): List { + val containerId = ContainerId.createFrom(topicId, gaeSubtopic) + return gaeSubtopic.title.checkTitleOrDescTextForHtml(containerId) + + gaeSubtopic.thumbnailFilename.checkThumbnailFilename(containerId) + + gaeSubtopic.thumbnailBgColor.checkBackgroundHexColor(containerId) + + checkHasRequiredWebTranslationsFor(containerId, defaultLanguage, TITLE) + } + + fun isStoryItselfCompatible(gaeStory: GaeStory): CompatibilityResult { + val containerId = ContainerId.createFrom(gaeStory) + val defaultLanguage = gaeStory.languageCode.resolveLanguageCode() + return CompatibilityResult.createFrom { + gaeStory.title.checkTitleOrDescTextForHtml(containerId) + + gaeStory.description.checkTitleOrDescTextForHtml(containerId) + + gaeStory.thumbnailFilename.checkThumbnailFilename(containerId) + + gaeStory.thumbnailBgColor.checkBackgroundHexColor(containerId) + + gaeStory.languageCode.checkDefaultLanguageCode(containerId) + + checkHasRequiredWebTranslationsFor(containerId, defaultLanguage, TITLE, DESCRIPTION) + + gaeStory.storyContents.nodes.flatMap { + checkStoryNodeCompatibility(gaeStory, it, defaultLanguage, containerId) + } + } + } + + private fun checkStoryNodeCompatibility( + gaeStory: GaeStory, + gaeStoryNode: GaeStoryNode, + defaultLanguage: LanguageType, + storyContainerId: ContainerId + ): List { + return ContainerId.createFrom(gaeStory, gaeStoryNode)?.let { containerId -> + return gaeStoryNode.title.checkTitleOrDescTextForHtml(containerId) + + gaeStoryNode.description.checkTitleOrDescTextForHtml(containerId) + + gaeStoryNode.thumbnailFilename.checkThumbnailFilename(containerId) + + gaeStoryNode.thumbnailBgColor.checkBackgroundHexColor(containerId) + + checkHasRequiredWebTranslationsFor(containerId, defaultLanguage, TITLE, DESCRIPTION) + } ?: listOf(CompatibilityFailure.StoryIsMissingExplorationId(gaeStory.id, storyContainerId)) + } + + fun isSubtopicPageItselfCompatible( + gaeSubtopicPage: GaeSubtopicPage, + correspondingGaeSubtopic: GaeSubtopic + ): CompatibilityResult { + val containerId = ContainerId.createFrom(gaeSubtopicPage, correspondingGaeSubtopic) + val expectedTranslatedContentIds = + subtitledHtmlCollector.collectSubtitles(gaeSubtopicPage).collectContentIds() + val defaultLanguage = gaeSubtopicPage.languageCode.resolveLanguageCode() + return CompatibilityResult.createFrom { + checkSubtopicPageContentsCompatibility( + containerId, gaeSubtopicPage.pageContents, expectedTranslatedContentIds, defaultLanguage + ) + gaeSubtopicPage.languageCode.checkDefaultLanguageCode(containerId) + } + } + + private fun checkSubtopicPageContentsCompatibility( + origin: ContainerId, + subtopicPageContents: GaeSubtopicPageContents, + expectedTranslatedContentIds: Set, + defaultLanguage: LanguageType + ): List { + return subtopicPageContents.subtitledHtml.checkHasValidHtml(origin) + + checkWrittenTranslationsCompatibility( + origin, + subtopicPageContents.writtenTranslations, + expectedTranslatedContentIds, + defaultLanguage + ) + checkRecordedVoiceoversCompatibility(origin, subtopicPageContents.recordedVoiceovers) + } + + fun isExplorationItselfCompatible(completeExploration: CompleteExploration): CompatibilityResult { + val containerId = ContainerId.createFrom(completeExploration.exploration) + val expectedTranslatedContentIds = + subtitledHtmlCollector.collectSubtitles(completeExploration).collectContentIds() + val defaultLanguage = completeExploration.exploration.languageCode.resolveLanguageCode() + return CompatibilityResult.createFrom { + checkAllEntityTranslationsCompatibility( + containerId, completeExploration.translations, expectedTranslatedContentIds, defaultLanguage + ) + checkExplorationCompatibility( + containerId, completeExploration.exploration, defaultLanguage + ) + } + } + + private fun checkExplorationCompatibility( + origin: ContainerId, + gaeExploration: GaeExploration, + defaultLanguage: LanguageType + ): List { + return gaeExploration.title.checkTitleOrDescTextForHtml(origin) + + gaeExploration.languageCode.checkDefaultLanguageCode(origin) + + checkHasRequiredWebTranslationsFor(origin, defaultLanguage, TITLE) + + gaeExploration.statesSchemaVersion.checkIsValidStateSchemaVersion(origin) + + gaeExploration.states.flatMap { (stateName, state) -> + checkStateCompatibility(origin, stateName, state) + } + } + + private fun checkStateCompatibility( + origin: ContainerId, + stateName: String, + gaeState: GaeState + ): List { + return gaeState.content.checkHasValidHtml(origin) + + checkInteractionInstanceCompatibility(origin, stateName, gaeState.interaction) + + checkRecordedVoiceoversCompatibility(origin, gaeState.recordedVoiceovers) + } + + private fun checkInteractionInstanceCompatibility( + origin: ContainerId, + stateName: String, + gaeInteractionInstance: GaeInteractionInstance + ): List { + return gaeInteractionInstance.id.checkIsValidInteractionId(stateName, origin) + + checkInteractionCustArgsCompatibility(origin, gaeInteractionInstance.customizationArgs) + + checkAnswerGroupsCompatibility(origin, gaeInteractionInstance.answerGroups) + + checkOutcomeCompatibility(origin, gaeInteractionInstance.defaultOutcome) + + checkHintsCompatibility(origin, gaeInteractionInstance.hints) + + checkSolutionCompatibility(origin, gaeInteractionInstance.solution) + } + + private fun checkInteractionCustArgsCompatibility( + origin: ContainerId, + gaeCustomizationArgs: GaeInteractionCustomizationArgsMap + ): List { + return gaeCustomizationArgs.customizationArgs.values.flatMap { argValue -> + when (argValue) { + is GaeCustomizationArgValue.GaeImageWithRegions, is GaeCustomizationArgValue.SingleBoolean, + is GaeCustomizationArgValue.SingleInteger, is GaeCustomizationArgValue.StringList -> + emptyList() + is GaeCustomizationArgValue.SubtitledUnicode -> argValue.value.checkHasNoValidHtml(origin) + is GaeCustomizationArgValue.SubtitledTextList -> + argValue.value.flatMap { it.checkHasValidHtml(origin) } + } + } + } + + private fun checkAnswerGroupsCompatibility( + origin: ContainerId, + gaeAnswerGroups: List + ) = gaeAnswerGroups.flatMap { checkOutcomeCompatibility(origin, it.outcome) } + + private fun checkOutcomeCompatibility(origin: ContainerId, gaeOutcome: GaeOutcome?) = + gaeOutcome?.feedback?.checkHasValidHtml(origin) ?: emptyList() + + private fun checkHintsCompatibility(origin: ContainerId, gaeHints: List) = + gaeHints.flatMap { it.hintContent.checkHasValidHtml(origin) } + + private fun checkSolutionCompatibility(origin: ContainerId, gaeSolution: GaeSolution?) = + gaeSolution?.explanation?.checkHasValidHtml(origin) ?: emptyList() + + fun isSkillItselfCompatible(gaeSkill: GaeSkill): CompatibilityResult { + val containerId = ContainerId.createFrom(gaeSkill) + val contentIdsToXlate = subtitledHtmlCollector.collectSubtitles(gaeSkill).collectContentIds() + val defaultLanguage = gaeSkill.languageCode.resolveLanguageCode() + return CompatibilityResult.createFrom { + // Note that Oppia wbe translations don't include skill descriptions, so they aren't checked. + gaeSkill.description.checkTitleOrDescTextForHtml(containerId) + + gaeSkill.languageCode.checkDefaultLanguageCode(containerId) + + checkSkillContentsCompatibility( + containerId, gaeSkill.skillContents, contentIdsToXlate, defaultLanguage + ) + } + } + + private fun checkSkillContentsCompatibility( + origin: ContainerId, + gaeSkillContents: GaeSkillContents, + expectedTranslatedContentIds: Set, + defaultLanguage: LanguageType + ): List { + return gaeSkillContents.explanation.checkHasValidHtml(origin) + + gaeSkillContents.workedExamples.flatMap { checkWorkedExampleCompatibility(origin, it) } + + checkWrittenTranslationsCompatibility( + origin, gaeSkillContents.writtenTranslations, expectedTranslatedContentIds, defaultLanguage + ) + checkRecordedVoiceoversCompatibility(origin, gaeSkillContents.recordedVoiceovers) + } + + private fun checkWorkedExampleCompatibility( + origin: ContainerId, + gaeWorkedExample: GaeWorkedExample + ): List { + return gaeWorkedExample.question.checkHasValidHtml(origin) + + gaeWorkedExample.explanation.checkHasValidHtml(origin) + } + + private fun checkWrittenTranslationsCompatibility( + origin: ContainerId, + gaeWrittenTranslations: GaeWrittenTranslations, + expectedContentIds: Set, + defaultLanguage: LanguageType + ): List { + val allExpectedContentIds = expectedContentIds + gaeWrittenTranslations.translationsMapping.keys + val contentIdLanguages = allExpectedContentIds.associateWith { + gaeWrittenTranslations.translationsMapping[it]?.keys ?: setOf() + } + return gaeWrittenTranslations.translationsMapping.flatMap { (contentId, contentMap) -> + contentMap.entries.flatMap { (languageCode, translation) -> + val languageType = languageCode.resolveLanguageCode() + checkWrittenTranslationCompatibility(origin, contentId, languageType, translation) + } + } + contentIdLanguages.flatMap { (contentId, languageCodes) -> + languageCodes.checkHasRequiredTranslations(origin, contentId, defaultLanguage) + } + } + + private fun checkWrittenTranslationCompatibility( + origin: ContainerId, + contentId: String, + languageType: LanguageType, + gaeWrittenTranslation: GaeWrittenTranslation + ): List { + return when (val translation = gaeWrittenTranslation.translation) { + is GaeWrittenTranslation.Translation.SingleString -> + translation.value.checkHasValidHtml(origin, contentId, languageType) + is GaeWrittenTranslation.Translation.StringList -> + translation.value.flatMap { it.checkHasValidHtml(origin, contentId, languageType) } + } + } + + private fun checkRecordedVoiceoversCompatibility( + origin: ContainerId, + gaeRecordedVoiceovers: GaeRecordedVoiceovers + ): List { + return gaeRecordedVoiceovers.voiceoversMapping.values.flatMap { contentMap -> + contentMap.values.flatMap { it.filename.checkAudioFilename(origin) } + } + } + + private fun checkAllEntityTranslationsCompatibility( + origin: ContainerId, + translations: Map>, + expectedContentIds: Set, + defaultLanguage: LanguageType + ): List { + val allExpectedContentIds = + expectedContentIds + translations.values.flatMap { it.payload.translations.keys } + val contentIdLanguages = allExpectedContentIds.associateWith { contentId -> + translations.filter { (_, entityTranslation) -> + contentId in entityTranslation.payload.translations + }.keys + } + return translations.flatMap { (languageType, translations) -> + checkEntityTranslationsCompatibility(origin, languageType, translations.payload) + } + contentIdLanguages.flatMap { (contentId, languageCodes) -> + languageCodes.checkHasRequiredTranslations(origin, contentId, defaultLanguage) + } + } + + private fun checkEntityTranslationsCompatibility( + origin: ContainerId, + languageType: LanguageType, + gaeEntityTranslations: GaeEntityTranslations + ): List { + return gaeEntityTranslations.translations.flatMap { (contentId, translatedContent) -> + checkTranslatedContentCompatibility(origin, contentId, languageType, translatedContent) + } + } + + private fun checkTranslatedContentCompatibility( + origin: ContainerId, + contentId: String, + languageType: LanguageType, + gaeTranslatedContent: GaeTranslatedContent + ): List { + return when (val translation = gaeTranslatedContent.contentValue) { + is GaeTranslatedContent.Translation.SingleString -> + translation.value.checkHasValidHtml(origin, contentId, languageType) + is GaeTranslatedContent.Translation.StringList -> + translation.value.flatMap { it.checkHasValidHtml(origin, contentId, languageType) } + } + } + + data class CompatibilityConstraints( + val supportedInteractionIds: Set, + val supportedDefaultLanguages: Set, + val requiredTranslationLanguages: Set, + val supportedImageFormats: Set, + val supportedAudioFormats: Set, + val supportedHtmlTags: Set, + val supportedStateSchemaVersion: Int, + val topicDependencies: Map>, + val forcedVersions: DownloadListVersions? + ) { + fun supportsImageWithExtension(extension: String): Boolean = + supportedImageFormats.any { it.equals(extension, ignoreCase = true) } + + fun supportsAudioWithExtension(extension: String): Boolean = + supportedAudioFormats.any { it.equals(extension, ignoreCase = true) } + + fun hasTopicDependencies(topicId: String): Boolean = topicId in topicDependencies + } + + sealed class CompatibilityResult { + object Compatible : CompatibilityResult() + + data class Incompatible(val failures: List) : CompatibilityResult() + + companion object { + fun createFrom(computeFailures: () -> List): CompatibilityResult = + computeFailures().takeIf { it.isNotEmpty() }?.let { Incompatible(it) } ?: Compatible + } + } + + sealed class CompatibilityFailure { + abstract val origin: ContainerId + + data class TopicHasNoKnownDependencies( + val topicId: String, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class HtmlInTitleOrDescription( + val text: String, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class TextHasInvalidTags( + val contentId: String, + val invalidTagNames: Set, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class TranslatedTextHasInvalidTags( + val contentId: String, + val invalidTagNames: Set, + val languageType: LanguageType, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class HtmlUnexpectedlyInUnicodeContent( + val contentId: String, + val text: String, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class ThumbnailHasInvalidImageFormat( + val imageFilename: String, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class TextReferencesInvalidImageFormat( + val contentId: String, + val imageFilename: String, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class TextUsesImageTagWithMissingFilePath( + val contentId: String, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class ThumbnailHasInvalidColor( + val color: String, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class AudioVoiceoverHasInvalidAudioFormat( + val audioFilename: String, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class UnsupportedDefaultLanguageCode( + val languageCode: String, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class MissingRequiredXlationLangForTitleOrDescFromWeb( + val contentContext: LocalizationTracker.ContentContext, + val missingLanguages: Set, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class MissingRequiredXlationLangForContentTranslation( + val contentId: String, + val missingLanguages: Set, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class StateSchemaVersionTooNew( + val schemaVersion: Int, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class StateHasInvalidInteractionId( + val stateName: String, + val interactionId: String?, + override val origin: ContainerId + ) : CompatibilityFailure() + + data class StoryIsMissingExplorationId( + val storyId: String, + override val origin: ContainerId + ) : CompatibilityFailure() + } + + private fun String.checkIsValidTopicId(origin: ContainerId): List { + return if (!constraints.hasTopicDependencies(topicId = this)) { + listOf(TopicHasNoKnownDependencies(topicId = this, origin)) + } else listOf() + } + + private fun String?.checkThumbnailFilename(origin: ContainerId): List { + return this?.substringAfter('.')?.takeUnless { + constraints.supportsImageWithExtension(it) + }?.let { listOf(ThumbnailHasInvalidImageFormat(imageFilename = this, origin)) } ?: emptyList() + } + + private fun String.checkImageFilename( + origin: ContainerId, + contentId: String + ): List { + return substringAfter('.').takeUnless { constraints.supportsImageWithExtension(it) }?.let { + listOf(TextReferencesInvalidImageFormat(contentId, imageFilename = this, origin)) + } ?: emptyList() + } + + private fun String?.checkAudioFilename(origin: ContainerId): List { + return this?.substringAfter('.')?.takeUnless { + constraints.supportsAudioWithExtension(it) + }?.let { + listOf(AudioVoiceoverHasInvalidAudioFormat(audioFilename = this, origin)) + } ?: emptyList() + } + + private fun String?.checkBackgroundHexColor(origin: ContainerId): List { + return if (this != null && this.parseColorRgb() == null) { + listOf(ThumbnailHasInvalidColor(color = this, origin)) + } else emptyList() + } + + private fun String.checkDefaultLanguageCode(origin: ContainerId): List { + return if (resolveLanguageCode() !in constraints.supportedDefaultLanguages) { + listOf(UnsupportedDefaultLanguageCode(languageCode = this, origin)) + } else emptyList() + } + + @JvmName("checkLanguageCodesHaveRequiredTranslations") + private fun Set.checkHasRequiredTranslations( + origin: ContainerId, + contentId: String, + defaultLanguage: LanguageType + ): List { + return map { + it.resolveLanguageCode() + }.toSet().checkHasRequiredTranslations(origin, contentId, defaultLanguage) + } + + private fun Set.checkHasRequiredTranslations( + origin: ContainerId, + contentId: String, + defaultLanguage: LanguageType + ): List { + // Translations are implied for the default language (since the GAE structures embed those + // values directly with references to content IDs). + val availableTranslationLanguages = this + defaultLanguage + val missingLanguages = constraints.requiredTranslationLanguages - availableTranslationLanguages + return if (missingLanguages.isNotEmpty()) { + listOf(MissingRequiredXlationLangForContentTranslation(contentId, missingLanguages, origin)) + } else emptyList() + } + + private fun checkHasRequiredWebTranslationsFor( + origin: ContainerId, + defaultLanguage: LanguageType, + vararg contentContexts: LocalizationTracker.ContentContext + ): List { + return contentContexts.flatMap { + checkHasRequiredWebTranslationsForSingleContext(origin, it, defaultLanguage) + } + } + + private fun checkHasRequiredWebTranslationsForSingleContext( + origin: ContainerId, + contentContext: LocalizationTracker.ContentContext, + defaultLanguage: LanguageType + ): List { + // See checkHasRequiredTranslations for the logic used here with defaultLanguage. + val availableTranslationLanguages = + localizationTracker.computeAvailableWebTranslations( + origin, contentContext + ).keys + defaultLanguage + val missingLanguages = constraints.requiredTranslationLanguages - availableTranslationLanguages + return if (missingLanguages.isNotEmpty()) { + listOf( + MissingRequiredXlationLangForTitleOrDescFromWeb(contentContext, missingLanguages, origin) + ) + } else emptyList() + } + + private fun GaeSubtitledHtml.checkHasValidHtml(origin: ContainerId): List = + text.checkHasValidHtml(origin, contentId, languageType = null) + + private fun GaeSubtitledUnicode.checkHasNoValidHtml( + origin: ContainerId + ): List = text.checkUnicodeTextForHtml(origin, contentId) + + private fun String.checkHasValidHtml( + origin: ContainerId, + contentId: String, + languageType: LanguageType? + ): List { + val extraTags = extractHtmlTags() - constraints.supportedHtmlTags + val tagFailures = if (extraTags.isNotEmpty()) { + val failure = languageType?.let { + TranslatedTextHasInvalidTags(contentId, extraTags, it, origin) + } ?: TextHasInvalidTags(contentId, extraTags, origin) + listOf(failure) + } else emptyList() + return tagFailures + checkHasValidImageReferences(origin, contentId) + } + + private fun String.checkHasValidImageReferences( + origin: ContainerId, + contentId: String + ): List { + val imageReferences = extractImageReferences() + return imageReferences.filterNotNull().flatMap { + it.checkImageFilename(origin, contentId) + } + listOfNotNull( + imageReferences.find { it == null }?.let { + TextUsesImageTagWithMissingFilePath(contentId, origin) + } + ) + } + + private fun Int.checkIsValidStateSchemaVersion(origin: ContainerId): List { + return if (this > constraints.supportedStateSchemaVersion) { + listOf(StateSchemaVersionTooNew(schemaVersion = this, origin)) + } else emptyList() + } + + private fun String?.checkIsValidInteractionId( + stateName: String, + origin: ContainerId + ): List { + return if (this !in constraints.supportedInteractionIds) { + listOf(StateHasInvalidInteractionId(stateName, interactionId = this, origin)) + } else emptyList() + } + + private companion object { + private val HTML_PRESENCE_REGEX = "".toRegex() + // This regex is a simplification of the standard: https://www.w3.org/TR/xml/#NT-NameStartChar. + private val HTML_TAG_REGEX = "<\\s*([^\\s/>]+)[^>]*?>".toRegex() + private val IMAGE_TAG_REGEX = "<\\s*oppia-noninteractive-image.+?>".toRegex() + private val IMAGE_FILE_PATH_REGEX = "filepath-with-value\\s*=\\s*\"(.+?)\"".toRegex() + + private fun String.checkTitleOrDescTextForHtml( + origin: ContainerId + ): List { + return if (HTML_PRESENCE_REGEX.containsMatchIn(this)) { + listOf(HtmlInTitleOrDescription(text = this, origin)) + } else emptyList() + } + + private fun String.checkUnicodeTextForHtml( + origin: ContainerId, + contentId: String + ): List { + return if (HTML_PRESENCE_REGEX.containsMatchIn(this)) { + listOf(HtmlUnexpectedlyInUnicodeContent(contentId, text = this, origin)) + } else emptyList() + } + + private fun String.extractHtmlTags(): Set = + HTML_TAG_REGEX.findAll(this).map { it.destructured }.map { (tagName) -> tagName }.toSet() + + // TODO: Move to common utility? + private fun String.extractImageReferences() = + IMAGE_TAG_REGEX.findAll(this).map { it.value.extractImageReferenceFromTag() }.toSet() + + private fun String.extractImageReferenceFromTag(): String? { + return IMAGE_FILE_PATH_REGEX.find(this)?.destructured?.let { (filePath) -> + filePath + }?.removeExtraEscapedQuotes() + } + + private fun Set.collectContentIds(): Set = + filterIsInstance().map { it.contentId }.toSet() + + // Some values are double-wrapped with quotes. + private fun String.removeExtraEscapedQuotes() = + removePrefix("&quot;").removeSuffix("&quot;") + } +} diff --git a/scripts/src/java/org/oppia/android/scripts/gae/compat/SubtitledHtmlCollector.kt b/scripts/src/java/org/oppia/android/scripts/gae/compat/SubtitledHtmlCollector.kt new file mode 100644 index 00000000000..e36db4c6c7f --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/gae/compat/SubtitledHtmlCollector.kt @@ -0,0 +1,207 @@ +package org.oppia.android.scripts.gae.compat + +import org.oppia.android.scripts.gae.json.GaeCustomizationArgValue +import org.oppia.android.scripts.gae.json.GaeEntityTranslations +import org.oppia.android.scripts.gae.json.GaeExploration +import org.oppia.android.scripts.gae.json.GaeHint +import org.oppia.android.scripts.gae.json.GaeInteractionCustomizationArgsMap +import org.oppia.android.scripts.gae.json.GaeInteractionInstance +import org.oppia.android.scripts.gae.json.GaeOutcome +import org.oppia.android.scripts.gae.json.GaeSkill +import org.oppia.android.scripts.gae.json.GaeSkillContents +import org.oppia.android.scripts.gae.json.GaeSolution +import org.oppia.android.scripts.gae.json.GaeState +import org.oppia.android.scripts.gae.json.GaeStory +import org.oppia.android.scripts.gae.json.GaeStoryNode +import org.oppia.android.scripts.gae.json.GaeSubtitledHtml +import org.oppia.android.scripts.gae.json.GaeSubtitledUnicode +import org.oppia.android.scripts.gae.json.GaeSubtopic +import org.oppia.android.scripts.gae.json.GaeSubtopicPage +import org.oppia.android.scripts.gae.json.GaeTopic +import org.oppia.android.scripts.gae.json.GaeTranslatedContent +import org.oppia.android.scripts.gae.json.GaeWorkedExample +import org.oppia.android.scripts.gae.json.GaeWrittenTranslation +import org.oppia.android.scripts.gae.json.GaeWrittenTranslations +import org.oppia.android.scripts.gae.proto.LocalizationTracker +import org.oppia.android.scripts.gae.proto.LocalizationTracker.ContentContext.DESCRIPTION +import org.oppia.android.scripts.gae.proto.LocalizationTracker.ContentContext.TITLE +import org.oppia.proto.v1.structure.LanguageType +import org.oppia.android.scripts.gae.json.GaeTranslatedContent.Translation.SingleString as TranslatedSingleString +import org.oppia.android.scripts.gae.json.GaeTranslatedContent.Translation.StringList as TranslatedStringList +import org.oppia.android.scripts.gae.json.GaeWrittenTranslation.Translation.SingleString as WrittenSingleString +import org.oppia.android.scripts.gae.json.GaeWrittenTranslation.Translation.StringList as WrittenStringList + +class SubtitledHtmlCollector(private val localizationTracker: LocalizationTracker) { + fun collectSubtitles(gaeTopic: GaeTopic): Set { + val localId = LocalizationTracker.ContainerId.createFrom(gaeTopic) + val title = setOf(gaeTopic.name.titleToSubtitle()) + val description = setOf(gaeTopic.description.descriptionToSubtitle()) + val titleXlations = localizationTracker.computeAvailableWebTranslations(localId, TITLE) + val descXlations = localizationTracker.computeAvailableWebTranslations(localId, DESCRIPTION) + val subtopicTexts = gaeTopic.subtopics.flatSet { it.collectSubtitles(gaeTopic.id) } + return title + description + titleXlations.translationsToSubtitles() + + descXlations.translationsToSubtitles() + subtopicTexts + } + + fun collectSubtitles(gaeSubtopicPage: GaeSubtopicPage): Set { + val mainContent = setOf(gaeSubtopicPage.pageContents.subtitledHtml.toSubtitle()) + val translations = gaeSubtopicPage.pageContents.writtenTranslations.collectSubtitles() + return mainContent + translations + } + + fun collectSubtitles(gaeStory: GaeStory): Set { + val localId = LocalizationTracker.ContainerId.createFrom(gaeStory) + val title = setOf(gaeStory.title.titleToSubtitle()) + val description = setOf(gaeStory.description.descriptionToSubtitle()) + val titleXlations = localizationTracker.computeAvailableWebTranslations(localId, TITLE) + val descXlations = localizationTracker.computeAvailableWebTranslations(localId, DESCRIPTION) + val chapterTexts = gaeStory.storyContents.nodes.flatSet { it.collectSubtitles(gaeStory) } + return title + description + titleXlations.translationsToSubtitles() + + descXlations.translationsToSubtitles() + chapterTexts + } + + fun collectSubtitles(completeExploration: CompleteExploration): Set { + return completeExploration.exploration.collectSubtitles() + + completeExploration.translations.values.flatSet { it.payload.collectSubtitles() } + } + + fun collectSubtitles(gaeSkill: GaeSkill): Set { + val description = setOf(gaeSkill.description.descriptionToSubtitle()) + val contentTexts = gaeSkill.skillContents.collectSubtitles() + return description + contentTexts + } + + private fun GaeSubtopic.collectSubtitles(topicId: String): Set { + val localId = LocalizationTracker.ContainerId.createFrom(topicId, this) + return localizationTracker.computeAvailableWebTranslations( + localId, TITLE + ).translationsToSubtitles() + } + + private fun GaeStoryNode.collectSubtitles(story: GaeStory): Set { + val title = setOf(title.titleToSubtitle()) + val description = setOf(description.descriptionToSubtitle()) + val xlations = LocalizationTracker.ContainerId.createFrom(story, this)?.let { localId -> + val titleXlations = localizationTracker.computeAvailableWebTranslations(localId, TITLE) + val descXlations = localizationTracker.computeAvailableWebTranslations(localId, DESCRIPTION) + return@let titleXlations + descXlations + } ?: emptyMap() + return title + description + xlations.translationsToSubtitles() + } + + private fun GaeExploration.collectSubtitles(): Set { + val localId = LocalizationTracker.ContainerId.createFrom(this) + val title = setOf(title.titleToSubtitle()) + val titleXlations = localizationTracker.computeAvailableWebTranslations(localId, TITLE) + val stateTexts = states.values.flatSet { it.collectSubtitles() } + return title + titleXlations.translationsToSubtitles() + stateTexts + } + + private fun GaeState.collectSubtitles(): Set = + setOf(content.toSubtitle()) + interaction.collectSubtitles() + + private fun GaeInteractionInstance.collectSubtitles(): Set { + val argTexts = customizationArgs.collectSubtitles() + val groupTexts = answerGroups.flatSet { it.outcome.collectSubtitles() } + val defaultOutcomeTexts = defaultOutcome?.collectSubtitles() ?: emptySet() + val hintTexts = hints.flatSet { it.collectSubtitles() } + val solutionTexts = solution?.collectSubtitles() ?: emptySet() + return argTexts + groupTexts + defaultOutcomeTexts + hintTexts + solutionTexts + } + + private fun GaeInteractionCustomizationArgsMap.collectSubtitles(): Set = + customizationArgs.values.flatSet { it.collectSubtitles() } + + private fun GaeCustomizationArgValue.collectSubtitles(): Set { + return when (this) { + is GaeCustomizationArgValue.GaeImageWithRegions, is GaeCustomizationArgValue.SingleBoolean, + is GaeCustomizationArgValue.SingleInteger -> emptySet() + is GaeCustomizationArgValue.StringList -> value.mapToSet { it.customArgValueToSubtitle() } + is GaeCustomizationArgValue.SubtitledUnicode -> setOf(value.toSubtitle()) + is GaeCustomizationArgValue.SubtitledTextList -> value.mapToSet { it.toSubtitle() } + } + } + + private fun GaeOutcome.collectSubtitles(): Set = setOf(feedback.toSubtitle()) + + private fun GaeHint.collectSubtitles(): Set = setOf(hintContent.toSubtitle()) + + private fun GaeSolution.collectSubtitles(): Set = setOf(explanation.toSubtitle()) + + private fun GaeEntityTranslations.collectSubtitles(): Set = + translations.values.flatSet { it.collectSubtitles() } + + private fun GaeTranslatedContent.collectSubtitles(): Set { + @Suppress("USELESS_CAST") // Cast is required due to cross-module builds. + return when (contentValue) { + is TranslatedSingleString -> + setOf((contentValue as TranslatedSingleString).value.translationToSubtitle()) + is TranslatedStringList -> + (contentValue as TranslatedStringList).value.mapToSet { it.translationToSubtitle() } + } + } + + private fun GaeSkillContents.collectSubtitles(): Set { + val explanationText = setOf(explanation.toSubtitle()) + val workedExampleTexts = workedExamples.flatSet { it.collectSubtitles() } + val translations = writtenTranslations.collectSubtitles() + return explanationText + workedExampleTexts + translations + } + + private fun GaeWorkedExample.collectSubtitles(): Set = + setOf(question.toSubtitle(), explanation.toSubtitle()) + + private fun GaeWrittenTranslations.collectSubtitles(): Set { + return translationsMapping.values.flatSet { + it.values.flatSet { translation -> translation.collectSubtitles() } + } + } + + private fun GaeWrittenTranslation.collectSubtitles(): Set { + // TODO: Add TODO with bug for this & other such casts by referencing https://youtrack.jetbrains.com/issue/KT-50534. + @Suppress("USELESS_CAST") // Cast is required due to cross-module builds. + return when (translation) { + is WrittenSingleString -> + setOf((translation as WrittenSingleString).value.translationToSubtitle()) + is WrittenStringList -> + (translation as WrittenStringList).value.mapToSet { it.translationToSubtitle() } + } + } + + sealed class SubtitledText { + abstract val text: String + + data class Title(override val text: String) : SubtitledText() + + data class Description(override val text: String) : SubtitledText() + + data class Translation(override val text: String) : SubtitledText() + + data class CustomizationArgValue(override val text: String) : SubtitledText() + + data class TextWithContentId(val contentId: String, override val text: String) : SubtitledText() + } + + companion object { + private fun Iterable.flatSet(transform: (I) -> Set): Set = + flatMapTo(mutableSetOf(), transform) + + private fun String.titleToSubtitle() = SubtitledText.Title(this) + + private fun String.descriptionToSubtitle() = SubtitledText.Description(this) + + private fun String.translationToSubtitle() = SubtitledText.Translation(this) + + private fun Map.translationsToSubtitles() = + values.mapToSet { it.translationToSubtitle() } + + private fun String.customArgValueToSubtitle() = SubtitledText.CustomizationArgValue(this) + + private fun GaeSubtitledHtml.toSubtitle() = SubtitledText.TextWithContentId(contentId, text) + + private fun GaeSubtitledUnicode.toSubtitle() = SubtitledText.TextWithContentId(contentId, text) + + private fun Iterable.mapToSet(transform: (I) -> O): Set = + mapTo(mutableSetOf(), transform) + } +} diff --git a/scripts/src/java/org/oppia/android/scripts/gae/compat/TopicPackRepository.kt b/scripts/src/java/org/oppia/android/scripts/gae/compat/TopicPackRepository.kt new file mode 100644 index 00000000000..cf31ef13317 --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/gae/compat/TopicPackRepository.kt @@ -0,0 +1,1012 @@ +package org.oppia.android.scripts.gae.compat + +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import org.oppia.android.scripts.gae.compat.LoadResult.Companion.combine +import org.oppia.android.scripts.gae.compat.LoadResult.Companion.flatten +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityFailure +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityResult +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityResult.Compatible +import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.CompatibilityResult.Incompatible +import org.oppia.android.scripts.gae.compat.TopicPackRepository.MetricCallbacks.DataGroupType +import org.oppia.android.scripts.gae.json.AndroidActivityHandlerService +import org.oppia.android.scripts.gae.json.GaeExploration +import org.oppia.android.scripts.gae.json.GaeSkill +import org.oppia.android.scripts.gae.json.GaeStory +import org.oppia.android.scripts.gae.json.GaeSubtopic +import org.oppia.android.scripts.gae.json.GaeSubtopicPage +import org.oppia.android.scripts.gae.json.GaeTopic +import org.oppia.android.scripts.gae.json.VersionedStructure +import org.oppia.android.scripts.gae.proto.LocalizationTracker +import org.oppia.android.scripts.gae.proto.LocalizationTracker.Companion.VALID_LANGUAGE_TYPES +import org.oppia.android.scripts.gae.proto.LocalizationTracker.Companion.resolveLanguageCode +import org.oppia.android.scripts.proto.DownloadListVersions +import org.oppia.proto.v1.structure.LanguageType +import org.oppia.proto.v1.structure.LanguageType.LANGUAGE_CODE_UNSPECIFIED +import org.oppia.proto.v1.structure.SubtopicPageIdDto +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock +import org.oppia.android.scripts.gae.compat.VersionedStructureReference.Exploration as VersionedExploration +import org.oppia.android.scripts.gae.compat.VersionedStructureReference.Skill as VersionedSkill +import org.oppia.android.scripts.gae.compat.VersionedStructureReference.Story as VersionedStory +import org.oppia.android.scripts.gae.compat.VersionedStructureReference.SubtopicPage as VersionedSubtopicPage +import org.oppia.android.scripts.gae.compat.VersionedStructureReference.Topic as VersionedTopic + +private typealias GenericStructureReference = + VersionedStructureReference +private typealias GenericLoadResult = LoadResult + +class TopicPackRepository( + private val androidService: AndroidActivityHandlerService, + private val coroutineDispatcher: CoroutineDispatcher, + localizationTracker: LocalizationTracker, + private val constraints: StructureCompatibilityChecker.CompatibilityConstraints +) { + private val textCollector by lazy { SubtitledHtmlCollector(localizationTracker) } + private val compatibilityChecker by lazy { + StructureCompatibilityChecker(constraints, localizationTracker, textCollector) + } + private val versionStructureMapManager: VersionStructureMapManager by lazy { + constraints.forcedVersions?.let { + VersionStructureMapManagerFixVersionsImpl(androidService, compatibilityChecker, it) + } ?: VersionStructureMapManagerTakeLatestImpl(androidService, compatibilityChecker) + } + + // TODO: We need to be able to retrieve assets irrespective of schemas... + fun downloadConstructedCompleteTopicAsync( + topicId: String, + metricCallbacks: MetricCallbacks + ): Deferred { + // TODO: + // Algorithm: pick the newest transitive closure of a topic & its dependencies such that all + // structures within the closure are compatible with the app. + // Per topic: + // - For a version, verify the topic structure itself is compatible. + // - If it isn't, try the previous version. + // - If no versions are, shortcircuit: the topic is wholly incompatible. + // - For a compatible version, collect the transitive closure of structures. + // - Verify that each structure is compatible. + // - If any structure is not compatible, try earlier versions one at-a-time until compatibility + // is found. + // - If no version is found compatible, this version of the topic is not compatible. Try a + // previous version. + // - If no version of the topic has a compatible closure, the topic is wholly incompatible. + return CoroutineScope(coroutineDispatcher).async { + when (val result = tryCreateCompatiblePack(topicId, metricCallbacks)) { + is LoadResult.Pending -> error("Pack result should not be pending for topic: $topicId.") + is LoadResult.Success -> result.value + is LoadResult.Failure -> { + error( + "Failed to load complete topic pack with ID: $topicId. Encountered failures:" + + "\n${result.computeFailureString()}." + ) + } + } + } + } + + private suspend fun tryCreateCompatiblePack( + topicId: String, + metricCallbacks: MetricCallbacks + ): LoadResult { + // Attempt to load a completely internally consistent topic pack for the latest topic version. + // If that fails, try the next previous version of the topic and continue until either no + // versions remain or one is found to be able to be loaded. + val result = tryCreatePackForLatestTrackedTopicVersion(topicId, metricCallbacks) + if (result is LoadResult.Failure) { + val structureId = StructureId.Topic(topicId) + metricCallbacks.resetAllGroupItemCounts() + if (versionStructureMapManager.lookUpResultCount(structureId) > 1) { + versionStructureMapManager.invalidateVersion( + structureId, versionStructureMapManager.findMostRecent(structureId) + ) + return tryCreateCompatiblePack(topicId, metricCallbacks) // Try again for the next version. + } + } + return result // The result either passed, or there are no more topics to try. + } + + private suspend fun tryCreatePackForLatestTrackedTopicVersion( + topicId: String, + metricCallbacks: MetricCallbacks + ): LoadResult { + // TODO: + // Algorithm: + // 1. Attempt to create a complete topic. If any constituent structures fail to import, the whole topic is unavailable. + // 2. Verify cross-structure compatibility. If any structure violates cross-structure consistency, back that structure up 1 version and try (1) again. + // 3. If at least one structure fails to ever be compatible, the topic isn't supported. Otherwise, it is. + + // First, try to create a complete topic. All structures must be available at at least one + // version. + return tryLoadTopic(topicId).transformAsync { gaeTopic -> + tryLoadPackFragments(gaeTopic, metricCallbacks).combine(TopicPackFragment::combineWith) + }.transform(TopicPackFragment::toTopicPack) + } + + private suspend fun tryLoadPackFragments( + gaeTopic: GaeTopic, + metricCallbacks: MetricCallbacks + ): List> { + // TODO: Batch different results? + val subtopicsResult = + tryLoadSubtopics(gaeTopic.id, gaeTopic.computeContainedSubtopicMap(), metricCallbacks) + val storiesResult = tryLoadStories(gaeTopic.computeReferencedStoryIds(), metricCallbacks) + val explorationsResult = storiesResult.transformAsync { storiesPack -> + tryLoadExplorations( + expIds = storiesPack.expectedStories.values.flatSet { + it.computeReferencedExplorationIds() + }, + metricCallbacks + ) + } + return listOf( + LoadResult.Success(TopicPackFragment(topic = gaeTopic)), + subtopicsResult, + storiesResult, + explorationsResult, + tryLoadSkillsClosureAsFragment( + gaeTopic, subtopicsResult, storiesResult, explorationsResult, metricCallbacks + ), + LoadResult.Success( + TopicPackFragment(defaultLanguage = gaeTopic.languageCode.resolveLanguageCode()) + ) + ) + } + + private suspend fun tryLoadSubtopics( + topicId: String, + subtopics: Map, + metricCallbacks: MetricCallbacks + ): LoadResult { + metricCallbacks.reportGroupItemCount(DataGroupType.SUBTOPIC, subtopics.size) + return subtopics.keys.map { subtopicIndex -> + SubtopicPageIdDto.newBuilder().apply { + this.topicId = topicId + this.subtopicIndex = subtopicIndex + }.build() + }.mapIndexed { index, subtopicId -> + CoroutineScope(coroutineDispatcher).async { + tryLoadSubtopicPage( + subtopicId.topicId, subtopicId.subtopicIndex, subtopics.getValue(subtopicId.subtopicIndex) + ).transform { subtopicId to it }.also { + metricCallbacks.reportItemDownloaded(DataGroupType.SUBTOPIC, index) + } + } + }.awaitAll().combine { subtopicPages -> + TopicPackFragment(subtopicPages = subtopicPages.toMap()) + } + } + + private suspend fun tryLoadStories( + storyIds: Set, + metricCallbacks: MetricCallbacks + ): LoadResult { + metricCallbacks.reportGroupItemCount(DataGroupType.STORY, storyIds.size) + return storyIds.mapIndexed { index, storyId -> + CoroutineScope(coroutineDispatcher).async { + tryLoadStory(storyId).also { + metricCallbacks.reportItemDownloaded(DataGroupType.STORY, index) + } + } + }.awaitAll().combine { stories -> + TopicPackFragment(stories = stories.associateBy(GaeStory::id)) + } + } + + private suspend fun tryLoadExplorations( + expIds: Set, + metricCallbacks: MetricCallbacks + ): LoadResult { + metricCallbacks.reportGroupItemCount(DataGroupType.EXPLORATION, expIds.size) + return expIds.mapIndexed { index, expId -> + CoroutineScope(coroutineDispatcher).async { + tryLoadExploration(expId).also { + metricCallbacks.reportItemDownloaded(DataGroupType.EXPLORATION, index) + } + } + }.awaitAll().combine { explorations -> + TopicPackFragment(explorations = explorations.associateBy { it.exploration.id }) + } + } + + private suspend fun tryLoadSkillsClosureAsFragment( + gaeTopic: GaeTopic, + subtopicsResult: LoadResult, + storiesResult: LoadResult, + explorationsResult: LoadResult, + metricCallbacks: MetricCallbacks + ): LoadResult { + // Use the topic & all loaded subtopics/stories/explorations to determine the initial set of + // skill IDs, then retrieve a complete skills list closure before constructing and returning a + // topic pack fragment. + return subtopicsResult.transformAsync { subtopicPagesFragment -> + storiesResult.transformAsync { storiesFragment -> + explorationsResult.transformAsync { explorationsFragment -> + val topicSkillIds = gaeTopic.collectSkillIds() + val subtopicSkillIds = + subtopicPagesFragment.expectedSubtopicPages.values.flatSet { it.collectSkillIds() } + val storySkillIds = + storiesFragment.expectedStories.values.flatSet { it.collectSkillIds() } + val expSkillIds = + explorationsFragment.expectedExplorations.values.flatSet { it.collectSkillIds() } + val initialSkillIds = topicSkillIds + subtopicSkillIds + storySkillIds + expSkillIds + tryLoadSkillsClosure(initialSkillIds, metricCallbacks) + } + } + }.transform { TopicPackFragment(referencedSkills = it.associateBy(GaeSkill::id)) } + } + + private suspend fun tryLoadSkillsClosure( + skillIds: Set, + metricCallbacks: MetricCallbacks + ): LoadResult> { + // Load skills in a loop until all known skills are loaded (since concept cards may themselves + // reference other skills not referenced elsewhere in a topic). + metricCallbacks.reportGroupItemCount(DataGroupType.SKILL, skillIds.size) + return tryLoadSkills(skillIds, metricCallbacks).transformAsync { skills -> + val allReferencedSkillIds = skillIds + skills.flatSet { it.collectSkillIds() } + if (allReferencedSkillIds != skillIds) { + metricCallbacks.resetGroupItemCount(DataGroupType.SKILL) + tryLoadSkillsClosure(allReferencedSkillIds, metricCallbacks) + } else LoadResult.Success(skills) + } + } + + private suspend fun tryLoadSkills( + skillIds: Set, + metricCallbacks: MetricCallbacks + ): LoadResult> { + return skillIds.mapIndexed { index, skillId -> + CoroutineScope(coroutineDispatcher).async { + tryLoadSkill(skillId).also { + metricCallbacks.reportItemDownloaded(DataGroupType.SKILL, index) + } + } + }.awaitAll().flatten() + } + + private suspend fun tryLoadTopic(topicId: String): LoadResult { + return tryLoadLatestStructure( + StructureId.Topic(topicId), + retrieveStructureVersion = GaeTopic::version, + createReference = ::VersionedTopic + ).safeCast() + } + + private suspend fun tryLoadSubtopicPage( + topicId: String, + subtopicIndex: Int, + correspondingGaeSubtopic: GaeSubtopic + ): LoadResult { + return tryLoadLatestStructure( + StructureId.Subtopic(topicId, subtopicIndex), + retrieveStructureVersion = GaeSubtopicPage::version + ) { id, version -> + VersionedSubtopicPage(id, version, correspondingGaeSubtopic) + }.safeCast() + } + + private suspend fun tryLoadStory(storyId: String): LoadResult { + return tryLoadLatestStructure( + StructureId.Story(storyId), + retrieveStructureVersion = GaeStory::version, + createReference = ::VersionedStory + ).safeCast() + } + + private suspend fun tryLoadExploration(expId: String): LoadResult { + return tryLoadLatestStructure( + StructureId.Exploration(expId), + retrieveStructureVersion = CompleteExploration::version + ) { id, version -> + VersionedExploration(id, version, coroutineDispatcher, constraints) + }.safeCast() + } + + private suspend fun tryLoadSkill(skillId: String): LoadResult { + return tryLoadLatestStructure( + StructureId.Skill(skillId), + retrieveStructureVersion = GaeSkill::version, + createReference = ::VersionedSkill + ).safeCast() + } + + private suspend fun tryLoadLatestStructure( + structureId: I, + retrieveStructureVersion: (S) -> Int, + createReference: (I, Int) -> VersionedStructureReference + ): GenericLoadResult { + versionStructureMapManager.ensureInitialized( + structureId, retrieveStructureVersion, createReference + ) + + // Start backwards from the most recent (known) version of the structure until one is found + // that's at least directly compatible with the import pipeline. No guarantees are made yet + // about cross-structure compatibility as that's checked later. + var checkedReference: GenericStructureReference? = + versionStructureMapManager.findMostRecent(structureId) + var lastInvalidReference: GenericStructureReference? = null + while (checkedReference != null) { + val result = tryLoadStructure(structureId, checkedReference) + if (lastInvalidReference != null) { + versionStructureMapManager.invalidateVersion(structureId, lastInvalidReference) + } + if (result is LoadResult.Success<*>) return result + lastInvalidReference = checkedReference // This structure isn't compatible. + checkedReference = checkedReference.toPreviousVersion() + } + + // If no versions match, return the failures of the oldest structure (since all others have been + // eliminated). + return tryLoadStructure(structureId, versionStructureMapManager.findMostRecent(structureId)) + } + + private suspend fun tryLoadStructure( + structureId: StructureId, + reference: GenericStructureReference + ): GenericLoadResult { + return when (val result = versionStructureMapManager.lookUp(structureId, reference)) { + is LoadResult.Pending -> { + versionStructureMapManager.update( + structureId, reference.loadVersioned(androidService, compatibilityChecker) + ) + // This should be present now. + versionStructureMapManager.lookUp(structureId, reference).also { + check(it !is LoadResult.Pending) { "Expected reference to be loaded: $it." } + } + } + is LoadResult.Success, is LoadResult.Failure -> result + } + } + + private inline fun GenericLoadResult.safeCast(): LoadResult { + return when (this) { + is LoadResult.Pending -> LoadResult.Pending() + is LoadResult.Success -> LoadResult.Success(value as S) + is LoadResult.Failure -> LoadResult.Failure(failures) + } + } + + private fun GaeTopic.collectSkillIds(): Set = + textCollector.collectSubtitles(this).extractSkillIds() + computeDirectlyReferencedSkillIds() + + private fun GaeSubtopicPage.collectSkillIds(): Set = + textCollector.collectSubtitles(this).extractSkillIds() + + private fun GaeStory.collectSkillIds(): Set = + textCollector.collectSubtitles(this).extractSkillIds() + computeDirectlyReferencedSkillIds() + + private fun CompleteExploration.collectSkillIds(): Set { + return textCollector.collectSubtitles(this).extractSkillIds() + + exploration.computeDirectlyReferencedSkillIds() + } + + private fun GaeSkill.collectSkillIds(): Set = + textCollector.collectSubtitles(this).extractSkillIds() + computeDirectlyReferencedSkillIds() + + // TODO: Document that the item count can be reported multiple times for the same type. It will + // only go up except when a new structure is found (which means reset will be called first). If + // the length increases, old indexes won't be passed to reportGroupItemDownloaded. + // TODO: Document that the string passed for types are meant to be GUIDs among all structures, but + // this invariant is not ensured. + data class MetricCallbacks( + val resetAllGroupItemCounts: () -> Unit, + val resetGroupItemCount: (DataGroupType) -> Unit, + val reportGroupItemCount: (DataGroupType, Int) -> Unit, + private val reportGroupItemDownloaded: suspend (DataGroupType, String) -> Unit + ) { + suspend fun reportItemDownloaded(type: DataGroupType, index: Int) { + reportGroupItemDownloaded(type, "${type.name}-$index") + } + + enum class DataGroupType { + STORY, + SUBTOPIC, + EXPLORATION, + SKILL, + } + } + + private data class TopicPackFragment( + val topic: GaeTopic? = null, + val subtopicPages: Map? = null, + val stories: Map? = null, + val explorations: Map? = null, + val referencedSkills: Map? = null, + val defaultLanguage: LanguageType? = null + ) { + val expectedTopic by lazy { checkNotNull(topic) { "Topic was not initialized." } } + val expectedSubtopicPages by lazy { + checkNotNull(subtopicPages) { "Subtopic pages were not initialized." } + } + val expectedStories by lazy { checkNotNull(stories) { "Stories were not initialized." } } + val expectedExplorations by lazy { + checkNotNull(explorations) { "Explorations were not initialized." } + } + val expectedReferencedSkills by lazy { + checkNotNull(referencedSkills) { "Skills were not initialized." } + } + val expectedDefaultLanguage by lazy { + checkNotNull(defaultLanguage) { "Default language was not initialized." } + } + + fun toTopicPack(): CompleteTopicPack { + return CompleteTopicPack( + topic = expectedTopic, + subtopicPages = expectedSubtopicPages, + stories = expectedStories, + explorations = expectedExplorations, + referencedSkills = expectedReferencedSkills, + defaultLanguage = expectedDefaultLanguage + ) + } + + fun combineWith(other: TopicPackFragment): TopicPackFragment { + return copy( + topic = expectOne(topic, other.topic), + subtopicPages = expectOne(subtopicPages, other.subtopicPages), + stories = expectOne(stories, other.stories), + explorations = expectOne(explorations, other.explorations), + referencedSkills = expectOne(referencedSkills, other.referencedSkills), + defaultLanguage = expectOne(defaultLanguage, other.defaultLanguage) + ) + } + + private companion object { + private fun expectOne(first: T?, second: T?): T? { + return when { + first != null && second == null -> first + first == null && second != null -> second + first == null && second == null -> null + else -> error("Expected to pick one of, not both: $first, $second.") + } + } + } + } + + private companion object { + private const val CONCEPT_CARD_TAG = "oppia-noninteractive-skillreview" + private const val SKILL_ID_ATTRIBUTE_NAME = "skill_id-with-value" + private val CONCEPT_CARD_PATTERN = "<$CONCEPT_CARD_TAG.+?".toRegex() + + private fun Iterable.flatSet(transform: (I) -> Set): Set = + flatMapTo(mutableSetOf(), transform) + + private fun Set.extractSkillIds(): Set = + map { it.text }.flatSet { it.extractSkillIds() } + + private fun String.extractSkillIds(): Set = + CONCEPT_CARD_PATTERN.findAll(this).map { it.value.extractSkillId() }.toSet() + + private fun String.extractSkillId(): String = + substringAfter("$SKILL_ID_ATTRIBUTE_NAME=\"").substringBefore("\"").replace("&quot;", "") + } +} + +private sealed class LoadResult { + fun combineWith(other: LoadResult, combine: (T, I) -> O): LoadResult { + return when (this) { + is Pending -> Pending() // At least one is pending. + is Success -> when (other) { + is Pending -> Pending() // At least one is pending. + is Success -> Success(combine(value, other.value)) // Both are successes. + is Failure -> Failure(other.failures) // At least one is failing. + } + is Failure -> when (other) { + is Pending -> Pending() // At least one is pending. + is Success -> Failure(failures) // At least one is failing. + is Failure -> Failure(failures + other.failures) // Both are failing. + } + } + } + + fun transform(operation: (T) -> O): LoadResult = transformAsync { Success(operation(it)) } + + inline fun transformAsync(operation: (T) -> LoadResult): LoadResult { + return when (this) { + is Pending -> Pending() + is Success -> operation(value) + is Failure -> Failure(failures) + } + } + + // Note that the 'unused' here helps to ensure that all instances of 'Pending' act like a + // singleton (as though it were an object) without losing its generic type safety. + data class Pending(val unused: Int = 0) : LoadResult() + + data class Success(val value: T) : LoadResult() + + data class Failure(val failures: List) : LoadResult() { + fun computeFailureString(): String = failures.joinToString(separator = "\n") { "- $it" } + } + + companion object { + fun List>.flatten(): LoadResult> = combine> { it } + + fun List>.combine(transform: (List) -> O): LoadResult { + return fold(Success(listOf()) as LoadResult>) { ongoing, newValue -> + ongoing.combineWith(newValue, Collection::plus) + }.transform(transform) + } + + fun List>.combine(combine: (T, T) -> T): LoadResult = + reduce { ongoing, newValue -> ongoing.combineWith(newValue, combine) } + } +} + +private interface VersionedStructureFetcher { + fun fetchLatestFromRemoteAsync( + id: I, + service: AndroidActivityHandlerService + ): Deferred> + + fun fetchMultiFromRemoteAsync( + id: I, + versions: List, + service: AndroidActivityHandlerService + ): Deferred>> +} + +private sealed class VersionedStructureReference { + // TODO: Try 50 or a higher number once multi-version fetching works on Oppia web (see https://github.com/oppia/oppia/issues/18241). + val defaultVersionFetchCount: Int = 1 + abstract val structureId: I + abstract val version: Int + abstract val fetcher: VersionedStructureFetcher + + abstract fun checkCompatibility( + checker: StructureCompatibilityChecker, + structure: S + ): CompatibilityResult + + abstract fun toNewVersion(newVersion: Int): VersionedStructureReference + + fun toPreviousVersion(): VersionedStructureReference? = + (version - 1).takeIf { it > 0 }?.let { toNewVersion(it) } + + suspend fun loadLatest( + service: AndroidActivityHandlerService, + checker: StructureCompatibilityChecker + ): Pair> { + val result = fetcher.fetchLatestFromRemoteAsync(structureId, service) + return result.await().payload to result.toLoadResult(checker) + } + + suspend fun loadVersioned( + service: AndroidActivityHandlerService, + checker: StructureCompatibilityChecker + ): Map, LoadResult> { + val oldestVersionToRequest = (version - defaultVersionFetchCount).coerceAtLeast(1) + val versionsToRequest = (oldestVersionToRequest until version).toList() + val structures = fetcher.fetchMultiFromRemoteAsync( + structureId, versionsToRequest, service + ).toLoadResult(checker) + return versionsToRequest.zip(structures).toMap().mapKeys { (version, _) -> + toNewVersion(version) + } + } + + private suspend fun Deferred>.toLoadResult( + checker: StructureCompatibilityChecker + ): LoadResult = await().toLoadResult(checker) + + @JvmName("listToLoadResult") + private suspend fun Deferred>>.toLoadResult( + checker: StructureCompatibilityChecker + ): List> = await().map { it.toLoadResult(checker) } + + private fun VersionedStructure.toLoadResult( + checker: StructureCompatibilityChecker + ): LoadResult { + return when (val compatibilityResult = checkCompatibility(checker, payload)) { + Compatible -> LoadResult.Success(payload) + is Incompatible -> LoadResult.Failure(compatibilityResult.failures).also { + // TODO: Remove. + error("Failed to load: $it.") + } + } + } + + data class Topic( + override val structureId: StructureId.Topic, + override val version: Int + ) : VersionedStructureReference() { + override val fetcher by lazy { TopicFetcher() } + + override fun toNewVersion(newVersion: Int) = copy(version = newVersion) + + override fun checkCompatibility(checker: StructureCompatibilityChecker, structure: GaeTopic) = + checker.isTopicItselfCompatible(structure) + } + + data class SubtopicPage( + override val structureId: StructureId.Subtopic, + override val version: Int, + val correspondingGaeSubtopic: GaeSubtopic + ) : VersionedStructureReference() { + override val fetcher by lazy { SubtopicFetcher() } + + override fun toNewVersion(newVersion: Int) = copy(version = newVersion) + + override fun checkCompatibility( + checker: StructureCompatibilityChecker, + structure: GaeSubtopicPage + ) = checker.isSubtopicPageItselfCompatible(structure, correspondingGaeSubtopic) + } + + data class Story( + override val structureId: StructureId.Story, + override val version: Int + ) : VersionedStructureReference() { + override val fetcher by lazy { StoryFetcher() } + + override fun toNewVersion(newVersion: Int) = copy(version = newVersion) + + override fun checkCompatibility(checker: StructureCompatibilityChecker, structure: GaeStory) = + checker.isStoryItselfCompatible(structure) + } + + data class Exploration( + override val structureId: StructureId.Exploration, + override val version: Int, + private val coroutineDispatcher: CoroutineDispatcher, + private val compatibilityConstraints: StructureCompatibilityChecker.CompatibilityConstraints + ) : VersionedStructureReference() { + override val fetcher by lazy { ExplorationFetcher(coroutineDispatcher) } + + override fun toNewVersion(newVersion: Int) = copy(version = newVersion) + + override fun checkCompatibility( + checker: StructureCompatibilityChecker, + structure: CompleteExploration + ) = checker.isExplorationItselfCompatible(structure) + } + + data class Skill( + override val structureId: StructureId.Skill, + override val version: Int + ) : VersionedStructureReference() { + override val fetcher by lazy { SkillFetcher() } + + override fun toNewVersion(newVersion: Int) = copy(version = newVersion) + + override fun checkCompatibility(checker: StructureCompatibilityChecker, structure: GaeSkill) = + checker.isSkillItselfCompatible(structure) + } + + companion object { + const val INVALID_VERSION = 0 + } +} + +private class TopicFetcher : VersionedStructureFetcher { + override fun fetchLatestFromRemoteAsync( + id: StructureId.Topic, + service: AndroidActivityHandlerService + ) = service.fetchLatestTopicAsync(id.id) + + override fun fetchMultiFromRemoteAsync( + id: StructureId.Topic, + versions: List, + service: AndroidActivityHandlerService + ) = service.fetchTopicByVersionsAsync(id.id, versions) +} + +private class StoryFetcher : VersionedStructureFetcher { + override fun fetchLatestFromRemoteAsync( + id: StructureId.Story, + service: AndroidActivityHandlerService + ) = service.fetchLatestStoryAsync(id.id) + + override fun fetchMultiFromRemoteAsync( + id: StructureId.Story, + versions: List, + service: AndroidActivityHandlerService + ) = service.fetchStoryByVersionsAsync(id.id, versions) +} + +private class SubtopicFetcher : VersionedStructureFetcher { + override fun fetchLatestFromRemoteAsync( + id: StructureId.Subtopic, + service: AndroidActivityHandlerService + ) = service.fetchLatestRevisionCardAsync(id.topicId, id.subtopicIndex) + + override fun fetchMultiFromRemoteAsync( + id: StructureId.Subtopic, + versions: List, + service: AndroidActivityHandlerService + ) = service.fetchRevisionCardByVersionsAsync(id.topicId, id.subtopicIndex, versions) +} + +private class SkillFetcher : VersionedStructureFetcher { + override fun fetchLatestFromRemoteAsync( + id: StructureId.Skill, + service: AndroidActivityHandlerService + ) = service.fetchLatestConceptCardAsync(id.id) + + override fun fetchMultiFromRemoteAsync( + id: StructureId.Skill, + versions: List, + service: AndroidActivityHandlerService + ) = service.fetchConceptCardByVersionsAsync(id.id, versions) +} + +private class ExplorationFetcher( + private val coroutineDispatcher: CoroutineDispatcher +) : VersionedStructureFetcher { + override fun fetchLatestFromRemoteAsync( + id: StructureId.Exploration, + service: AndroidActivityHandlerService + ): Deferred> { + return CoroutineScope(coroutineDispatcher).async { + val latestExp = service.fetchLatestExplorationAsync(id.id).await() + return@async latestExp.copyWithNewPayload(service.downloadExploration(id, latestExp.payload)) + } + } + + override fun fetchMultiFromRemoteAsync( + id: StructureId.Exploration, + versions: List, + service: AndroidActivityHandlerService + ): Deferred>> { + return CoroutineScope(coroutineDispatcher).async { + service.fetchExplorationByVersionsAsync(id.id, versions).await().map { + it.copyWithNewPayload(service.downloadExploration(id, it.payload)) + } + } + } + + private companion object { + private suspend fun AndroidActivityHandlerService.downloadExploration( + id: StructureId.Exploration, + exploration: GaeExploration + ): CompleteExploration { + val translations = VALID_LANGUAGE_TYPES.map { languageType -> + fetchExplorationTranslationsAsync( + id.id, exploration.version, languageType.toContentLanguageCode() + ) + }.awaitAll() + return CompleteExploration( + exploration, + translations.associateBy { + it.languageCode?.resolveLanguageCode() ?: LANGUAGE_CODE_UNSPECIFIED + } + ) + } + + private fun LanguageType.toContentLanguageCode(): String { + return when (this) { + LanguageType.ENGLISH -> "en" + LanguageType.ARABIC -> "ar" + LanguageType.HINDI -> "hi" + LanguageType.HINGLISH -> "hi-en" + // Note: Oppia web doesn't support pt-br specific content translations yet. + LanguageType.BRAZILIAN_PORTUGUESE -> "pt" + LanguageType.SWAHILI -> "sw" + LanguageType.NIGERIAN_PIDGIN -> "pcm" + LANGUAGE_CODE_UNSPECIFIED, LanguageType.UNRECOGNIZED -> + error("Unsupported language type: $this.") + } + } + } +} + +private sealed class StructureId { + data class Topic(val id: String) : StructureId() + + data class Subtopic(val topicId: String, val subtopicIndex: Int) : StructureId() + + data class Story(val id: String) : StructureId() + + data class Exploration(val id: String) : StructureId() + + data class Skill(val id: String) : StructureId() +} + +private interface VersionStructureMapManager { + fun lookUp( + structureId: StructureId, + genericReference: GenericStructureReference + ): GenericLoadResult + + fun lookUpResultCount(structureId: StructureId): Int + + suspend fun ensureInitialized( + structureId: I, + retrieveStructureVersion: (S) -> Int, + createReference: (I, Int) -> VersionedStructureReference + ) + + fun findMostRecent(structureId: StructureId): GenericStructureReference + + fun invalidateVersion(structureId: StructureId, reference: GenericStructureReference) + + fun update( + structureId: StructureId, + loadResults: Map + ) +} + +private class VersionStructureMapManagerTakeLatestImpl( + private val androidService: AndroidActivityHandlerService, + private val compatibilityChecker: StructureCompatibilityChecker +) : VersionStructureMapManager { + private val cachedStructures = + mutableMapOf>() + // TODO: Move over to an actor pattern to be more cooperative with coroutines. + private val lock = ReentrantLock() + + override fun lookUp( + structureId: StructureId, + genericReference: GenericStructureReference + ) = lock.withLock { cachedStructures.getValue(structureId).getValue(genericReference) } + + override fun lookUpResultCount(structureId: StructureId) = + lock.withLock { cachedStructures.getValue(structureId).size } + + override suspend fun ensureInitialized( + structureId: I, + retrieveStructureVersion: (S) -> Int, + createReference: (I, Int) -> VersionedStructureReference + ) { + // Note that these operations aren't atomic, but fetching and checking a structure is idempotent + // so multiple operations can kick-off and the last result taken for future caching. + if (structureId !in lock.withLock { cachedStructures }) { + // If no version of this structure has been loaded yet, preload the latest version and pending + // results for all previous versions. + val versionedRef = createReference(structureId, VersionedStructureReference.INVALID_VERSION) + val (structure, result) = versionedRef.loadLatest(androidService, compatibilityChecker) + val latestVersion = versionedRef.toNewVersion(retrieveStructureVersion(structure)) + val structureMap = mutableMapOf() + structureMap[latestVersion] = result + for (it in 1 until latestVersion.version) { + structureMap[versionedRef.toNewVersion(it)] = LoadResult.Pending() + } + lock.withLock { cachedStructures[structureId] = structureMap } + } + } + + override fun findMostRecent(structureId: StructureId): GenericStructureReference { + val references = + lock.withLock { cachedStructures[structureId]?.keys } + ?: error("Structure hasn't yet been initialized: $structureId.") + return checkNotNull(references.maxByOrNull { it.version }) { + "Failed to find most recent structure reference in map: $this for ID: $structureId." + } + } + + override fun invalidateVersion(structureId: StructureId, reference: GenericStructureReference) { + lock.withLock { + val structureMap = cachedStructures.getValue(structureId) + require(reference == findMostRecent(reference.structureId)) { + "Can only invalidate the most recent version of a structure." + } + check(structureMap.size > 1) { "Cannot remove the final structure." } + structureMap.remove(reference) + } + } + + override fun update( + structureId: StructureId, + loadResults: Map + ) { + lock.withLock { loadResults.forEach(cachedStructures.getValue(structureId)::put) } + } +} + +private class VersionStructureMapManagerFixVersionsImpl( + private val androidService: AndroidActivityHandlerService, + private val compatibilityChecker: StructureCompatibilityChecker, + private val forcedVersions: DownloadListVersions +) : VersionStructureMapManager { + // Only at most one structure can be loaded per ID. + private val cachedStructures = + mutableMapOf>() + private val structureVersions by lazy { forcedVersions.toPairs().toMap() } + // TODO: Move over to an actor pattern to be more cooperative with coroutines. + private val lock = ReentrantLock() + + override fun lookUp( + structureId: StructureId, + genericReference: GenericStructureReference + ): GenericLoadResult { + val (ref, result) = lookUp(structureId) + check(ref == genericReference) { + "Reference doesn't match forced version. Received:\n$genericReference\nExpected:\n$ref" + } + return result + } + + override fun lookUpResultCount(structureId: StructureId): Int { + // Verify that the structure ID is known. + lookUp(structureId) + return 1 // Supported IDs always have exactly 1 result since versions are fixed. + } + + override suspend fun ensureInitialized( + structureId: I, + retrieveStructureVersion: (S) -> Int, + createReference: (I, Int) -> VersionedStructureReference + ) { + val fixedVersion = + structureVersions[structureId] ?: error("No fixed version configured for ID: $structureId.") + if (structureId !in lock.withLock { cachedStructures }) { + // If the fixed version hasn't been loaded yet, ensure it's loaded. + val versionedRef = createReference(structureId, fixedVersion) + val (_, result) = versionedRef.loadLatest(androidService, compatibilityChecker) + check(result is LoadResult.Success) { + "Expected loading structure by ID $structureId for version $fixedVersion to succeed." + } + lock.withLock { cachedStructures[structureId] = versionedRef to result } + } + } + + // There's only at most one version per ID, so that's always the 'latest'. + override fun findMostRecent(structureId: StructureId) = lookUp(structureId).first + + override fun invalidateVersion(structureId: StructureId, reference: GenericStructureReference) { + error("Cannot invalidate versions when versions are fixed, for reference:\n$reference") + } + + override fun update( + structureId: StructureId, + loadResults: Map + ) { + val (expectedRef, _) = lookUp(structureId) + check(loadResults.size == 1) { "Expected one result for fixed version ID: $structureId." } + + val (encounteredRef, newResult) = loadResults.entries.single() + check(encounteredRef == expectedRef) { + "Expected structure reference:\n$expectedRef\nEncountered:\n$encounteredRef" + } + + lock.withLock { cachedStructures[structureId] = expectedRef to newResult } + } + + private fun lookUp(structureId: StructureId): Pair { + return lock.withLock { cachedStructures[structureId] } + ?: error("ID is not part of forced versions: $structureId.") + } + + private companion object { + private fun DownloadListVersions.toPairs() = + trackedTopicInfoList.toPairs() + trackedSkillInfoList.toPairs() + + private fun DownloadListVersions.TopicInfo.toPair() = StructureId.Topic(id) to contentVersion + + private fun DownloadListVersions.TopicInfo.toPairs() = + storyInfoList.toPairs() + subtopicInfoList.toPairs(id) + toPair() + + @JvmName("topicInfoIterableToPairs") + private fun Iterable.toPairs() = flatMap { it.toPairs() } + + private fun DownloadListVersions.StoryInfo.toPair() = StructureId.Story(id) to contentVersion + + private fun DownloadListVersions.StoryInfo.toPairs() = + chapterInfoList.map { it.toPair() } + toPair() + + @JvmName("storyInfoIterableToPairs") + private fun Iterable.toPairs() = flatMap { it.toPairs() } + + private fun DownloadListVersions.ChapterInfo.toPair() = + StructureId.Exploration(explorationId) to explorationContentVersion + + private fun DownloadListVersions.SubtopicInfo.toPair(topicId: String) = + StructureId.Subtopic(topicId, index) to contentVersion + + @JvmName("subtopicInfoIterableToPairs") + private fun Iterable.toPairs(topicId: String) = + map { it.toPair(topicId) } + + private fun DownloadListVersions.SkillInfo.toPair() = StructureId.Skill(id) to contentVersion + + @JvmName("skillInfoIterableToPairs") + private fun Iterable.toPairs() = map { it.toPair() } + } +} + +private fun VersionedStructure.copyWithNewPayload(newPayload: O): VersionedStructure = + VersionedStructure(id, newPayload, languageCode, version) diff --git a/scripts/src/java/org/oppia/android/scripts/proto/BUILD.bazel b/scripts/src/java/org/oppia/android/scripts/proto/BUILD.bazel index 7265ff0efc3..0de13397742 100644 --- a/scripts/src/java/org/oppia/android/scripts/proto/BUILD.bazel +++ b/scripts/src/java/org/oppia/android/scripts/proto/BUILD.bazel @@ -21,6 +21,17 @@ java_proto_library( deps = [":affected_tests_proto"], ) +oppia_proto_library( + name = "download_list_versions_proto", + srcs = ["download_list_versions.proto"], +) + +java_proto_library( + name = "download_list_versions_java_proto", + visibility = ["//scripts:oppia_script_library_visibility"], + deps = [":download_list_versions_proto"], +) + oppia_proto_library( name = "filename_pattern_validation_checks_proto", srcs = ["filename_pattern_validation_checks.proto"], diff --git a/scripts/src/java/org/oppia/android/scripts/proto/download_list_versions.proto b/scripts/src/java/org/oppia/android/scripts/proto/download_list_versions.proto new file mode 100644 index 00000000000..1991afee6c1 --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/proto/download_list_versions.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package proto; + +option java_package = "org.oppia.android.scripts.proto"; +option java_multiple_files = true; + +message DownloadListVersions { + repeated TopicInfo tracked_topic_info = 1; + repeated SkillInfo tracked_skill_info = 2; + + message TopicInfo { + string id = 1; + int32 content_version = 2; + repeated StoryInfo story_info = 3; + repeated SubtopicInfo subtopic_info = 4; + } + + message StoryInfo { + string id = 1; + int32 content_version = 2; + repeated ChapterInfo chapter_info = 3; + } + + message ChapterInfo { + string exploration_id = 1; + int32 exploration_content_version = 2; + } + + message SubtopicInfo { + int32 index = 1; + int32 content_version = 2; + } + + message SkillInfo { + string id = 1; + int32 content_version = 2; + } +}