diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b66cef7a6..46d826b1c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -36,6 +36,10 @@ jobs: username: ${{ secrets.FLYTE_BOT_USERNAME }} password: ${{ secrets.FLYTE_BOT_PAT }} + - name: Pack with SBT + if: ${{ github.ref != 'refs/heads/master' }} + run: cd flytekit-examples-scala && sbt compile test pack + - name: Verify with Maven if: ${{ github.ref != 'refs/heads/master' }} run: mvn --batch-mode verify -Pci diff --git a/flytekit-examples-scala/build.sbt b/flytekit-examples-scala/build.sbt new file mode 100644 index 000000000..4ab8c85b6 --- /dev/null +++ b/flytekit-examples-scala/build.sbt @@ -0,0 +1,10 @@ +import org.flyte.flytekitscala.FlytekitScalaPlugin + +ThisBuild / version := "0.4.60-SNAPSHOT" + +ThisBuild / scalaVersion := "2.13.14" + +lazy val root = (project in file(".")) + .settings( + name := "flytekit-examples-scala_2.13", + ).enablePlugins(FlytekitScalaPlugin) diff --git a/flytekit-examples-scala/pom.xml b/flytekit-examples-scala/pom.xml deleted file mode 100644 index 933af4e08..000000000 --- a/flytekit-examples-scala/pom.xml +++ /dev/null @@ -1,137 +0,0 @@ - - - - 4.0.0 - - - org.flyte - flytekit-parent - 0.4.60-SNAPSHOT - - - flytekit-examples-scala_2.13 - - Flytekit Java Examples in Scala - Examples of Tasks, Workflows and Launch plans written in Scala. - - - - true - - - - - - org.scala-lang - scala-library - ${scala213.version} - - - org.scala-lang - scala-reflect - ${scala213.version} - - - - - - - org.flyte - flytekit-api - - - org.scala-lang - scala-library - - - org.scala-lang - scala-reflect - - - org.flyte - flytekit-scala_2.13 - - - - - - - maven-jar-plugin - - ${project.build.directory}/lib - - - - maven-dependency-plugin - - - copy-compile - - copy-dependencies - - prepare-package - - ${project.build.directory}/lib - false - false - true - runtime - - - - - - net.alchim31.maven - scala-maven-plugin - - - -language:experimental.macros - - - - - - compile - testCompile - - - - attach-javadocs - - doc-jar - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - none - - - - - - diff --git a/flytekit-examples-scala/project/FlytekitScalaPlugin.scala b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala new file mode 100644 index 000000000..9684d0d98 --- /dev/null +++ b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala @@ -0,0 +1,155 @@ +/* + * Copyright 2020-2023 Flyte Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.flyte.flytekitscala + +import io.github.classgraph.{ClassGraph, ClassInfo, ClassInfoList, ScanResult} +import sbt.Keys.* +import sbt.* +import xerial.sbt.pack.PackPlugin +import xerial.sbt.pack.PackPlugin.autoImport.{packCopyDependenciesTarget, packDir, packLibJars, packResourceDir, packTargetDir} + +import scala.collection.JavaConverters.* + +object FlytekitScalaPlugin extends AutoPlugin { + val autoImport = FlytekitJavaKeys + import autoImport._ + + private val MetaInfoServiceFileNames = Seq( + "org.flyte.flytekit.SdkRunnableTask", + "org.flyte.flytekit.SdkDynamicWorkflowTask", + "org.flyte.flytekit.SdkPluginTask", + "org.flyte.flytekit.SdkContainerTask", + "org.flyte.flytekit.SdkWorkflow", + "org.flyte.flytekit.SdkLaunchPlanRegistry" + ) + + override def trigger: PluginTrigger = noTrigger + override def requires: Plugins = PackPlugin + + + + override lazy val projectSettings: Seq[Def.Setting[_]] = Seq( + flyteVersion := "0.4.58", + libraryDependencies ++= + Seq( + "org.flyte" % "flytekit-api" % flyteVersion.value, + "org.flyte" %% "flytekit-scala" % flyteVersion.value, + "org.flyte" % "flytekit-testing" % flyteVersion.value % Test + ), + // add flyte generated services after compilation as a jar resource + // note that we first have to remove potentially duplicated META-INF/services + // files to address a failure path like: + // $ sbt clean pack + // And then build project in IntelliJ (where generated files are copied to target/classes folder) + // $ sbt pack # this will result duplicated files and fail the build + Compile / packageBin / mappings := + (Compile / packageBin / mappings).value + .filterNot(v => MetaInfoServiceFileNames.contains(v._1.getName)) ++ + flyteGenerateServicesTask(Compile) + .map(_.map(f => (f, s"META-INF/services/${f.getName}"))) + .value, + // add flyte generated services after compilation as a test resource + Test / resourceGenerators += flyteGenerateServicesTask(Test) + ) + + private def flyteGenerateServicesTask(configKey: ConfigKey) = Def.task { + val log = (configKey / streams).value.log + val classPath = (Runtime / fullClasspath).value.map(_.data.getAbsolutePath) + val classGraph = new ClassGraph().overrideClasspath(classPath: _*) + val result = classGraph.enableMethodInfo().scan() + try { + MetaInfoServiceFileNames + .filter(fileName => result.getClassInfo(fileName) != null) // in case old version of flytekit-java + .map { fileName => + val impls = getClassesImplementingOrExtending(result, fileName, log) + impls.foreach(x => log.info(s"Discovered $fileName: $x")) + val services = impls.mkString("\n") + val file = (configKey / classDirectory).value / "META-INF" / "services" / fileName + IO.write(file, services) + file + } + } finally { + result.close() + } + } + + private def getClassesImplementingOrExtending( + result: ScanResult, + className: String, + log: Logger + ): List[String] = { + val classesOrInterfaces = + if (result.getClassInfo(className).isInterface) { + result.getClassesImplementing(className) + } else { + result.getSubclasses(className) + } + + warnAnonymousClasses(classesOrInterfaces, log) + + val subClasses = + classesOrInterfaces + .filter(x => !x.isAbstract && !x.isAnonymousInnerClass) + + failIfMissingDefaultConstructor(subClasses, log) + + val subInterfaces = classesOrInterfaces.getInterfaces.getNames.asScala.toList + val subAbstractClasses = classesOrInterfaces.filter(_.isAbstract).getNames.asScala.toList + + val all = subClasses.getNames.asScala.toList ++ + subInterfaces.flatMap(getClassesImplementingOrExtending(result, _, log)) ++ + subAbstractClasses.flatMap(getClassesImplementingOrExtending(result, _, log)) + + all.distinct + } + + private def warnAnonymousClasses( + classesOrInterfaces: ClassInfoList, + log: Logger + ): Unit = { + classesOrInterfaces + .filter(_.isAnonymousInnerClass) + .forEach( + x => + log.warn( + s"Anonymous class ${x.getName} cannot be used to implement Flyte entities" + ) + ) + } + + private def failIfMissingDefaultConstructor(classes: ClassInfoList, log: Logger): Unit = { + val classesMissingDefaultConstructor = classes.filter(hasNoDefaultConstructor) + + if (!classesMissingDefaultConstructor.isEmpty) { + classesMissingDefaultConstructor.forEach( + x => log.error(s"Class ${x.getName} has no default constructor defined") + ) + + throw new MessageOnlyException( + "One or more classes implementing Flyte entity have no default constructor defined" + ) + } + } + + private def hasNoDefaultConstructor(clazz: ClassInfo): Boolean = + clazz.getDeclaredConstructorInfo.filter(_.getParameterInfo.isEmpty).isEmpty +} + +object FlytekitJavaKeys { + // don't override defaults for these settings unless you want to use unstable version + lazy val flyteVersion = settingKey[String]("Flyte version") +} diff --git a/flytekit-examples-scala/project/build.properties b/flytekit-examples-scala/project/build.properties new file mode 100644 index 000000000..136f452e0 --- /dev/null +++ b/flytekit-examples-scala/project/build.properties @@ -0,0 +1 @@ +sbt.version = 1.10.1 diff --git a/flytekit-examples-scala/project/build.sbt b/flytekit-examples-scala/project/build.sbt new file mode 100644 index 000000000..0bb4c6400 --- /dev/null +++ b/flytekit-examples-scala/project/build.sbt @@ -0,0 +1,6 @@ +import scala.collection.immutable.Seq + + +libraryDependencies ++= Seq( + "io.github.classgraph" % "classgraph" % "4.8.87" +) diff --git a/flytekit-examples-scala/project/plugins.sbt b/flytekit-examples-scala/project/plugins.sbt new file mode 100644 index 000000000..467be7cdd --- /dev/null +++ b/flytekit-examples-scala/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.20") \ No newline at end of file diff --git a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkLaunchPlanRegistry b/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkLaunchPlanRegistry deleted file mode 100644 index acd3fc633..000000000 --- a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkLaunchPlanRegistry +++ /dev/null @@ -1 +0,0 @@ -org.flyte.examples.flytekitscala.LaunchPlanRegistry diff --git a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkRunnableTask b/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkRunnableTask deleted file mode 100644 index 201af9a92..000000000 --- a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkRunnableTask +++ /dev/null @@ -1,7 +0,0 @@ -org.flyte.examples.flytekitscala.HelloWorldTask -org.flyte.examples.flytekitscala.SumTask -org.flyte.examples.flytekitscala.GreetTask -org.flyte.examples.flytekitscala.AddQuestionTask -org.flyte.examples.flytekitscala.NoInputsTask -org.flyte.examples.flytekitscala.NestedIOTask -org.flyte.examples.flytekitscala.NestedIOTaskNoop diff --git a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkWorkflow b/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkWorkflow deleted file mode 100644 index 844fdc040..000000000 --- a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkWorkflow +++ /dev/null @@ -1,3 +0,0 @@ -org.flyte.examples.flytekitscala.FibonacciWorkflow -org.flyte.examples.flytekitscala.WelcomeWorkflow -org.flyte.examples.flytekitscala.NestedIOWorkflow diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 1d6d4f6c1..1346ad4f0 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -63,11 +63,6 @@ flytekit-examples test - - org.flyte - flytekit-examples-scala_2.13 - test - org.flyte jflyte diff --git a/integration-tests/src/test/java/org/flyte/AdditionalIT.java b/integration-tests/src/test/java/org/flyte/AdditionalIT.java index 5355ddb71..ea3f65d2d 100644 --- a/integration-tests/src/test/java/org/flyte/AdditionalIT.java +++ b/integration-tests/src/test/java/org/flyte/AdditionalIT.java @@ -16,14 +16,11 @@ */ package org.flyte; -import static org.flyte.examples.FlyteEnvironment.STAGING_DOMAIN; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import flyteidl.core.Literals; -import flyteidl.core.Literals.LiteralMap; import org.flyte.utils.Literal; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -31,6 +28,7 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) class AdditionalIT extends Fixtures { + @ParameterizedTest @CsvSource({ "0,0,0,0,a == b && c == d", @@ -70,12 +68,4 @@ void testStructs(String name, boolean expected) { assertThat(output, equalTo(Literal.ofBooleanMap(ImmutableMap.of("exists", expected)))); } - - @Test - void testStructsScala() { - Literals.LiteralMap output = - CLIENT.createExecution("NestedIOWorkflowLaunchPlan", STAGING_DOMAIN); - - assertThat(output, equalTo(LiteralMap.getDefaultInstance())); - } } diff --git a/integration-tests/src/test/java/org/flyte/Fixtures.java b/integration-tests/src/test/java/org/flyte/Fixtures.java index d80bec77d..40f68a864 100644 --- a/integration-tests/src/test/java/org/flyte/Fixtures.java +++ b/integration-tests/src/test/java/org/flyte/Fixtures.java @@ -26,6 +26,6 @@ class Fixtures { static { CLIENT.registerWorkflows("integration-tests/target/lib"); CLIENT.registerWorkflows("flytekit-examples/target/lib"); - CLIENT.registerWorkflows("flytekit-examples-scala/target/lib", STAGING_DOMAIN); + CLIENT.registerWorkflows("flytekit-examples-scala/target/pack/lib", STAGING_DOMAIN); } } diff --git a/pom.xml b/pom.xml index 9233acf93..6a040f237 100644 --- a/pom.xml +++ b/pom.xml @@ -55,7 +55,6 @@ flytekit-scala-tests flytekit-testing flytekit-examples - flytekit-examples-scala flytekit-local-engine flyteidl-protos jflyte-api