From 577397da9fa18fd52394b654d640d1c5c954d3d1 Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 12:10:23 +0200 Subject: [PATCH 01/12] Scala Examples sbt Signed-off-by: Rafael Raposo --- flytekit-examples-scala/.bsp/sbt.json | 1 + flytekit-examples-scala/build.sbt | 10 ++ flytekit-examples-scala/pom.xml | 137 ------------------ .../project/FlytekitScalaPlugin.scala | 136 +++++++++++++++++ .../project/build.properties | 1 + flytekit-examples-scala/project/build.sbt | 6 + .../org.flyte.flytekit.SdkLaunchPlanRegistry | 1 - .../org.flyte.flytekit.SdkRunnableTask | 7 - .../services/org.flyte.flytekit.SdkWorkflow | 3 - .../flytekitscala/FlytekitScalaPlugin.scala | 136 +++++++++++++++++ 10 files changed, 290 insertions(+), 148 deletions(-) create mode 100644 flytekit-examples-scala/.bsp/sbt.json create mode 100644 flytekit-examples-scala/build.sbt delete mode 100644 flytekit-examples-scala/pom.xml create mode 100644 flytekit-examples-scala/project/FlytekitScalaPlugin.scala create mode 100644 flytekit-examples-scala/project/build.properties create mode 100644 flytekit-examples-scala/project/build.sbt delete mode 100644 flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkLaunchPlanRegistry delete mode 100644 flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkRunnableTask delete mode 100644 flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkWorkflow create mode 100644 flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala diff --git a/flytekit-examples-scala/.bsp/sbt.json b/flytekit-examples-scala/.bsp/sbt.json new file mode 100644 index 000000000..631bc3d92 --- /dev/null +++ b/flytekit-examples-scala/.bsp/sbt.json @@ -0,0 +1 @@ +{"name":"sbt","version":"1.10.1","bspVersion":"2.1.0-M1","languages":["scala"],"argv":["/Users/rafaelraposo/.sdkman/candidates/java/11.0.22-amzn/bin/java","-Xms100m","-Xmx100m","-classpath","/Users/rafaelraposo/Library/Application Support/JetBrains/IntelliJIdea2024.1/plugins/Scala/launcher/sbt-launch.jar","-Dsbt.script=/Users/rafaelraposo/.sdkman/candidates/sbt/current/bin/sbt","xsbt.boot.Boot","-bsp"]} \ No newline at end of file diff --git a/flytekit-examples-scala/build.sbt b/flytekit-examples-scala/build.sbt new file mode 100644 index 000000000..4ab8c85b6 --- /dev/null +++ b/flytekit-examples-scala/build.sbt @@ -0,0 +1,10 @@ +import org.flyte.flytekitscala.FlytekitScalaPlugin + +ThisBuild / version := "0.4.60-SNAPSHOT" + +ThisBuild / scalaVersion := "2.13.14" + +lazy val root = (project in file(".")) + .settings( + name := "flytekit-examples-scala_2.13", + ).enablePlugins(FlytekitScalaPlugin) diff --git a/flytekit-examples-scala/pom.xml b/flytekit-examples-scala/pom.xml deleted file mode 100644 index 933af4e08..000000000 --- a/flytekit-examples-scala/pom.xml +++ /dev/null @@ -1,137 +0,0 @@ - - - - 4.0.0 - - - org.flyte - flytekit-parent - 0.4.60-SNAPSHOT - - - flytekit-examples-scala_2.13 - - Flytekit Java Examples in Scala - Examples of Tasks, Workflows and Launch plans written in Scala. - - - - true - - - - - - org.scala-lang - scala-library - ${scala213.version} - - - org.scala-lang - scala-reflect - ${scala213.version} - - - - - - - org.flyte - flytekit-api - - - org.scala-lang - scala-library - - - org.scala-lang - scala-reflect - - - org.flyte - flytekit-scala_2.13 - - - - - - - maven-jar-plugin - - ${project.build.directory}/lib - - - - maven-dependency-plugin - - - copy-compile - - copy-dependencies - - prepare-package - - ${project.build.directory}/lib - false - false - true - runtime - - - - - - net.alchim31.maven - scala-maven-plugin - - - -language:experimental.macros - - - - - - compile - testCompile - - - - attach-javadocs - - doc-jar - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - none - - - - - - diff --git a/flytekit-examples-scala/project/FlytekitScalaPlugin.scala b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala new file mode 100644 index 000000000..e567af65f --- /dev/null +++ b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala @@ -0,0 +1,136 @@ +package org.flyte.flytekitscala + +import io.github.classgraph.{ClassGraph, ClassInfo, ClassInfoList, ScanResult} +import sbt.Keys._ +import sbt._ + +import scala.collection.JavaConverters.* + +object FlytekitScalaPlugin extends AutoPlugin { + val autoImport = FlytekitJavaKeys + import autoImport._ + + private val MetaInfoServiceFileNames = Seq( + "org.flyte.flytekit.SdkRunnableTask", + "org.flyte.flytekit.SdkDynamicWorkflowTask", + "org.flyte.flytekit.SdkPluginTask", + "org.flyte.flytekit.SdkContainerTask", + "org.flyte.flytekit.SdkWorkflow", + "org.flyte.flytekit.SdkLaunchPlanRegistry" + ) + + override def trigger: PluginTrigger = noTrigger + + + + override lazy val projectSettings: Seq[Def.Setting[_]] = Seq( + flyteVersion := "0.4.58", + libraryDependencies ++= + Seq( + "org.flyte" % "flytekit-api" % flyteVersion.value, + "org.flyte" %% "flytekit-scala" % flyteVersion.value, + "org.flyte" % "flytekit-testing" % flyteVersion.value % Test + ), + // add flyte generated services after compilation as a jar resource + // note that we first have to remove potentially duplicated META-INF/services + // files to address a failure path like: + // $ sbt clean pack + // And then build project in IntelliJ (where generated files are copied to target/classes folder) + // $ sbt pack # this will result duplicated files and fail the build + Compile / packageBin / mappings := + (Compile / packageBin / mappings).value + .filterNot(v => MetaInfoServiceFileNames.contains(v._1.getName)) ++ + flyteGenerateServicesTask(Compile) + .map(_.map(f => (f, s"META-INF/services/${f.getName}"))) + .value, + // add flyte generated services after compilation as a test resource + Test / resourceGenerators += flyteGenerateServicesTask(Test) + ) + + private def flyteGenerateServicesTask(configKey: ConfigKey) = Def.task { + val log = (configKey / streams).value.log + val classPath = (Runtime / fullClasspath).value.map(_.data.getAbsolutePath) + val classGraph = new ClassGraph().overrideClasspath(classPath: _*) + val result = classGraph.enableMethodInfo().scan() + try { + MetaInfoServiceFileNames + .filter(fileName => result.getClassInfo(fileName) != null) // in case old version of flytekit-java + .map { fileName => + val impls = getClassesImplementingOrExtending(result, fileName, log) + impls.foreach(x => log.info(s"Discovered $fileName: $x")) + val services = impls.mkString("\n") + val file = (configKey / classDirectory).value / "META-INF" / "services" / fileName + IO.write(file, services) + file + } + } finally { + result.close() + } + } + + private def getClassesImplementingOrExtending( + result: ScanResult, + className: String, + log: Logger + ): List[String] = { + val classesOrInterfaces = + if (result.getClassInfo(className).isInterface) { + result.getClassesImplementing(className) + } else { + result.getSubclasses(className) + } + + warnAnonymousClasses(classesOrInterfaces, log) + + val subClasses = + classesOrInterfaces + .filter(x => !x.isAbstract && !x.isAnonymousInnerClass) + + failIfMissingDefaultConstructor(subClasses, log) + + val subInterfaces = classesOrInterfaces.getInterfaces.getNames.asScala.toList + val subAbstractClasses = classesOrInterfaces.filter(_.isAbstract).getNames.asScala.toList + + val all = subClasses.getNames.asScala.toList ++ + subInterfaces.flatMap(getClassesImplementingOrExtending(result, _, log)) ++ + subAbstractClasses.flatMap(getClassesImplementingOrExtending(result, _, log)) + + all.distinct + } + + private def warnAnonymousClasses( + classesOrInterfaces: ClassInfoList, + log: Logger + ): Unit = { + classesOrInterfaces + .filter(_.isAnonymousInnerClass) + .forEach( + x => + log.warn( + s"Anonymous class ${x.getName} cannot be used to implement Flyte entities" + ) + ) + } + + private def failIfMissingDefaultConstructor(classes: ClassInfoList, log: Logger): Unit = { + val classesMissingDefaultConstructor = classes.filter(hasNoDefaultConstructor) + + if (!classesMissingDefaultConstructor.isEmpty) { + classesMissingDefaultConstructor.forEach( + x => log.error(s"Class ${x.getName} has no default constructor defined") + ) + + throw new MessageOnlyException( + "One or more classes implementing Flyte entity have no default constructor defined" + ) + } + } + + private def hasNoDefaultConstructor(clazz: ClassInfo): Boolean = + clazz.getDeclaredConstructorInfo.filter(_.getParameterInfo.isEmpty).isEmpty +} + +object FlytekitJavaKeys { + // don't override defaults for these settings unless you want to use unstable version + lazy val flyteVersion = settingKey[String]("Flyte version") +} \ No newline at end of file diff --git a/flytekit-examples-scala/project/build.properties b/flytekit-examples-scala/project/build.properties new file mode 100644 index 000000000..136f452e0 --- /dev/null +++ b/flytekit-examples-scala/project/build.properties @@ -0,0 +1 @@ +sbt.version = 1.10.1 diff --git a/flytekit-examples-scala/project/build.sbt b/flytekit-examples-scala/project/build.sbt new file mode 100644 index 000000000..6a03c2289 --- /dev/null +++ b/flytekit-examples-scala/project/build.sbt @@ -0,0 +1,6 @@ +import scala.collection.immutable.Seq + + +libraryDependencies ++=Seq( + "io.github.classgraph" % "classgraph" % "4.8.87" +) \ No newline at end of file diff --git a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkLaunchPlanRegistry b/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkLaunchPlanRegistry deleted file mode 100644 index acd3fc633..000000000 --- a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkLaunchPlanRegistry +++ /dev/null @@ -1 +0,0 @@ -org.flyte.examples.flytekitscala.LaunchPlanRegistry diff --git a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkRunnableTask b/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkRunnableTask deleted file mode 100644 index 201af9a92..000000000 --- a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkRunnableTask +++ /dev/null @@ -1,7 +0,0 @@ -org.flyte.examples.flytekitscala.HelloWorldTask -org.flyte.examples.flytekitscala.SumTask -org.flyte.examples.flytekitscala.GreetTask -org.flyte.examples.flytekitscala.AddQuestionTask -org.flyte.examples.flytekitscala.NoInputsTask -org.flyte.examples.flytekitscala.NestedIOTask -org.flyte.examples.flytekitscala.NestedIOTaskNoop diff --git a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkWorkflow b/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkWorkflow deleted file mode 100644 index 844fdc040..000000000 --- a/flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkWorkflow +++ /dev/null @@ -1,3 +0,0 @@ -org.flyte.examples.flytekitscala.FibonacciWorkflow -org.flyte.examples.flytekitscala.WelcomeWorkflow -org.flyte.examples.flytekitscala.NestedIOWorkflow diff --git a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala new file mode 100644 index 000000000..e567af65f --- /dev/null +++ b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala @@ -0,0 +1,136 @@ +package org.flyte.flytekitscala + +import io.github.classgraph.{ClassGraph, ClassInfo, ClassInfoList, ScanResult} +import sbt.Keys._ +import sbt._ + +import scala.collection.JavaConverters.* + +object FlytekitScalaPlugin extends AutoPlugin { + val autoImport = FlytekitJavaKeys + import autoImport._ + + private val MetaInfoServiceFileNames = Seq( + "org.flyte.flytekit.SdkRunnableTask", + "org.flyte.flytekit.SdkDynamicWorkflowTask", + "org.flyte.flytekit.SdkPluginTask", + "org.flyte.flytekit.SdkContainerTask", + "org.flyte.flytekit.SdkWorkflow", + "org.flyte.flytekit.SdkLaunchPlanRegistry" + ) + + override def trigger: PluginTrigger = noTrigger + + + + override lazy val projectSettings: Seq[Def.Setting[_]] = Seq( + flyteVersion := "0.4.58", + libraryDependencies ++= + Seq( + "org.flyte" % "flytekit-api" % flyteVersion.value, + "org.flyte" %% "flytekit-scala" % flyteVersion.value, + "org.flyte" % "flytekit-testing" % flyteVersion.value % Test + ), + // add flyte generated services after compilation as a jar resource + // note that we first have to remove potentially duplicated META-INF/services + // files to address a failure path like: + // $ sbt clean pack + // And then build project in IntelliJ (where generated files are copied to target/classes folder) + // $ sbt pack # this will result duplicated files and fail the build + Compile / packageBin / mappings := + (Compile / packageBin / mappings).value + .filterNot(v => MetaInfoServiceFileNames.contains(v._1.getName)) ++ + flyteGenerateServicesTask(Compile) + .map(_.map(f => (f, s"META-INF/services/${f.getName}"))) + .value, + // add flyte generated services after compilation as a test resource + Test / resourceGenerators += flyteGenerateServicesTask(Test) + ) + + private def flyteGenerateServicesTask(configKey: ConfigKey) = Def.task { + val log = (configKey / streams).value.log + val classPath = (Runtime / fullClasspath).value.map(_.data.getAbsolutePath) + val classGraph = new ClassGraph().overrideClasspath(classPath: _*) + val result = classGraph.enableMethodInfo().scan() + try { + MetaInfoServiceFileNames + .filter(fileName => result.getClassInfo(fileName) != null) // in case old version of flytekit-java + .map { fileName => + val impls = getClassesImplementingOrExtending(result, fileName, log) + impls.foreach(x => log.info(s"Discovered $fileName: $x")) + val services = impls.mkString("\n") + val file = (configKey / classDirectory).value / "META-INF" / "services" / fileName + IO.write(file, services) + file + } + } finally { + result.close() + } + } + + private def getClassesImplementingOrExtending( + result: ScanResult, + className: String, + log: Logger + ): List[String] = { + val classesOrInterfaces = + if (result.getClassInfo(className).isInterface) { + result.getClassesImplementing(className) + } else { + result.getSubclasses(className) + } + + warnAnonymousClasses(classesOrInterfaces, log) + + val subClasses = + classesOrInterfaces + .filter(x => !x.isAbstract && !x.isAnonymousInnerClass) + + failIfMissingDefaultConstructor(subClasses, log) + + val subInterfaces = classesOrInterfaces.getInterfaces.getNames.asScala.toList + val subAbstractClasses = classesOrInterfaces.filter(_.isAbstract).getNames.asScala.toList + + val all = subClasses.getNames.asScala.toList ++ + subInterfaces.flatMap(getClassesImplementingOrExtending(result, _, log)) ++ + subAbstractClasses.flatMap(getClassesImplementingOrExtending(result, _, log)) + + all.distinct + } + + private def warnAnonymousClasses( + classesOrInterfaces: ClassInfoList, + log: Logger + ): Unit = { + classesOrInterfaces + .filter(_.isAnonymousInnerClass) + .forEach( + x => + log.warn( + s"Anonymous class ${x.getName} cannot be used to implement Flyte entities" + ) + ) + } + + private def failIfMissingDefaultConstructor(classes: ClassInfoList, log: Logger): Unit = { + val classesMissingDefaultConstructor = classes.filter(hasNoDefaultConstructor) + + if (!classesMissingDefaultConstructor.isEmpty) { + classesMissingDefaultConstructor.forEach( + x => log.error(s"Class ${x.getName} has no default constructor defined") + ) + + throw new MessageOnlyException( + "One or more classes implementing Flyte entity have no default constructor defined" + ) + } + } + + private def hasNoDefaultConstructor(clazz: ClassInfo): Boolean = + clazz.getDeclaredConstructorInfo.filter(_.getParameterInfo.isEmpty).isEmpty +} + +object FlytekitJavaKeys { + // don't override defaults for these settings unless you want to use unstable version + lazy val flyteVersion = settingKey[String]("Flyte version") +} \ No newline at end of file From 8e8dc3180b5c2e7e031bcd9b3c927bd0ec230f37 Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 12:48:53 +0200 Subject: [PATCH 02/12] fmt Signed-off-by: Rafael Raposo --- .../main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala index e567af65f..68e14f7bc 100644 --- a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala +++ b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala @@ -1,5 +1,4 @@ package org.flyte.flytekitscala - import io.github.classgraph.{ClassGraph, ClassInfo, ClassInfoList, ScanResult} import sbt.Keys._ import sbt._ From a604f32c28fc3db876b6a57f766b6c81fd514c23 Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 12:55:24 +0200 Subject: [PATCH 03/12] boiler and module Signed-off-by: Rafael Raposo --- .../project/FlytekitScalaPlugin.scala | 16 +++ .../flytekitscala/FlytekitScalaPlugin.scala | 135 ------------------ pom.xml | 1 - 3 files changed, 16 insertions(+), 136 deletions(-) delete mode 100644 flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala diff --git a/flytekit-examples-scala/project/FlytekitScalaPlugin.scala b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala index e567af65f..3c34b9770 100644 --- a/flytekit-examples-scala/project/FlytekitScalaPlugin.scala +++ b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2020-2023 Flyte Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.flyte.flytekitscala import io.github.classgraph.{ClassGraph, ClassInfo, ClassInfoList, ScanResult} diff --git a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala deleted file mode 100644 index 68e14f7bc..000000000 --- a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/FlytekitScalaPlugin.scala +++ /dev/null @@ -1,135 +0,0 @@ -package org.flyte.flytekitscala -import io.github.classgraph.{ClassGraph, ClassInfo, ClassInfoList, ScanResult} -import sbt.Keys._ -import sbt._ - -import scala.collection.JavaConverters.* - -object FlytekitScalaPlugin extends AutoPlugin { - val autoImport = FlytekitJavaKeys - import autoImport._ - - private val MetaInfoServiceFileNames = Seq( - "org.flyte.flytekit.SdkRunnableTask", - "org.flyte.flytekit.SdkDynamicWorkflowTask", - "org.flyte.flytekit.SdkPluginTask", - "org.flyte.flytekit.SdkContainerTask", - "org.flyte.flytekit.SdkWorkflow", - "org.flyte.flytekit.SdkLaunchPlanRegistry" - ) - - override def trigger: PluginTrigger = noTrigger - - - - override lazy val projectSettings: Seq[Def.Setting[_]] = Seq( - flyteVersion := "0.4.58", - libraryDependencies ++= - Seq( - "org.flyte" % "flytekit-api" % flyteVersion.value, - "org.flyte" %% "flytekit-scala" % flyteVersion.value, - "org.flyte" % "flytekit-testing" % flyteVersion.value % Test - ), - // add flyte generated services after compilation as a jar resource - // note that we first have to remove potentially duplicated META-INF/services - // files to address a failure path like: - // $ sbt clean pack - // And then build project in IntelliJ (where generated files are copied to target/classes folder) - // $ sbt pack # this will result duplicated files and fail the build - Compile / packageBin / mappings := - (Compile / packageBin / mappings).value - .filterNot(v => MetaInfoServiceFileNames.contains(v._1.getName)) ++ - flyteGenerateServicesTask(Compile) - .map(_.map(f => (f, s"META-INF/services/${f.getName}"))) - .value, - // add flyte generated services after compilation as a test resource - Test / resourceGenerators += flyteGenerateServicesTask(Test) - ) - - private def flyteGenerateServicesTask(configKey: ConfigKey) = Def.task { - val log = (configKey / streams).value.log - val classPath = (Runtime / fullClasspath).value.map(_.data.getAbsolutePath) - val classGraph = new ClassGraph().overrideClasspath(classPath: _*) - val result = classGraph.enableMethodInfo().scan() - try { - MetaInfoServiceFileNames - .filter(fileName => result.getClassInfo(fileName) != null) // in case old version of flytekit-java - .map { fileName => - val impls = getClassesImplementingOrExtending(result, fileName, log) - impls.foreach(x => log.info(s"Discovered $fileName: $x")) - val services = impls.mkString("\n") - val file = (configKey / classDirectory).value / "META-INF" / "services" / fileName - IO.write(file, services) - file - } - } finally { - result.close() - } - } - - private def getClassesImplementingOrExtending( - result: ScanResult, - className: String, - log: Logger - ): List[String] = { - val classesOrInterfaces = - if (result.getClassInfo(className).isInterface) { - result.getClassesImplementing(className) - } else { - result.getSubclasses(className) - } - - warnAnonymousClasses(classesOrInterfaces, log) - - val subClasses = - classesOrInterfaces - .filter(x => !x.isAbstract && !x.isAnonymousInnerClass) - - failIfMissingDefaultConstructor(subClasses, log) - - val subInterfaces = classesOrInterfaces.getInterfaces.getNames.asScala.toList - val subAbstractClasses = classesOrInterfaces.filter(_.isAbstract).getNames.asScala.toList - - val all = subClasses.getNames.asScala.toList ++ - subInterfaces.flatMap(getClassesImplementingOrExtending(result, _, log)) ++ - subAbstractClasses.flatMap(getClassesImplementingOrExtending(result, _, log)) - - all.distinct - } - - private def warnAnonymousClasses( - classesOrInterfaces: ClassInfoList, - log: Logger - ): Unit = { - classesOrInterfaces - .filter(_.isAnonymousInnerClass) - .forEach( - x => - log.warn( - s"Anonymous class ${x.getName} cannot be used to implement Flyte entities" - ) - ) - } - - private def failIfMissingDefaultConstructor(classes: ClassInfoList, log: Logger): Unit = { - val classesMissingDefaultConstructor = classes.filter(hasNoDefaultConstructor) - - if (!classesMissingDefaultConstructor.isEmpty) { - classesMissingDefaultConstructor.forEach( - x => log.error(s"Class ${x.getName} has no default constructor defined") - ) - - throw new MessageOnlyException( - "One or more classes implementing Flyte entity have no default constructor defined" - ) - } - } - - private def hasNoDefaultConstructor(clazz: ClassInfo): Boolean = - clazz.getDeclaredConstructorInfo.filter(_.getParameterInfo.isEmpty).isEmpty -} - -object FlytekitJavaKeys { - // don't override defaults for these settings unless you want to use unstable version - lazy val flyteVersion = settingKey[String]("Flyte version") -} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 9233acf93..6a040f237 100644 --- a/pom.xml +++ b/pom.xml @@ -55,7 +55,6 @@ flytekit-scala-tests flytekit-testing flytekit-examples - flytekit-examples-scala flytekit-local-engine flyteidl-protos jflyte-api From 88b0bcadd677272d84362aeb942647c3f0cb261a Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 12:59:26 +0200 Subject: [PATCH 04/12] boiler and module Signed-off-by: Rafael Raposo --- flytekit-examples-scala/.bsp/sbt.json | 1 - flytekit-examples-scala/project/FlytekitScalaPlugin.scala | 2 +- flytekit-examples-scala/project/build.sbt | 4 ++-- integration-tests/pom.xml | 5 ----- 4 files changed, 3 insertions(+), 9 deletions(-) delete mode 100644 flytekit-examples-scala/.bsp/sbt.json diff --git a/flytekit-examples-scala/.bsp/sbt.json b/flytekit-examples-scala/.bsp/sbt.json deleted file mode 100644 index 631bc3d92..000000000 --- a/flytekit-examples-scala/.bsp/sbt.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"sbt","version":"1.10.1","bspVersion":"2.1.0-M1","languages":["scala"],"argv":["/Users/rafaelraposo/.sdkman/candidates/java/11.0.22-amzn/bin/java","-Xms100m","-Xmx100m","-classpath","/Users/rafaelraposo/Library/Application Support/JetBrains/IntelliJIdea2024.1/plugins/Scala/launcher/sbt-launch.jar","-Dsbt.script=/Users/rafaelraposo/.sdkman/candidates/sbt/current/bin/sbt","xsbt.boot.Boot","-bsp"]} \ No newline at end of file diff --git a/flytekit-examples-scala/project/FlytekitScalaPlugin.scala b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala index 3c34b9770..081022dfe 100644 --- a/flytekit-examples-scala/project/FlytekitScalaPlugin.scala +++ b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala @@ -149,4 +149,4 @@ object FlytekitScalaPlugin extends AutoPlugin { object FlytekitJavaKeys { // don't override defaults for these settings unless you want to use unstable version lazy val flyteVersion = settingKey[String]("Flyte version") -} \ No newline at end of file +} diff --git a/flytekit-examples-scala/project/build.sbt b/flytekit-examples-scala/project/build.sbt index 6a03c2289..0bb4c6400 100644 --- a/flytekit-examples-scala/project/build.sbt +++ b/flytekit-examples-scala/project/build.sbt @@ -1,6 +1,6 @@ import scala.collection.immutable.Seq -libraryDependencies ++=Seq( +libraryDependencies ++= Seq( "io.github.classgraph" % "classgraph" % "4.8.87" -) \ No newline at end of file +) diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 1d6d4f6c1..1346ad4f0 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -63,11 +63,6 @@ flytekit-examples test - - org.flyte - flytekit-examples-scala_2.13 - test - org.flyte jflyte From 589ad827dc1243029f66b68563b7d555f9b52ff2 Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 14:33:52 +0200 Subject: [PATCH 05/12] remove from it test Signed-off-by: Rafael Raposo --- integration-tests/src/test/java/org/flyte/Fixtures.java | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/src/test/java/org/flyte/Fixtures.java b/integration-tests/src/test/java/org/flyte/Fixtures.java index d80bec77d..d73f76aac 100644 --- a/integration-tests/src/test/java/org/flyte/Fixtures.java +++ b/integration-tests/src/test/java/org/flyte/Fixtures.java @@ -26,6 +26,5 @@ class Fixtures { static { CLIENT.registerWorkflows("integration-tests/target/lib"); CLIENT.registerWorkflows("flytekit-examples/target/lib"); - CLIENT.registerWorkflows("flytekit-examples-scala/target/lib", STAGING_DOMAIN); } } From ba5a56d929053f538db25642e999115d823f194f Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 14:53:29 +0200 Subject: [PATCH 06/12] fmt Signed-off-by: Rafael Raposo --- integration-tests/src/test/java/org/flyte/Fixtures.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration-tests/src/test/java/org/flyte/Fixtures.java b/integration-tests/src/test/java/org/flyte/Fixtures.java index d73f76aac..07aeb824d 100644 --- a/integration-tests/src/test/java/org/flyte/Fixtures.java +++ b/integration-tests/src/test/java/org/flyte/Fixtures.java @@ -16,8 +16,6 @@ */ package org.flyte; -import static org.flyte.examples.FlyteEnvironment.STAGING_DOMAIN; - import org.flyte.utils.FlyteSandboxClient; class Fixtures { From ca733ed5147587514f099ac05f32214db21b6a65 Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 15:56:45 +0200 Subject: [PATCH 07/12] scala-examples test Signed-off-by: Rafael Raposo --- .../src/test/java/org/flyte/AdditionalIT.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/integration-tests/src/test/java/org/flyte/AdditionalIT.java b/integration-tests/src/test/java/org/flyte/AdditionalIT.java index 5355ddb71..51e335919 100644 --- a/integration-tests/src/test/java/org/flyte/AdditionalIT.java +++ b/integration-tests/src/test/java/org/flyte/AdditionalIT.java @@ -70,12 +70,4 @@ void testStructs(String name, boolean expected) { assertThat(output, equalTo(Literal.ofBooleanMap(ImmutableMap.of("exists", expected)))); } - - @Test - void testStructsScala() { - Literals.LiteralMap output = - CLIENT.createExecution("NestedIOWorkflowLaunchPlan", STAGING_DOMAIN); - - assertThat(output, equalTo(LiteralMap.getDefaultInstance())); - } } From 50989c9cb72b28fe5793bf0853ec4e63326479bf Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 16:18:22 +0200 Subject: [PATCH 08/12] fmt Signed-off-by: Rafael Raposo --- .../src/test/java/org/flyte/AdditionalIT.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/integration-tests/src/test/java/org/flyte/AdditionalIT.java b/integration-tests/src/test/java/org/flyte/AdditionalIT.java index 51e335919..6f4b63732 100644 --- a/integration-tests/src/test/java/org/flyte/AdditionalIT.java +++ b/integration-tests/src/test/java/org/flyte/AdditionalIT.java @@ -31,17 +31,18 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) class AdditionalIT extends Fixtures { + @ParameterizedTest @CsvSource({ - "0,0,0,0,a == b && c == d", - "0,1,0,0,a < b && c == d", - "1,0,0,0,a > b && c == d", - "0,0,1,0,a == b && c > d", - "0,1,1,0,a < b && c > d", - "1,0,1,0,a > b && c > d", - "0,0,0,1,a == b && c < d", - "0,1,0,1,a < b && c < d", - "1,0,0,1,a > b && c < d", + "0,0,0,0,a == b && c == d", + "0,1,0,0,a < b && c == d", + "1,0,0,0,a > b && c == d", + "0,0,1,0,a == b && c > d", + "0,1,1,0,a < b && c > d", + "1,0,1,0,a > b && c > d", + "0,0,0,1,a == b && c < d", + "0,1,0,1,a < b && c < d", + "1,0,0,1,a > b && c < d", }) void testBranchNodeWorkflow(long a, long b, long c, long d, String expected) { Literals.LiteralMap output = @@ -59,8 +60,8 @@ void testBranchNodeWorkflow(long a, long b, long c, long d, String expected) { @ParameterizedTest @CsvSource({ - "table-exists,true", - "non-existent,false", + "table-exists,true", + "non-existent,false", }) void testStructs(String name, boolean expected) { Literals.LiteralMap output = From 28f1d07ef0976af4bc5bcff79e2d1f5c49f9b45c Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 16:19:14 +0200 Subject: [PATCH 09/12] fmt Signed-off-by: Rafael Raposo --- .../src/test/java/org/flyte/AdditionalIT.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/integration-tests/src/test/java/org/flyte/AdditionalIT.java b/integration-tests/src/test/java/org/flyte/AdditionalIT.java index 6f4b63732..ea3f65d2d 100644 --- a/integration-tests/src/test/java/org/flyte/AdditionalIT.java +++ b/integration-tests/src/test/java/org/flyte/AdditionalIT.java @@ -16,14 +16,11 @@ */ package org.flyte; -import static org.flyte.examples.FlyteEnvironment.STAGING_DOMAIN; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import flyteidl.core.Literals; -import flyteidl.core.Literals.LiteralMap; import org.flyte.utils.Literal; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -34,15 +31,15 @@ class AdditionalIT extends Fixtures { @ParameterizedTest @CsvSource({ - "0,0,0,0,a == b && c == d", - "0,1,0,0,a < b && c == d", - "1,0,0,0,a > b && c == d", - "0,0,1,0,a == b && c > d", - "0,1,1,0,a < b && c > d", - "1,0,1,0,a > b && c > d", - "0,0,0,1,a == b && c < d", - "0,1,0,1,a < b && c < d", - "1,0,0,1,a > b && c < d", + "0,0,0,0,a == b && c == d", + "0,1,0,0,a < b && c == d", + "1,0,0,0,a > b && c == d", + "0,0,1,0,a == b && c > d", + "0,1,1,0,a < b && c > d", + "1,0,1,0,a > b && c > d", + "0,0,0,1,a == b && c < d", + "0,1,0,1,a < b && c < d", + "1,0,0,1,a > b && c < d", }) void testBranchNodeWorkflow(long a, long b, long c, long d, String expected) { Literals.LiteralMap output = @@ -60,8 +57,8 @@ void testBranchNodeWorkflow(long a, long b, long c, long d, String expected) { @ParameterizedTest @CsvSource({ - "table-exists,true", - "non-existent,false", + "table-exists,true", + "non-existent,false", }) void testStructs(String name, boolean expected) { Literals.LiteralMap output = From 67d36d97d7a9afcfbb479483d14f012533692ffa Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Mon, 5 Aug 2024 16:47:55 +0200 Subject: [PATCH 10/12] cd Signed-off-by: Rafael Raposo --- .github/workflows/build.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b66cef7a6..5ca04dc5d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -36,6 +36,10 @@ jobs: username: ${{ secrets.FLYTE_BOT_USERNAME }} password: ${{ secrets.FLYTE_BOT_PAT }} + - name: Pack with SBT + if: ${{ github.ref != 'refs/heads/master' }} + run: cd flytekit-examples-scala && sbt clean package + - name: Verify with Maven if: ${{ github.ref != 'refs/heads/master' }} run: mvn --batch-mode verify -Pci From d90af650d88fdd4664a1e97939e845fcc0efab0b Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Tue, 6 Aug 2024 10:56:21 +0200 Subject: [PATCH 11/12] pack and IT Signed-off-by: Rafael Raposo --- .../project/FlytekitScalaPlugin.scala | 9 ++++++--- flytekit-examples-scala/project/plugins.sbt | 1 + integration-tests/src/test/java/org/flyte/Fixtures.java | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 flytekit-examples-scala/project/plugins.sbt diff --git a/flytekit-examples-scala/project/FlytekitScalaPlugin.scala b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala index 081022dfe..9684d0d98 100644 --- a/flytekit-examples-scala/project/FlytekitScalaPlugin.scala +++ b/flytekit-examples-scala/project/FlytekitScalaPlugin.scala @@ -17,8 +17,10 @@ package org.flyte.flytekitscala import io.github.classgraph.{ClassGraph, ClassInfo, ClassInfoList, ScanResult} -import sbt.Keys._ -import sbt._ +import sbt.Keys.* +import sbt.* +import xerial.sbt.pack.PackPlugin +import xerial.sbt.pack.PackPlugin.autoImport.{packCopyDependenciesTarget, packDir, packLibJars, packResourceDir, packTargetDir} import scala.collection.JavaConverters.* @@ -36,6 +38,7 @@ object FlytekitScalaPlugin extends AutoPlugin { ) override def trigger: PluginTrigger = noTrigger + override def requires: Plugins = PackPlugin @@ -47,7 +50,7 @@ object FlytekitScalaPlugin extends AutoPlugin { "org.flyte" %% "flytekit-scala" % flyteVersion.value, "org.flyte" % "flytekit-testing" % flyteVersion.value % Test ), - // add flyte generated services after compilation as a jar resource + // add flyte generated services after compilation as a jar resource // note that we first have to remove potentially duplicated META-INF/services // files to address a failure path like: // $ sbt clean pack diff --git a/flytekit-examples-scala/project/plugins.sbt b/flytekit-examples-scala/project/plugins.sbt new file mode 100644 index 000000000..467be7cdd --- /dev/null +++ b/flytekit-examples-scala/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.20") \ No newline at end of file diff --git a/integration-tests/src/test/java/org/flyte/Fixtures.java b/integration-tests/src/test/java/org/flyte/Fixtures.java index 07aeb824d..40f68a864 100644 --- a/integration-tests/src/test/java/org/flyte/Fixtures.java +++ b/integration-tests/src/test/java/org/flyte/Fixtures.java @@ -16,6 +16,8 @@ */ package org.flyte; +import static org.flyte.examples.FlyteEnvironment.STAGING_DOMAIN; + import org.flyte.utils.FlyteSandboxClient; class Fixtures { @@ -24,5 +26,6 @@ class Fixtures { static { CLIENT.registerWorkflows("integration-tests/target/lib"); CLIENT.registerWorkflows("flytekit-examples/target/lib"); + CLIENT.registerWorkflows("flytekit-examples-scala/target/pack/lib", STAGING_DOMAIN); } } From 12da18a1a0b06f51a863490811b677d1b4cd19e6 Mon Sep 17 00:00:00 2001 From: Rafael Raposo Date: Tue, 6 Aug 2024 10:59:44 +0200 Subject: [PATCH 12/12] sbt compile test pack Signed-off-by: Rafael Raposo --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5ca04dc5d..46d826b1c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -38,7 +38,7 @@ jobs: - name: Pack with SBT if: ${{ github.ref != 'refs/heads/master' }} - run: cd flytekit-examples-scala && sbt clean package + run: cd flytekit-examples-scala && sbt compile test pack - name: Verify with Maven if: ${{ github.ref != 'refs/heads/master' }}