From 932c7ddf272a9a8fa851659df6289dc1b47358cc Mon Sep 17 00:00:00 2001 From: jamie-albert Date: Fri, 17 Jan 2025 02:36:23 -0800 Subject: [PATCH 1/2] spark-3.5-scala-2.13/GHSA-8qv5-68g4-248j-fix --- spark-3.5-scala-2.13.yaml | 4 +- .../GHSA-8qv5-68g4-248j-fix.patch | 104 ++++++++++++++++++ spark-3.5-scala-2.13/pombump-properties.yaml | 4 +- 3 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 spark-3.5-scala-2.13/GHSA-8qv5-68g4-248j-fix.patch diff --git a/spark-3.5-scala-2.13.yaml b/spark-3.5-scala-2.13.yaml index c945bd1371c..fbe3c8d911d 100644 --- a/spark-3.5-scala-2.13.yaml +++ b/spark-3.5-scala-2.13.yaml @@ -1,7 +1,7 @@ package: name: spark-3.5-scala-2.13 version: 3.5.4 - epoch: 0 + epoch: 1 description: Unified engine for large-scale data analytics copyright: - license: Apache-2.0 @@ -50,7 +50,7 @@ pipeline: # These are not CVE patches, but patches to make the build work. - uses: patch with: - patches: make-distribution.patch mvn.patch + patches: make-distribution.patch mvn.patch GHSA-8qv5-68g4-248j-fix.patch - runs: | ./dev/change-scala-version.sh 2.13 diff --git a/spark-3.5-scala-2.13/GHSA-8qv5-68g4-248j-fix.patch b/spark-3.5-scala-2.13/GHSA-8qv5-68g4-248j-fix.patch new file mode 100644 index 00000000000..ef9ae7fd6e6 --- /dev/null +++ b/spark-3.5-scala-2.13/GHSA-8qv5-68g4-248j-fix.patch @@ -0,0 +1,104 @@ +From 55764a9ae8ef92512ef65229deac221c1c6afa95 Mon Sep 17 00:00:00 2001 +From: yangjie01 +Date: Fri, 15 Sep 2023 18:54:05 -0700 +Subject: [PATCH] [SPARK-40497][BUILD] Re-upgrade Scala to 2.13.11 + +### What changes were proposed in this pull request? +This PR aims to re-upgrade Scala to 2.13.11, after SPARK-45144 was merged, the build issues mentioned in https://github.com/apache/spark/pull/41943 should no longer exist. + +- https://www.scala-lang.org/news/2.13.11 + +Additionally, this pr adds a new suppression rule for warning message: `Implicit definition should have explicit type`, this is a new compile check introduced by https://github.com/scala/scala/pull/10083, we must fix it when we upgrading to use Scala 3 + +### Why are the changes needed? +This release improves collections, adds support for JDK 20 and 21, adds support for JDK 17 `sealed`: +- https://github.com/scala/scala/pull/10363 +- https://github.com/scala/scala/pull/10184 +- https://github.com/scala/scala/pull/10397 +- https://github.com/scala/scala/pull/10348 +- https://github.com/scala/scala/pull/10105 + +There are 2 known issues in this version: + +- https://github.com/scala/bug/issues/12800 +- https://github.com/scala/bug/issues/12799 + +For the first one, there is no compilation warning messages related to `match may not be exhaustive` in Spark compile log, and for the second one, there is no case of `method.isAnnotationPresent(Deprecated.class)` in Spark code, there is just + +https://github.com/apache/spark/blob/8c84d2c9349d7b607db949c2e114df781f23e438/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala#L130 + +in Spark Code, and I checked `javax.annotation.Nonnull` no this issue. + +So I think These two issues will not affect Spark itself, but this doesn't mean it won't affect the code written by end users themselves + +The full release notes as follows: + +- https://github.com/scala/scala/releases/tag/v2.13.11 + +### Does this PR introduce _any_ user-facing change? +Yes, this is a Scala version change. + +### How was this patch tested? +- Existing Test + +### Was this patch authored or co-authored using generative AI tooling? +No + +Closes #42918 from LuciferYang/SPARK-40497-2. + +Authored-by: yangjie01 +Signed-off-by: Dongjoon Hyun +--- + pom.xml | 6 +++++- + project/SparkBuild.scala | 4 +++- + 2 files changed, 8 insertions(+), 2 deletions(-) + +diff --git a/pom.xml b/pom.xml +index 91d93e8c3cc2c..779f9e64f1dcb 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -177,7 +177,7 @@ + 2.7.0 + 2.2.0 + +- 4.8.0 ++ 4.7.1 + false + 2.16.0 + +@@ -3630,7 +3630,7 @@ + + scala-2.13 + +- 2.13.8 ++ 2.13.11 + 2.13 + + +@@ -3689,6 +3689,10 @@ + --> + -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBase.scala:s + -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBaseOps.scala:s ++ ++ -Wconf:msg=Implicit definition should have explicit type:s + + + +diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala +index f67be83561d35..dc2e20c644b2a 100644 +--- a/project/SparkBuild.scala ++++ b/project/SparkBuild.scala +@@ -286,7 +286,9 @@ object SparkBuild extends PomBuild { + // TODO(SPARK-43850): Remove the following suppression rules and remove `import scala.language.higherKinds` + // from the corresponding files when Scala 2.12 is no longer supported. + "-Wconf:cat=unused-imports&src=org\\/apache\\/spark\\/graphx\\/impl\\/VertexPartitionBase.scala:s", +- "-Wconf:cat=unused-imports&src=org\\/apache\\/spark\\/graphx\\/impl\\/VertexPartitionBaseOps.scala:s" ++ "-Wconf:cat=unused-imports&src=org\\/apache\\/spark\\/graphx\\/impl\\/VertexPartitionBaseOps.scala:s", ++ // SPARK-40497 Upgrade Scala to 2.13.11 and suppress `Implicit definition should have explicit type` ++ "-Wconf:msg=Implicit definition should have explicit type:s" + ) + } + } + diff --git a/spark-3.5-scala-2.13/pombump-properties.yaml b/spark-3.5-scala-2.13/pombump-properties.yaml index 6e83b7b97c3..56c1136e97b 100644 --- a/spark-3.5-scala-2.13/pombump-properties.yaml +++ b/spark-3.5-scala-2.13/pombump-properties.yaml @@ -23,7 +23,7 @@ properties: value: "3.25.5" - property: woodstox-core.version value: "5.4.0" - - property: scala.version - value: "2.13.10" - property: zookeeper.version value: "3.7.2" # Changed from 3.7.3 to an existing version + - property: ivy.version + value: "2.5.3" # Changed from 3.7.3 to an existing version From fab5e0c9c667be9169c503c21a72eda6a800599c Mon Sep 17 00:00:00 2001 From: jamie-albert Date: Fri, 17 Jan 2025 02:46:57 -0800 Subject: [PATCH 2/2] Remove incorrect comment Signed-off-by: jamie-albert --- spark-3.5-scala-2.13/pombump-properties.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark-3.5-scala-2.13/pombump-properties.yaml b/spark-3.5-scala-2.13/pombump-properties.yaml index 56c1136e97b..d208bca3077 100644 --- a/spark-3.5-scala-2.13/pombump-properties.yaml +++ b/spark-3.5-scala-2.13/pombump-properties.yaml @@ -26,4 +26,4 @@ properties: - property: zookeeper.version value: "3.7.2" # Changed from 3.7.3 to an existing version - property: ivy.version - value: "2.5.3" # Changed from 3.7.3 to an existing version + value: "2.5.3"