update branch

lyft · Nov 9, 2023 · ca508a2 · ca508a2
2 parents d2fadf2 + c93bba8
commit ca508a2
Show file tree

Hide file tree

Showing 845 changed files with 41,225 additions and 90,605 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
diff --git a/.github/workflows/build_main.yml → .github/workflows/build_and_test_ansi.yml b/.github/workflows/build_main.yml → .github/workflows/build_and_test_ansi.yml
@@ -17,11 +17,16 @@
 # under the License.
 #
 
+<<<<<<<< HEAD:.github/workflows/build_main.yml
 name: "Build"
+========
+name: "Build and test (ANSI)"
+>>>>>>>> c93bba8b9d4823c0d891561e041eaec91be0c11b:.github/workflows/build_and_test_ansi.yml
 
 on:
   push:
     branches:
+<<<<<<<< HEAD:.github/workflows/build_main.yml
     - '**'
 
 jobs:
@@ -30,3 +35,15 @@ jobs:
       packages: write
     name: Run
     uses: ./.github/workflows/build_and_test.yml
+========
+      - branch-3.3
+
+jobs:
+  call-build-and-test:
+    name: Call main build
+    uses: ./.github/workflows/build_and_test.yml
+    if: github.repository == 'apache/spark'
+    with:
+      ansi_enabled: true
+
+>>>>>>>> c93bba8b9d4823c0d891561e041eaec91be0c11b:.github/workflows/build_and_test_ansi.yml
diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml
@@ -31,12 +31,9 @@ jobs:
   notify:
     name: Notify test workflow
     runs-on: ubuntu-20.04
-    permissions:
-      actions: read
-      checks: write
     steps:
       - name: "Notify test workflow"
-        uses: actions/github-script@v6
+        uses: actions/github-script@f05a81df23035049204b043b50c3322045ce7eb3 # pin@v3
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -49,7 +46,7 @@ jobs:
             const params = {
               owner: context.payload.pull_request.head.repo.owner.login,
               repo: context.payload.pull_request.head.repo.name,
-              id: 'build_main.yml',
+              id: 'build_and_test.yml',
               branch: context.payload.pull_request.head.ref,
             }
             const check_run_params = {
@@ -72,15 +69,15 @@ jobs:
               // Assume that runs were not found.
             }
 
-            const name = 'Build'
+            const name = 'Build and test'
             const head_sha = context.payload.pull_request.head.sha
             let status = 'queued'
 
             if (!runs || runs.data.workflow_runs.length === 0) {
               status = 'completed'
               const conclusion = 'action_required'
 
-              github.rest.checks.create({
+              github.checks.create({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 name: name,
@@ -116,7 +113,7 @@ jobs:
 
               // Here we get check run ID to provide Check run view instead of Actions view, see also SPARK-37879.
               const check_runs = await github.request(check_run_endpoint, check_run_params)
-              const check_run_head = check_runs.data.check_runs.filter(r => r.name === "Run / Check changes")[0]
+              const check_run_head = check_runs.data.check_runs.filter(r => r.name === "Configure jobs")[0]
 
               if (check_run_head.head_sha != context.payload.pull_request.head.sha) {
                 throw new Error('There was a new unsynced commit pushed. Please retrigger the workflow.');
@@ -132,7 +129,7 @@ jobs:
                 + '/actions/runs/'
                 + run_id
 
-              github.rest.checks.create({
+              github.checks.create({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 name: name,

diff --git a/.github/workflows/test_report.yml b/.github/workflows/test_report.yml
@@ -20,13 +20,12 @@
 name: Report test results
 on:
   workflow_run:
-    workflows: ["Build"]
+    workflows: ["Build and test", "Build and test (ANSI)"]
     types:
       - completed
 
 jobs:
   test_report:
-    if: github.event.workflow_run.conclusion != 'skipped'
     runs-on: ubuntu-latest
     steps:
     - name: Download test results to report

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.4.1
+Version: 3.3.1
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R:

diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.4.1</version>
+    <version>3.3.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -152,16 +152,6 @@
         </dependency>
       </dependencies>
     </profile>
-    <profile>
-      <id>connect</id>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-connect_${scala.binary.version}</artifactId>
-          <version>${project.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
     <profile>
       <id>kubernetes</id>
       <dependencies>

diff --git a/bin/pyspark b/bin/pyspark
@@ -50,7 +50,7 @@ export PYSPARK_DRIVER_PYTHON_OPTS
 
 # Add the PySpark classes to the Python path:
 export PYTHONPATH="${SPARK_HOME}/python/:$PYTHONPATH"
-export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH"
+export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9.5-src.zip:$PYTHONPATH"
 
 # Load the PySpark shell.py script when ./pyspark is used interactively:
 export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"

diff --git a/bin/pyspark2.cmd b/bin/pyspark2.cmd
@@ -30,7 +30,7 @@ if "x%PYSPARK_DRIVER_PYTHON%"=="x" (
 )
 
 set PYTHONPATH=%SPARK_HOME%\python;%PYTHONPATH%
-set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.9.7-src.zip;%PYTHONPATH%
+set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.9.5-src.zip;%PYTHONPATH%
 
 set OLD_PYTHONSTARTUP=%PYTHONSTARTUP%
 set PYTHONSTARTUP=%SPARK_HOME%\python\pyspark\shell.py

diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.4.1</version>
+    <version>3.3.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -89,7 +89,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j2-impl</artifactId>
+      <artifactId>log4j-slf4j-impl</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.4.1</version>
+    <version>3.3.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -42,46 +42,20 @@
     </dependency>
 
     <!-- Core dependencies -->
-    <!-- Netty Begin -->
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-all</artifactId>
     </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-transport-native-epoll</artifactId>
-      <classifier>linux-x86_64</classifier>
-    </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-transport-native-epoll</artifactId>
-      <classifier>linux-aarch_64</classifier>
-    </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-transport-native-kqueue</artifactId>
-      <classifier>osx-aarch_64</classifier>
-    </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-transport-native-kqueue</artifactId>
-      <classifier>osx-x86_64</classifier>
-    </dependency>
-    <!-- Netty End -->
-
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
     </dependency>
+
     <dependency>
       <groupId>${leveldbjni.group}</groupId>
       <artifactId>leveldbjni-all</artifactId>
       <version>1.8</version>
     </dependency>
-    <dependency>
-      <groupId>org.rocksdb</groupId>
-      <artifactId>rocksdbjni</artifactId>
-    </dependency>
 
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
@@ -144,13 +118,14 @@
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j2-impl</artifactId>
+      <artifactId>log4j-slf4j-impl</artifactId>
       <scope>test</scope>
     </dependency>
 
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
+      <scope>test</scope>
    </dependency>
 
     <!--

diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.4.1</version>
+    <version>3.3.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -104,7 +104,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j2-impl</artifactId>
+      <artifactId>log4j-slf4j-impl</artifactId>
       <scope>test</scope>
     </dependency>
 

diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.4.1</version>
+    <version>3.3.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -136,11 +136,6 @@
             <goals>
               <goal>shade</goal>
             </goals>
-            <configuration>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-              </transformers>
-            </configuration>
           </execution>
         </executions>
       </plugin>
@@ -179,10 +174,6 @@
                           tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_epoll_x86_64.so" />
                     <move file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_kqueue_x86_64.jnilib"
                           tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_kqueue_x86_64.jnilib" />
-                    <move file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_epoll_aarch_64.so"
-                          tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_epoll_aarch_64.so" />
-                    <move file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_kqueue_aarch_64.jnilib"
-                          tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_kqueue_aarch_64.jnilib" />
                     <jar destfile="${shuffle.jar}" basedir="${project.build.directory}/exploded" />
                 </target>
             </configuration>