MLE-31165 Bumping to Spark 4.2.0

build.gradle

@@ -77,7 +77,7 @@ subprojects {
          // This oddly does not have a 2.21.0 or 2.21.1 release, just 2.21.
          details.useVersion "2.21"
        } else {
-         details.useVersion "2.21.4"
+         details.useVersion jacksonVersion
        }
        details.because "Force Jackson to 2.21.x to align with Java Client 8.1.0."
      }

flux-cli/build.gradle

@@ -14,6 +14,9 @@ configurations {
     resolutionStrategy {
       // Resolves a medium CVE that comes from json-unit-assertj .
       force "net.minidev:json-smart:2.5.2"
+
+      // Resolves a CVE in 1.11.0
+      force "at.yawk.lz4:lz4-java:1.11.1"
     }
   }
 }
@@ -22,15 +25,8 @@ dependencies {
   implementation("org.apache.spark:spark-sql_2.13:${sparkVersion}") {
     // The rocksdbjni dependency weighs in at 50mb and so far does not appear necessary for our use of Spark.
     exclude module: "rocksdbjni"
-
-    // As of Spark 4.1.1, Spark is depending on a 1.8.x version of lz4-java that has two
-    // CVEs. According to https://github.com/yawkat/lz4-java, patched versions should now be obtained from
-    // that repository. Once Spark uses a patched version, this exclusion can be removed.
-    exclude module: "lz4-java"
   }
 
-  implementation "at.yawk.lz4:lz4-java:1.10.4"
-
   implementation "com.marklogic:marklogic-spark-connector:${connectorVersion}"
   implementation "info.picocli:picocli:${picocliVersion}"
 
@@ -50,7 +46,6 @@ dependencies {
   }
 
   // Based on testing so far, this appears to suffice for allowing hadoop-aws to read from and write to S3.
-  // Using 2.29.52, which is what Hadoop 3.4.2 depends on.
   implementation "software.amazon.awssdk:s3-transfer-manager:${awssdkVersion}"
 
   // Needed for S3 authentication with an SSO profile.

flux-cli/src/test/java/com/marklogic/flux/api/GenericFilesImporterTest.java

@@ -1,9 +1,10 @@
 /*
- * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
+ * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
  */
 package com.marklogic.flux.api;
 
 import com.marklogic.flux.AbstractTest;
+import com.marklogic.spark.ConnectorException;
 import org.junit.jupiter.api.Test;
 
 import java.util.stream.Stream;
@@ -91,7 +92,11 @@ void abortOnWriteFailure() {
                 .abortOnWriteFailure(true)
                 .permissionsString("not-a-real-role,update"));
 
-        FluxException ex = assertThrows(FluxException.class, command::execute);
+        // This is currently catching a ConnectorException instead of a FluxException due to a change in the Spark
+        // connector via PR 683 for the Spark connector. That change should be undone with the real fix being that
+        // FailedRequest should become serializable in the Java Client.
+//        FluxException ex = assertThrows(FluxException.class, command::execute);
+        ConnectorException ex = assertThrows(ConnectorException.class, command::execute);
         assertTrue(ex.getMessage().contains("Role does not exist"), "Unexpected error: " + ex.getMessage());
     }
 

flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportOrcFilesTest.java

@@ -171,7 +171,7 @@ void abortOnReadFailure() {
             "--permissions", DEFAULT_PERMISSIONS
         );
 
-        assertTrue(stderr.contains("Command failed") && stderr.contains("Could not read footer"),
+        assertTrue(stderr.contains("Command failed") && stderr.contains("footer is too large"),
             "The command should have failed because Spark could not read the footer of the invalid Avro file; " +
                 "stderr: " + stderr);
     }

flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportParquetFilesTest.java

@@ -171,7 +171,7 @@ void invalidParquetFile() {
         );
 
         assertTrue(
-            stderr.contains("Error: [FAILED_READ_FILE.CANNOT_READ_FILE_FOOTER]"),
+            stderr.contains("is not a Parquet file"),
             "Sometimes Spark will throw a SparkException that wraps a SparkException, and it's the wrapped exception " +
                 "that has the useful message in it. This test verifies that we use the message from the wrapped " +
                 "SparkException, which is far more helpful for this particular failure. Unexpected stderr: " + stderr
@@ -226,7 +226,7 @@ void abortOnReadFailure() {
             "--permissions", DEFAULT_PERMISSIONS
         );
 
-        assertTrue(stderr.contains("Command failed") && stderr.contains("Could not read footer"),
+        assertTrue(stderr.contains("Command failed") && stderr.contains("footer is too large"),
             "The command should have failed because Spark could not read the footer of the invalid Avro file; " +
                 "stderr: " + stderr);
     }

gradle.properties

@@ -6,9 +6,16 @@ connectorVersion=3.1.2-SNAPSHOT
 picocliVersion=4.7.7
 
 # Aligned with our Spark connector.
-sparkVersion=4.1.1
-hadoopVersion=3.4.2
-awssdkVersion=2.29.52
+sparkVersion=4.2.0
+hadoopVersion=3.5.0
+
+# We want the minor version to match Spark, but the patch version can be higher. This is the case with 2.21.x, where
+# LangChain4j is using 2.21.4 and Spark 4.2.0 is using 2.21.2, but 2.21.5 is available.
+jacksonVersion=2.21.5
+
+# This needs to match the version used by Hadoop.
+awssdkVersion=2.35.4
+
 nettyVersion=4.2.16.Final
 langchain4jVersion=1.17.2
 tikaVersion=3.3.1