diff --git a/build.gradle b/build.gradle index 4d6901dc..54d58dca 100644 --- a/build.gradle +++ b/build.gradle @@ -77,7 +77,7 @@ subprojects { // This oddly does not have a 2.21.0 or 2.21.1 release, just 2.21. details.useVersion "2.21" } else { - details.useVersion "2.21.4" + details.useVersion jacksonVersion } details.because "Force Jackson to 2.21.x to align with Java Client 8.1.0." } diff --git a/flux-cli/build.gradle b/flux-cli/build.gradle index f4602828..38893d97 100644 --- a/flux-cli/build.gradle +++ b/flux-cli/build.gradle @@ -14,6 +14,9 @@ configurations { resolutionStrategy { // Resolves a medium CVE that comes from json-unit-assertj . force "net.minidev:json-smart:2.5.2" + + // Resolves a CVE in 1.11.0 + force "at.yawk.lz4:lz4-java:1.11.1" } } } @@ -22,15 +25,8 @@ dependencies { implementation("org.apache.spark:spark-sql_2.13:${sparkVersion}") { // The rocksdbjni dependency weighs in at 50mb and so far does not appear necessary for our use of Spark. exclude module: "rocksdbjni" - - // As of Spark 4.1.1, Spark is depending on a 1.8.x version of lz4-java that has two - // CVEs. According to https://github.com/yawkat/lz4-java, patched versions should now be obtained from - // that repository. Once Spark uses a patched version, this exclusion can be removed. - exclude module: "lz4-java" } - implementation "at.yawk.lz4:lz4-java:1.10.4" - implementation "com.marklogic:marklogic-spark-connector:${connectorVersion}" implementation "info.picocli:picocli:${picocliVersion}" @@ -50,7 +46,6 @@ dependencies { } // Based on testing so far, this appears to suffice for allowing hadoop-aws to read from and write to S3. - // Using 2.29.52, which is what Hadoop 3.4.2 depends on. implementation "software.amazon.awssdk:s3-transfer-manager:${awssdkVersion}" // Needed for S3 authentication with an SSO profile. diff --git a/flux-cli/src/test/java/com/marklogic/flux/api/GenericFilesImporterTest.java b/flux-cli/src/test/java/com/marklogic/flux/api/GenericFilesImporterTest.java index b972ad06..25ab016e 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/api/GenericFilesImporterTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/api/GenericFilesImporterTest.java @@ -1,9 +1,10 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.api; import com.marklogic.flux.AbstractTest; +import com.marklogic.spark.ConnectorException; import org.junit.jupiter.api.Test; import java.util.stream.Stream; @@ -91,7 +92,11 @@ void abortOnWriteFailure() { .abortOnWriteFailure(true) .permissionsString("not-a-real-role,update")); - FluxException ex = assertThrows(FluxException.class, command::execute); + // This is currently catching a ConnectorException instead of a FluxException due to a change in the Spark + // connector via PR 683 for the Spark connector. That change should be undone with the real fix being that + // FailedRequest should become serializable in the Java Client. +// FluxException ex = assertThrows(FluxException.class, command::execute); + ConnectorException ex = assertThrows(ConnectorException.class, command::execute); assertTrue(ex.getMessage().contains("Role does not exist"), "Unexpected error: " + ex.getMessage()); } diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportOrcFilesTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportOrcFilesTest.java index 42af6297..906bab8b 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportOrcFilesTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportOrcFilesTest.java @@ -171,7 +171,7 @@ void abortOnReadFailure() { "--permissions", DEFAULT_PERMISSIONS ); - assertTrue(stderr.contains("Command failed") && stderr.contains("Could not read footer"), + assertTrue(stderr.contains("Command failed") && stderr.contains("footer is too large"), "The command should have failed because Spark could not read the footer of the invalid Avro file; " + "stderr: " + stderr); } diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportParquetFilesTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportParquetFilesTest.java index 2d00148c..6ec801b9 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportParquetFilesTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportParquetFilesTest.java @@ -171,7 +171,7 @@ void invalidParquetFile() { ); assertTrue( - stderr.contains("Error: [FAILED_READ_FILE.CANNOT_READ_FILE_FOOTER]"), + stderr.contains("is not a Parquet file"), "Sometimes Spark will throw a SparkException that wraps a SparkException, and it's the wrapped exception " + "that has the useful message in it. This test verifies that we use the message from the wrapped " + "SparkException, which is far more helpful for this particular failure. Unexpected stderr: " + stderr @@ -226,7 +226,7 @@ void abortOnReadFailure() { "--permissions", DEFAULT_PERMISSIONS ); - assertTrue(stderr.contains("Command failed") && stderr.contains("Could not read footer"), + assertTrue(stderr.contains("Command failed") && stderr.contains("footer is too large"), "The command should have failed because Spark could not read the footer of the invalid Avro file; " + "stderr: " + stderr); } diff --git a/gradle.properties b/gradle.properties index 9d172dfb..20f27da8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,9 +6,16 @@ connectorVersion=3.1.2-SNAPSHOT picocliVersion=4.7.7 # Aligned with our Spark connector. -sparkVersion=4.1.1 -hadoopVersion=3.4.2 -awssdkVersion=2.29.52 +sparkVersion=4.2.0 +hadoopVersion=3.5.0 + +# We want the minor version to match Spark, but the patch version can be higher. This is the case with 2.21.x, where +# LangChain4j is using 2.21.4 and Spark 4.2.0 is using 2.21.2, but 2.21.5 is available. +jacksonVersion=2.21.5 + +# This needs to match the version used by Hadoop. +awssdkVersion=2.35.4 + nettyVersion=4.2.16.Final langchain4jVersion=1.17.2 tikaVersion=3.3.1