diff --git a/dist/licenses/hadoop-client-3.1.2.jar.sha1 b/dist/licenses/hadoop-client-3.1.2.jar.sha1 deleted file mode 100644 index 65c0c4952..000000000 --- a/dist/licenses/hadoop-client-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6a815ff6298839c4fb1b96859c583c69f0f7c80c \ No newline at end of file diff --git a/dist/licenses/hadoop-common-3.1.2.jar.sha1 b/dist/licenses/hadoop-common-3.1.2.jar.sha1 deleted file mode 100644 index e9f82b869..000000000 --- a/dist/licenses/hadoop-common-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4742708e260bbdee3dda478fd440966c0b0967c0 \ No newline at end of file diff --git a/dist/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1 b/dist/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1 deleted file mode 100644 index a4041b593..000000000 --- a/dist/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4e7b3fe8875c01c9264f2e31043b04503e3f7022 \ No newline at end of file diff --git a/dist/licenses/spark-core_2.11-2.3.0.jar.sha1 b/dist/licenses/spark-core_2.11-2.3.0.jar.sha1 deleted file mode 100644 index 44379c147..000000000 --- a/dist/licenses/spark-core_2.11-2.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -09e2bc021bd38b06da2e0a56fdd9d13935503d94 \ No newline at end of file diff --git a/dist/licenses/spark-sql_2.11-2.3.0.jar.sha1 b/dist/licenses/spark-sql_2.11-2.3.0.jar.sha1 deleted file mode 100644 index e519c1764..000000000 --- a/dist/licenses/spark-sql_2.11-2.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e19282137294c8a889917c05006931fbcd8d66d2 \ No newline at end of file diff --git a/dist/licenses/spark-streaming_2.11-2.3.0.jar.sha1 b/dist/licenses/spark-streaming_2.11-2.3.0.jar.sha1 deleted file mode 100644 index 133766241..000000000 --- a/dist/licenses/spark-streaming_2.11-2.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -57da1135f7192a2be85987f1708abf94887f7323 \ No newline at end of file diff --git a/dist/licenses/spark-yarn_2.11-2.3.0.jar.sha1 b/dist/licenses/spark-yarn_2.11-2.3.0.jar.sha1 deleted file mode 100644 index 1877f6a67..000000000 --- a/dist/licenses/spark-yarn_2.11-2.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dffb93bc7154b1242155585fd117425c40d70bd4 \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 91eb15c4e..4f87f973b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -10,26 +10,26 @@ org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JA ## Dependecies Version # Logging -log4jVersion = 2.17.1 +log4jVersion = 2.17.2 # Hadoop versions -hadoop3Version = 3.1.2 -hadoop2Version = 2.7.6 -hadoop22Version = 2.2.0 +hadoop3Version = 3.3.2 +hadoop2Version = 2.10.6 +#hadoop22Version = 2.2.0 # Common libraries hiveVersion = 3.1.2 pigVersion = 0.15.0 # pig 0.15 depends on Joda but doesn't declare it in its pom -jodaVersion = 2.9.3 +jodaVersion = 2.10.3 # note the versions here are tied to the ones in Hadoop distro - 1.8.8 jacksonVersion = 1.8.8 # Spark -spark13Version = 1.6.2 -spark20Version = 2.3.0 +spark13Version = 1.6.3 +spark20Version = 2.4.7 spark22Version = 2.2.3 -spark24Version = 2.4.4 +spark24Version = 2.4.7 spark30Version = 3.2.1 # same as Spark's @@ -45,12 +45,12 @@ scala213MajorVersion = 2.13 stormVersion = 1.0.6 # Testing -junitVersion = 4.11 -mockitoVersion = 1.8.5 +junitVersion = 4.13.2 +mockitoVersion = 4.3.1 hamcrestVersion = 1.3 -minikdcVersion = 2.7.7 +minikdcVersion = 3.3.2 # Hive 0.11 finally updated antlr to 3.4 so there are no more conflicts with Pig antlrVersion = 3.4 thriftVersion = 0.5.0 -groovyVersion = 2.4.4 +groovyVersion = 2.5.15 diff --git a/licenses/commons-logging-1.1.1.jar.sha1 b/licenses/commons-logging-1.1.1.jar.sha1 deleted file mode 100644 index 66a2bd0ee..000000000 --- a/licenses/commons-logging-1.1.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5043bfebc3db072ed80fbd362e7caf00e885d8ae \ No newline at end of file diff --git a/licenses/hadoop-client-3.1.2.jar.sha1 b/licenses/hadoop-client-3.1.2.jar.sha1 deleted file mode 100644 index 65c0c4952..000000000 --- a/licenses/hadoop-client-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6a815ff6298839c4fb1b96859c583c69f0f7c80c \ No newline at end of file diff --git a/licenses/hadoop-common-3.1.2.jar.sha1 b/licenses/hadoop-common-3.1.2.jar.sha1 deleted file mode 100644 index e9f82b869..000000000 --- a/licenses/hadoop-common-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4742708e260bbdee3dda478fd440966c0b0967c0 \ No newline at end of file diff --git a/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1 b/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1 deleted file mode 100644 index a4041b593..000000000 --- a/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4e7b3fe8875c01c9264f2e31043b04503e3f7022 \ No newline at end of file diff --git a/licenses/protobuf-java-2.5.0.jar.sha1 b/licenses/protobuf-java-2.5.0.jar.sha1 deleted file mode 100644 index 71f918819..000000000 --- a/licenses/protobuf-java-2.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a10732c76bfacdbd633a7eb0f7968b1059a65dfa \ No newline at end of file diff --git a/mr/licenses/hadoop-client-3.1.2.jar.sha1 b/mr/licenses/hadoop-client-3.1.2.jar.sha1 deleted file mode 100644 index 65c0c4952..000000000 --- a/mr/licenses/hadoop-client-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6a815ff6298839c4fb1b96859c583c69f0f7c80c \ No newline at end of file diff --git a/mr/licenses/hadoop-common-3.1.2.jar.sha1 b/mr/licenses/hadoop-common-3.1.2.jar.sha1 deleted file mode 100644 index e9f82b869..000000000 --- a/mr/licenses/hadoop-common-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4742708e260bbdee3dda478fd440966c0b0967c0 \ No newline at end of file diff --git a/mr/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1 b/mr/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1 deleted file mode 100644 index a4041b593..000000000 --- a/mr/licenses/hadoop-mapreduce-client-core-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4e7b3fe8875c01c9264f2e31043b04503e3f7022 \ No newline at end of file diff --git a/spark/core/build.gradle b/spark/core/build.gradle index efee6dc04..feeea321a 100644 --- a/spark/core/build.gradle +++ b/spark/core/build.gradle @@ -49,16 +49,15 @@ sparkVariants { } add(variant.configuration('implementation'), project(":elasticsearch-hadoop-mr")) - add(variant.configuration('implementation'), "commons-logging:commons-logging:1.1.1") + add(variant.configuration('implementation'), "commons-logging:commons-logging:1.2") - add(variant.configuration('compileOnly'), "com.fasterxml.jackson.module:jackson-module-scala_${variant.scalaMajorVersion}:2.9.10") - add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.6.7") - add(variant.configuration('compileOnly'), "com.google.guava:guava:14.0.1") - add(variant.configuration('compileOnly'), "com.google.protobuf:protobuf-java:2.5.0") - add(variant.configuration('compileOnly'), "org.slf4j:slf4j-api:1.7.6") + add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.13.1") + add(variant.configuration('compileOnly'), "com.google.guava:guava:31.1-jre") + add(variant.configuration('compileOnly'), "com.google.protobuf:protobuf-java:3.19.4") + add(variant.configuration('compileOnly'), "org.slf4j:slf4j-api:1.7.36") add(variant.configuration('test', 'implementation'), project(":test:shared")) - add(variant.configuration('test', 'implementation'), "com.esotericsoftware.kryo:kryo:2.21") + add(variant.configuration('test', 'implementation'), "com.esotericsoftware.kryo:kryo:2.24.0") add(variant.configuration('test', 'implementation'), "org.apache.spark:spark-core_${variant.scalaMajorVersion}:${variant.sparkVersion}") { exclude group: 'javax.servlet' exclude group: 'org.apache.hadoop' @@ -68,19 +67,21 @@ sparkVariants { add(variant.configuration('test', 'implementation'), "org.elasticsearch:securemock:1.2") if (variant.scalaMajorVersion == '2.10') { + add(variant.configuration('compileOnly'), "com.fasterxml.jackson.module:jackson-module-scala_${variant.scalaMajorVersion}:2.11.4") add(variant.configuration('implementation'), "org.apache.spark:spark-unsafe_${variant.scalaMajorVersion}:${variant.sparkVersion}") - add(variant.configuration('implementation'), "org.apache.avro:avro:1.7.7") - add(variant.configuration('implementation'), "log4j:log4j:1.2.17") - add(variant.configuration('implementation'), "com.google.code.findbugs:jsr305:2.0.1") - add(variant.configuration('implementation'), "org.json4s:json4s-ast_2.10:3.2.10") - add(variant.configuration('implementation'), "com.esotericsoftware.kryo:kryo:2.21") + add(variant.configuration('implementation'), "org.apache.avro:avro:1.11.0") + add(variant.configuration('implementation'), "com.google.code.findbugs:jsr305:3.0.2") + add(variant.configuration('implementation'), "org.json4s:json4s-ast_2.10:3.6.11") + add(variant.configuration('implementation'), "com.esotericsoftware.kryo:kryo:2.24.0") add(variant.configuration('compileOnly'), "org.apache.hadoop:hadoop-annotations:${project.ext.hadoopVersion}") add(variant.configuration('compileOnly'), "org.codehaus.jackson:jackson-core-asl:${project.ext.jacksonVersion}") add(variant.configuration('compileOnly'), "org.codehaus.jackson:jackson-mapper-asl:${project.ext.jacksonVersion}") - add(variant.configuration('compileOnly'), "org.codehaus.woodstox:stax2-api:3.1.4") + add(variant.configuration('compileOnly'), "org.codehaus.woodstox:stax2-api:4.2.1") if (variant.sparkVersion == spark22Version) { add(variant.configuration('compileOnly'), "org.apache.spark:spark-tags_${variant.scalaMajorVersion}:${variant.sparkVersion}") } + } else { + add(variant.configuration('compileOnly'), "com.fasterxml.jackson.module:jackson-module-scala_${variant.scalaMajorVersion}:2.13.1") } add(variant.configuration('additionalSources'), project(":elasticsearch-hadoop-mr")) diff --git a/spark/core/licenses/commons-logging-1.1.1.jar.sha1 b/spark/core/licenses/commons-logging-1.1.1.jar.sha1 deleted file mode 100644 index 66a2bd0ee..000000000 --- a/spark/core/licenses/commons-logging-1.1.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5043bfebc3db072ed80fbd362e7caf00e885d8ae \ No newline at end of file diff --git a/spark/core/licenses/spark-core_2.12-2.4.4.jar.sha1 b/spark/core/licenses/spark-core_2.12-2.4.4.jar.sha1 deleted file mode 100644 index 2954d17f0..000000000 --- a/spark/core/licenses/spark-core_2.12-2.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e606150eb038cccf319869de3c0cd32e6f9d2ff \ No newline at end of file diff --git a/spark/sql-13/build.gradle b/spark/sql-13/build.gradle index 0a97d75a6..5a4d71740 100644 --- a/spark/sql-13/build.gradle +++ b/spark/sql-13/build.gradle @@ -70,27 +70,28 @@ sparkVariants { add(variant.configuration('implementation'), "org.apache.spark:spark-streaming_${variant.scalaMajorVersion}:$variant.sparkVersion") { exclude group: 'org.apache.hadoop' } - add(variant.configuration('implementation'), "commons-logging:commons-logging:1.1.1") + add(variant.configuration('implementation'), "commons-logging:commons-logging:1.2") add(variant.configuration('implementation'), "javax.xml.bind:jaxb-api:2.3.1") add(variant.configuration('implementation'), "org.apache.spark:spark-catalyst_${variant.scalaMajorVersion}:$variant.sparkVersion") // Scala compiler needs these for arcane reasons, but they are not used in the api nor the runtime - add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.6.7") - add(variant.configuration('compileOnly'), "com.google.guava:guava:16.0.1") - add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:3.2.11") - add(variant.configuration('compileOnly'), "org.slf4j:slf4j-api:1.7.6") + add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.13.1") + add(variant.configuration('compileOnly'), "com.google.guava:guava:31.1-jre") + add(variant.configuration('compileOnly'), "org.slf4j:slf4j-api:1.7.36") if ('2.10'.equals(scalaMajorVersion)) { add(variant.configuration('implementation'), "org.apache.spark:spark-unsafe_${variant.scalaMajorVersion}:$variant.sparkVersion") - add(variant.configuration('implementation'), "org.apache.avro:avro:1.7.7") - add(variant.configuration('implementation'), "log4j:log4j:1.2.17") - add(variant.configuration('implementation'), "com.google.code.findbugs:jsr305:2.0.1") - add(variant.configuration('implementation'), "org.json4s:json4s-ast_2.10:3.2.10") - add(variant.configuration('implementation'), "com.esotericsoftware.kryo:kryo:2.21") + add(variant.configuration('implementation'), "org.apache.avro:avro:1.11.0") + add(variant.configuration('implementation'), "com.google.code.findbugs:jsr305:3.0.2") + add(variant.configuration('implementation'), "org.json4s:json4s-ast_2.10:3.6.11") + add(variant.configuration('implementation'), "com.esotericsoftware.kryo:kryo:2.24.0") add(variant.configuration('compileOnly'), "org.apache.hadoop:hadoop-annotations:${project.ext.hadoopVersion}") add(variant.configuration('compileOnly'), "org.codehaus.jackson:jackson-core-asl:${project.ext.jacksonVersion}") add(variant.configuration('compileOnly'), "org.codehaus.jackson:jackson-mapper-asl:${project.ext.jacksonVersion}") - add(variant.configuration('compileOnly'), "org.codehaus.woodstox:stax2-api:3.1.4") + add(variant.configuration('compileOnly'), "org.codehaus.woodstox:stax2-api:4.2.1") + add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:3.6.12") + } else { + add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:4.0.4") } add(variant.configuration('test', 'implementation'), project(":test:shared")) diff --git a/spark/sql-13/licenses/commons-logging-1.1.1.jar.sha1 b/spark/sql-13/licenses/commons-logging-1.1.1.jar.sha1 deleted file mode 100644 index 66a2bd0ee..000000000 --- a/spark/sql-13/licenses/commons-logging-1.1.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5043bfebc3db072ed80fbd362e7caf00e885d8ae \ No newline at end of file diff --git a/spark/sql-13/licenses/spark-catalyst_2.11-1.6.2.jar.sha1 b/spark/sql-13/licenses/spark-catalyst_2.11-1.6.2.jar.sha1 deleted file mode 100644 index 5f012f611..000000000 --- a/spark/sql-13/licenses/spark-catalyst_2.11-1.6.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da5b94e90e0da2d425652cf5bdbf89832d94983f \ No newline at end of file diff --git a/spark/sql-13/licenses/spark-core_2.11-1.6.2.jar.sha1 b/spark/sql-13/licenses/spark-core_2.11-1.6.2.jar.sha1 deleted file mode 100644 index 1cd44c19b..000000000 --- a/spark/sql-13/licenses/spark-core_2.11-1.6.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b12fbb980cfa5a4401cb98db338a01f76f483347 \ No newline at end of file diff --git a/spark/sql-13/licenses/spark-sql_2.11-1.6.2.jar.sha1 b/spark/sql-13/licenses/spark-sql_2.11-1.6.2.jar.sha1 deleted file mode 100644 index fea2e296f..000000000 --- a/spark/sql-13/licenses/spark-sql_2.11-1.6.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76686db5bdeee0ca6b6e51ff39b796869f74483c \ No newline at end of file diff --git a/spark/sql-13/licenses/spark-streaming_2.11-1.6.2.jar.sha1 b/spark/sql-13/licenses/spark-streaming_2.11-1.6.2.jar.sha1 deleted file mode 100644 index 19fabfe99..000000000 --- a/spark/sql-13/licenses/spark-streaming_2.11-1.6.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0ec8196dd1e04499e38e3fd9a7c914b6cec7a145 \ No newline at end of file diff --git a/spark/sql-20/build.gradle b/spark/sql-20/build.gradle index b2a7dac06..4f0644fd4 100644 --- a/spark/sql-20/build.gradle +++ b/spark/sql-20/build.gradle @@ -66,35 +66,36 @@ sparkVariants { add(variant.configuration('implementation'), "org.apache.spark:spark-streaming_${variant.scalaMajorVersion}:$variant.sparkVersion") { exclude group: 'org.apache.hadoop' } - add(variant.configuration('implementation'), "org.slf4j:slf4j-api:1.7.6") { + add(variant.configuration('implementation'), "org.slf4j:slf4j-api:1.7.36") { because 'spark exposes slf4j components in traits that we need to extend' } - add(variant.configuration('implementation'), "commons-logging:commons-logging:1.1.1") + add(variant.configuration('implementation'), "commons-logging:commons-logging:1.2") add(variant.configuration('implementation'), "javax.xml.bind:jaxb-api:2.3.1") - add(variant.configuration('implementation'), "com.google.protobuf:protobuf-java:2.5.0") + add(variant.configuration('implementation'), "com.google.protobuf:protobuf-java:3.19.4") add(variant.configuration('implementation'), "org.apache.spark:spark-catalyst_${variant.scalaMajorVersion}:$variant.sparkVersion") add(variant.configuration('implementation'), "org.apache.spark:spark-yarn_${variant.scalaMajorVersion}:$variant.sparkVersion") { exclude group: 'org.apache.hadoop' } // Scala compiler needs these for arcane reasons, but they are not used in the api nor the runtime - add(variant.configuration('compileOnly'), "org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.0.0") - add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.6.7") - add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:3.2.11") + add(variant.configuration('compileOnly'), "org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1") + add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.13.1") add(variant.configuration('compileOnly'), "org.apache.spark:spark-tags_${variant.scalaMajorVersion}:$variant.sparkVersion") if ('2.10' == scalaMajorVersion) { add(variant.configuration('implementation'), "org.apache.spark:spark-unsafe_${variant.scalaMajorVersion}:$variant.sparkVersion") - add(variant.configuration('implementation'), "org.apache.avro:avro:1.7.7") - add(variant.configuration('implementation'), "log4j:log4j:1.2.17") - add(variant.configuration('implementation'), "com.google.code.findbugs:jsr305:2.0.1") - add(variant.configuration('implementation'), "org.json4s:json4s-ast_2.10:3.2.10") - add(variant.configuration('implementation'), "com.esotericsoftware.kryo:kryo:2.21") + add(variant.configuration('implementation'), "org.apache.avro:avro:1.11.0") + add(variant.configuration('implementation'), "com.google.code.findbugs:jsr305:3.0.2") + add(variant.configuration('implementation'), "org.json4s:json4s-ast_2.10:3.6.11") + add(variant.configuration('implementation'), "com.esotericsoftware.kryo:kryo:2.24.0") add(variant.configuration('compileOnly'), "org.apache.hadoop:hadoop-annotations:${project.ext.hadoopVersion}") add(variant.configuration('compileOnly'), "org.codehaus.jackson:jackson-core-asl:${project.ext.jacksonVersion}") add(variant.configuration('compileOnly'), "org.codehaus.jackson:jackson-mapper-asl:${project.ext.jacksonVersion}") - add(variant.configuration('compileOnly'), "org.codehaus.woodstox:stax2-api:3.1.4") - add(variant.configuration('compileOnly'), "org.apache.htrace:htrace-core4:4.1.0-incubating") + add(variant.configuration('compileOnly'), "org.codehaus.woodstox:stax2-api:4.2.1") + add(variant.configuration('compileOnly'), "org.apache.htrace:htrace-core4:4.2.0-incubating") + add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:3.6.12") + } else { + add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:4.0.4") } add(variant.configuration('test', 'implementation'), project(":test:shared")) diff --git a/spark/sql-20/licenses/commons-logging-1.1.1.jar.sha1 b/spark/sql-20/licenses/commons-logging-1.1.1.jar.sha1 deleted file mode 100644 index 66a2bd0ee..000000000 --- a/spark/sql-20/licenses/commons-logging-1.1.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5043bfebc3db072ed80fbd362e7caf00e885d8ae \ No newline at end of file diff --git a/spark/sql-20/licenses/protobuf-java-2.5.0.jar.sha1 b/spark/sql-20/licenses/protobuf-java-2.5.0.jar.sha1 deleted file mode 100644 index 71f918819..000000000 --- a/spark/sql-20/licenses/protobuf-java-2.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a10732c76bfacdbd633a7eb0f7968b1059a65dfa \ No newline at end of file diff --git a/spark/sql-20/licenses/spark-catalyst_2.12-2.4.4.jar.sha1 b/spark/sql-20/licenses/spark-catalyst_2.12-2.4.4.jar.sha1 deleted file mode 100644 index 86b852e43..000000000 --- a/spark/sql-20/licenses/spark-catalyst_2.12-2.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -050172ab29cae575c85216348cf22a544f7d7205 \ No newline at end of file diff --git a/spark/sql-20/licenses/spark-core_2.12-2.4.4.jar.sha1 b/spark/sql-20/licenses/spark-core_2.12-2.4.4.jar.sha1 deleted file mode 100644 index 2954d17f0..000000000 --- a/spark/sql-20/licenses/spark-core_2.12-2.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e606150eb038cccf319869de3c0cd32e6f9d2ff \ No newline at end of file diff --git a/spark/sql-20/licenses/spark-sql_2.12-2.4.4.jar.sha1 b/spark/sql-20/licenses/spark-sql_2.12-2.4.4.jar.sha1 deleted file mode 100644 index e41e1214f..000000000 --- a/spark/sql-20/licenses/spark-sql_2.12-2.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec1d629508b27dd126a3b24ce8af91e72778e16f \ No newline at end of file diff --git a/spark/sql-20/licenses/spark-streaming_2.12-2.4.4.jar.sha1 b/spark/sql-20/licenses/spark-streaming_2.12-2.4.4.jar.sha1 deleted file mode 100644 index 4220cec02..000000000 --- a/spark/sql-20/licenses/spark-streaming_2.12-2.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5051a3786b6125f505b968666d2c56d01aef0095 \ No newline at end of file diff --git a/spark/sql-20/licenses/spark-yarn_2.12-2.4.4.jar.sha1 b/spark/sql-20/licenses/spark-yarn_2.12-2.4.4.jar.sha1 deleted file mode 100644 index fd31875d1..000000000 --- a/spark/sql-20/licenses/spark-yarn_2.12-2.4.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -640cef8f1bfa6772ddf6a940905a3aeecd8d7663 \ No newline at end of file diff --git a/spark/sql-30/build.gradle b/spark/sql-30/build.gradle index e8512aae7..fc1406cf6 100644 --- a/spark/sql-30/build.gradle +++ b/spark/sql-30/build.gradle @@ -65,26 +65,26 @@ sparkVariants { add(variant.configuration('implementation'), "org.apache.spark:spark-streaming_${variant.scalaMajorVersion}:$variant.sparkVersion") { exclude group: 'org.apache.hadoop' } - add(variant.configuration('implementation'), "org.slf4j:slf4j-api:1.7.6") { + add(variant.configuration('implementation'), "org.slf4j:slf4j-api:1.7.36") { because 'spark exposes slf4j components in traits that we need to extend' } - add(variant.configuration('implementation'), "commons-logging:commons-logging:1.1.1") + add(variant.configuration('implementation'), "commons-logging:commons-logging:1.2") add(variant.configuration('implementation'), "javax.xml.bind:jaxb-api:2.3.1") - add(variant.configuration('implementation'), "com.google.protobuf:protobuf-java:2.5.0") + add(variant.configuration('implementation'), "com.google.protobuf:protobuf-java:3.19.4") add(variant.configuration('implementation'), "org.apache.spark:spark-catalyst_${variant.scalaMajorVersion}:$variant.sparkVersion") add(variant.configuration('implementation'), "org.apache.spark:spark-yarn_${variant.scalaMajorVersion}:$variant.sparkVersion") { exclude group: 'org.apache.hadoop' } // Scala compiler needs these for arcane reasons, but they are not used in the api nor the runtime - add(variant.configuration('compileOnly'), "org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.0.0") - add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.6.7") - add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:3.6.6") - add(variant.configuration('compileOnly'), "org.json4s:json4s-ast_${variant.scalaMajorVersion}:3.6.6") + add(variant.configuration('compileOnly'), "org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1") + add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.13.1") + add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:4.0.4") + add(variant.configuration('compileOnly'), "org.json4s:json4s-ast_${variant.scalaMajorVersion}:4.0.4") add(variant.configuration('compileOnly'), "org.apache.spark:spark-tags_${variant.scalaMajorVersion}:$variant.sparkVersion") add(variant.configuration('test', 'implementation'), project(":test:shared")) - add(variant.configuration('test', 'implementation'), "jakarta.servlet:jakarta.servlet-api:4.0.3") + add(variant.configuration('test', 'implementation'), "jakarta.servlet:jakarta.servlet-api:5.0.0") add(variant.configuration('test', 'implementation'), "org.elasticsearch:securemock:1.2") add(variant.configuration('test', 'implementation'), "org.apache.spark:spark-core_${variant.scalaMajorVersion}:$variant.sparkVersion") { exclude group: 'javax.servlet' @@ -95,7 +95,7 @@ sparkVariants { } add(variant.configuration('itest', 'implementation'), project(":test:shared")) - add(variant.configuration('itest', 'implementation'), "jakarta.servlet:jakarta.servlet-api:4.0.3") + add(variant.configuration('itest', 'implementation'), "jakarta.servlet:jakarta.servlet-api:5.0.0") add(variant.configuration('itest', 'implementation'), "org.apache.spark:spark-yarn_${variant.scalaMajorVersion}:$variant.sparkVersion") { exclude group: 'org.apache.hadoop' } diff --git a/spark/sql-30/licenses/commons-logging-1.1.1.jar.sha1 b/spark/sql-30/licenses/commons-logging-1.1.1.jar.sha1 deleted file mode 100644 index 66a2bd0ee..000000000 --- a/spark/sql-30/licenses/commons-logging-1.1.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5043bfebc3db072ed80fbd362e7caf00e885d8ae \ No newline at end of file diff --git a/spark/sql-30/licenses/protobuf-java-2.5.0.jar.sha1 b/spark/sql-30/licenses/protobuf-java-2.5.0.jar.sha1 deleted file mode 100644 index 71f918819..000000000 --- a/spark/sql-30/licenses/protobuf-java-2.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a10732c76bfacdbd633a7eb0f7968b1059a65dfa \ No newline at end of file