Created
November 24, 2019 14:52
-
-
Save suztomo/643baceebcbfb84fb1e4d53c88ba946b to your computer and use it in GitHub Desktop.
Attempt to use Hadoop Client 2.8.5 in Beam https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/1028#issuecomment-557717962
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
suztomo@suxtomo24:~/beam6$ git diff 95e1a77f | |
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | |
index 2414880b02..6ef367709b 100644 | |
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | |
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | |
@@ -374,8 +374,8 @@ class BeamModulePlugin implements Plugin<Project> { | |
def google_cloud_core_version = "1.91.1" | |
def google_cloud_spanner_version = "1.38.0" | |
def grpc_version = "1.23.0" | |
- def guava_version = "28.1-jre" | |
- def hadoop_version = "2.7.3" | |
+ def guava_version = "25.1-jre" | |
+ def hadoop_version = "2.8.4" | |
def hamcrest_version = "2.1" | |
def jackson_version = "2.9.10" | |
def jaxb_api_version = "2.2.12" | |
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java | |
index 9d16a41531..45a1986607 100644 | |
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java | |
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.java | |
@@ -36,7 +36,7 @@ import org.apache.beam.sdk.options.PipelineOptions; | |
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; | |
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; | |
import org.apache.hadoop.conf.Configuration; | |
-import org.apache.hadoop.hdfs.DFSConfigKeys; | |
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; | |
/** {@link AutoService} registrar for the {@link HadoopFileSystem}. */ | |
@AutoService(FileSystemRegistrar.class) | |
@@ -75,7 +75,7 @@ public class HadoopFileSystemRegistrar implements FileSystemRegistrar { | |
builder.add(new HadoopFileSystem(scheme, configuration)); | |
registeredSchemes.add(scheme); | |
} | |
- final String nameServices = configuration.get(DFSConfigKeys.DFS_NAMESERVICES); | |
+ final String nameServices = configuration.get(HdfsClientConfigKeys.DFS_NAMESERVICES); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment