Last active
December 3, 2017 18:53
-
-
Save rohangarg/7aa44c4292704eddfe3f03d6c38867e1 to your computer and use it in GitHub Desktop.
HADOOP-14294
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java | |
index 790902c6229..afbafd668d8 100644 | |
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java | |
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java | |
@@ -23,6 +23,10 @@ | |
import org.apache.hadoop.conf.Configuration; | |
import org.apache.hadoop.conf.Configuration.DeprecationDelta; | |
+import java.util.ArrayList; | |
+import java.util.HashMap; | |
+import java.util.Map; | |
+ | |
/** | |
* Constants. | |
*/ | |
@@ -121,6 +125,42 @@ public static void addDeprecatedKeys() { | |
Configuration.reloadExistingConfigurations(); | |
} | |
+ /** | |
+ * Adds deprecated keys to configuration as mentioned in deprecatedKeys param | |
+ * @param deprecatedKeys map where map-key is the deprecated key name and map-value is the updated key name | |
+ */ | |
+ public static void addDeprecatedKeys(Map<String, String> deprecatedKeys) { | |
+ if (deprecatedKeys == null || deprecatedKeys.isEmpty()) { | |
+ return; | |
+ } | |
+ ArrayList<DeprecationDelta> deprecationDeltas = new ArrayList<>(); | |
+ for(String deprecatedKey : deprecatedKeys.keySet()) { | |
+ String newKey = deprecatedKeys.get(deprecatedKey); | |
+ deprecationDeltas.add(new DeprecationDelta(deprecatedKey, newKey)); | |
+ } | |
+ | |
+ Configuration.addDeprecations(deprecationDeltas.toArray(new DeprecationDelta[deprecationDeltas.size()])); | |
+ Configuration.reloadExistingConfigurations(); | |
+ } | |
+ | |
+ public static void addDeprecatedMountPointHostnameKeys(String hostname) { | |
+ String deprecatedADLPrefix = "dfs.adls."; | |
+ String newADLPrefix = "fs.adl."; | |
+ | |
+ // hostname keys | |
+ String deprecatedHostNameProperty = deprecatedADLPrefix + hostname + ".hostname"; | |
+ String hostNameProperty = newADLPrefix + hostname + ".hostname"; | |
+ | |
+ // mountpoint keys | |
+ String deprecatedMountPointProperty = deprecatedADLPrefix + hostname + ".mountpoint"; | |
+ String mountPointProperty = newADLPrefix + hostname + ".mountpoint"; | |
+ | |
+ Map<String, String> deprecatedKeys = new HashMap<>(); | |
+ deprecatedKeys.put(deprecatedHostNameProperty, hostNameProperty); | |
+ deprecatedKeys.put(deprecatedMountPointProperty, mountPointProperty); | |
+ addDeprecatedKeys(deprecatedKeys); | |
+ } | |
+ | |
private AdlConfKeys() { | |
} | |
} | |
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java | |
index a4965959dee..7358e206d67 100644 | |
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java | |
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java | |
@@ -147,8 +147,9 @@ public void initialize(URI storeUri, Configuration conf) throws IOException { | |
String hostname = storeUri.getHost(); | |
if (!hostname.contains(".") && !hostname.equalsIgnoreCase( | |
"localhost")) { // this is a symbolic name. Resolve it. | |
- String hostNameProperty = "dfs.adls." + hostname + ".hostname"; | |
- String mountPointProperty = "dfs.adls." + hostname + ".mountpoint"; | |
+ AdlConfKeys.addDeprecatedMountPointHostnameKeys(hostname); | |
+ String hostNameProperty = "fs.adl." + hostname + ".hostname"; | |
+ String mountPointProperty = "fs.adl." + hostname + ".mountpoint"; | |
accountFQDN = getNonEmptyVal(conf, hostNameProperty); | |
mountPoint = getNonEmptyVal(conf, mountPointProperty); | |
} else { | |
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java | |
index 3d51b42111e..4d2eb78d544 100644 | |
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java | |
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java | |
@@ -68,6 +68,9 @@ | |
*/ | |
public class TestValidateConfiguration { | |
+ String DUMMY_MOUNTPOINT_KEY = "fs.adl.dummy.mountpoint"; | |
+ String DUMMY_HOSTNAME_KEY = "fs.adl.dummy.hostname"; | |
+ | |
@Test | |
public void validateConfigurationKeys() { | |
Assert | |
@@ -124,6 +127,7 @@ public void testSetDeprecatedKeys() throws ClassNotFoundException { | |
// Force AdlFileSystem static initialization to register deprecated keys. | |
Class.forName(AdlFileSystem.class.getName()); | |
+ AdlConfKeys.addDeprecatedMountPointHostnameKeys("dummy"); | |
assertDeprecatedKeys(conf); | |
} | |
@@ -148,6 +152,7 @@ public void testLoadDeprecatedKeys() | |
// Force AdlFileSystem static initialization to register deprecated keys. | |
Class.forName(AdlFileSystem.class.getName()); | |
+ AdlConfKeys.addDeprecatedMountPointHostnameKeys("dummy"); | |
assertDeprecatedKeys(conf); | |
} | |
@@ -160,6 +165,8 @@ private void setDeprecatedKeys(Configuration conf) { | |
conf.set("dfs.adls.oauth2.credential", "dummyCredential"); | |
conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass"); | |
conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker"); | |
+ conf.set("dfs.adls.dummy.mountpoint", "dummyMountPoint"); | |
+ conf.set("dfs.adls.dummy.hostname", "dummyHostname"); | |
} | |
private void assertDeprecatedKeys(Configuration conf) { | |
@@ -177,5 +184,9 @@ private void assertDeprecatedKeys(Configuration conf) { | |
conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY)); | |
Assert.assertEquals("dummyTracker", | |
conf.get(LATENCY_TRACKER_KEY)); | |
+ Assert.assertEquals("dummyMountPoint", | |
+ conf.get(DUMMY_MOUNTPOINT_KEY)); | |
+ Assert.assertEquals("dummyHostname", | |
+ conf.get(DUMMY_HOSTNAME_KEY)); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment