Skip to content

Instantly share code, notes, and snippets.

@rohangarg
Last active December 3, 2017 18:53
Show Gist options
  • Save rohangarg/7aa44c4292704eddfe3f03d6c38867e1 to your computer and use it in GitHub Desktop.
Save rohangarg/7aa44c4292704eddfe3f03d6c38867e1 to your computer and use it in GitHub Desktop.
HADOOP-14294
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index 790902c6229..afbafd668d8 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -23,6 +23,10 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configuration.DeprecationDelta;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
/**
* Constants.
*/
@@ -121,6 +125,42 @@ public static void addDeprecatedKeys() {
Configuration.reloadExistingConfigurations();
}
+ /**
+ * Adds deprecated keys to configuration as mentioned in deprecatedKeys param
+ * @param deprecatedKeys map where map-key is the deprecated key name and map-value is the updated key name
+ */
+ public static void addDeprecatedKeys(Map<String, String> deprecatedKeys) {
+ if (deprecatedKeys == null || deprecatedKeys.isEmpty()) {
+ return;
+ }
+ ArrayList<DeprecationDelta> deprecationDeltas = new ArrayList<>();
+ for(String deprecatedKey : deprecatedKeys.keySet()) {
+ String newKey = deprecatedKeys.get(deprecatedKey);
+ deprecationDeltas.add(new DeprecationDelta(deprecatedKey, newKey));
+ }
+
+ Configuration.addDeprecations(deprecationDeltas.toArray(new DeprecationDelta[deprecationDeltas.size()]));
+ Configuration.reloadExistingConfigurations();
+ }
+
+ public static void addDeprecatedMountPointHostnameKeys(String hostname) {
+ String deprecatedADLPrefix = "dfs.adls.";
+ String newADLPrefix = "fs.adl.";
+
+ // hostname keys
+ String deprecatedHostNameProperty = deprecatedADLPrefix + hostname + ".hostname";
+ String hostNameProperty = newADLPrefix + hostname + ".hostname";
+
+ // mountpoint keys
+ String deprecatedMountPointProperty = deprecatedADLPrefix + hostname + ".mountpoint";
+ String mountPointProperty = newADLPrefix + hostname + ".mountpoint";
+
+ Map<String, String> deprecatedKeys = new HashMap<>();
+ deprecatedKeys.put(deprecatedHostNameProperty, hostNameProperty);
+ deprecatedKeys.put(deprecatedMountPointProperty, mountPointProperty);
+ addDeprecatedKeys(deprecatedKeys);
+ }
+
private AdlConfKeys() {
}
}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index a4965959dee..7358e206d67 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -147,8 +147,9 @@ public void initialize(URI storeUri, Configuration conf) throws IOException {
String hostname = storeUri.getHost();
if (!hostname.contains(".") && !hostname.equalsIgnoreCase(
"localhost")) { // this is a symbolic name. Resolve it.
- String hostNameProperty = "dfs.adls." + hostname + ".hostname";
- String mountPointProperty = "dfs.adls." + hostname + ".mountpoint";
+ AdlConfKeys.addDeprecatedMountPointHostnameKeys(hostname);
+ String hostNameProperty = "fs.adl." + hostname + ".hostname";
+ String mountPointProperty = "fs.adl." + hostname + ".mountpoint";
accountFQDN = getNonEmptyVal(conf, hostNameProperty);
mountPoint = getNonEmptyVal(conf, mountPointProperty);
} else {
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
index 3d51b42111e..4d2eb78d544 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
@@ -68,6 +68,9 @@
*/
public class TestValidateConfiguration {
+ String DUMMY_MOUNTPOINT_KEY = "fs.adl.dummy.mountpoint";
+ String DUMMY_HOSTNAME_KEY = "fs.adl.dummy.hostname";
+
@Test
public void validateConfigurationKeys() {
Assert
@@ -124,6 +127,7 @@ public void testSetDeprecatedKeys() throws ClassNotFoundException {
// Force AdlFileSystem static initialization to register deprecated keys.
Class.forName(AdlFileSystem.class.getName());
+ AdlConfKeys.addDeprecatedMountPointHostnameKeys("dummy");
assertDeprecatedKeys(conf);
}
@@ -148,6 +152,7 @@ public void testLoadDeprecatedKeys()
// Force AdlFileSystem static initialization to register deprecated keys.
Class.forName(AdlFileSystem.class.getName());
+ AdlConfKeys.addDeprecatedMountPointHostnameKeys("dummy");
assertDeprecatedKeys(conf);
}
@@ -160,6 +165,8 @@ private void setDeprecatedKeys(Configuration conf) {
conf.set("dfs.adls.oauth2.credential", "dummyCredential");
conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass");
conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker");
+ conf.set("dfs.adls.dummy.mountpoint", "dummyMountPoint");
+ conf.set("dfs.adls.dummy.hostname", "dummyHostname");
}
private void assertDeprecatedKeys(Configuration conf) {
@@ -177,5 +184,9 @@ private void assertDeprecatedKeys(Configuration conf) {
conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY));
Assert.assertEquals("dummyTracker",
conf.get(LATENCY_TRACKER_KEY));
+ Assert.assertEquals("dummyMountPoint",
+ conf.get(DUMMY_MOUNTPOINT_KEY));
+ Assert.assertEquals("dummyHostname",
+ conf.get(DUMMY_HOSTNAME_KEY));
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment