Tests that hdfs-site.xml is correctly loaded in the master branch.

TestProvidedConfiguration.java

/*
 * Copyright 2013 Cloudera Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.kitesdk.data.spi.filesystem;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URLClassLoader;
import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.util.StringUtils;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
import org.kitesdk.compat.DynMethods;
import org.kitesdk.compat.Hadoop;
import org.kitesdk.data.DatasetDescriptor;
import org.kitesdk.data.Datasets;
import org.kitesdk.data.spi.DefaultConfiguration;

public class TestProvidedConfiguration {

  private static DynMethods.UnboundMethod getFS = new DynMethods
      .Builder("getFileSystem")
      .impl(MiniDFSCluster.class)
      .build();

  @Test
  public void test() throws IOException {
    Assume.assumeTrue(!Hadoop.isHadoop1()); // hdfs-site.xml is not used

    // get the default config
    Configuration c = DefaultConfiguration.get();

    // verify that the hdfs-site.xml file has not been loaded
    Assert.assertFalse(
        "Configuration has already loaded hdfs-site.xml from the classpath",
        c.getBoolean("kite.config-is-present", false));

    // drop a Configuration into the CLASSPATH
    Configuration hdfsSite = new Configuration(false);
    hdfsSite.set("fs.defaultFS", "hdfs://localhost:18020");
    hdfsSite.setBoolean("kite.config-is-present", true);

    try {
      System.err.println("URLs: " + StringUtils.join(",", Arrays.asList(
          ((URLClassLoader) TestProvidedConfiguration.class.getClassLoader()).getURLs())));
      // /tmp/test-classpath is added to the maven-surefire-plugin
      new File("/tmp/test-classpath").mkdir();
      OutputStream out = new FileOutputStream("/tmp/test-classpath/hdfs-site.xml");
      hdfsSite.writeXml(out);
      out.close();

      // force FileSystem to load hdfs-site.xml
      FileSystem.getLocal(c);

      Assert.assertTrue(
          "Configuration did not load hdfs-site.xml from the classpath",
          c.getBoolean("kite.config-is-present", false));

      // if this fails with "The directory is already locked" set umask to 0022
      MiniDFSCluster cluster = new MiniDFSCluster(18020, c, 1, true, true, null, null);
      //cluster = new MiniDFSCluster.Builder(new Configuration()).build();
      FileSystem dfs = getFS.invoke(cluster);

      Datasets.create("dataset:hdfs:/tmp/data/ns/test",
          new DatasetDescriptor.Builder()
              .schemaLiteral("\"string\"")
              .build());

      Assert.assertTrue("Should create the dataset in HDFS",
          dfs.exists(new Path("/tmp/data/ns/test/.metadata")));

      Datasets.delete("dataset:hdfs:/tmp/data/ns/test");

      cluster.shutdown();
    } finally {
      // ensure this does not affect other tests
      new File("/tmp/test-classpath/hdfs-site.xml").delete();
    }
  }
}