spark-shell, embedded-redis and ClassLoader

create_temp_binary.scala

import java.util.EnumSet
import java.nio.file.{Files, Path, Paths}
import java.nio.file.StandardCopyOption
import java.nio.file.attribute.{PosixFilePermission, PosixFilePermissions}

// Exception handling omitted for brevity
def createTempRedisBinary(binaryName: String = "redis-server-2.8.19"): Path = {
  val classLoader = Thread.currentThread.getContextClassLoader
  val inputStream = classLoader.getResourceAsStream(binaryName)
  val tempBinary = Files.createTempFile("binaryPrefix-", "-binarySuffix")
  Files.copy(inputStream, tempBinary, StandardCopyOption.REPLACE_EXISTING)
  Files.setPosixFilePermissions(
    tempBinary,
    EnumSet.of(PosixFilePermission.OWNER_READ,
               PosixFilePermission.OWNER_WRITE,
               PosixFilePermission.OWNER_EXECUTE))
  // Clean up on termination
  tempBinary.toFile().deleteOnExit()
  inputStream.close()
  tempBinary
}

// Following code will not work
// 1. Attributes will be overridden by StandardCopyOption.REPLACE_EXISTING
val fileAttributes = PosixFilePermissions
  .asFileAttribute(
    EnumSet.of(PosixFilePermission.OWNER_READ,
               PosixFilePermission.OWNER_WRITE, 
               PosixFilePermission.OWNER_EXECUTE))
val tempBinary = Files.createTempFile("prefix-", "-suffix", fileAttributes)
// 2. Why not use StandardCopyOption.COPY_ATTRIBUTES?
//    Cos you are not allowed to use that optinon when copying from archive/jar file
Files.copy(inputStream, tempBinary, StandardCopyOption.COPY_ATTRIBUTES)

guava-14.0-vs-18.0-classloader.java

// Guava 14.0 - bundled with Spark
// https://github.com/google/guava/blob/v14.0/guava/src/com/google/common/io/Resources.java#L191
/**
 * Returns a {@code URL} pointing to {@code resourceName} if the resource is
 * found in the class path. {@code Resources.class.getClassLoader()} is used
 * to locate the resource.
 */
public static URL getResource(String resourceName) {
    URL url = Resources.class.getClassLoader().getResource(resourceName);
    checkArgument(url != null, "resource %s not found.", resourceName);
    return url;
}

// --------------------------------------------------------------------------------------------------------------------------
// Guava 18.0 - dependency for embedded-redis
// https://github.com/google/guava/blob/v18.0/guava/src/com/google/common/io/Resources.java

/**
 * Returns a {@code URL} pointing to {@code resourceName} if the resource is
 * found using the {@linkplain Thread#getContextClassLoader() context class
 * loader}. In simple environments, the context class loader will find
 * resources from the class path. In environments where different threads can
 * have different class loaders, for example app servers, the context class
 * loader will typically have been set to an appropriate loader for the
 * current thread.
 *
 * <p>In the unusual case where the context class loader is null, the class
 * loader that loaded this class ({@code Resources}) will be used instead.
 * 
 */
public static URL getResource(String resourceName) {
  ClassLoader loader = MoreObjects.firstNonNull(
      Thread.currentThread().getContextClassLoader(),
      Resources.class.getClassLoader());
  URL url = loader.getResource(resourceName);
  checkArgument(url != null, "resource %s not found.", resourceName);
  return url;
}

issue.scala

$ spark-shell --packages "com.github.kstyrc:embedded-redis:0.6,redis.clients:jedis:2.9.0"

scala> import redis.embedded.RedisServer
import redis.embedded.RedisServer

scala> val rds = new RedisServer()
java.lang.IllegalArgumentException: resource redis-server-2.8.19 not found.
  at com.google.common.base.Preconditions.checkArgument(Preconditions.java:119)
  at com.google.common.io.Resources.getResource(Resources.java:191)
  at redis.embedded.util.JarUtil.extractExecutableFromJar(JarUtil.java:17)

temp.scala

    sys.addShutdownHook {
	        hostPortPool.values.foreach { pool =>  pool.close() }
	    }


import redis.embedded.ports.EphemeralPortProvider
val port = new EphemeralPortProvider().next()

scala> val cl = ClassLoader.getSystemClassLoader
cl: ClassLoader = sun.misc.Launcher$AppClassLoader@3581c5f3

scala> cl.asInstanceOf[java.net.URLClassLoader].getURLs.foreach(println)

scala> val cl = ClassLoader.getSystemClassLoader
cl: ClassLoader = sun.misc.Launcher$AppClassLoader@3581c5f3

scala> cl.get
getClass   getParent   getResource   getResourceAsStream   getResources

scala> cl.getResource("redis-server-2.8.19.app")
res37: java.net.URL = null

scala> Thread.currentThread().getContextClassLoader()
res38: ClassLoader = scala.tools.nsc.interpreter.IMain$TranslatingClassLoader@30564ae8


scala> getClass.getClassLoader()
res39: ClassLoader = scala.tools.nsc.interpreter.IMain$TranslatingClassLoader@30564ae8