Created
November 1, 2014 12:43
-
-
Save ftom/fa61ece18bdd627fe8c4 to your computer and use it in GitHub Desktop.
How Spark determines the default IP to use for the Spark driver
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* Get the local host's IP address in dotted-quad format (e.g. 1.2.3.4). | |
* Note, this is typically not used from within core spark. | |
*/ | |
lazy val localIpAddress: String = findLocalIpAddress() | |
lazy val localIpAddressHostname: String = getAddressHostName(localIpAddress) | |
private def findLocalIpAddress(): String = { | |
val defaultIpOverride = System.getenv("SPARK_LOCAL_IP") | |
if (defaultIpOverride != null) { | |
defaultIpOverride | |
} else { | |
val address = InetAddress.getLocalHost | |
if (address.isLoopbackAddress) { | |
// Address resolves to something like 127.0.1.1, which happens on Debian; try to find | |
// a better address using the local network interfaces | |
// getNetworkInterfaces returns ifs in reverse order compared to ifconfig output order | |
// on unix-like system. On windows, it returns in index order. | |
// It's more proper to pick ip address following system output order. | |
val activeNetworkIFs = NetworkInterface.getNetworkInterfaces.toList | |
val reOrderedNetworkIFs = if (isWindows) activeNetworkIFs else activeNetworkIFs.reverse | |
for (ni <- reOrderedNetworkIFs) { | |
for (addr <- ni.getInetAddresses if !addr.isLinkLocalAddress && | |
!addr.isLoopbackAddress && addr.isInstanceOf[Inet4Address]) { | |
// We've found an address that looks reasonable! | |
logWarning("Your hostname, " + InetAddress.getLocalHost.getHostName + " resolves to" + | |
" a loopback address: " + address.getHostAddress + "; using " + addr.getHostAddress + | |
" instead (on interface " + ni.getName + ")") | |
logWarning("Set SPARK_LOCAL_IP if you need to bind to another address") | |
return addr.getHostAddress | |
} | |
} | |
logWarning("Your hostname, " + InetAddress.getLocalHost.getHostName + " resolves to" + | |
" a loopback address: " + address.getHostAddress + ", but we couldn't find any" + | |
" external IP address!") | |
logWarning("Set SPARK_LOCAL_IP if you need to bind to another address") | |
} | |
address.getHostAddress | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment