Skip to content

Instantly share code, notes, and snippets.

@ottomata
Created May 23, 2016 18:43
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ottomata/6caf158d3b787a1c3439d936a1e28916 to your computer and use it in GitHub Desktop.
Save ottomata/6caf158d3b787a1c3439d936a1e28916 to your computer and use it in GitHub Desktop.
# build
javac -cp '/usr/share/druid/hadoop-dependencies/hadoop-client/2.6.0/*' SnappyNativeTest.java && jar -cf SnappyNativeTest.jar SnappyNativeTest.class
# run without LD_LIBRARY_PATH
java -cp /usr/share/druid/lib/*:/usr/share/druid/extensions/druid-hdfs-storage/*:/usr/share/druid/hadoop-dependencies/hadoop-client/2.6.0/*:SnappyNativeTest.jar SnappyNativeTest
java.library.path = /usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib
:( native hadoop NOT manually loaded: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path
:( native snappy NOT manually loaded: java.lang.UnsatisfiedLinkError: no snappy in java.library.path
2016-05-23T18:41:23,813 WARN [main] org.apache.hadoop.util.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
:( native hadoop NOT loaded according to Hadoop
Exception in thread "main" java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy()Z
at org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy(Native Method)
at SnappyNativeTest.main(SnappyNativeTest.java:40)
ls /usr/lib/hadoop/lib/native
libhadoop.a libhadooppipes.a libhadoop.so libhadoop.so.1.0.0 libhadooputils.a libhdfs.a libnativetask.a libnativetask.so libnativetask.so.1.0.0 libsnappy.a libsnappy.so libsnappy.so.1 libsnappy.so.1.1.3
# run with LD_LIBRARY_PATH
LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native java -cp /usr/share/druid/lib/*:/usr/share/druid/extensions/druid-hdfs-storage/*:/usr/share/druid/hadoop-dependencies/hadoop-client/2.6.0/*:SnappyNativeTest.jar SnappyNativeTest
java.library.path = /usr/lib/hadoop/lib/native:/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib
:) native hadoop manually loaded
:) native snappy manually loaded
:) native hadoop loaded according to Hadoop
:) native snappy loaded according to Hadoop
:) native snappy loaded according to Hadoop SnappyCodec
import java.util.Enumeration;
import java.util.Properties;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.io.compress.SnappyCodec;
public class SnappyNativeTest
{
public static void main(String [] args)
{
System.out.println("java.library.path = " + System.getProperty("java.library.path"));
// Try to manually load hadoop native
try {
System.loadLibrary("hadoop");
System.out.println(":) native hadoop manually loaded");
} catch (Throwable t) {
System.out.println(":( native hadoop NOT manually loaded: " + t);
}
// Try to manually load snappy native
try {
System.loadLibrary("snappy");
System.out.println(":) native snappy manually loaded");
} catch (UnsatisfiedLinkError ex) {
System.out.println(":( native snappy NOT manually loaded: " + ex.toString());
}
// Check if Hadoop thinks hadoop native is loaded
if (NativeCodeLoader.isNativeCodeLoaded()) {
System.out.println(":) native hadoop loaded according to Hadoop");
}
else {
System.out.println(":( native hadoop NOT loaded according to Hadoop");
}
// Check if Hadoop thinks snappy native is loaded
if (NativeCodeLoader.buildSupportsSnappy()) {
System.out.println(":) native snappy loaded according to Hadoop");
}
else {
System.out.println(":( native snappy NOT loaded according to Hadoop");
}
// Check if Hadoop SnappyCodec thinks snappy native is loaded.
// This will throw if it is not
SnappyCodec.checkNativeCodeLoaded();
System.out.println(":) native snappy loaded according to Hadoop SnappyCodec");
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment