Created
May 23, 2016 18:43
-
-
Save ottomata/6caf158d3b787a1c3439d936a1e28916 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# build | |
javac -cp '/usr/share/druid/hadoop-dependencies/hadoop-client/2.6.0/*' SnappyNativeTest.java && jar -cf SnappyNativeTest.jar SnappyNativeTest.class | |
# run without LD_LIBRARY_PATH | |
java -cp /usr/share/druid/lib/*:/usr/share/druid/extensions/druid-hdfs-storage/*:/usr/share/druid/hadoop-dependencies/hadoop-client/2.6.0/*:SnappyNativeTest.jar SnappyNativeTest | |
java.library.path = /usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib | |
:( native hadoop NOT manually loaded: java.lang.UnsatisfiedLinkError: no hadoop in java.library.path | |
:( native snappy NOT manually loaded: java.lang.UnsatisfiedLinkError: no snappy in java.library.path | |
2016-05-23T18:41:23,813 WARN [main] org.apache.hadoop.util.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable | |
:( native hadoop NOT loaded according to Hadoop | |
Exception in thread "main" java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy()Z | |
at org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy(Native Method) | |
at SnappyNativeTest.main(SnappyNativeTest.java:40) | |
ls /usr/lib/hadoop/lib/native | |
libhadoop.a libhadooppipes.a libhadoop.so libhadoop.so.1.0.0 libhadooputils.a libhdfs.a libnativetask.a libnativetask.so libnativetask.so.1.0.0 libsnappy.a libsnappy.so libsnappy.so.1 libsnappy.so.1.1.3 | |
# run with LD_LIBRARY_PATH | |
LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native java -cp /usr/share/druid/lib/*:/usr/share/druid/extensions/druid-hdfs-storage/*:/usr/share/druid/hadoop-dependencies/hadoop-client/2.6.0/*:SnappyNativeTest.jar SnappyNativeTest | |
java.library.path = /usr/lib/hadoop/lib/native:/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib | |
:) native hadoop manually loaded | |
:) native snappy manually loaded | |
:) native hadoop loaded according to Hadoop | |
:) native snappy loaded according to Hadoop | |
:) native snappy loaded according to Hadoop SnappyCodec |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.util.Enumeration; | |
import java.util.Properties; | |
import org.apache.hadoop.util.NativeCodeLoader; | |
import org.apache.hadoop.io.compress.SnappyCodec; | |
public class SnappyNativeTest | |
{ | |
public static void main(String [] args) | |
{ | |
System.out.println("java.library.path = " + System.getProperty("java.library.path")); | |
// Try to manually load hadoop native | |
try { | |
System.loadLibrary("hadoop"); | |
System.out.println(":) native hadoop manually loaded"); | |
} catch (Throwable t) { | |
System.out.println(":( native hadoop NOT manually loaded: " + t); | |
} | |
// Try to manually load snappy native | |
try { | |
System.loadLibrary("snappy"); | |
System.out.println(":) native snappy manually loaded"); | |
} catch (UnsatisfiedLinkError ex) { | |
System.out.println(":( native snappy NOT manually loaded: " + ex.toString()); | |
} | |
// Check if Hadoop thinks hadoop native is loaded | |
if (NativeCodeLoader.isNativeCodeLoaded()) { | |
System.out.println(":) native hadoop loaded according to Hadoop"); | |
} | |
else { | |
System.out.println(":( native hadoop NOT loaded according to Hadoop"); | |
} | |
// Check if Hadoop thinks snappy native is loaded | |
if (NativeCodeLoader.buildSupportsSnappy()) { | |
System.out.println(":) native snappy loaded according to Hadoop"); | |
} | |
else { | |
System.out.println(":( native snappy NOT loaded according to Hadoop"); | |
} | |
// Check if Hadoop SnappyCodec thinks snappy native is loaded. | |
// This will throw if it is not | |
SnappyCodec.checkNativeCodeLoaded(); | |
System.out.println(":) native snappy loaded according to Hadoop SnappyCodec"); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment