Skip to content

Instantly share code, notes, and snippets.

@jdevoo
Created March 17, 2015 16:21
Show Gist options
  • Save jdevoo/d862141575498286e716 to your computer and use it in GitHub Desktop.
Save jdevoo/d862141575498286e716 to your computer and use it in GitHub Desktop.
Cascalog on cygwin
Leiningen 2.5.0
javac 1.7.0_15 (for hadoop)
javac 1.6.0_45 (for cascalog)
OpenSSH_6.7p1, OpenSSL 1.0.1k 8 Jan 2015
---my mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>mapred.job.tracker</name>
<value>L3300-000001:10002</value>
</property>
</configuration>
---my hdfs-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>
---my core-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://L3300-000001:10001</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/cygwin/tmp</value>
</property>
<property>
<name>fs.file.impl</name>
<value>com.conga.services.hadoop.patch.HADOOP_7682.WinLocalFileSystem</value>
</property>
</configuration>
---my hadoop-env.sh
export JAVA_HOME=/cygdrive/c/jdk1.7.0_15
export HADOOP_OPTS=-Djava.net.preferIPv4Stack=true
place patch-hadoop_7682-1.0.x-win.jar inside lib
---project.clj for cascalog
(defproject spatialog "0.1.0-SNAPSHOT"
:repositories {"conjars" "http://conjars.org/repo"}
:dependencies [[org.clojure/clojure "1.5.1"]
[cascalog "1.10.0"]
[cascalog/cascalog-core "2.1.1"]
[local/patch-hadoop_7682 "1.0.x-win"]]
:jvm-opts ["-Dhadoop.tmp.dir=/cygwin/tmp"]
:profiles {:provided {:dependencies [[org.apache.hadoop/hadoop-core "1.2.1"]]}})
mvn install:install-file -Durl=file:repo -DgroupId=local -DartifactId=patch-hadoop_7682 -Dversion=1.0.x-win -Dpackaging=jar -Dfile=/home/P3300009/lib/hadoop-1.2.1/lib/patch-hadoop_7682-1.0.x-win.jar
---lein resources/core-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>fs.file.impl</name>
<value>com.conga.services.hadoop.patch.HADOOP_7682.WinLocalFileSystem</value>
</property>
</configuration>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment