Skip to content

Instantly share code, notes, and snippets.

@erikkerber
Created February 18, 2013 02:55
Show Gist options
  • Save erikkerber/4974871 to your computer and use it in GitHub Desktop.
Save erikkerber/4974871 to your computer and use it in GitHub Desktop.
Hadoop build script.
<!DOCTYPE project>
<project name="SEIS785Project" default="main" basedir=".">
<!-- load properties file -->
<property file="build.properties" />
<!-- Sets variables which can later be used. -->
<!-- The value of a property is accessed via ${} -->
<property name="src.dir" location="src" />
<property name="build.dir" location="build" />
<property name="dist.dir" location="dist" />
<property name="docs.dir" location="docs" />
<!-- Deletes the existing build, docs and dist directory-->
<target name="clean" description="Delete the existing build, docs and dist directory.">
<delete dir="${build.dir}" />
<delete dir="${docs.dir}" />
<delete dir="${dist.dir}" />
</target>
<!-- Creates the build, docs and dist directory-->
<target name="makedirs" description="Create the build, docs and dist directory.">
<mkdir dir="${build.dir}" />
<mkdir dir="${docs.dir}" />
<mkdir dir="${dist.dir}" />
</target>
<!-- Compiles the Java code -->
<target name="compile" depends="clean, makedirs" description="Compile the Java code.">
<javac
srcdir="${src.dir}"
destdir="${build.dir}"
classpath="${hadoop-classpath}"
includeantruntime="false"
fork="yes"
executable="${javac-location}"
>
</javac>
</target>
<!-- Creates Javadoc -->
<target name="docs" depends="compile" description="Create javadoc.">
<javadoc packagenames="src" sourcepath="${src.dir}" destdir="${docs.dir}">
<!-- Define which files / directory should get included, we include all -->
<fileset dir="${src.dir}">
<include name="**" />
</fileset>
</javadoc>
</target>
<!--Creates the deployable jar file -->
<target name="jar" depends="compile" description="Create jar file.">
<jar destfile="${dist.dir}\${ant.project.name}.jar" basedir="${build.dir}">
</jar>
</target>
<!-- Securely upload jar file and Shell Script file to server -->
<target name="pscp-jar" depends="jar" description="Securely upload jar file to server.">
<echo message="Uploading jar file to server using pscp" />
<exec executable="${pscp-location}">
<arg value="-v" />
<arg value="-pw" />
<arg value="${ust-cluster-pass}" />
<arg value="${dist.dir}/${ant.project.name}.jar" />
<arg value="${ust-username}@${ust-host}:${ust-remote-dir}/${ant.project.name}.jar" />
</exec>
<echo message="Done uploading jar file to server using pscp" />
</target>
<target name="pscp-script" description="Securely upload Shell Script file to server.">
<echo message="Uploading Shell Script file to server using pscp" />
<exec executable="${pscp-location}">
<arg value="-v" />
<arg value="-pw" />
<arg value="${ust-cluster-pass}" />
<arg value="${ust-job-script}" />
<arg value="${ust-username}@${ust-host}:${ust-remote-dir}/${ust-job-script}" />
</exec>
<echo message="Done uploading Shell Script file to server using pscp" />
<echo message="Set mode of Shell Script to 0755 and run dos2unix on this file"/>
<exec executable="${plink-location}">
<arg value="-v" />
<arg value="-pw" />
<arg value="${ust-cluster-pass}" />
<arg value="${ust-username}@${ust-host}" />
<arg value="chmod 0755 ${ust-remote-dir}/${ust-job-script}; dos2unix ${ust-remote-dir}/${ust-job-script}" />
</exec>
<echo message="Done setting mode of Shell Script to 0755 and ran dos2unix on this file."/>
</target>
<!-- Run Shell Script -->
<target name="run" depends="pscp-jar" description="Runs Shell Script that in turn runs Hadoop job.">
<echo>Running Hadoop job</echo>
<exec executable="${plink-location}">
<arg value="-v" />
<arg value="-pw" />
<arg value="${ust-cluster-pass}" />
<arg value="${ust-username}@${ust-host}" />
<arg value="${ust-remote-dir}/${ust-job-script}" />
</exec>
<echo>Done running Hadoop job</echo>
</target>
<!-- NEED TO MOVE pscp and run on its own target so they can be run separately for bit jobs.
Do not need to copy and run every time the project is compiled. -->
<target name="main" depends="compile, jar, pscp-jar, pscp-script, run">
<description>Main target</description>
</target>
</project>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment