Skip to content

Instantly share code, notes, and snippets.

@chriswhite199
Created November 11, 2013 13:51
Show Gist options
  • Save chriswhite199/7413473 to your computer and use it in GitHub Desktop.
Save chriswhite199/7413473 to your computer and use it in GitHub Desktop.
Reflection hack to get the successful map task attempt hostnames - which isn't available via the JobClient public API. This is for Hadoop 1.2.1 and may not be forwards or backwards compatible.
package sandbox.hadoop;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.JobTracker;
import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class JobClientTaskHostDriver extends Configured implements Tool {
public static void main(String args[]) throws Exception {
ToolRunner.run(new JobClientTaskHostDriver(), args);
}
@Override
public int run(String[] args) throws Exception {
Configuration conf = getConf();
JobClient client = new JobClient(new JobConf(conf));
Method method = JobClient.class.getDeclaredMethod("createRPCProxy", InetSocketAddress.class,
Configuration.class);
method.setAccessible(true);
Object rpcClientSubProtocol = method.invoke(client, JobTracker.getAddress(conf), conf);
Method completeEventsMethod = rpcClientSubProtocol.getClass().getDeclaredMethod("getTaskCompletionEvents",
JobID.class, int.class, int.class);
for (Object tceObj : ((Object[]) completeEventsMethod.invoke(rpcClientSubProtocol,
JobID.forName("job_201311110747_0001"), 0, 100))) {
TaskCompletionEvent tce = (TaskCompletionEvent) tceObj;
if (tce.isMapTask()) {
URI uri = new URI(tce.getTaskTrackerHttp());
System.err.println(tce.getTaskAttemptId() + " @ " + uri.getHost());
}
}
return 0;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment