Skip to content

Instantly share code, notes, and snippets.

@jiacai2050

jiacai2050/cn.ucloud.hadoop.ApplicationMaster.java

Last active August 29, 2015 14:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jiacai2050/1c5666a5e27e673493ca to your computer and use it in GitHub Desktop.
Save jiacai2050/1c5666a5e27e673493ca to your computer and use it in GitHub Desktop.
Download ZIP
my yarn app -distributed shell
Raw
cn.ucloud.hadoop.ApplicationMaster.java
package cn.ucloud.hadoop;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.NMClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.Records;
public class ApplicationMaster {
private static ByteBuffer allTokens;
public static void main(String[] args) throws Exception {
System.out.println("Here IN ---------------------------------------");
Credentials credentials = UserGroupInformation.getCurrentUser()
.getCredentials();
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
// Now remove the AM->RM token so that containers cannot access it.
Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
while (iter.hasNext()) {
Token<?> token = iter.next();
if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
iter.remove();
}
}
allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
final String command = args[0];
final int n = Integer.valueOf(args[1]);
// Initialize clients to ResourceManager and NodeManagers
Configuration conf = new YarnConfiguration();
AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
rmClient.init(conf);
rmClient.start();
NMClient nmClient = NMClient.createNMClient();
nmClient.init(conf);
nmClient.start();
// Register with ResourceManager
System.out.println("registerApplicationMaster 0");
rmClient.registerApplicationMaster("", 0, "");
System.out.println("registerApplicationMaster 1");
// Priority for worker containers - priorities are intra-application
Priority priority = Records.newRecord(Priority.class);
priority.setPriority(0);
// Resource requirements for worker containers
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(128);
capability.setVirtualCores(1);
// Make container requests to ResourceManager
for (int i = 0; i < n; ++i) {
ContainerRequest containerAsk = new ContainerRequest(capability,
null, null, priority);
System.out.println("Making res-req " + i);
rmClient.addContainerRequest(containerAsk);
}
// Obtain allocated containers, launch and check for responses
int responseId = 0;
int completedContainers = 0;
while (completedContainers < n) {
AllocateResponse response = rmClient.allocate(responseId++);
for (Container container : response.getAllocatedContainers()) {
// Launch container by create ContainerLaunchContext
ContainerLaunchContext ctx = Records
.newRecord(ContainerLaunchContext.class);
ctx.setTokens(allTokens.duplicate());
ctx.setCommands(Collections.singletonList(command + " 1>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stdout" + " 2>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stderr"));
System.out.println("Launching container " + container.getId());
nmClient.startContainer(container, ctx);
}
for (ContainerStatus status : response
.getCompletedContainersStatuses()) {
++completedContainers;
System.out.println("Completed container "
+ status.getContainerId());
}
Thread.sleep(100);
}
// Un-register with ResourceManager
rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
"", "");
}
}
Raw
cn.ucloud.hadoop.Client.java
package cn.ucloud.hadoop;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
public class AppClient {
String command = "ls";
int n = 2;
public AppClient() {
Configuration conf = new YarnConfiguration();
YarnClient client = YarnClient.createYarnClient();
client.init(conf);
client.start();
try {
YarnClusterMetrics metrics = client.getYarnClusterMetrics();
System.out.println("numNodeManagers = "
+ metrics.getNumNodeManagers());
List<NodeReport> nodeReports = client
.getNodeReports(NodeState.RUNNING);
for (NodeReport node : nodeReports) {
System.out.println("nodeId=" + node.getNodeId()
+ ", nodeAddress=" + node.getHttpAddress()
+ ", nodeRackName=" + node.getRackName()
+ ", nodeNumContainers=" + node.getNumContainers());
}
QueueInfo queueInfo = client.getQueueInfo("default");
System.out.println("Queue info" + ", queueName="
+ queueInfo.getQueueName() + ", queueCurrentCapacity="
+ queueInfo.getCurrentCapacity() + ", queueMaxCapacity="
+ queueInfo.getMaximumCapacity()
+ ", queueApplicationCount="
+ queueInfo.getApplications().size()
+ ", queueChildQueueCount="
+ queueInfo.getChildQueues().size());
List<QueueUserACLInfo> listAclInfo = client.getQueueAclsInfo();
for (QueueUserACLInfo aclInfo : listAclInfo) {
for (QueueACL userAcl : aclInfo.getUserAcls()) {
System.out.println("User ACL Info for Queue"
+ ", queueName=" + aclInfo.getQueueName()
+ ", userAcl=" + userAcl.name());
}
}
YarnClientApplication app = client.createApplication();
GetNewApplicationResponse appResponse = app
.getNewApplicationResponse();
System.out.println("appId = " + appResponse.getApplicationId());
System.out.println("maxMemory = "
+ appResponse.getMaximumResourceCapability().getMemory());
ApplicationSubmissionContext appContext = app
.getApplicationSubmissionContext();
appContext.setApplicationName("helloyarn");
ContainerLaunchContext amContainer = Records
.newRecord(ContainerLaunchContext.class);
// 设置Local资源
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
initResource(conf, localResources);
amContainer.setLocalResources(localResources);
// 设置环境变量
Map<String, String> environment = new HashMap<String, String>();
initEnvironment(conf, environment);
amContainer.setEnvironment(environment);
amContainer.setCommands(Collections
.singletonList(Environment.JAVA_HOME.$$() + "/bin/java"
+ " cn.ucloud.hadoop.ApplicationMaster" + " "
+ command + " " + String.valueOf(n) + " 1>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stdout" + " 2>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stderr"));
// Setup security tokens
if (UserGroupInformation.isSecurityEnabled()) {
Credentials credentials = new Credentials();
String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
if (tokenRenewer == null || tokenRenewer.length() == 0) {
throw new IOException(
"Can't get Master Kerberos principal for the RM to use as renewer");
}
FileSystem fs = FileSystem.get(conf);
// For now, only getting tokens for the default file-system.
final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer,
credentials);
if (tokens != null) {
for (Token<?> token : tokens) {
System.out.println("Got dt for " + fs.getUri() + "; "
+ token);
}
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0,
dob.getLength());
amContainer.setTokens(fsTokens);
}
appContext.setAMContainerSpec(amContainer);
// 设置内存资源
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(200);
capability.setVirtualCores(1);
appContext.setResource(capability);
Priority priortiy = Records.newRecord(Priority.class);
priortiy.setPriority(1);
appContext.setPriority(priortiy);
appContext.setQueue("default");
// Submit application
ApplicationId appId = appContext.getApplicationId();
System.out.println("Submitting application " + appId);
System.out.println(client.submitApplication(appContext));
ApplicationReport appReport = client.getApplicationReport(appId);
YarnApplicationState appState = appReport.getYarnApplicationState();
while (appState != YarnApplicationState.FINISHED
&& appState != YarnApplicationState.KILLED
&& appState != YarnApplicationState.FAILED) {
Thread.sleep(100);
appReport = client.getApplicationReport(appId);
appState = appReport.getYarnApplicationState();
}
System.out.println(appReport.getDiagnostics());
System.out
.println("Application " + appId + " finished with"
+ " state " + appState + " at "
+ appReport.getFinishTime());
} catch (YarnException | IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public void initResource(Configuration conf,
Map<String, LocalResource> localResources) {
try {
LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
FileSystem fs = FileSystem.get(conf);
Path dst = new Path("/simpleapp.jar");
FileStatus destStatus = fs.getFileStatus(dst);
amJarRsrc.setType(LocalResourceType.FILE);
amJarRsrc.setVisibility(LocalResourceVisibility.PUBLIC);
amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
amJarRsrc.setTimestamp(destStatus.getModificationTime());
amJarRsrc.setSize(destStatus.getLen());
// The framework will create a symlink called simpleapp.jar in the
// working directory that will be linked back to the actual file.
// The ApplicationMaster, if needs to reference the jar file, would
// need to use the symlink filename.
localResources.put("simpleapp.jar", amJarRsrc);
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
public void initEnvironment(Configuration conf,
Map<String, String> environment) {
StringBuilder classPathEnv = new StringBuilder(
Environment.CLASSPATH.$$()).append(
ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
for (String c : conf
.getStrings(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
classPathEnv.append(c.trim());
}
// add the runtime classpath needed for tests to work
if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
classPathEnv.append(':');
classPathEnv.append(System.getProperty("java.class.path"));
}
environment.put("CLASSPATH", classPathEnv.toString());
}
public static void main(String[] args) {
new AppClient();
}
}
Raw
resourcemanager err log
2014-10-25 20:04:42,572 INFO org.apache.hadoop.yarn.server.resourcemanager.ClientRMService: Allocated new applicationId: 4
2014-10-25 20:04:43,349 WARN org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: The specific max attempts: 0 for application: 4 is invalid, because it is out of the range [1, 2]. Use the global max attempts instead.
2014-10-25 20:04:43,349 INFO org.apache.hadoop.yarn.server.resourcemanager.ClientRMService: Application with id 4 submitted by user liujiacai
2014-10-25 20:04:43,349 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: Storing application with id application_1414233374278_0004
2014-10-25 20:04:43,349 INFO org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger: USER=liujiacai IP=127.0.0.1 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_1414233374278_0004
2014-10-25 20:04:43,349 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: application_1414233374278_0004 State change from NEW to NEW_SAVING
2014-10-25 20:04:43,349 INFO org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore: Storing info for app: application_1414233374278_0004
2014-10-25 20:04:43,350 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: application_1414233374278_0004 State change from NEW_SAVING to SUBMITTED
2014-10-25 20:04:43,350 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler: Accepted application application_1414233374278_0004 from user: liujiacai, in queue: default, currently num of applications: 1
2014-10-25 20:04:43,351 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: application_1414233374278_0004 State change from SUBMITTED to ACCEPTED
2014-10-25 20:04:43,351 INFO org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService: Registering app attempt : appattempt_1414233374278_0004_000001
2014-10-25 20:04:43,351 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000001 State change from NEW to SUBMITTED
2014-10-25 20:04:43,351 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler: Added Application Attempt appattempt_1414233374278_0004_000001 to scheduler from user: liujiacai
2014-10-25 20:04:43,352 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000001 State change from SUBMITTED to SCHEDULED
2014-10-25 20:04:44,201 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1414233374278_0004_01_000001 Container Transitioned from NEW to ALLOCATED
2014-10-25 20:04:44,201 INFO org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger: USER=liujiacai OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1414233374278_0004 CONTAINERID=container_1414233374278_0004_01_000001
2014-10-25 20:04:44,201 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode: Assigned container container_1414233374278_0004_01_000001 of capacity <memory:1024, vCores:1> on host localhost:50221, which has 1 containers, <memory:1024, vCores:1> used and <memory:7168, vCores:7> available after allocation
2014-10-25 20:04:44,202 INFO org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM: Sending NMToken for nodeId : localhost:50221 for container : container_1414233374278_0004_01_000001
2014-10-25 20:04:44,202 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1414233374278_0004_01_000001 Container Transitioned from ALLOCATED to ACQUIRED
2014-10-25 20:04:44,202 INFO org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM: Clear node set for appattempt_1414233374278_0004_000001
2014-10-25 20:04:44,202 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: Storing attempt: AppId: application_1414233374278_0004 AttemptId: appattempt_1414233374278_0004_000001 MasterContainer: Container: [ContainerId: container_1414233374278_0004_01_000001, NodeId: localhost:50221, NodeHttpAddress: localhost:8042, Resource: <memory:1024, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: 127.0.0.1:50221 }, ]
2014-10-25 20:04:44,202 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000001 State change from SCHEDULED to ALLOCATED_SAVING
2014-10-25 20:04:44,202 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000001 State change from ALLOCATED_SAVING to ALLOCATED
2014-10-25 20:04:44,203 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Launching masterappattempt_1414233374278_0004_000001
2014-10-25 20:04:44,204 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Setting up container Container: [ContainerId: container_1414233374278_0004_01_000001, NodeId: localhost:50221, NodeHttpAddress: localhost:8042, Resource: <memory:1024, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: 127.0.0.1:50221 }, ] for AM appattempt_1414233374278_0004_000001
2014-10-25 20:04:44,204 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Command to launch container container_1414233374278_0004_01_000001 : {{JAVA_HOME}}/bin/java cn.ucloud.hadoop.ApplicationMaster ls 2 1><LOG_DIR>/stdout 2><LOG_DIR>/stderr
2014-10-25 20:04:44,204 INFO org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager: Create AMRMToken for ApplicationAttempt: appattempt_1414233374278_0004_000001
2014-10-25 20:04:44,204 INFO org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager: Creating password for appattempt_1414233374278_0004_000001
2014-10-25 20:04:44,213 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Done launching container Container: [ContainerId: container_1414233374278_0004_01_000001, NodeId: localhost:50221, NodeHttpAddress: localhost:8042, Resource: <memory:1024, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: 127.0.0.1:50221 }, ] for AM appattempt_1414233374278_0004_000001
2014-10-25 20:04:44,213 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000001 State change from ALLOCATED to LAUNCHED
2014-10-25 20:04:45,204 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1414233374278_0004_01_000001 Container Transitioned from ACQUIRED to COMPLETED
2014-10-25 20:04:45,204 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt: Completed container: container_1414233374278_0004_01_000001 in state: COMPLETED event:FINISHED
2014-10-25 20:04:45,204 INFO org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger: USER=liujiacai OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1414233374278_0004 CONTAINERID=container_1414233374278_0004_01_000001
2014-10-25 20:04:45,205 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode: Released container container_1414233374278_0004_01_000001 of capacity <memory:1024, vCores:1> on host localhost:50221, which currently has 0 containers, <memory:0, vCores:0> used and <memory:8192, vCores:8> available, release resources=true
2014-10-25 20:04:45,205 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: Updating application attempt appattempt_1414233374278_0004_000001 with final state: FAILED, and exit status: -1000
2014-10-25 20:04:45,205 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler: Application attempt appattempt_1414233374278_0004_000001 released container container_1414233374278_0004_01_000001 on node: host: localhost:50221 #containers=0 available=8192 used=0 with event: FINISHED
2014-10-25 20:04:45,205 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000001 State change from LAUNCHED to FINAL_SAVING
2014-10-25 20:04:45,205 INFO org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService: Unregistering app attempt : appattempt_1414233374278_0004_000001
2014-10-25 20:04:45,206 INFO org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager: Application finished, removing password for appattempt_1414233374278_0004_000001
2014-10-25 20:04:45,206 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000001 State change from FINAL_SAVING to FAILED
2014-10-25 20:04:45,207 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler: Application appattempt_1414233374278_0004_000001 is done. finalState=FAILED
2014-10-25 20:04:45,207 INFO org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService: Registering app attempt : appattempt_1414233374278_0004_000002
2014-10-25 20:04:45,207 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo: Application application_1414233374278_0004 requests cleared
2014-10-25 20:04:45,207 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000002 State change from NEW to SUBMITTED
2014-10-25 20:04:45,207 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler: Added Application Attempt appattempt_1414233374278_0004_000002 to scheduler from user: liujiacai
2014-10-25 20:04:45,208 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000002 State change from SUBMITTED to SCHEDULED
2014-10-25 20:04:46,205 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler: Null container completed...
2014-10-25 20:04:46,205 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1414233374278_0004_02_000001 Container Transitioned from NEW to ALLOCATED
2014-10-25 20:04:46,206 INFO org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger: USER=liujiacai OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1414233374278_0004 CONTAINERID=container_1414233374278_0004_02_000001
2014-10-25 20:04:46,206 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode: Assigned container container_1414233374278_0004_02_000001 of capacity <memory:1024, vCores:1> on host localhost:50221, which has 1 containers, <memory:1024, vCores:1> used and <memory:7168, vCores:7> available after allocation
2014-10-25 20:04:46,206 INFO org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM: Sending NMToken for nodeId : localhost:50221 for container : container_1414233374278_0004_02_000001
2014-10-25 20:04:46,206 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1414233374278_0004_02_000001 Container Transitioned from ALLOCATED to ACQUIRED
2014-10-25 20:04:46,206 INFO org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM: Clear node set for appattempt_1414233374278_0004_000002
2014-10-25 20:04:46,206 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: Storing attempt: AppId: application_1414233374278_0004 AttemptId: appattempt_1414233374278_0004_000002 MasterContainer: Container: [ContainerId: container_1414233374278_0004_02_000001, NodeId: localhost:50221, NodeHttpAddress: localhost:8042, Resource: <memory:1024, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: 127.0.0.1:50221 }, ]
2014-10-25 20:04:46,207 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000002 State change from SCHEDULED to ALLOCATED_SAVING
2014-10-25 20:04:46,207 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000002 State change from ALLOCATED_SAVING to ALLOCATED
2014-10-25 20:04:46,207 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Launching masterappattempt_1414233374278_0004_000002
2014-10-25 20:04:46,209 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Setting up container Container: [ContainerId: container_1414233374278_0004_02_000001, NodeId: localhost:50221, NodeHttpAddress: localhost:8042, Resource: <memory:1024, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: 127.0.0.1:50221 }, ] for AM appattempt_1414233374278_0004_000002
2014-10-25 20:04:46,209 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Command to launch container container_1414233374278_0004_02_000001 : {{JAVA_HOME}}/bin/java cn.ucloud.hadoop.ApplicationMaster ls 2 1><LOG_DIR>/stdout 2><LOG_DIR>/stderr
2014-10-25 20:04:46,210 INFO org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager: Create AMRMToken for ApplicationAttempt: appattempt_1414233374278_0004_000002
2014-10-25 20:04:46,210 INFO org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager: Creating password for appattempt_1414233374278_0004_000002
2014-10-25 20:04:46,228 INFO org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher: Done launching container Container: [ContainerId: container_1414233374278_0004_02_000001, NodeId: localhost:50221, NodeHttpAddress: localhost:8042, Resource: <memory:1024, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: 127.0.0.1:50221 }, ] for AM appattempt_1414233374278_0004_000002
2014-10-25 20:04:46,229 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000002 State change from ALLOCATED to LAUNCHED
2014-10-25 20:04:47,208 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1414233374278_0004_02_000001 Container Transitioned from ACQUIRED to COMPLETED
2014-10-25 20:04:47,208 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt: Completed container: container_1414233374278_0004_02_000001 in state: COMPLETED event:FINISHED
2014-10-25 20:04:47,209 INFO org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger: USER=liujiacai OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1414233374278_0004 CONTAINERID=container_1414233374278_0004_02_000001
2014-10-25 20:04:47,209 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode: Released container container_1414233374278_0004_02_000001 of capacity <memory:1024, vCores:1> on host localhost:50221, which currently has 0 containers, <memory:0, vCores:0> used and <memory:8192, vCores:8> available, release resources=true
2014-10-25 20:04:47,209 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: Updating application attempt appattempt_1414233374278_0004_000002 with final state: FAILED, and exit status: -1000
2014-10-25 20:04:47,209 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler: Application attempt appattempt_1414233374278_0004_000002 released container container_1414233374278_0004_02_000001 on node: host: localhost:50221 #containers=0 available=8192 used=0 with event: FINISHED
2014-10-25 20:04:47,209 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000002 State change from LAUNCHED to FINAL_SAVING
2014-10-25 20:04:47,210 INFO org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService: Unregistering app attempt : appattempt_1414233374278_0004_000002
2014-10-25 20:04:47,211 INFO org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager: Application finished, removing password for appattempt_1414233374278_0004_000002
2014-10-25 20:04:47,211 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl: appattempt_1414233374278_0004_000002 State change from FINAL_SAVING to FAILED
2014-10-25 20:04:47,211 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: Updating application application_1414233374278_0004 with final state: FAILED
2014-10-25 20:04:47,211 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: application_1414233374278_0004 State change from ACCEPTED to FINAL_SAVING
2014-10-25 20:04:47,211 INFO org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore: Updating info for app: application_1414233374278_0004
2014-10-25 20:04:47,211 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler: Application appattempt_1414233374278_0004_000002 is done. finalState=FAILED
2014-10-25 20:04:47,212 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo: Application application_1414233374278_0004 requests cleared
2014-10-25 20:04:47,259 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: Application application_1414233374278_0004 failed 2 times due to AM Container for appattempt_1414233374278_0004_000002 exited with exitCode: -1000 due to: .Failing this attempt.. Failing the application.
2014-10-25 20:04:47,259 INFO org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl: application_1414233374278_0004 State change from FINAL_SAVING to FAILED
2014-10-25 20:04:47,259 WARN org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger: USER=liujiacai OPERATION=Application Finished - Failed TARGET=RMAppManager RESULT=FAILURE DESCRIPTION=App failed with state: FAILED PERMISSIONS=Application application_1414233374278_0004 failed 2 times due to AM Container for appattempt_1414233374278_0004_000002 exited with exitCode: -1000 due to: .Failing this attempt.. Failing the application. APPID=application_1414233374278_0004
2014-10-25 20:04:47,260 INFO org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary: appId=application_1414233374278_0004,name=helloyarn,user=liujiacai,queue=root.liujiacai,state=FAILED,trackingUrl=http://localhost:8088/cluster/app/application_1414233374278_0004,appMasterHost=N/A,startTime=1414238683349,finishTime=1414238687211,finalStatus=FAILED
2014-10-25 20:04:48,209 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler: Null container completed...
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment