Skip to content

Instantly share code, notes, and snippets.

@charleech

charleech/directory-of-tez.txt

Last active March 10, 2020 14:06
Show Gist options
  • Save charleech/8cd615c0236626882c7a3ea7532ad4d5 to your computer and use it in GitHub Desktop.
Save charleech/8cd615c0236626882c7a3ea7532ad4d5 to your computer and use it in GitHub Desktop.
Download ZIP
The YARN application log for launch_container.sh.
Raw
directory-of-tez.txt
/opt/tez
root@hive-on-tez:/opt/tez# ls -la
total 6632
drwxr-xr-x. 1 1000 root 4096 Mar 2 08:20 .
drwxr-xr-x. 1 root root 4096 Mar 2 08:26 ..
-rw-r--r--. 1 1000 root 17257 Mar 20 2019 LICENSE
-rw-r--r--. 1 1000 root 1465 Mar 20 2019 LICENSE-BSD-3clause
-rw-r--r--. 1 1000 root 16792 Mar 20 2019 LICENSE-CDDLv1.0
-rw-r--r--. 1 1000 root 36261 Mar 20 2019 LICENSE-CDDLv1.1-GPLv2_withCPE
-rw-r--r--. 1 1000 root 1045 Mar 20 2019 LICENSE-MIT
-rw-r--r--. 1 1000 root 4128 Mar 20 2019 LICENSE-SIL_OpenFontLicense-v1.1
-rw-r--r--. 1 1000 root 1509 Mar 20 2019 NOTICE
drwxr-xr-x. 1 1000 root 4096 Mar 10 20:37 conf
-rw-r--r--. 1 1000 root 8843 Mar 20 2019 hadoop-shim-0.9.2.jar
-rw-r--r--. 1 1000 root 5493 Mar 20 2019 hadoop-shim-2.7-0.9.2.jar
drwxr-xr-x. 2 1000 root 4096 Mar 2 08:20 lib
drwxr-xr-x. 2 1000 root 4096 Mar 20 2019 share
-rw-r--r--. 1 1000 root 1032313 Mar 20 2019 tez-api-0.9.2.jar
-rw-r--r--. 1 1000 root 5405 Mar 20 2019 tez-build-tools-0.9.2.jar
-rw-r--r--. 1 1000 root 85901 Mar 20 2019 tez-common-0.9.2.jar
-rw-r--r--. 1 1000 root 1412256 Mar 20 2019 tez-dag-0.9.2.jar
-rw-r--r--. 1 1000 root 56834 Mar 20 2019 tez-examples-0.9.2.jar
-rw-r--r--. 1 1000 root 107753 Mar 20 2019 tez-ext-service-tests-0.9.2.jar
-rw-r--r--. 1 1000 root 79136 Mar 20 2019 tez-history-parser-0.9.2.jar
-rw-r--r--. 1 1000 root 15303 Mar 20 2019 tez-javadoc-tools-0.9.2.jar
-rw-r--r--. 1 1000 root 78979 Mar 20 2019 tez-job-analyzer-0.9.2.jar
-rw-r--r--. 1 1000 root 296867 Mar 20 2019 tez-mapreduce-0.9.2.jar
-rw-r--r--. 1 1000 root 76070 Mar 20 2019 tez-protobuf-history-plugin-0.9.2.jar
-rw-r--r--. 1 1000 root 198820 Mar 20 2019 tez-runtime-internals-0.9.2.jar
-rw-r--r--. 1 1000 root 768660 Mar 20 2019 tez-runtime-library-0.9.2.jar
-rw-r--r--. 1 1000 root 158992 Mar 20 2019 tez-tests-0.9.2.jar
-rw-r--r--. 1 1000 root 2200884 Mar 20 2019 tez-ui-0.9.2.war
-rw-r--r--. 1 1000 root 28202 Mar 20 2019 tez-yarn-timeline-history-0.9.2.jar
-rw-r--r--. 1 1000 root 7784 Mar 20 2019 tez-yarn-timeline-history-with-acls-0.9.2.jar
/opt/tez/lib
root@hive-on-tez:/opt/tez/lib# ls -la
total 11336
drwxr-xr-x. 2 1000 root 4096 Mar 2 08:20 .
drwxr-xr-x. 1 1000 root 4096 Mar 2 08:20 ..
-rw-r--r--. 1 1000 root 206280 Feb 14 2019 RoaringBitmap-0.5.21.jar
-rw-r--r--. 1 1000 root 535731 Nov 20 2018 async-http-client-1.8.16.jar
-rw-r--r--. 1 1000 root 41123 Nov 20 2018 commons-cli-1.2.jar
-rw-r--r--. 1 1000 root 58160 Nov 20 2018 commons-codec-1.4.jar
-rw-r--r--. 1 1000 root 588337 Nov 20 2018 commons-collections-3.2.2.jar
-rw-r--r--. 1 1000 root 751238 Nov 20 2018 commons-collections4-4.1.jar
-rw-r--r--. 1 1000 root 185140 Nov 20 2018 commons-io-2.4.jar
-rw-r--r--. 1 1000 root 284220 Nov 20 2018 commons-lang-2.6.jar
-rw-r--r--. 1 1000 root 1599627 Nov 20 2018 commons-math3-3.1.1.jar
-rw-r--r--. 1 1000 root 1648200 Nov 20 2018 guava-11.0.2.jar
-rw-r--r--. 1 1000 root 753123 Nov 20 2018 hadoop-mapreduce-client-common-2.7.2.jar
-rw-r--r--. 1 1000 root 1531485 Nov 20 2018 hadoop-mapreduce-client-core-2.7.2.jar
-rw-r--r--. 1 1000 root 130458 Nov 20 2018 jersey-client-1.9.jar
-rw-r--r--. 1 1000 root 147952 Nov 20 2018 jersey-json-1.9.jar
-rw-r--r--. 1 1000 root 81743 Nov 20 2018 jettison-1.3.4.jar
-rw-r--r--. 1 1000 root 520813 Nov 21 2018 jetty-server-9.3.24.v20180605.jar
-rw-r--r--. 1 1000 root 458642 Nov 21 2018 jetty-util-9.3.24.v20180605.jar
-rw-r--r--. 1 1000 root 33031 Nov 20 2018 jsr305-3.0.0.jar
-rw-r--r--. 1 1000 root 111908 Nov 20 2018 metrics-core-3.1.0.jar
-rw-r--r--. 1 1000 root 1199572 Nov 20 2018 netty-3.6.2.Final.jar
-rw-r--r--. 1 1000 root 533455 Nov 20 2018 protobuf-java-2.5.0.jar
-rw-r--r--. 1 1000 root 105112 Nov 20 2018 servlet-api-2.5.jar
-rw-r--r--. 1 1000 root 32119 Nov 20 2018 slf4j-api-1.7.10.jar
-rw-r--r--. 1 1000 root 8866 Nov 20 2018 slf4j-log4j12-1.7.10.jar
/opt/tez/conf
root@hive-on-tez:/opt/tez/conf# ls -la
total 92
drwxr-xr-x. 1 1000 root 4096 Mar 10 20:37 .
drwxr-xr-x. 1 1000 root 4096 Mar 2 08:20 ..
-rw-r--r--. 1 1000 root 55975 Mar 20 2019 tez-default-template.xml
-rw-r--r--. 1 1000 root 13448 Mar 20 2019 tez-runtime-default-template.xml
-rw-r--r--. 1 root root 2974 Mar 10 14:50 tez-site.xml
/opt/tez/share
root@hive-on-tez:/opt/tez/share# ls -la
total 45184
drwxr-xr-x. 2 1000 root 4096 Mar 20 2019 .
drwxr-xr-x. 1 1000 root 4096 Mar 2 08:20 ..
-rw-r--r--. 1 1000 root 46254263 Mar 20 2019 tez.tar.gz
Raw
launch_container.sh
#!/bin/bash
set -o pipefail -e
export PRELAUNCH_OUT="/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/prelaunch.out"
exec >"${PRELAUNCH_OUT}"
export PRELAUNCH_ERR="/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/prelaunch.err"
exec 2>"${PRELAUNCH_ERR}"
echo "Setting up env variables"
export JAVA_HOME=${JAVA_HOME:-"/usr/local/openjdk-8"}
export HADOOP_COMMON_HOME=${HADOOP_COMMON_HOME:-"/opt/hadoop"}
export HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME:-"/opt/hadoop"}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/hadoop/etc/hadoop"}
export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-"/opt/hadoop"}
export HADOOP_HOME=${HADOOP_HOME:-"/opt/hadoop"}
export PATH=${PATH:-"/usr/local/openjdk-8/bin:/opt/hadoop/bin:/opt/hadoop/sbin:/opt/spark/bin:/opt/spark/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}
export LANG=${LANG:-"C.UTF-8"}
export TZ=${TZ:-"Asia/Bangkok"}
export HADOOP_TOKEN_FILE_LOCATION="/tmp/hadoop-root/nm-local-dir/usercache/root/appcache/application_1583471849377_0040/container_1583471849377_0040_01_000001/container_tokens"
export CONTAINER_ID="container_1583471849377_0040_01_000001"
export NM_PORT="61673"
export NM_HOST="node-slave3"
export NM_HTTP_PORT="61693"
export LOCAL_DIRS="/tmp/hadoop-root/nm-local-dir/usercache/root/appcache/application_1583471849377_0040"
export LOCAL_USER_DIRS="/tmp/hadoop-root/nm-local-dir/usercache/root/"
export LOG_DIRS="/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001"
export USER="root"
export LOGNAME="root"
export HOME="/home/"
export PWD="/tmp/hadoop-root/nm-local-dir/usercache/root/appcache/application_1583471849377_0040/container_1583471849377_0040_01_000001"
export JVM_PID="$$"
export MALLOC_ARENA_MAX="4"
export NM_AUX_SERVICE_mapreduce_shuffle="AAA0+gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
export APPLICATION_WEB_PROXY_BASE="/proxy/application_1583471849377_0040"
export TEZ_CLIENT_VERSION="0.9.2"
export CLASSPATH="$PWD:$PWD/*:$PWD/tezlib/*:$PWD/tezlib/lib/*:"
export APP_SUBMIT_TIME_ENV="1583847502305"
export LD_LIBRARY_PATH="$PWD:$PWD:/opt/hadoop/lib/native/"
export HADOOP_MAPRED_HOME="/opt/hadoop"
echo "Setting up job resources"
ln -sf -- "/tmp/hadoop-root/nm-local-dir/usercache/root/filecache/10/hive-exec-3.1.2-50b0545693ae9392559584116a172ee32922e57ae9ea0eb7dc120290c217c2d5.jar" "hive-exec-3.1.2-50b0545693ae9392559584116a172ee32922e57ae9ea0eb7dc120290c217c2d5.jar"
ln -sf -- "/tmp/hadoop-root/nm-local-dir/usercache/root/appcache/application_1583471849377_0040/filecache/10/tez.session.local-resources.pb" "tez.session.local-resources.pb"
ln -sf -- "/tmp/hadoop-root/nm-local-dir/usercache/root/appcache/application_1583471849377_0040/filecache/11/tez-conf.pb" "tez-conf.pb"
ln -sf -- "/tmp/hadoop-root/nm-local-dir/filecache/102/tez.tar.gz" "tezlib"
echo "Copying debugging information"
# Creating copy of launch script
cp "launch_container.sh" "/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/launch_container.sh"
chmod 640 "/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/launch_container.sh"
# Determining directory contents
echo "ls -l:" 1>"/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/directory.info"
ls -l 1>>"/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/directory.info"
echo "find -L . -maxdepth 5 -ls:" 1>>"/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/directory.info"
find -L . -maxdepth 5 -ls 1>>"/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/directory.info"
echo "broken symlinks(find -L . -maxdepth 5 -type l -ls):" 1>>"/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/directory.info"
find -L . -maxdepth 5 -type l -ls 1>>"/opt/hadoop/logs/userlogs/application_1583471849377_0040/container_1583471849377_0040_01_000001/directory.info"
echo "Launching container"
# The following line is wrapped for readable.
exec /bin/bash -c "$JAVA_HOME/bin/java \
-Xmx1228m \
-Djava.io.tmpdir=$PWD/tmp \
-server \
-Djava.net.preferIPv4Stack=true \
-Dhadoop.metrics.log.level=WARN \
400 \ # <---- Suspected which may be related to the above error.
-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator \
-Dlog4j.configuration=tez-container-log4j.properties \
-Dyarn.app.container.log.dir=/opt/hadoop/logs/userlogs/application_1583471849377_0012/container_1583471849377_0012_02_000001 \
-Dtez.root.logger=INFO,CLA \
-Dsun.nio.ch.bugLevel='' \
org.apache.tez.dag.app.DAGAppMaster \
--session \
1>/opt/hadoop/logs/userlogs/application_1583471849377_0012/container_1583471849377_0012_02_000001/stdout \
2>/opt/hadoop/logs/userlogs/application_1583471849377_0012/container_1583471849377_0012_02_000001/stderr "
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment