Spark Docker Build 3

spark-docker-build-3.log

$ git diff
diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
index 72563551e5..61959ca2a3 100755
--- a/bin/docker-image-tool.sh
+++ b/bin/docker-image-tool.sh
@@ -182,7 +182,7 @@ PYDOCKERFILE=
 RDOCKERFILE=
 NOCACHEARG=
 BUILD_PARAMS=
-while getopts f:p:R:mr:t:n:b: option
+while getopts f:p:R:mr:t:nb: option
 do
  case "${option}"
  in
 
$ ./bin/docker-image-tool.sh -n -R hello-world.dockerfile -r docker.io/slarkin -t v2.4.0-rc4 build
Sending build context to Docker daemon  1.408GB
Step 1/17 : FROM openjdk:8-alpine
 ---> 54ae553cb104
Step 2/17 : ARG spark_jars=jars
 ---> Running in 6d25abf66d60
Removing intermediate container 6d25abf66d60
 ---> b32ac4be92e0
Step 3/17 : ARG example_jars=examples/jars
 ---> Running in 0c7bea5e4591
Removing intermediate container 0c7bea5e4591
 ---> 18b7195df3b5
Step 4/17 : ARG img_path=kubernetes/dockerfiles
 ---> Running in ec745a875eda
Removing intermediate container ec745a875eda
 ---> 5537baaf801b
Step 5/17 : ARG k8s_tests=kubernetes/tests
 ---> Running in 87477743729a
Removing intermediate container 87477743729a
 ---> 72b80ff36c4f
Step 6/17 : RUN set -ex &&     apk upgrade --no-cache &&     apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs &&     mkdir -p /opt/spark &&     mkdir -p /opt/spark/examples &&     mkdir -p /opt/spark/work-dir &&     touch /opt/spark/RELEASE &&     rm /bin/sh &&     ln -sv /bin/bash /bin/sh &&     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su &&     chgrp root /etc/passwd && chmod ug+rw /etc/passwd
 ---> Running in 93af685c4f6a
+ apk upgrade --no-cache
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/APKINDEX.tar.gz
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/community/x86_64/APKINDEX.tar.gz
(1/5) Upgrading libx11 (1.6.5-r1 -> 1.6.6-r0)
(2/5) Upgrading libgcc (6.4.0-r8 -> 6.4.0-r9)
(3/5) Upgrading libjpeg-turbo (1.5.3-r1 -> 1.5.3-r3)
(4/5) Upgrading libstdc++ (6.4.0-r8 -> 6.4.0-r9)
(5/5) Upgrading nss (3.36.1-r0 -> 3.36.1-r1)
OK: 101 MiB in 53 packages
+ apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/APKINDEX.tar.gz
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/community/x86_64/APKINDEX.tar.gz
(1/10) Installing ncurses-terminfo-base (6.1_p20180818-r1)
(2/10) Installing ncurses-terminfo (6.1_p20180818-r1)
(3/10) Installing ncurses-libs (6.1_p20180818-r1)
(4/10) Installing readline (7.0.003-r0)
(5/10) Installing bash (4.4.19-r1)
Executing bash-4.4.19-r1.post-install
(6/10) Installing e2fsprogs-libs (1.44.2-r0)
(7/10) Installing krb5 (1.15.3-r0)
(8/10) Installing libc6-compat (1.1.19-r10)
(9/10) Installing linux-pam (1.3.0-r0)
(10/10) Installing tini (0.18.0-r0)
Executing busybox-1.28.4-r1.trigger
OK: 111 MiB in 63 packages
+ mkdir -p /opt/spark
+ mkdir -p /opt/spark/examples
+ mkdir -p /opt/spark/work-dir
+ touch /opt/spark/RELEASE
+ rm /bin/sh
+ ln -sv /bin/bash /bin/sh
+ echo 'auth required pam_wheel.so use_uid'
+ chgrp root /etc/passwd
'/bin/sh' -> '/bin/bash'
+ chmod ug+rw /etc/passwd
Removing intermediate container 93af685c4f6a
 ---> bf70d9cc327e
Step 7/17 : COPY ${spark_jars} /opt/spark/jars
 ---> 9858b81aed31
Step 8/17 : COPY bin /opt/spark/bin
 ---> a89292b13757
Step 9/17 : COPY sbin /opt/spark/sbin
 ---> 09abbd08d06f
Step 10/17 : COPY ${img_path}/spark/entrypoint.sh /opt/
 ---> 8f157397f946
Step 11/17 : COPY ${example_jars} /opt/spark/examples/jars
 ---> 6b8e6df95074
Step 12/17 : COPY examples/src /opt/spark/examples/src
 ---> 3c79d1d89d56
Step 13/17 : COPY ${k8s_tests} /opt/spark/tests
 ---> c0c14109d99b
Step 14/17 : COPY data /opt/spark/data
 ---> 78b1ec86c27f
Step 15/17 : ENV SPARK_HOME /opt/spark
 ---> Running in 79c51141d18a
Removing intermediate container 79c51141d18a
 ---> 56c6c4a9ca5d
Step 16/17 : WORKDIR /opt/spark/work-dir
 ---> Running in e6ece49b0729
Removing intermediate container e6ece49b0729
 ---> 2ec55ae6cf66
Step 17/17 : ENTRYPOINT [ "/opt/entrypoint.sh" ]
 ---> Running in 0ba8c86a0e48
Removing intermediate container 0ba8c86a0e48
 ---> 4d6364fc8a59
Successfully built 4d6364fc8a59
Successfully tagged slarkin/spark:v2.4.0-rc4
Sending build context to Docker daemon  1.408GB
Step 1/9 : ARG base_img
Step 2/9 : FROM $base_img
 ---> 4d6364fc8a59
Step 3/9 : WORKDIR /
 ---> Running in f979c921aa75
Removing intermediate container f979c921aa75
 ---> 58ba0e2458f5
Step 4/9 : RUN mkdir ${SPARK_HOME}/python
 ---> Running in 27dd8ee8c1f2
Removing intermediate container 27dd8ee8c1f2
 ---> 46db2408ff42
Step 5/9 : RUN apk add --no-cache python &&     apk add --no-cache python3 &&     python -m ensurepip &&     python3 -m ensurepip &&     rm -r /usr/lib/python*/ensurepip &&     pip install --upgrade pip setuptools &&     rm -r /root/.cache
 ---> Running in 792aa2dc53e7
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/APKINDEX.tar.gz
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/community/x86_64/APKINDEX.tar.gz
(1/3) Installing expat (2.2.5-r0)
(2/3) Installing gdbm (1.13-r1)
(3/3) Installing python2 (2.7.15-r1)
Executing busybox-1.28.4-r1.trigger
OK: 149 MiB in 66 packages
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/APKINDEX.tar.gz
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/community/x86_64/APKINDEX.tar.gz
(1/2) Installing xz-libs (5.2.4-r0)
(2/2) Installing python3 (3.6.6-r0)
Executing busybox-1.28.4-r1.trigger
OK: 202 MiB in 68 packages
Collecting setuptools
Collecting pip
Installing collected packages: setuptools, pip
Successfully installed pip-9.0.3 setuptools-39.0.1
Looking in links: /tmp/tmpyrtkgne6
Requirement already satisfied: setuptools in /usr/lib/python3.6/site-packages (39.0.1)
Requirement already satisfied: pip in /usr/lib/python3.6/site-packages (10.0.1)
Collecting pip
  Downloading https://files.pythonhosted.org/packages/c2/d7/90f34cb0d83a6c5631cf71dfe64cc1054598c843a92b400e55675cc2ac37/pip-18.1-py2.py3-none-any.whl (1.3MB)
Collecting setuptools
  Downloading https://files.pythonhosted.org/packages/96/06/c8ee69628191285ddddffb277bd5abdf769166e7a14b867c2a172f0175b1/setuptools-40.4.3-py2.py3-none-any.whl (569kB)
Installing collected packages: pip, setuptools
  Found existing installation: pip 9.0.3
    Uninstalling pip-9.0.3:
      Successfully uninstalled pip-9.0.3
  Found existing installation: setuptools 39.0.1
    Uninstalling setuptools-39.0.1:
      Successfully uninstalled setuptools-39.0.1
Successfully installed pip-18.1 setuptools-40.4.3
Removing intermediate container 792aa2dc53e7
 ---> 56ef5bff7b34
Step 6/9 : COPY python/lib ${SPARK_HOME}/python/lib
 ---> a261d4655cd8
Step 7/9 : ENV PYTHONPATH ${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-*.zip
 ---> Running in d9d9371c9b6d
Removing intermediate container d9d9371c9b6d
 ---> 2143a5606735
Step 8/9 : WORKDIR /opt/spark/work-dir
 ---> Running in d0b16187d751
Removing intermediate container d0b16187d751
 ---> ca19f22fde5f
Step 9/9 : ENTRYPOINT [ "/opt/entrypoint.sh" ]
 ---> Running in dee205bc7c02
Removing intermediate container dee205bc7c02
 ---> 2c7a50453469
Successfully built 2c7a50453469
Successfully tagged slarkin/spark-py:v2.4.0-rc4
Sending build context to Docker daemon  1.408GB
Step 1/1 : FROM hello-world
 ---> 4ab4c602aa5e
[Warning] One or more build-args [base_img] were not consumed
Successfully built 4ab4c602aa5e
Successfully tagged slarkin/spark-r:v2.4.0-rc4