Spark Docker Build 1

spark-docker-build-1.log

$ ./bin/docker-image-tool.sh -R hello-world.dockerfile -r docker.io/slarkin -t v2.4.0-rc4 build
Sending build context to Docker daemon  1.408GB
Step 1/17 : FROM openjdk:8-alpine
8-alpine: Pulling from library/openjdk
4fe2ade4980c: Pull complete
6fc58a8d4ae4: Pull complete
fe815adf554b: Pull complete
Digest: sha256:a2d7b02891b158d01523e26ad069d40d5eb2c14d6943cf4df969b097acaa77d3
Status: Downloaded newer image for openjdk:8-alpine
 ---> 54ae553cb104
Step 2/17 : ARG spark_jars=jars
 ---> Running in 95be490a6d5c
Removing intermediate container 95be490a6d5c
 ---> c8cb5df70f94
Step 3/17 : ARG example_jars=examples/jars
 ---> Running in 3ec962c24abe
Removing intermediate container 3ec962c24abe
 ---> b8cdb89a5076
Step 4/17 : ARG img_path=kubernetes/dockerfiles
 ---> Running in f034d86794df
Removing intermediate container f034d86794df
 ---> dd4c9b02796d
Step 5/17 : ARG k8s_tests=kubernetes/tests
 ---> Running in e5ea32c89f4e
Removing intermediate container e5ea32c89f4e
 ---> 9c3d9f275d56
Step 6/17 : RUN set -ex &&     apk upgrade --no-cache &&     apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs &&     mkdir -p /opt/spark &&     mkdir -p /opt/spark/examples &&     mkdir -p /opt/spark/work-dir &&     touch /opt/spark/RELEASE &&     rm /bin/sh &&     ln -sv /bin/bash /bin/sh &&     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su &&     chgrp root /etc/passwd && chmod ug+rw /etc/passwd
 ---> Running in 6667435f9400
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/APKINDEX.tar.gz
+ apk upgrade --no-cache
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/community/x86_64/APKINDEX.tar.gz
(1/5) Upgrading libx11 (1.6.5-r1 -> 1.6.6-r0)
(2/5) Upgrading libgcc (6.4.0-r8 -> 6.4.0-r9)
(3/5) Upgrading libjpeg-turbo (1.5.3-r1 -> 1.5.3-r3)
(4/5) Upgrading libstdc++ (6.4.0-r8 -> 6.4.0-r9)
(5/5) Upgrading nss (3.36.1-r0 -> 3.36.1-r1)
OK: 101 MiB in 53 packages
+ apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/APKINDEX.tar.gz
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/community/x86_64/APKINDEX.tar.gz
(1/10) Installing ncurses-terminfo-base (6.1_p20180818-r1)
(2/10) Installing ncurses-terminfo (6.1_p20180818-r1)
(3/10) Installing ncurses-libs (6.1_p20180818-r1)
(4/10) Installing readline (7.0.003-r0)
(5/10) Installing bash (4.4.19-r1)
Executing bash-4.4.19-r1.post-install
(6/10) Installing e2fsprogs-libs (1.44.2-r0)
(7/10) Installing krb5 (1.15.3-r0)
(8/10) Installing libc6-compat (1.1.19-r10)
(9/10) Installing linux-pam (1.3.0-r0)
(10/10) Installing tini (0.18.0-r0)
Executing busybox-1.28.4-r1.trigger
OK: 111 MiB in 63 packages
+ mkdir -p /opt/spark
+ mkdir -p /opt/spark/examples
+ mkdir -p /opt/spark/work-dir
+ touch /opt/spark/RELEASE
+ rm /bin/sh
+ ln -sv /bin/bash /bin/sh
+ echo 'auth required pam_wheel.so use_uid'
+ chgrp root /etc/passwd
'/bin/sh' -> '/bin/bash'
+ chmod ug+rw /etc/passwd
Removing intermediate container 6667435f9400
 ---> bda859b91565
Step 7/17 : COPY ${spark_jars} /opt/spark/jars
 ---> 046b3f384d9d
Step 8/17 : COPY bin /opt/spark/bin
 ---> 8f1da11ea179
Step 9/17 : COPY sbin /opt/spark/sbin
 ---> 79c7cb6076a2
Step 10/17 : COPY ${img_path}/spark/entrypoint.sh /opt/
 ---> fa5e6859fdff
Step 11/17 : COPY ${example_jars} /opt/spark/examples/jars
 ---> a6ebeacb4bb4
Step 12/17 : COPY examples/src /opt/spark/examples/src
 ---> e3c59fa9fa2f
Step 13/17 : COPY ${k8s_tests} /opt/spark/tests
 ---> fefc89fb4b6d
Step 14/17 : COPY data /opt/spark/data
 ---> 59be5a146a55
Step 15/17 : ENV SPARK_HOME /opt/spark
 ---> Running in f680e500a372
Removing intermediate container f680e500a372
 ---> f2a2793ec4cd
Step 16/17 : WORKDIR /opt/spark/work-dir
 ---> Running in cad91c067f53
Removing intermediate container cad91c067f53
 ---> dd4c2b7511cf
Step 17/17 : ENTRYPOINT [ "/opt/entrypoint.sh" ]
 ---> Running in 57296f448fd2
Removing intermediate container 57296f448fd2
 ---> 9ea3563428b8
Successfully built 9ea3563428b8
Successfully tagged slarkin/spark:v2.4.0-rc4
Sending build context to Docker daemon  1.408GB
Step 1/9 : ARG base_img
Step 2/9 : FROM $base_img
 ---> 9ea3563428b8
Step 3/9 : WORKDIR /
 ---> Running in 40b5b2634c34
Removing intermediate container 40b5b2634c34
 ---> cb43029d45cd
Step 4/9 : RUN mkdir ${SPARK_HOME}/python
 ---> Running in 6d830034dd4c
Removing intermediate container 6d830034dd4c
 ---> 63d9d04f0b90
Step 5/9 : RUN apk add --no-cache python &&     apk add --no-cache python3 &&     python -m ensurepip &&     python3 -m ensurepip &&     rm -r /usr/lib/python*/ensurepip &&     pip install --upgrade pip setuptools &&     rm -r /root/.cache
 ---> Running in 1b2f1d540ceb
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/APKINDEX.tar.gz
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/community/x86_64/APKINDEX.tar.gz
(1/3) Installing expat (2.2.5-r0)
(2/3) Installing gdbm (1.13-r1)
(3/3) Installing python2 (2.7.15-r1)
Executing busybox-1.28.4-r1.trigger
OK: 149 MiB in 66 packages
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/APKINDEX.tar.gz
fetch http://dl-cdn.alpinelinux.org/alpine/v3.8/community/x86_64/APKINDEX.tar.gz
(1/2) Installing xz-libs (5.2.4-r0)
(2/2) Installing python3 (3.6.6-r0)
Executing busybox-1.28.4-r1.trigger
OK: 202 MiB in 68 packages
Collecting setuptools
Collecting pip
Installing collected packages: setuptools, pip
Successfully installed pip-9.0.3 setuptools-39.0.1
Looking in links: /tmp/tmp7meci7hq
Requirement already satisfied: setuptools in /usr/lib/python3.6/site-packages (39.0.1)
Requirement already satisfied: pip in /usr/lib/python3.6/site-packages (10.0.1)
Collecting pip
  Downloading https://files.pythonhosted.org/packages/c2/d7/90f34cb0d83a6c5631cf71dfe64cc1054598c843a92b400e55675cc2ac37/pip-18.1-py2.py3-none-any.whl (1.3MB)
Collecting setuptools
  Downloading https://files.pythonhosted.org/packages/96/06/c8ee69628191285ddddffb277bd5abdf769166e7a14b867c2a172f0175b1/setuptools-40.4.3-py2.py3-none-any.whl (569kB)
Installing collected packages: pip, setuptools
  Found existing installation: pip 9.0.3
    Uninstalling pip-9.0.3:
      Successfully uninstalled pip-9.0.3
  Found existing installation: setuptools 39.0.1
    Uninstalling setuptools-39.0.1:
      Successfully uninstalled setuptools-39.0.1
Successfully installed pip-18.1 setuptools-40.4.3
Removing intermediate container 1b2f1d540ceb
 ---> eb03b0ce17d5
Step 6/9 : COPY python/lib ${SPARK_HOME}/python/lib
 ---> e1be327bd4ce
Step 7/9 : ENV PYTHONPATH ${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-*.zip
 ---> Running in c388dfb71c95
Removing intermediate container c388dfb71c95
 ---> 7030b30a741e
Step 8/9 : WORKDIR /opt/spark/work-dir
 ---> Running in 2ee580858692
Removing intermediate container 2ee580858692
 ---> 01c61b37ff01
Step 9/9 : ENTRYPOINT [ "/opt/entrypoint.sh" ]
 ---> Running in 6a46060cec30
Removing intermediate container 6a46060cec30
 ---> 1668cd0591a6
Successfully built 1668cd0591a6
Successfully tagged slarkin/spark-py:v2.4.0-rc4
Sending build context to Docker daemon  1.408GB
Step 1/1 : FROM hello-world
latest: Pulling from library/hello-world
d1725b59e92d: Pull complete
Digest: sha256:0add3ace90ecb4adbf7777e9aacf18357296e799f81cabc9fde470971e499788
Status: Downloaded newer image for hello-world:latest
 ---> 4ab4c602aa5e
[Warning] One or more build-args [base_img] were not consumed
Successfully built 4ab4c602aa5e
Successfully tagged slarkin/spark-r:v2.4.0-rc4