-
-
Save codope/3dd986de5e54f0650dd74b6032e4456c to your computer and use it in GitHub Desktop.
Hudi Docker ARM64
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/docker/compose/docker-compose_hadoop284_hive233_spark244.yml b/docker/compose/docker-compose_hadoop284_hive233_spark244.yml | |
| index 05790963b..31e3b6dac 100644 | |
| --- a/docker/compose/docker-compose_hadoop284_hive233_spark244.yml | |
| +++ b/docker/compose/docker-compose_hadoop284_hive233_spark244.yml | |
| @@ -19,6 +19,7 @@ services: | |
| namenode: | |
| image: apachehudi/hudi-hadoop_2.8.4-namenode:latest | |
| + platform: linux/arm64 | |
| hostname: namenode | |
| container_name: namenode | |
| environment: | |
| @@ -36,6 +37,7 @@ services: | |
| datanode1: | |
| image: apachehudi/hudi-hadoop_2.8.4-datanode:latest | |
| + platform: linux/arm64 | |
| container_name: datanode1 | |
| hostname: datanode1 | |
| environment: | |
| @@ -58,6 +60,7 @@ services: | |
| historyserver: | |
| image: apachehudi/hudi-hadoop_2.8.4-history:latest | |
| + platform: linux/arm64 | |
| hostname: historyserver | |
| container_name: historyserver | |
| environment: | |
| @@ -79,7 +82,8 @@ services: | |
| - historyserver:/hadoop/yarn/timeline | |
| hive-metastore-postgresql: | |
| - image: bde2020/hive-metastore-postgresql:2.3.0 | |
| + image: menorah84/hive-metastore-postgresql:2.3.0 | |
| + platform: linux/arm64 | |
| + environment: | |
| + - POSTGRES_HOST_AUTH_METHOD=trust | |
| volumes: | |
| - hive-metastore-postgresql:/var/lib/postgresql | |
| hostname: hive-metastore-postgresql | |
| @@ -87,6 +91,7 @@ services: | |
| hivemetastore: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:latest | |
| + platform: linux/arm64 | |
| hostname: hivemetastore | |
| container_name: hivemetastore | |
| links: | |
| @@ -110,6 +115,7 @@ services: | |
| hiveserver: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:latest | |
| + platform: linux/arm64 | |
| hostname: hiveserver | |
| container_name: hiveserver | |
| env_file: | |
| @@ -129,6 +135,7 @@ services: | |
| sparkmaster: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_2.4.4:latest | |
| + platform: linux/arm64 | |
| hostname: sparkmaster | |
| container_name: sparkmaster | |
| env_file: | |
| @@ -146,6 +153,7 @@ services: | |
| spark-worker-1: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_2.4.4:latest | |
| + platform: linux/arm64 | |
| hostname: spark-worker-1 | |
| container_name: spark-worker-1 | |
| env_file: | |
| @@ -163,7 +171,8 @@ services: | |
| - "namenode" | |
| zookeeper: | |
| - image: 'bitnami/zookeeper:3.4.12-r68' | |
| + image: 'arm64v8/zookeeper:3.4.12' | |
| + platform: linux/arm64 | |
| hostname: zookeeper | |
| container_name: zookeeper | |
| ports: | |
| @@ -172,7 +181,8 @@ services: | |
| - ALLOW_ANONYMOUS_LOGIN=yes | |
| kafka: | |
| - image: 'bitnami/kafka:2.0.0' | |
| + image: 'wurstmeister/kafka:2.12-2.0.1' | |
| + platform: linux/arm64 | |
| hostname: kafkabroker | |
| container_name: kafkabroker | |
| ports: | |
| @@ -185,6 +195,7 @@ services: | |
| container_name: presto-coordinator-1 | |
| hostname: presto-coordinator-1 | |
| image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.217:latest | |
| + platform: linux/arm64 | |
| ports: | |
| - '8090:8090' | |
| environment: | |
| @@ -204,6 +215,7 @@ services: | |
| container_name: presto-worker-1 | |
| hostname: presto-worker-1 | |
| image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.217:latest | |
| + platform: linux/arm64 | |
| depends_on: ["presto-coordinator-1"] | |
| environment: | |
| - PRESTO_JVM_MAX_HEAP=512M | |
| @@ -221,17 +233,18 @@ services: | |
| - ${HUDI_WS}:/var/hoodie/ws | |
| command: worker | |
| - graphite: | |
| - container_name: graphite | |
| - hostname: graphite | |
| - image: graphiteapp/graphite-statsd | |
| - ports: | |
| - - 80:80 | |
| - - 2003-2004:2003-2004 | |
| - - 8126:8126 | |
| +# graphite: | |
| +# container_name: graphite | |
| +# hostname: graphite | |
| +# image: graphiteapp/graphite-statsd | |
| +# ports: | |
| +# - 80:80 | |
| +# - 2003-2004:2003-2004 | |
| +# - 8126:8126 | |
| adhoc-1: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.4.4:latest | |
| + platform: linux/arm64 | |
| hostname: adhoc-1 | |
| container_name: adhoc-1 | |
| env_file: | |
| @@ -253,6 +266,7 @@ services: | |
| adhoc-2: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.4.4:latest | |
| + platform: linux/arm64 | |
| hostname: adhoc-2 | |
| container_name: adhoc-2 | |
| env_file: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/docker/compose/docker-compose_hadoop284_hive233_spark244.yml b/docker/compose/docker-compose_hadoop284_hive233_spark244.yml | |
| index 05790963b..31e3b6dac 100644 | |
| --- a/docker/compose/docker-compose_hadoop284_hive233_spark244.yml | |
| +++ b/docker/compose/docker-compose_hadoop284_hive233_spark244.yml | |
| @@ -19,6 +19,7 @@ services: | |
| namenode: | |
| image: apachehudi/hudi-hadoop_2.8.4-namenode:latest | |
| + platform: linux/arm64 | |
| hostname: namenode | |
| container_name: namenode | |
| environment: | |
| @@ -36,6 +37,7 @@ services: | |
| datanode1: | |
| image: apachehudi/hudi-hadoop_2.8.4-datanode:latest | |
| + platform: linux/arm64 | |
| container_name: datanode1 | |
| hostname: datanode1 | |
| environment: | |
| @@ -58,6 +60,7 @@ services: | |
| historyserver: | |
| image: apachehudi/hudi-hadoop_2.8.4-history:latest | |
| + platform: linux/arm64 | |
| hostname: historyserver | |
| container_name: historyserver | |
| environment: | |
| @@ -79,7 +82,8 @@ services: | |
| - historyserver:/hadoop/yarn/timeline | |
| hive-metastore-postgresql: | |
| - image: bde2020/hive-metastore-postgresql:2.3.0 | |
| + image: menorah84/hive-metastore-postgresql:2.3.0 | |
| + platform: linux/arm64 | |
| volumes: | |
| - hive-metastore-postgresql:/var/lib/postgresql | |
| hostname: hive-metastore-postgresql | |
| @@ -87,6 +91,7 @@ services: | |
| hivemetastore: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:latest | |
| + platform: linux/arm64 | |
| hostname: hivemetastore | |
| container_name: hivemetastore | |
| links: | |
| @@ -110,6 +115,7 @@ services: | |
| hiveserver: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:latest | |
| + platform: linux/arm64 | |
| hostname: hiveserver | |
| container_name: hiveserver | |
| env_file: | |
| @@ -129,6 +135,7 @@ services: | |
| sparkmaster: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_2.4.4:latest | |
| + platform: linux/arm64 | |
| hostname: sparkmaster | |
| container_name: sparkmaster | |
| env_file: | |
| @@ -146,6 +153,7 @@ services: | |
| spark-worker-1: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_2.4.4:latest | |
| + platform: linux/arm64 | |
| hostname: spark-worker-1 | |
| container_name: spark-worker-1 | |
| env_file: | |
| @@ -163,7 +171,8 @@ services: | |
| - "namenode" | |
| zookeeper: | |
| - image: 'bitnami/zookeeper:3.4.12-r68' | |
| + image: 'arm64v8/zookeeper:3.4.12' | |
| + platform: linux/arm64 | |
| hostname: zookeeper | |
| container_name: zookeeper | |
| ports: | |
| @@ -172,7 +181,8 @@ services: | |
| - ALLOW_ANONYMOUS_LOGIN=yes | |
| kafka: | |
| - image: 'bitnami/kafka:2.0.0' | |
| + image: 'wurstmeister/kafka:2.12-2.0.1' | |
| + platform: linux/arm64 | |
| hostname: kafkabroker | |
| container_name: kafkabroker | |
| ports: | |
| @@ -185,6 +195,7 @@ services: | |
| container_name: presto-coordinator-1 | |
| hostname: presto-coordinator-1 | |
| image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.217:latest | |
| + platform: linux/arm64 | |
| ports: | |
| - '8090:8090' | |
| environment: | |
| @@ -204,6 +215,7 @@ services: | |
| container_name: presto-worker-1 | |
| hostname: presto-worker-1 | |
| image: apachehudi/hudi-hadoop_2.8.4-prestobase_0.217:latest | |
| + platform: linux/arm64 | |
| depends_on: ["presto-coordinator-1"] | |
| environment: | |
| - PRESTO_JVM_MAX_HEAP=512M | |
| @@ -221,17 +233,18 @@ services: | |
| - ${HUDI_WS}:/var/hoodie/ws | |
| command: worker | |
| - graphite: | |
| - container_name: graphite | |
| - hostname: graphite | |
| - image: graphiteapp/graphite-statsd | |
| - ports: | |
| - - 80:80 | |
| - - 2003-2004:2003-2004 | |
| - - 8126:8126 | |
| +# graphite: | |
| +# container_name: graphite | |
| +# hostname: graphite | |
| +# image: graphiteapp/graphite-statsd | |
| +# ports: | |
| +# - 80:80 | |
| +# - 2003-2004:2003-2004 | |
| +# - 8126:8126 | |
| adhoc-1: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.4.4:latest | |
| + platform: linux/arm64 | |
| hostname: adhoc-1 | |
| container_name: adhoc-1 | |
| env_file: | |
| @@ -253,6 +266,7 @@ services: | |
| adhoc-2: | |
| image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.4.4:latest | |
| + platform: linux/arm64 | |
| hostname: adhoc-2 | |
| container_name: adhoc-2 | |
| env_file: | |
| diff --git a/docker/hoodie/hadoop/base/Dockerfile b/docker/hoodie/hadoop/base/Dockerfile | |
| index 2c98ce624..280c38146 100644 | |
| --- a/docker/hoodie/hadoop/base/Dockerfile | |
| +++ b/docker/hoodie/hadoop/base/Dockerfile | |
| @@ -15,7 +15,7 @@ | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| -FROM openjdk:8u212-jdk-slim-stretch | |
| +FROM arm64v8/openjdk:8u212-jdk-slim-stretch | |
| MAINTAINER Hoodie | |
| USER root | |
| diff --git a/docker/hoodie/hadoop/datanode/Dockerfile b/docker/hoodie/hadoop/datanode/Dockerfile | |
| index 79dd798f7..8afa2cf2c 100644 | |
| --- a/docker/hoodie/hadoop/datanode/Dockerfile | |
| +++ b/docker/hoodie/hadoop/datanode/Dockerfile | |
| @@ -17,7 +17,7 @@ | |
| ARG HADOOP_VERSION=2.8.4 | |
| ARG HADOOP_DN_PORT=50075 | |
| -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest | |
| +FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest | |
| ENV HADOOP_DN_PORT ${HADOOP_DN_PORT} | |
| diff --git a/docker/hoodie/hadoop/historyserver/Dockerfile b/docker/hoodie/hadoop/historyserver/Dockerfile | |
| index e08adbb05..91fa552af 100644 | |
| --- a/docker/hoodie/hadoop/historyserver/Dockerfile | |
| +++ b/docker/hoodie/hadoop/historyserver/Dockerfile | |
| @@ -17,7 +17,7 @@ | |
| ARG HADOOP_VERSION=2.8.4 | |
| ARG HADOOP_HISTORY_PORT=8188 | |
| -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest | |
| +FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest | |
| ENV HADOOP_HISTORY_PORT ${HADOOP_HISTORY_PORT} | |
| diff --git a/docker/hoodie/hadoop/hive_base/Dockerfile b/docker/hoodie/hadoop/hive_base/Dockerfile | |
| index 8d85fd5b5..01ff0cf11 100644 | |
| --- a/docker/hoodie/hadoop/hive_base/Dockerfile | |
| +++ b/docker/hoodie/hadoop/hive_base/Dockerfile | |
| @@ -16,7 +16,7 @@ | |
| # limitations under the License. | |
| ARG HADOOP_VERSION=2.8.4 | |
| -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest | |
| +FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest | |
| ENV HIVE_HOME /opt/hive | |
| ENV PATH $HIVE_HOME/bin:$PATH | |
| @@ -33,7 +33,7 @@ ENV HIVE_URL ${HIVE_URL} | |
| RUN echo "Hive URL is :${HIVE_URL}" && wget ${HIVE_URL} -O hive.tar.gz && \ | |
| tar -xzvf hive.tar.gz && mv *hive*-bin hive && \ | |
| ln -s /usr/share/java/mysql-connector-java.jar $HIVE_HOME/lib/mysql-connector-java.jar && \ | |
| - wget https://jdbc.postgresql.org/download/postgresql-9.4.1212.jar -O $HIVE_HOME/lib/postgresql-jdbc.jar && \ | |
| + wget --no-check-certificate https://jdbc.postgresql.org/download/postgresql-9.4.1212.jar -O $HIVE_HOME/lib/postgresql-jdbc.jar && \ | |
| rm hive.tar.gz && mkdir -p /var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/ | |
| #Spark should be compiled with Hive to be able to use it | |
| diff --git a/docker/hoodie/hadoop/namenode/Dockerfile b/docker/hoodie/hadoop/namenode/Dockerfile | |
| index d89c30eff..757e895ac 100644 | |
| --- a/docker/hoodie/hadoop/namenode/Dockerfile | |
| +++ b/docker/hoodie/hadoop/namenode/Dockerfile | |
| @@ -17,7 +17,7 @@ | |
| ARG HADOOP_VERSION=2.8.4 | |
| ARG HADOOP_WEBHDFS_PORT=50070 | |
| -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest | |
| +FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest | |
| ENV HADOOP_WEBHDFS_PORT ${HADOOP_WEBHDFS_PORT} | |
| diff --git a/docker/hoodie/hadoop/prestobase/Dockerfile b/docker/hoodie/hadoop/prestobase/Dockerfile | |
| index 43b989e6b..3a22e8c88 100644 | |
| --- a/docker/hoodie/hadoop/prestobase/Dockerfile | |
| +++ b/docker/hoodie/hadoop/prestobase/Dockerfile | |
| @@ -20,7 +20,7 @@ | |
| ARG HADOOP_VERSION=2.8.4 | |
| ARG HIVE_VERSION=2.3.3 | |
| -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest as hadoop-base | |
| +FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest as hadoop-base | |
| ARG PRESTO_VERSION=0.217 | |
| diff --git a/docker/hoodie/hadoop/spark_base/Dockerfile b/docker/hoodie/hadoop/spark_base/Dockerfile | |
| index 7eeab093a..91c53e8d8 100644 | |
| --- a/docker/hoodie/hadoop/spark_base/Dockerfile | |
| +++ b/docker/hoodie/hadoop/spark_base/Dockerfile | |
| @@ -17,7 +17,7 @@ | |
| ARG HADOOP_VERSION=2.8.4 | |
| ARG HIVE_VERSION=2.3.3 | |
| -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION} | |
| +FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION} | |
| ENV ENABLE_INIT_DAEMON true | |
| ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon | |
| diff --git a/docker/hoodie/hadoop/sparkadhoc/Dockerfile b/docker/hoodie/hadoop/sparkadhoc/Dockerfile | |
| index a114cf0fb..b387a30b4 100644 | |
| --- a/docker/hoodie/hadoop/sparkadhoc/Dockerfile | |
| +++ b/docker/hoodie/hadoop/sparkadhoc/Dockerfile | |
| @@ -18,7 +18,7 @@ | |
| ARG HADOOP_VERSION=2.8.4 | |
| ARG HIVE_VERSION=2.3.3 | |
| ARG SPARK_VERSION=2.4.4 | |
| -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} | |
| +FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} | |
| ARG PRESTO_VERSION=0.217 | |
| COPY adhoc.sh /opt/spark | |
| diff --git a/docker/hoodie/hadoop/sparkmaster/Dockerfile b/docker/hoodie/hadoop/sparkmaster/Dockerfile | |
| index aaeb03f39..3e5e7eddf 100644 | |
| --- a/docker/hoodie/hadoop/sparkmaster/Dockerfile | |
| +++ b/docker/hoodie/hadoop/sparkmaster/Dockerfile | |
| @@ -18,7 +18,7 @@ | |
| ARG HADOOP_VERSION=2.8.4 | |
| ARG HIVE_VERSION=2.3.3 | |
| ARG SPARK_VERSION=2.4.4 | |
| -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} | |
| +FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} | |
| COPY master.sh /opt/spark | |
| diff --git a/docker/hoodie/hadoop/sparkworker/Dockerfile b/docker/hoodie/hadoop/sparkworker/Dockerfile | |
| index ba867f2d3..81f2013c5 100644 | |
| --- a/docker/hoodie/hadoop/sparkworker/Dockerfile | |
| +++ b/docker/hoodie/hadoop/sparkworker/Dockerfile | |
| @@ -18,7 +18,7 @@ | |
| ARG HADOOP_VERSION=2.8.4 | |
| ARG HIVE_VERSION=2.3.3 | |
| ARG SPARK_VERSION=2.4.4 | |
| -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} | |
| +FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION} | |
| COPY worker.sh /opt/spark | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment