README.md

Put the files to the root of the hadoop directory, and after a -Pdist build just do a docker-compose up -d

For scaling up: docker-compose scale datanode=3

docker-compose.yaml

version: "3"
services:
   namenode:
      image: flokkr/hadoop-hdfs-namenode:latest
      hostname: namenode
      volumes:
         - ./hadoop-dist/target/hadoop-3.0.0-beta1-SNAPSHOT:/opt/hadoop
      ports:
         - 50070:50070
         - 9870:9870
      environment:
          ENSURE_NAMENODE_DIR: /data/namenode
      env_file:
         - ./docker-config
   datanode:
      image: flokkr/hadoop-hdfs-datanode:latest
      volumes:
        - ./hadoop-dist/target/hadoop-3.0.0-beta1-SNAPSHOT:/opt/hadoop
   ksm:
      image: flokkr/hadoop:latest
      volumes:
         - ./hadoop-dist/target/hadoop-3.0.0-beta1-SNAPSHOT:/opt/hadoop
      ports:
         - 9874
      env_file:
          - ./docker-config
      command: ["/opt/hadoop/bin/hdfs","ksm"]
   scm:
      image: flokkr/hadoop:latest
      volumes:
         - ./hadoop-dist/target/hadoop-3.0.0-beta1-SNAPSHOT:/opt/hadoop
      ports:
         - 9876
      env_file:
          - ./docker-config
      command: ["/opt/hadoop/bin/hdfs","scm"]

docker-config

CORE-SITE.XML_fs.defaultFS: "hdfs://namenode:9000"
OZONE-SITE.XML_ozone.ksm.address=localhost
OZONE-SITE.XML_ozone.scm.names=localhost
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id=/tmp/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=localhost
OZONE-SITE.XML_ozone.container.metadata.dirs=/tmp/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=localhost
HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
HDFS-SITE.XML_dfs.namenode.http-bind-host=0.0.0.0
MAPRED-SITE.XML_mapreduce.framework.name=yarn
YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager
YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600
YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n