Hive 2.3.3 with Hadoop 3.1.0 on Mac OS

.bash_profile

export HADOOP_HOME=/Users/username/Tools/hadoop-3.1.0
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
 
alias hstart=$HADOOP_HOME/sbin/start-all.sh
alias hstop=$HADOOP_HOME/sbin/stop-all.sh
 
export HIVE_HOME=/Users/username/Tools/apache-hive-2.3.3-bin
export PATH=$PATH:$HIVE_HOME/bin

Readme.md

Format the name Node

/Users/username/Tools/hadoop-3.1.0/bin/hadoop namenode -format

Use hstart to start all services. Use hstop to start all services.

Init the Hive metastore.

/Users/username/Tools/apache-hive-2.3.3-bin/bin/schematool -initSchema -dbType mysql

Create HDFS Directories

hdfs dfs -mkdir -p /user/hive/warehouse

hdfs dfs -chmod g+w /user/hive/warehouse

hdfs dfs -mkdir -p /tmp

hdfs dfs -chmod g+w /tmp

core-site.xml

<!-- /Users/username/Tools/hadoop-3.1.0/etc/hadoop/core-site.xml -->

<configuration>
	<property>
    <name>fs.defaultFS</name>
    <value>hdfs://localhost:9000</value>
  </property>
</configuration>

download.sh

# Download Hadoop 3.1.0
curl -O http://www-eu.apache.org/dist/hadoop/common/hadoop-3.1.0/hadoop-3.1.0.tar.gz
tar -xzf hadoop-3.1.0.tar.gz

# Create data node and name node directories.
mkdir -p hadoop-3.1.0/data-node hadoop-3.1.0/name-node

# Download Apache Hive 2.3.3
curl -O http://www-eu.apache.org/dist/hive/hive-2.3.3/apache-hive-2.3.3-bin.tar.gz
tar -xzf apache-hive-2.3.3-bin.tar.gz

# Download MySQL driver.
curl -O https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-8.0.11.tar.gz
tar -xzf mysql-connector-java-8.0.11.tar.gz

cp mysql-connector-java-8.0.11/mysql-connector-java-8.0.11.jar /Users/username/Tools/apache-hive-2.3.3-bin/lib

# Create hive-site.xml
cd apache-hive-2.3.3-bin/conf
cp hive-default.xml.template hive-site.xml

# Check that you can ssh to the localhost without a passphrase:
ssh localhost

# If you cannot ssh to localhost without a passphrase, execute the following commands:
ssh-keygen -t dsa -P “” -f ~/.ssh/id_dsa
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

# Try to ssh again to your localhost and if you are still getting a password prompt then.
chmod go-w $HOME $HOME/.ssh
chmod 600 $HOME/.ssh/authorized_keys
chown `whoami` $HOME/.ssh/authorized_keys

hdfs-site.xml

<!-- /Users/username/Tools/hadoop-3.1.0/etc/hadoop/hdfs-site.xml -->

<configuration>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>/Users/username/Tools/hadoop-3.1.0/name-node</value>
  </property>
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>/Users/username/Tools/hadoop-3.1.0/data-node</value>
  </property>
</configuration>

hive-site.xml

<!-- /Users/username/Tools/apache-hive-2.3.3-bin/conf/hive-site.xml -->

<property>
  <name>javax.jdo.option.ConnectionURL</name>
  <value>jdbc:mysql://localhost/metastore?useSSL=false</value>
</property>
<property>
  <name>javax.jdo.option.ConnectionDriverName</name>
  <value>com.mysql.cj.jdbc.Driver</value>
</property>
<property>
  <name>javax.jdo.option.ConnectionUserName</name>
  <value>hiveuser</value>
</property>
<property>
  <name>javax.jdo.option.ConnectionPassword</name>
  <value>hivepassword</value>
</property>
<property>
    <name>hive.exec.local.scratchdir</name>
    <value>/tmp/hive</value>
    <description>Local scratch space for Hive jobs</description>
</property>
<property>
    <name>hive.downloaded.resources.dir</name>
    <value>/tmp/hive</value>
    <description>Temporary local directory for added resources in the remote file system.</description>
</property>
<property>
    <name>hive.querylog.location</name>
    <value>/tmp/hive</value>
    <description>Location of Hive run time structured log file</description>
</property>
<property>
    <name>hive.metastore.schema.verification</name>
    <value>false</value>
    <description>
      Enforce metastore schema version consistency.
      True: Verify that version information stored in is compatible with one from Hive jars.  Also disable automatic
            schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures
            proper metastore schema migration. (Default)
      False: Warn if the version information stored in metastore doesn't match with one from in Hive jars.
    </description>
  </property>

mapred-site.xml

<!-- /Users/username/Tools/hadoop-3.1.0/etc/hadoop/mapred-site.xml -->

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.admin.user.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_COMMON_HOME</value>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_COMMON_HOME</value>
    </property>
</configuration>

mysql shell

CREATE DATABASE metastore;
CREATE USER 'hiveuser'@'%' IDENTIFIED BY 'hivepassword'; 
GRANT ALL ON metastore.* TO 'hiveuser'@localhost IDENTIFIED BY 'hivepassword';
FLUSH PRIVILEGES;

yarn-site.xml

<!-- /Users/username/Tools/hadoop-3.1.0/etc/hadoop/yarn-site.xml -->

<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
</configuration>