Skip to content

Instantly share code, notes, and snippets.

@y-lan

y-lan/_use_hive_0_13_in_cdh4.md

Last active August 29, 2015 13:58
Show Gist options
  • Save y-lan/9968806 to your computer and use it in GitHub Desktop.
Save y-lan/9968806 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
_use_hive_0_13_in_cdh4.md

Hive v0.13 use protobuf 2.5.0 while CDH4.* depends on protobuf v2.4.0a, so it requires a little hack to make Hive v0.13 working under CDH4.*.

1) prepare

# install protobuf compiler 2.4.* : https://code.google.com/p/protobuf/
$protoc --version
libprotoc 2.4.1

2) build Hive v0.13 using protobuf v2.4.0a

$git clone https://github.com/apache/hive.git -b branch-0.13 apache-hive-0.13
cd apache-hive-0.13

## change protobuf dependency to v2.4.0a ( as same as CDH4.5
sed -i 's/<protobuf.version>2.5.0<\/protobuf.version>/<protobuf.version>2.4.0a<\/protobuf.version>/g' pom.xml

## re-compile hbase storage handler
protoc --java_out=hcatalog/storage-handlers/hbase/src/gen-java -I=hcatalog/storage-handlers/hbase/src/protobuf/org/apache/hcatalog/hbase/snapshot hcatalog/storage-handlers/hbase/src/protobuf/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.proto

## edit pom files for hadoop versions
## or patch as I did
## patch -p1  < hadoop_version_hack_for_cdh4_mr1.patch

## let maven handle the others
export MAVEN_OPTS="-Xmx2048m"
mvn clean package -DskipTests -Phadoop-2,protobuf -Pdist -e

3) deploy the new tarball

# packaging/target/apache-hive-0.13.0-bin.tar.gz

4) upgrade metastore

# backup
mysqldump --default-character-set=utf8 -h${host} -P${port} -u ${user} -p ${meta_db} > meta_hive_backup.sql

# execute upgrade script
cd scripts/metastore/upgrade/mysql
mysql -h${host} -P${port} -u ${user} -p < upgrade-0.12.0-to-0.13.0.mysql.sql

5) execute

Raw
hadoop_version_hack_for_cdh4_mr1.patch
From 21375ae6d743f1ba24010830a9b1a5b5ef7be6db Mon Sep 17 00:00:00 2001
From: lan <lan@localhost>
Date: Mon, 7 Apr 2014 16:01:21 +0900
Subject: [PATCH] shims hack for use mr1-cdh4.5.0
---
cli/pom.xml | 2 +-
hbase-handler/pom.xml | 9 ++++++-
hcatalog/storage-handlers/hbase/pom.xml | 8 ++++-
pom.xml | 25 ++++++++++++++-----
ql/pom.xml | 6 ++--
shims/0.20S/pom.xml | 18 ++++++++++++++
.../apache/hadoop/hive/shims/Hadoop20SShims.java | 14 ++++++----
shims/0.23/pom.xml | 20 ++++++++--------
shims/common-secure/pom.xml | 6 ++++
.../org/apache/hadoop/hive/shims/ShimLoader.java | 1 +
10 files changed, 79 insertions(+), 30 deletions(-)
diff --git a/cli/pom.xml b/cli/pom.xml
index 5a6d121..effd509 100644
--- a/cli/pom.xml
+++ b/cli/pom.xml
@@ -146,7 +146,7 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
+ <artifactId>hbase</artifactId>
<version>${hbase.hadoop2.version}</version>
<scope>test</scope>
</dependency>
diff --git a/hbase-handler/pom.xml b/hbase-handler/pom.xml
index 1112eeb..0f471af 100644
--- a/hbase-handler/pom.xml
+++ b/hbase-handler/pom.xml
@@ -165,10 +165,16 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop2-compat</artifactId>
+ <artifactId>hbase</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
<version>${hbase.hadoop2.version}</version>
<classifier>tests</classifier>
</dependency>
+ <!--
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop2-compat</artifactId>
@@ -215,6 +221,7 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ -->
</dependencies>
</profile>
</profiles>
diff --git a/hcatalog/storage-handlers/hbase/pom.xml b/hcatalog/storage-handlers/hbase/pom.xml
index 8972a34..b3fe214 100644
--- a/hcatalog/storage-handlers/hbase/pom.xml
+++ b/hcatalog/storage-handlers/hbase/pom.xml
@@ -174,9 +174,10 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
+ <artifactId>hbase</artifactId>
<version>${hbase.hadoop2.version}</version>
</dependency>
+ <!--
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
@@ -197,6 +198,7 @@
<artifactId>hbase-hadoop2-compat</artifactId>
<version>${hbase.hadoop2.version}</version>
</dependency>
+ -->
<!-- test -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -234,11 +236,12 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
+ <artifactId>hbase</artifactId>
<version>${hbase.hadoop2.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <!--
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop-compat</artifactId>
@@ -260,6 +263,7 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ -->
</dependencies>
</profile>
</profiles>
diff --git a/pom.xml b/pom.xml
index fdcc47f..80618c4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,10 +101,11 @@
<guava.version>11.0.2</guava.version>
<groovy.version>2.1.6</groovy.version>
<hadoop-20.version>0.20.2</hadoop-20.version>
- <hadoop-20S.version>1.2.1</hadoop-20S.version>
- <hadoop-23.version>2.3.0</hadoop-23.version>
+ <hadoop-20S.version>2.0.0-mr1-cdh4.5.0</hadoop-20S.version>
+ <hadoop-23.version>2.0.0-cdh4.5.0</hadoop-23.version>
+ <hadoop-23o.version>2.3.0</hadoop-23o.version>
<hbase.hadoop1.version>0.96.0-hadoop1</hbase.hadoop1.version>
- <hbase.hadoop2.version>0.96.0-hadoop2</hbase.hadoop2.version>
+ <hbase.hadoop2.version>0.94.6-cdh4.5.0</hbase.hadoop2.version>
<!-- httpcomponents are not always in version sync -->
<httpcomponents.client.version>4.2.5</httpcomponents.client.version>
<httpcomponents.core.version>4.2.5</httpcomponents.core.version>
@@ -134,7 +135,7 @@
<netty.version>3.4.0.Final</netty.version>
<parquet.version>1.3.2</parquet.version>
<pig.version>0.12.0</pig.version>
- <protobuf.version>2.5.0</protobuf.version>
+ <protobuf.version>2.4.0a</protobuf.version>
<stax.version>1.0.1</stax.version>
<slf4j.version>1.7.5</slf4j.version>
<ST4.version>4.0.4</ST4.version>
@@ -150,6 +151,14 @@
<repositories>
<repository>
+ <id>cdh.releases.repo</id>
+ <url>https://repository.cloudera.com/content/groups/cdh-releases-rcs</url>
+ <name>CDH Releases Repository</name>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ <repository>
<id>datanucleus</id>
<name>datanucleus maven repository</name>
<url>http://www.datanucleus.org/downloads/maven2</url>
@@ -548,8 +557,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven.compiler.plugin.version}</version>
<configuration>
- <source>1.6</source>
- <target>1.6</target>
+ <source>1.7</source>
+ <target>1.7</target>
</configuration>
</plugin>
<plugin>
@@ -963,9 +972,10 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
+ <artifactId>hbase</artifactId>
<version>${hbase.hadoop2.version}</version>
</dependency>
+ <!--
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop-compat</artifactId>
@@ -981,6 +991,7 @@
<artifactId>hbase-server</artifactId>
<version>${hbase.hadoop2.version}</version>
</dependency>
+ -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
diff --git a/ql/pom.xml b/ql/pom.xml
index b9ed8ab..cf96288 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -359,19 +359,19 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<!-- Test dependencies -->
diff --git a/shims/0.20S/pom.xml b/shims/0.20S/pom.xml
index 58ed136..47bf761 100644
--- a/shims/0.20S/pom.xml
+++ b/shims/0.20S/pom.xml
@@ -43,12 +43,30 @@
<!-- inter-project -->
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop-20S.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>${hadoop-20S.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-test</artifactId>
<version>${hadoop-20S.version}</version>
<optional>true</optional>
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index b1d7cb0..7113cc8 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -57,6 +57,8 @@
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.mapred.lib.TotalOrderPartitioner;
@@ -82,20 +84,20 @@ public String getTaskAttemptLogUrl(JobConf conf,
@Override
public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception {
JobTrackerState state;
- switch (clusterStatus.getJobTrackerState()) {
+ switch (clusterStatus.getJobTrackerStatus()) {
case INITIALIZING:
return JobTrackerState.INITIALIZING;
case RUNNING:
return JobTrackerState.RUNNING;
default:
- String errorMsg = "Unrecognized JobTracker state: " + clusterStatus.getJobTrackerState();
+ String errorMsg = "Unrecognized JobTracker state: " + clusterStatus.getJobTrackerStatus();
throw new Exception(errorMsg);
}
}
@Override
public org.apache.hadoop.mapreduce.TaskAttemptContext newTaskAttemptContext(Configuration conf, final Progressable progressable) {
- return new org.apache.hadoop.mapreduce.TaskAttemptContext(conf, new TaskAttemptID()) {
+ return new TaskAttemptContextImpl(conf, new TaskAttemptID()) {
@Override
public void progress() {
progressable.progress();
@@ -110,7 +112,7 @@ public TaskAttemptID newTaskAttemptID(JobID jobId, boolean isMap, int taskId, in
@Override
public org.apache.hadoop.mapreduce.JobContext newJobContext(Job job) {
- return new org.apache.hadoop.mapreduce.JobContext(job.getConfiguration(), job.getJobID());
+ return new JobContextImpl(job.getConfiguration(), job.getJobID());
}
@Override
@@ -257,7 +259,7 @@ public TaskAttemptID createTaskAttemptID() {
@Override
public TaskAttemptContext createTaskAttemptContext(Configuration conf, TaskAttemptID taskId) {
- return new TaskAttemptContext(conf, taskId);
+ return new TaskAttemptContextImpl(conf, taskId);
}
@Override
@@ -279,7 +281,7 @@ public TaskAttemptContext createTaskAttemptContext(Configuration conf, TaskAttem
@Override
public JobContext createJobContext(Configuration conf,
JobID jobId) {
- return new JobContext(conf, jobId);
+ return new JobContextImpl(conf, jobId);
}
@Override
diff --git a/shims/0.23/pom.xml b/shims/0.23/pom.xml
index 5ad0ee6..ea71c29 100644
--- a/shims/0.23/pom.xml
+++ b/shims/0.23/pom.xml
@@ -54,32 +54,32 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<type>test-jar</type>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<type>test-jar</type>
<optional>true</optional>
</dependency>
@@ -110,25 +110,25 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
</dependency>
<dependency>
@@ -141,7 +141,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
- <version>${hadoop-23.version}</version>
+ <version>${hadoop-23o.version}</version>
<optional>true</optional>
<type>test-jar</type>
</dependency>
diff --git a/shims/common-secure/pom.xml b/shims/common-secure/pom.xml
index 7a622dc..880cae0 100644
--- a/shims/common-secure/pom.xml
+++ b/shims/common-secure/pom.xml
@@ -64,6 +64,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools</artifactId>
<version>${hadoop-20S.version}</version>
<optional>true</optional>
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
index bf9c84f..50c1dbd 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
@@ -111,6 +111,7 @@ public static synchronized HadoopThriftAuthBridge getHadoopThriftAuthBridge() {
private static <T> T loadShims(Map<String, String> classMap, Class<T> xface) {
String vers = getMajorVersion();
+ vers = "0.20S";
String className = classMap.get(vers);
return createShim(className, xface);
}
--
1.7.2.5
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment