Skip to content

Instantly share code, notes, and snippets.

View dbist's full-sized avatar

Artem Ervits dbist

51 followers · 65 following
View GitHub Profile
@dbist
dbist / create_hbase_hive_table.ddl
Created November 9, 2015 21:06
sample HBase-mapped Hive table with timestamp for column family
CREATE EXTERNAL TABLE hbase_json_table(key string, json string, time timestamp)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:json,:timestamp")
TBLPROPERTIES ("hbase.table.name" = "jsontable");
@dbist
dbist / install_dmx.sh
Created July 18, 2017 18:06
DATE=`date +%F-%H%M%S`
export SYNCSORT_HOME=/home/centos
wget https://www.dropbox.com/s/mh5bbb6f7m6l84f/syncsort.zip?dl=0 -O $SYNCSORT_HOME/syncsort.zip
unzip syncsort.zip
cd $SYNCSORT_HOME/syncsort
gzip dmexpress_9-0_en_linux_2-6_x86-64_64bit.tar
tar xvzf dmexpress_9-0_en_linux_2-6_x86-64_64bit.tar.gz
cd $SYNCSORT_HOME/syncsort/dmexpress
./install -s $SYNCSORT_HOME/syncsort/silent-install.txt -slog $SYNCSORT_HOME/install-$DATE.log
@dbist
dbist / vim-highlight-search
Last active July 19, 2017 19:25
@dbist
dbist / mvn-findbugs-report
Last active July 19, 2017 19:40
# generate findbugs report in target of project as per https://gualtierotesta.wordpress.com/2015/06/14/tutorial-using-findbugs-with-maven/
# run findbugs maven goal first
mvn findbugs:findbugs
# then run site plugin goal
mvn clean install site
# once done, find the html report in target dir
@dbist
dbist / Generate passwordless ssh Centos7
Created August 15, 2017 12:40
ssh-keygen -t rsa
<enter>
<enter>
<enter>
cat .ssh/id_rsa.pub > .ssh/authorized_keys
chmod 400 .ssh/authorized_keys
@dbist
dbist / rebase patch
Last active October 2, 2017 21:32
```bash
> cd metron
> git checkout pr-branch
# make sure i have all the master changes
> git fetch apache
# rebase on to master
> git rebase -i apache/master
..... resolve conflicts .....
> git rebase --continue
# verify build
@dbist
dbist / build_oozie_4.3.1.sh
Last active March 28, 2018 14:36
build and deploy oozie
wget http://people.apache.org/\~satishsaley/oozie-4.3.1-rc3/oozie-4.3.1.tar.gz
$ bin/mkdistro.sh -DskipTests -Puber -DtargetJavaVersion=1.8 -DjavaVersion=1.8 -Dhadoop.version=2.7.5
$ cd distro/target/oozie-4.3.1-distro/oozie-4.3.1/
$ ls -lrt
total 871232
drwxr-xr-x 4 asasvari staff 136 Jan 30 11:16 oozie-core
-rw-r--r-- 1 asasvari staff 121306053 Jan 30 11:16 oozie.war
-rw-r--r-- 1 asasvari staff 46437 Jan 30 11:16 oozie-examples.tar.gz
-rw-r--r-- 1 asasvari staff 9140548 Jan 30 11:16 oozie-client-4.3.1.tar.gz
@dbist
dbist / verify_signatures.sh
Last active March 21, 2018 16:59
verify project signatures
# import relevant KEYS from project
curl https://dist.apache.org/repos/dist/release/accumulo/KEYS | gpg --import
# verify signature against downloaded file
gpg --verify hbase-thirdparty-2.1.0-src.tar.gz.asc hbase-thirdparty-2.1.0-src.tar.gz
# verify sha512 checksum on a file
gpg --print-md sha512 hbase-thirdparty-2.1.0-src.tar.gz | diff hbase-thirdparty-2.1.0-src.tar.gz.sha512 -
# verify sha512 checksum with case insensitivity but it should work either way (diff -i)
@dbist
dbist / create_schema_registry_and_sam_mysql_users.sh
Created May 2, 2018 19:02
Create MySQL users for Streaming Analytics Manager and Schema Registry
CREATE DATABASE streamline;
CREATE USER 'streamline'@'%';
GRANT ALL PRIVILEGES ON streamline.* to 'streamline'@'%' WITH GRANT OPTION;
SET PASSWORD FOR 'registry'@'%' = PASSWORD('R12$%34qw');
FLUSH PRIVILEGES;
CREATE DATABASE registry;
CREATE USER 'registry'@'%';
GRANT ALL PRIVILEGES ON registry.* to 'registry'@'%' WITH GRANT OPTION;
SET PASSWORD FOR 'registry'@'%' = PASSWORD('R12$%34qw');
@dbist
dbist / build_oozie_5.x.sh
Created May 23, 2018 17:01
hdfs dfs -mkdir /apps
hdfs dfs -chmod 777 /apps
hdfs dfs -mkdir /tmp
hdfs dfs -chmod -R 777 /tmp
hdfs dfs -mkdir -p /mr-history/tmp
hdfs dfs -mkdir -p /mr-history/done
hdfs dfs -chmod -R 1777 /mr-history
hdfs dfs -mkdir -p /tmp/hadoop-yarn/staging
hdfs dfs -chmod -R 777 /tmp/hadoop-yarn/staging
hdfs dfs -mkdir -p /user/vagrant