Skip to content

Instantly share code, notes, and snippets.

View jeongho's full-sized avatar

Jeongho Park jeongho

27 followers · 91 following
  • Deception Island, Antarctica
View GitHub Profile
@jeongho
jeongho / run_spark_pi_job.sh
Last active May 24, 2017 00:02
Spark pi job
export SPARK_CLASSPATH=$(hadoop classpath)
export SPARK_HOME=/opt/cloudera/parcels/CDH/lib/spark
spark-submit --class org.apache.spark.examples.SparkPi \
--master yarn \
--deploy-mode client \
$SPARK_HOME/lib/spark-examples.jar 10
spark-submit --class org.apache.spark.examples.SparkPi \
--master yarn \
@jeongho
jeongho / erlang_install.txt
Last active February 10, 2017 06:15
erlang install with erlang-solutions distro
#https://www.erlang-solutions.com/resources/download.html
#READ Installation using repository
#http://elixir-lang.org/install.html#raspberry-pi
# verify installation
which erl
/usr/local/bin/erl
# check version
erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell
@jeongho
jeongho / cm_config.sh
Last active October 1, 2016 03:02
Export and Import CM config
#!/usr/bin/env bash
# https://cloudera.github.io/cm_api/docs/quick-start/
# https://www.cloudera.com/documentation/enterprise/latest/topics/cm_intro_api.html
set -o pipefail
set +o errexit
set -x
username=${username:-admin}
@jeongho
jeongho / cm_migration.txt
Created September 13, 2016 18:18
CM Server Migration/Reinstallation
https://www.evernote.com/shard/s31/sh/039cec4d-694c-4a66-b2cb-f5ba6976974b/d31566b087605f32361007b3c6555fb4
@jeongho
jeongho / raid1_lvm.txt
Last active September 2, 2016 18:36
RAID1 + LVM install
--- raid 1
[root@localhost ~]# yum install -y mdadm
[root@localhost ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
sda 8:0 0 20G 0 disk
├─sda1 8:1 0 500M 0 part /boot
└─sda2 8:2 0 19.5G 0 part
├─VolGroup-lv_root (dm-0) 253:0 0 17.6G 0 lvm /
@jeongho
jeongho / pin_centos6.7.txt
Created August 3, 2016 16:05
Pin CentOS repository to 6.7 to prevent yum update goes to 6.8
1. disable Base repo
sed -i.bak '/^gpgcheck=1/ a enabled=0 ' /etc/yum.repos.d/CentOS-Base.repo
2. append Vault repo with CentOS 6.7
#-----------------
[C6.7-base]
name=CentOS-6.7 - Base
baseurl=http://vault.centos.org/6.7/os/$basearch/
gpgcheck=1
@jeongho
jeongho / cgroup_config.txt
Last active August 3, 2016 17:36
cgroup configuration
https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide
sudo yum install libcgroup
sudo service cgconfig start
sudo chkconfig cgconfig on
lscgroup
cat /etc/cgconfig.d/antarcticatec-server
group antarcticatec-server {
cpu {
@jeongho
jeongho / hdfs_tmp_cleanup.sh
Last active November 2, 2021 22:55
hdfs tmp folder cleanup
#!/usr/bin/env bash
#remove files older than X days:
#based off the hadoop fs -ls
#days=5; for f in $(cutoff=$(echo $(date +%s)"-$days*24*60*60" | bc); hadoop fs -ls -R /tmp 2>/dev/null|grep ^- |awk '{ print "echo $(date -d \""$6,$7"\" +%s)" , $8}'| bash | awk -v cutoff=$cutoff '$1 < cutoff'| sort -n | cut -f2 -d" "|grep ^$d); do hadoop fs -rm $f; done
#remove files older than X days:
days=5;
for f in $(cutoff=$(echo $(date +%s)"-$days*24*60*60" | bc);
hadoop fs -ls -R /tmp 2>/dev/null | grep ^- | \
@jeongho
jeongho / hadoop_client_setup.sh
Created February 25, 2016 17:43
Using the CLI to access the cluster from your own host
#!/usr/bin/env bash
#Using the CLI to access the cluster from your own host
#Step 1. Setup your Hadoop config
#Cloudera Manager UI, Services>All Services>Client Configuration URLs
#Step 2. Download CDH4 and setup your environment
#1. Point your browser at CDH Tarballs
#2. Click on CDH4 tarballs and download hadoop-2-x
#3. Update your environments (~/.bash_profile is a good bet)
@jeongho
jeongho / check_public_ip.sh
Created February 25, 2016 17:40
check public ip
#!/usr/bin/env bash
curl -s checkip.dyndns.org|sed -e 's/.*Current IP Address: //' -e 's/<.*$//'
#wget -qO- http://ipecho.net/plain ; echo