Skip to content

Instantly share code, notes, and snippets.

Sho Shimauchi shiumachi

Block or report user

Report or block shiumachi

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
@shiumachi
shiumachi / datagen.py
Created Nov 28, 2018
data generator for Hive / Impala demo
View datagen.py
import argparse
import random
usage = """\
%(prog)s [options]
"""
def init_parser():
View kafka-kudu-demo.py
from kafka import KafkaConsumer
from kafka.client import KafkaClient
import kudu
from kudu.client import Partitioning
import argparse
def init_argumentparser():
parser = argparse.ArgumentParser()
parser.add_argument('--kudu_master_address', default='', type=str, required=True)
parser.add_argument('--kudu_master_port', default='7051', type=str)
View wait_seconds.sh
#!/bin/bash
# utility functions
# wait_seconds N
# sleep N seconds
#
function wait_seconds()
{
func_name="wait_seconds"
if ! expr "$1" : '[0-9]*' > /dev/null ;
View hadoop-logaggr-timeline.sh
#!/bin/bash
aggregate_min=30 # default value
usage()
{
echo "hadoop-logaggr-timeline.sh [-h] [-t N] file" >&2
echo " -t N[min]: must be integer (default:30) " >&2
echo " aggregates logs each N minutes." >&2
echo " -h: help (this message)" >&2
View hadoop-logaggr.sh
#!/bin/bash
usage()
{
echo "hadoop-logaggr.sh [-h] file" >&2
echo " -h: help (this message)" >&2
exit 0
}
TEMP=`getopt h $*`
View mapreduce-test.sh
#!/bin/sh
TMP_DIR=/tmp
HADOOP_BIN_DIR=${HADOOP_MAPRED_HOME}/bin
HADOOP_COMMAND=${HADOOP_BIN_DIR}/hadoop
HADOOP_JAR_COMMAND="${HADOOP_COMMAND} jar"
MAPRED_SHARE_HOME=${HADOOP_MAPRED_HOME}/share/hadoop/mapreduce
HADOOP_EXAMPLE_JAR=${MAPRED_SHARE_HOME}/hadoop*examples*.jar
HADOOP_EXAMPLE_COMMAND="${HADOOP_JAR_COMMAND} ${HADOOP_EXAMPLE_JAR}"
View init-hdfs.sh
#!/bin/sh
source util.sh
TMP_DIR=/tmp
HADOOP_BIN_DIR=${HADOOP_HOME}/bin
HADOOP_COMMAND=${HADOOP_BIN_DIR}/hadoop
HADOOP_START_HDFS_COMMAND=`which start-dfs.sh`
HADOOP_STOP_HDFS_COMMAND=`which stop-dfs.sh`
View hbase-log-checker.sh
#!/bin/bash
usage(){
printf "%s:\n" "usage" 1>&2
printf "%s file \n" `basename $0` 1>&2
printf " %-15s: help (this message)" "-h" >&2
}
TEMP=`getopt h $*`
View gzip_logs.sh
#!/bin/bash
usage()
{
echo "usage: zip_logs.sh [directory] [-prod]" >&2
exit 0
}
TEMP=`getopt :h $*`
@shiumachi
shiumachi / alternatives-hadoop.sh
Created Nov 28, 2018
alternatives like script for hadoop
View alternatives-hadoop.sh
#!/bin/sh
HOME_LIB_DIR=${HOME}/lib
# symlink list
HADOOP_SYMLINK=${HOME_LIB_DIR}/hadoop
HBASE_SYMLINK=${HOME_LIB_DIR}/hbase
ZOOKEEPER_SYMLINK=${HOME_LIB_DIR}/zookeeper
HIVE_SYMLINK=${HOME_LIB_DIR}/hive
PIG_SYMLINK=${HOME_LIB_DIR}/pig
You can’t perform that action at this time.