Jazz Yao-Tsung Wang jazzwang

## gitlab
#!/bin/bash
case "$1" in
new)
	echo "[INFO] Creating new gitlab docker instances ..."
	docker create --name='gitlab' -it -e 'GITLAB_PORT=10080' -e 'GITLAB_SSH_PORT=10022' -p 10022:22 -p 10080:80 -v /var/run/docker.sock:/run/docker.sock -v $(which docker):/bin/docker -v $(pwd):/vagrant sameersbn/gitlab:latest
	;;
start)
	echo "[INFO] Starting gitlab docker instances ..."
	docker start gitlab
	;;

## cdh5
#!/bin/bash
case "$1" in
new)
        echo "[INFO] Creating new CDH5 docker instances ..."
        docker create -it --name CDH5 -h cdh5.etu.im -v ${PWD}:/vagrant caioquirino/docker-cloudera-quickstart /usr/bin/cdh_startup_script.sh
        ;;
start)
        echo "[INFO] Starting CDH5 docker instances ..."
        docker start CDH5
        ;;

## update-docker-images
#!/bin/bash
for i in $(docker images | grep ago | awk '{ print $1":"$2 }'); do docker pull $i; done
for i in $(docker images -f 'dangling=true' -q);  do docker rmi $i;  done

## column_limit_4K.sql
CREATE DATABASE IF NOT EXISTS test;
USE test;
CREATE TABLE IF NOT EXISTS 4K_columns ( col1 int, col2 int, col3 int, col4 int, col5 int, col6 int, col7 int, col8 int, col9 int, col10 int, col11 int, col12 int, col13 int, col14 int, col15 int, col16 int, col17 int, col18 int, col19 int, col20 int, col21 int, col22 int, col23 int, col24 int, col25 int, col26 int, col27 int, col28 int, col29 int, col30 int, col31 int, col32 int, col33 int, col34 int, col35 int, col36 int, col37 int, col38 int, col39 int, col40 int, col41 int, col42 int, col43 int, col44 int, col45 int, col46 int, col47 int, col48 int, col49 int, col50 int, col51 int, col52 int, col53 int, col54 int, col55 int, col56 int, col57 int, col58 int, col59 int, col60 int, col61 int, col62 int, col63 int, col64 int, col65 int, col66 int, col67 int, col68 int, col69 int, col70 int, col71 int, col72 int, col73 int, col74 int, col75 int, col76 int, col77 int, col78 int, col79 int, col80 int, col81 int, col82 int, col83 int, col84 int, col85 int, col86 int, c

## column_limit_5K.sql
CREATE DATABASE IF NOT EXISTS test;
USE test;
CREATE TABLE IF NOT EXISTS 10K_columns ( col1 int, col2 int, col3 int, col4 int, col5 int, col6 int, col7 int, col8 int, col9 int, col10 int, col11 int, col12 int, col13 int, col14 int, col15 int, col16 int, col17 int, col18 int, col19 int, col20 int, col21 int, col22 int, col23 int, col24 int, col25 int, col26 int, col27 int, col28 int, col29 int, col30 int, col31 int, col32 int, col33 int, col34 int, col35 int, col36 int, col37 int, col38 int, col39 int, col40 int, col41 int, col42 int, col43 int, col44 int, col45 int, col46 int, col47 int, col48 int, col49 int, col50 int, col51 int, col52 int, col53 int, col54 int, col55 int, col56 int, col57 int, col58 int, col59 int, col60 int, col61 int, col62 int, col63 int, col64 int, col65 int, col66 int, col67 int, col68 int, col69 int, col70 int, col71 int, col72 int, col73 int, col74 int, col75 int, col76 int, col77 int, col78 int, col79 int, col80 int, col81 int, col82 int, col83 int, col84 int, col85 int, col86 int,

## gist:45261be60c15d353aa79

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                jazzwang
                / gist:45261be60c15d353aa79
            
            
              Last active
              September 9, 2015 15:04
                — forked from debasishg/gist:8172796
            
              
                A collection of links for streaming algorithms and data structures
              
          
General Background and Overview


Probabilistic Data Structures for Web Analytics and Data Mining : A great overview of the space of probabilistic data structures and how they are used in approximation algorithm implementation.
Models and Issues in Data Stream Systems
Philippe Flajolet’s contribution to streaming algorithms : A presentation by Jérémie Lumbroso that visits some of the hostorical perspectives and how it all began with Flajolet
Approximate Frequency Counts over Data Streams by Gurmeet Singh Manku & Rajeev Motwani : One of the early papers on the subject.
[Methods for Finding Frequent Items in Data Streams](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.187.9800&amp;rep


## mahout 0.10 canopy test
root@mahout:~# wget http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data
root@mahout:~# hadoop fs -put synthetic_control.data testdata
root@mahout:~# hadoop fs -ls /user/root
Found 1 items
-rw-r--r--   1 root supergroup     288374 2015-12-20 07:39 /user/root/testdata
root@mahout:~# hadoop fs -cat /user/root/testdata | head -n 2
28.7812 34.4632 31.3381 31.2834 28.9207 33.7596 25.3969 27.7849 35.2479 27.1159 32.8717 29.2171 36.0253 32.337  34.5249 32.8717 34.1173 26.5235 27.6623 26.3693 25.7744 29.27   30.7326 29.5054 33.0292 25.04   28.9167 24.3437 26.1203 34.9424 25.0293 26.6311 35.6541 28.4353 29.1495 28.1584 26.1927 33.3182 30.9772 27.0443 35.5344 26.2353 28.9964 32.0036 31.0558 34.2553 28.0721 28.9402 35.4973 29.747  31.4333 24.5556 33.7431 25.0466 34.9318 34.9879 32.4721 33.3759 25.4652 25.8717
24.8923 25.741  27.5532 32.8217 27.8789 31.5926 31.4861 35.5469 27.9516 31.6595 27.5415 31.1887 27.4867 31.391  27.811  24.488  27.5918 35.6273 35.4102 31.4167 30.7447 24.1311 35.1422 3

## Mahout 0.10 Docker Image
## 先準備範例資料集
jazz@jazzbook:~$ wget http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data
## 我的筆電是 Ubuntu 14.04.3 LTS
jazz@jazzbook:~$ lsb_release -a
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu 14.04.3 LTS
Release:	14.04
Codename:	trusty
## 硬體規格是 16GB

## bigtop-hdfs NFS mount
## Test Environment

+---------- Host OS --------------+
|                   +------------+|
| Ubuntu 14.04 LTS  | Debian 8.2 ||
|                   +------------+|
+---------------------------------+

jazz@jazzbook:~$ lsb_release -a
No LSB modules are available.

## bash_color.sh
#!/bin/bash
#
#   This file echoes a bunch of color codes to the
#   terminal to demonstrate what's available.  Each
#   line is the color code of one forground color,
#   out of 17 (default + 16 escapes), followed by a
#   test use of that color on all nine background
#   colors (default + 8 escapes).
#
	#!/bin/bash
	case "$1" in
	new)
	echo "[INFO] Creating new gitlab docker instances ..."
	docker create --name='gitlab' -it -e 'GITLAB_PORT=10080' -e 'GITLAB_SSH_PORT=10022' -p 10022:22 -p 10080:80 -v /var/run/docker.sock:/run/docker.sock -v $(which docker):/bin/docker -v $(pwd):/vagrant sameersbn/gitlab:latest
	;;
	start)
	echo "[INFO] Starting gitlab docker instances ..."
	docker start gitlab
	;;
	#!/bin/bash
	for i in $(docker images \| grep ago \| awk '{ print $1":"$2 }'); do docker pull $i; done
	for i in $(docker images -f 'dangling=true' -q); do docker rmi $i; done
	CREATE DATABASE IF NOT EXISTS test;
	USE test;
	CREATE TABLE IF NOT EXISTS 4K_columns ( col1 int, col2 int, col3 int, col4 int, col5 int, col6 int, col7 int, col8 int, col9 int, col10 int, col11 int, col12 int, col13 int, col14 int, col15 int, col16 int, col17 int, col18 int, col19 int, col20 int, col21 int, col22 int, col23 int, col24 int, col25 int, col26 int, col27 int, col28 int, col29 int, col30 int, col31 int, col32 int, col33 int, col34 int, col35 int, col36 int, col37 int, col38 int, col39 int, col40 int, col41 int, col42 int, col43 int, col44 int, col45 int, col46 int, col47 int, col48 int, col49 int, col50 int, col51 int, col52 int, col53 int, col54 int, col55 int, col56 int, col57 int, col58 int, col59 int, col60 int, col61 int, col62 int, col63 int, col64 int, col65 int, col66 int, col67 int, col68 int, col69 int, col70 int, col71 int, col72 int, col73 int, col74 int, col75 int, col76 int, col77 int, col78 int, col79 int, col80 int, col81 int, col82 int, col83 int, col84 int, col85 int, col86 int, c
	CREATE DATABASE IF NOT EXISTS test;
	USE test;
	CREATE TABLE IF NOT EXISTS 10K_columns ( col1 int, col2 int, col3 int, col4 int, col5 int, col6 int, col7 int, col8 int, col9 int, col10 int, col11 int, col12 int, col13 int, col14 int, col15 int, col16 int, col17 int, col18 int, col19 int, col20 int, col21 int, col22 int, col23 int, col24 int, col25 int, col26 int, col27 int, col28 int, col29 int, col30 int, col31 int, col32 int, col33 int, col34 int, col35 int, col36 int, col37 int, col38 int, col39 int, col40 int, col41 int, col42 int, col43 int, col44 int, col45 int, col46 int, col47 int, col48 int, col49 int, col50 int, col51 int, col52 int, col53 int, col54 int, col55 int, col56 int, col57 int, col58 int, col59 int, col60 int, col61 int, col62 int, col63 int, col64 int, col65 int, col66 int, col67 int, col68 int, col69 int, col70 int, col71 int, col72 int, col73 int, col74 int, col75 int, col76 int, col77 int, col78 int, col79 int, col80 int, col81 int, col82 int, col83 int, col84 int, col85 int, col86 int,
	root@mahout:~# wget http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data
	root@mahout:~# hadoop fs -put synthetic_control.data testdata
	root@mahout:~# hadoop fs -ls /user/root
	Found 1 items
	-rw-r--r-- 1 root supergroup 288374 2015-12-20 07:39 /user/root/testdata
	root@mahout:~# hadoop fs -cat /user/root/testdata \| head -n 2
	28.7812 34.4632 31.3381 31.2834 28.9207 33.7596 25.3969 27.7849 35.2479 27.1159 32.8717 29.2171 36.0253 32.337 34.5249 32.8717 34.1173 26.5235 27.6623 26.3693 25.7744 29.27 30.7326 29.5054 33.0292 25.04 28.9167 24.3437 26.1203 34.9424 25.0293 26.6311 35.6541 28.4353 29.1495 28.1584 26.1927 33.3182 30.9772 27.0443 35.5344 26.2353 28.9964 32.0036 31.0558 34.2553 28.0721 28.9402 35.4973 29.747 31.4333 24.5556 33.7431 25.0466 34.9318 34.9879 32.4721 33.3759 25.4652 25.8717
	24.8923 25.741 27.5532 32.8217 27.8789 31.5926 31.4861 35.5469 27.9516 31.6595 27.5415 31.1887 27.4867 31.391 27.811 24.488 27.5918 35.6273 35.4102 31.4167 30.7447 24.1311 35.1422 3
	## 先準備範例資料集
	jazz@jazzbook:~$ wget http://archive.ics.uci.edu/ml/databases/synthetic_control/synthetic_control.data
	## 我的筆電是 Ubuntu 14.04.3 LTS
	jazz@jazzbook:~$ lsb_release -a
	No LSB modules are available.
	Distributor ID: Ubuntu
	Description: Ubuntu 14.04.3 LTS
	Release: 14.04
	Codename: trusty
	## 硬體規格是 16GB
	## Test Environment

	+---------- Host OS --------------+
	\| +------------+\|
	\| Ubuntu 14.04 LTS \| Debian 8.2 \|\|
	\| +------------+\|
	+---------------------------------+

	jazz@jazzbook:~$ lsb_release -a
	No LSB modules are available.
	#!/bin/bash
	#
	# This file echoes a bunch of color codes to the
	# terminal to demonstrate what's available. Each
	# line is the color code of one forground color,
	# out of 17 (default + 16 escapes), followed by a
	# test use of that color on all nine background
	# colors (default + 8 escapes).
	#