ZedYeung

#!/bin/bash
# Spark
mkdir ~/spark && cd ~/spark && \
    wget http://apache.claz.org/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz -O spark.tgz && \
    tar zxvf spark.tgz --strip 1 && \
    rm spark.tgz
    
pip install pyspark
pip install kafka-python

ZedYeung

Movies Recommendation:

• MovieLens - Movie Recommendation Data Sets http://www.grouplens.org/node/73
• Yahoo! - Movie, Music, and Images Ratings Data Sets http://webscope.sandbox.yahoo.com/catalog.php?datatype=r
• Jester - Movie Ratings Data Sets (Collaborative Filtering Dataset) http://www.ieor.berkeley.edu/~goldberg/jester-data/
• Cornell University - Movie-review data for use in sentiment-analysis experiments http://www.cs.cornell.edu/people/pabo/movie-review-data/

Music Recommendation:

• Last.fm - Music Recommendation Data Sets http://www.dtic.upf.edu/~ocelma/MusicRecommendationDataset/index.html

ZedYeung

import re
import glob

# https://www.elastic.co/blog/strings-are-dead-long-live-strings
files = glob.glob('./*19_mapping.json')
for file in files:
    print(file)
    with open(file) as f:
        text = f.read()

ZedYeung

#!/bin/bash
# npm install elasticdump -g
# Copy an index from production to staging with analyzer and mapping:
SRC=http://yue.yang:Bgyqhjbf1746@10.100.121.210:9200
DEST=http://yue.yang:Bgyqhjbf1746@35.224.243.158:9200
# INDICES="billing_data_v2-2018.07.20,platform_nlog_v9-2018.07.20,platform_vfw_v1-2018.07.20,platform_cna_v1-2018.07.20"
SIZE=10000
LIMIT=10000
INDICES="billing_data_v2-2018.07.19"

ZedYeung

#!/bin/bash
# CA
# https://help.ubuntu.com/community/VirtualBox/SharedFolders
sudo mkdir ca_shared
sudo mount -t vboxsf ca_shared ~/ca_shared -o uid=akb,gid=akb
# need to specify the uid to your user and gid to your group, otherwise the mount device would belong to root:root
# That means you would have permission problem to access this shared folder
# When you import req from this root owned shared folder, you would see this error:
# Note: using Easy-RSA configuration from: ./vars

ZedYeung

#!/bin/bash

# shadowsocks-libev
sudo apt install shadowsocks-libev
sudo ufw allow 8389

sudo tee /etc/shadowsocks-libev/config.json > /dev/null<<EOF
{
 "server":"192.168.1.109",
 "server_port":8389,

ZedYeung

plugins=(
  git
  docker
  encode64
  history
  systemd
)

# docker compose autocompletion
# autoload -Uz compinit && compinit -i

ZedYeung

#!/bin/bash

sudo apt update
sudo apt-get install apt-transport-https

# Java
sudo apt-get install openjdk-8-jdk

# oracle
# sudo add-apt-repository ppa:webupd8team/java

ZedYeung

USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b

POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b

ZedYeung

You can ssh into the VM by finding the IP (from kubectl config view) and using username "docker" password "tcuser":

ssh docker@192.168.XX.XX