Skip to content

Instantly share code, notes, and snippets.

View spark-sql.CMD
screen # 以防万一SSH断了
docker exec -ti dev bash
# 1. build spark
export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m"
./dev/make-distribution.sh --name spark-2.1.0-hadoop2.6.5 --tgz --mvn "$(which mvn)" \
-Psparkr -Phadoop-2.6 -Dhadoop.version=2.6.5 -Phive -Phive-thriftserver -Pyarn -DskipTests -Dmaven.javadoc.skip=true
[root@680108a35829 spark-2.1.0]# ll spark-2.1.0-bin-hadoop2.6.5.tgz
-rw-r--r-- 1 root root 185594381 Apr 21 09:48 spark-2.1.0-bin-hadoop2.6.5.tgz
@winse
winse / hadoop-2.6.5-CMD.sh
Created Apr 19, 2017
build hadoop-2.6.5 with snappy on docker Centos6
View hadoop-2.6.5-CMD.sh
tar zxvf snappy-1.1.3.tar.gz
cd snappy-1.1.3
./configure
make && make install
cd hadoop-2.6.5-src
mvn clean package -Pdist,native -Dmaven.javadoc.skip=true -DskipTests -Drequire.snappy=true -Dbundle.snappy=true -Dsnappy.lib=/usr/local/lib
View zeppelin-0.7.1-CMD.sh
tar zxvf zeppelin-0.7.1.tgz
cd zeppelin-0.7.1
./dev/change_scala_version.sh 2.11
# fix some network problems
MAVEN_REPO=/opt/repo
rm -rf $MAVEN_REPO/org/apache/pig/pig/0.16.0/*
rm -rf $MAVEN_REPO/org/apache/tez/tez-api/0.7.0/*
wget -P $MAVEN_REPO/org/apache/pig/pig/0.16.0/ http://nexus-private.hortonworks.com:8081/nexus/content/shadows/central-m1/org.apache.pig/jars/pig-0.16.0-h2.jar
wget -P $MAVEN_REPO/com/github/eirslett/yarn/0.18.1/yarn-0.18.1./ https://github.com/yarnpkg/yarn/releases/download/v0.18.1/yarn-v0.18.1.tar.gz
@winse
winse / Answers.cs
Created Sep 28, 2016
bookofanswers
View Answers.cs
using cn.bmob.api;
using cn.bmob.io;
using cn.bmob.json;
using cn.bmob.tools;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
@winse
winse / example-du.svg
Last active May 11, 2016
磁盘使用情况火焰图
View example-du.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View redis.sort.log
127.0.0.1:8888> sadd mylist 8 13 9 1 4 888 14 143
(integer) 8
127.0.0.1:8888> SCARD mylist
(integer) 8
127.0.0.1:8888> SMEMBERS mylist
1) "1"
2) "4"
3) "8"
4) "9"
5) "13"
@winse
winse / fileserver.tomcat.changed.diff
Last active Aug 29, 2015
file explorer server on tomcat
View fileserver.tomcat.changed.diff
[root@docker apache-tomcat-7.0.37]# git diff --cached
diff --git a/bin/catalina.sh b/bin/catalina.sh
old mode 100644
new mode 100755
diff --git a/bin/configtest.sh b/bin/configtest.sh
old mode 100644
new mode 100755
diff --git a/bin/daemon.sh b/bin/daemon.sh
old mode 100644
new mode 100755
@winse
winse / install-lrzsz.md
Created Oct 2, 2014
install lrzsz logs.
View install-lrzsz.md

Package Manager

yum install lrzsz

Compile

# https://ohse.de/uwe/software/lrzsz.html
@winse
winse / HelloScalaMapRed.scala
Created Sep 13, 2014
Scala on Hadoop sample。
View HelloScalaMapRed.scala
package com.github.winse.hadoop
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.Reducer
import org.apache.hadoop.io.Text
import org.apache.hadoop.io.IntWritable
import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.mapreduce.Mapper
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
@winse
winse / HelloSubmitMR.java
Last active Aug 29, 2015
windows下,直接对mapreduce程度打包jar,然后提交给resourcemanager执行的主程序的父类。
View HelloSubmitMR.java
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;