SPARK_CLASSPATH=/opt/spark-1.0.0/extlib/spark-liblinear-1.94.jar ./bin/spark-shell
sc.addJar("./extlib/spark-liblinear-1.94.jar")
import tw.edu.ntu.csie.liblinear._
public final class TrainNewsGroups { | |
public static void main(String[] args) throws IOException { | |
File base = new File(args[0]); | |
Multiset<String> overallCounts = HashMultiset.create(); | |
int leakType = 0; | |
if (args.length > 1) { | |
leakType = Integer.parseInt(args[1]); |
import org.apache.spark.mllib.util.MLUtils | |
import org.apache.spark.mllib.classification.LogisticRegressionWithSGD | |
val training = MLUtils.loadLibSVMFile(sc, "hdfs://dm01:8020/user/hive/warehouse/kdd12track2.db/training_libsvmfmt_10k", multiclass = false, numFeatures = 16777216, minPartitions = 64) | |
//val training = MLUtils.loadLibSVMFile(sc, "hdfs://dm01:8020/user/hive/warehouse/kdd12track2.db/training_libsvmfmt_10k", multiclass = false) | |
val model = LogisticRegressionWithSGD.train(training, numIterations = 1) | |
//val model = LogisticRegressionWithSGD.train(training, numIterations = 20) |
SPARK_CLASSPATH=/opt/spark-1.0.0/extlib/spark-liblinear-1.94.jar ./bin/spark-shell
sc.addJar("./extlib/spark-liblinear-1.94.jar")
import tw.edu.ntu.csie.liblinear._
AMConfiguration should be remained public because it is called by Hive in org.apache.hadoop.hive.ql.exec.tez.TezSessionState.
The following exception caused when using Tez-trunk on Hive 0.13.1/Hadoop 2.4.1.
Exception in thread "main" java.lang.IllegalAccessError: tried to access class org.apache.tez.client.AMConfiguration fro
m class org.apache.hadoop.hive.ql.exec.tez.TezSessionState
at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:184)
at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:123)
at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:355)
at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:681)
1|Justice, in its broadest context, includes both the attainment of that which is just and the philosophical discussion of that which is just. The concept of justice is based on numerous fields, and many differing viewpoints and perspectives including the concepts of moral rightness based on ethics, rationality, law, religion, equity and fairness. Often, the general discussion of justice is divided into the realm of social justice as found in philosophy, theology and religion, and, procedural justice as found in the study and application of the law. | |
2|Wisdom (sophia) is the ability to think and act using knowledge, experience, understanding, common sense, and insight.[1] Wisdom has been regarded as one of four cardinal virtues; and as a virtue, it is a habit or disposition to perform the action with the highest degree of adequacy under any given circumstance. This implies a possession of knowledge or the seeking thereof in order to apply it to the given circumstance. This involves an understanding of people, |
hdfs dfsadmin -report
# プールの作成
hdfs cacheadmin -addPool expr
#プールの削除
hdfs cacheadmin -removePool expr
# プールの確認
https://github.com/myui/hivemall/wiki/KDDCup-2012-track-2-CTR-prediction-dataset
create table training_libsvmfmt
ROW FORMAT DELIMITED
LINES TERMINATED BY "\n"
STORED AS TEXTFILE
AS
select
concat(label," ",features[1],":1.0 ",features[2],":1.0 ",features[3],":1.0 ",features[4],":1.0 ",features[5],":1.0 ",features[6],":1.0 ",features[7],":1.0 ",features[8],":1.0 ",features[9],":1.0 ",features[10],":1.0 ",features[11],":1.0 ",features[12],":1.0") as line
#!/bin/bash | |
nfolds=10 | |
evalsql="" | |
cat /dev/null > generate_cv.sql | |
echo "-- ${nfolds}-folds cross validation" >> generate_cv.sql | |
for i in `seq 1 ${nfolds}`; | |
do |
-- 10-folds cross validation | |
create or replace view training1 | |
as | |
select | |
userid, movieid, rating | |
from | |
ratings_groupded | |
where | |
gid != 1; |