anuragkh

#include <iostream>
#include <cstdio>
#include <fstream>
#include <cstdlib>
#include <string>
#include <cstring>
#include <ctime>
#include <chrono>

using namespace std;

anuragkh

!#/bin/bash
# Install openvswitch w/DPDK Ubuntu 14.04.3
# Author: Oleg Slavkin (oleg@slavkin.info)

echo "-------------------------------"
echo "Step 1. Install git package"
echo "-------------------------------"
sudo apt-get update
sudo apt-get -y install git

anuragkh

import edu.berkeley.cs.succinct.annot._

val succinctAnnot =  AnnotatedSuccinctRDD(sc, "/path/to/succinct/data")
// The following is equivalent to {ge^sentence} OVER /remains|still unknown|unclear|uncertain/
// The output is an RDD of Annotation objects, which is just a wrapper holding the annotation data (id, ranges, etc.)
val res = succinctAnnot.regexOver("(remains|is|still) (unknown|unclear|uncertain)", "ge", "sentence")
res.take(20)          // Take first 20 results
res.count             // Iterate through all results

anuragkh

import edu.berkeley.cs.succinct.annot._

val in = ... // Read the input as RDD[(String, String, String)]
val succinctAnnot = AnnotatedSuccinctRDD(in)
succinctAnnot.save("/path/to/output")

anuragkh

# On the master node

git clone https://github.com/amplab/succinct.git
cd succinct
git checkout annot-search
mvn clean -DskipTests package

cd ../spark         # Or wherever spark is installed (i.e., $SPARK_HOME)
cp ../succinct/assembly/target/succinct-0.1.7-SNAPSHOT.jar .        # Or wherever succinct is installed (i.e., $SUCCINCT_HOME)
./bin/spark-shell --jars succinct-0.1.7-SNAPSHOT.jar

anuragkh

import edu.berkeley.cs.succinct._

// "Succinct"-ify the text
val text = sc.textFile("/tmp/tmp.txt").map(_.getBytes)
val succinctText = text.succinct

// Read and filter annotations for "sentence" type
val annot = sc.textFile("/tmp/tmp.annot")
val sentenceAnnot = annot.filter(a => a contains "sentence")
val sentenceRanges = sentenceAnnot.map(a => a split '^').map(e => (e(0), (e(3).toLong, e(4).toLong)))

anuragkh

/* Spark Shell Executed */
./bin/spark-shell --master spark://servername:7077 --driver-class-path $CLASSPATH


/* Output */
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 1.4.0

anuragkh

while read server
do
  scp -i cqlkeypair.pem -r -oStrictHostKeyChecking=no $1 ubuntu@$server:~/ &
done < servers
wait

anuragkh

# ** List all indexes, all pretty-like
curl http://localhost:9200/_aliases?pretty=1

# ** Create an index, with specified settings
# Sample settings shown here; creates an index named "wiki"
# with 8 shards. Also creates a mapping called "articles"
# which specifies the index type for its properties ("text",
# "url" and "title") as "not_analyzed".
# See https://www.elastic.co/guide/en/elasticsearch/guide/master/index-doc.html

anuragkh

sudo wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo -O /etc/yum.repos.d/epel-apache-maven.repo
sudo sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo
sudo yum install -y apache-maven
mvn --version