mingwei-li

boolean updateValue(Event event) {
    long userInitiatedTimestamp = event.getUserInitiatedTimestamp();
    String userId = event.getUserId();
    User user = database.query(userId);
    long lastUserInitiatedTimestamp = user.getLastUserInitiatedTimestamp();
     
    // event arrives in correct order
    if(userInitiatedTimestamp > lastUserInitiatedTimestamp) {
        // register with latest timestamp
        user.setLastUserInitiatedTimestamp(userInitiatedTimestamp);

mingwei-li

hs.refreshIndex("index1")
hs.deleteIndex("index1")
hs.restoreIndex("index1")
hs.deleteIndex("index2")
hs.vacuumIndex("index2")

mingwei-li

== Physical Plan ==
*(1) Project [name#11]
+- *(1) Filter (isnotnull(id#10) && (id#10 = 1))
   +- *(1) FileScan parquet [id#10,name#11] Batched: true, Format: Parquet, Location: InMemoryFileIndex[file:/Users/mingwli/Dev/lib/hadoop-2.9.2/bin/spark-warehouse/indexes/index/v__=0], PartitionFilters: [], PushedFilters: [IsNotNull(id), EqualTo(id,1)], ReadSchema: struct<id:int,name:string>

mingwei-li

spark.enableHyperspace
query.show()

mingwei-li

=============================================================
Plan with indexes:
=============================================================
Project [name#11]
+- Filter (isnotnull(id#10) && (id#10 = 1))
   <----+- FileScan parquet [id#10,name#11] Batched: true, Format: Parquet, Location: InMemoryFileIndex[file:/Users/mingwli/Dev/lib/hadoop-2.9.2/bin/spark-warehouse/indexes/index/v__=0], PartitionFilters: [], PushedFilters: [IsNotNull(id), EqualTo(id,1)], ReadSchema: struct<id:int,name:string>---->
 
=============================================================
Plan without indexes:
=============================================================

mingwei-li

+-----+--------------+---------------+----------+--------------------+--------------------+--------------------+------+
| name|indexedColumns|includedColumns|numBuckets|              schema|       indexLocation|           queryPlan| state|
+-----+--------------+---------------+----------+--------------------+--------------------+--------------------+------+
|index|          [id]|         [name]|       200|{"type":"struct",...|file:/Users/mingw...|Relation[id#10,na...|ACTIVE|
+-----+--------------+---------------+----------+--------------------+--------------------+--------------------+------+

mingwei-li

val hs = new Hyperspace(spark)
hs.createIndex(df, IndexConfig("index", indexedColumns = Seq("id"), includedColumns = Seq("name")))
hs.indexes.show()

mingwei-li

import com.microsoft.hyperspace._
import com.microsoft.hyperspace.index._

mingwei-li

val df = spark.read.format("csv").option("header", "true").option("inferSchema", "true").load("hdfs://localhost:9000/hyperspace_test/customers.csv")
df.show()

mingwei-li