animeshtrivedi

/*
This is just a rough sketch how a the code should look like. While 
impelmenting the actual logic you may find that new classes or abstractions
might be required. 
*/
class CrailStreamProducerExample {
      public static void main(String[] args) {
        /* you need to implement CrailBroker, CrailProducer, CrailStreamWriter classes */
        BorkerProducer broker = new BorkerProducer();
        broker.connect("hostanme", port, someThingMore);

animeshtrivedi

// https://webcache.googleusercontent.com/search?q=cache:YymJdLRJt40J:https://github.com/apache/arrow/blob/master/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java+&cd=1&hl=en&ct=clnk&gl=ch&client=ubuntu
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *

animeshtrivedi

// http://www.jofre.de/?p=1459
package de.jofre.test;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.convert.GroupRecordConverter;
import org.apache.parquet.format.converter.ParquetMetadataConverter;

animeshtrivedi

About PrintCompilation

This note tries to document the output of PrintCompilation flag in HotSpot VM. It was originally intended to be a reply to a blog post on PrintCompilation from Stephen Colebourne. It's kind of grown too big to fit as a reply, so I'm putting it here.

Written by: Kris Mok rednaxelafx@gmail.com

Most of the contents in this note are based on my reading of HotSpot source code from OpenJDK and experimenting with the VM flags; otheres come from HotSpot mailing lists and other reading materials listed in the "References" section.

This

animeshtrivedi

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *

animeshtrivedi

/**
  * Created by atr on 19.09.17.
  */
import java.io.{BufferedWriter, File, FileWriter}

import com.ibm.crail.CrailFS
import com.ibm.crail.conf.CrailConfiguration
import org.apache.spark.SparkContext

import scala.util.Random

animeshtrivedi

package org.apache.spark.sql.execution.datasources.parquet

/**
  * Created by atr on 23.06.17.
  */
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0

animeshtrivedi

import java.nio.ByteBuffer
import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicLong}
import java.util.concurrent.{ConcurrentHashMap, Future, LinkedBlockingQueue, TimeUnit}

import com.ibm.crail._
import com.ibm.crail.conf.CrailConfiguration
import com.ibm.crail.utils.CrailImmediateOperation
import org.apache.spark._
import org.apache.spark.common._
import org.apache.spark.executor.ShuffleWriteMetrics

animeshtrivedi

# RDD settings 
spark.rdd.compress              false

# Shuffle settings 
spark.shuffle.manager sort
spark.shuffle.compress  false
spark.shuffle.spill false
spark.shuffle.spill.compress  false
spark.shuffle.sort.initialBufferSize  4194304
spark.shuffle.sort.bypassMergeThreshold 200

animeshtrivedi

public static void copyTest(String[] args) {
        Unsafe us = null;
        try {
            Field f = Unsafe.class.getDeclaredField("theUnsafe");

            f.setAccessible(true);
            us = (Unsafe) f.get(null);
            System.err.println(" oh my, I have it : " + us);
        } catch (Exception e) {
            e.printStackTrace();