Skip to content

Instantly share code, notes, and snippets.

View apivovarov's full-sized avatar

Alexander Pivovarov apivovarov

  • Amazon Web Services
  • Santa Clara, CA
  • 17:31 (UTC -07:00)
  • LinkedIn in/pivovaal
View GitHub Profile
@apivovarov
apivovarov / CSV.scala
Last active December 3, 2015 01:55
CSV parser in Scala
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
import scala.util.{Failure, Success, Try}
val f1: (Int) => Try[String] = x => Try(s"f1($x)")
val f2: (String) => Try[String] = y => Try(s"f2($y)")
val f3: (String) => Try[String] = z => Try(s"f3($z)")
val x = 10
val res: Try[String] = f1(x).flatMap(y => f2(y).flatMap(z => f3(z)))
println(res)
// Success(f3(f2(f1(10))))
package org.apache.hadoop.mapreduce.lib.output;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
package org.apache.hadoop.mapred;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileOutputCommitter;
import org.apache.hadoop.mapred.FileOutputFormat;
package org.apache.hadoop.mapred;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCommitter;
@apivovarov
apivovarov / Arm.scala
Last active June 5, 2016 05:12
Automatic Resource Management
/**
* Monad to work with AutoClosables
* Automatic Resource Management
*
* Example:
* val lines = Arm(new FileInputStream("/my.txt")).map { stream =>
* Source.fromInputStream(stream)("UTF-8").getLines
* }
*
*/
package org.x4444.onakka
import java.io.InputStream
import java.util.concurrent._
/**
* RunAll
*/
object RunAll {
@apivovarov
apivovarov / MapMerger.scala
Last active December 6, 2016 22:55
MapMerger
import scala.collection._
import scala.collection.immutable.HashMap
def time(times: Int)(fn: => Unit)(name: String): Unit = {
val t1 = System.currentTimeMillis()
var i = 0
while (i < times) {
fn
i += 1
}
package aaa;
import java.util.HashMap;
/**
* Use -Xmx30G -Xms30G for 100,000,000 rows
*/
public class MapUpdater {
public static void main(String[] args) throws InterruptedException {
package aaa;
import java.util.HashMap;
/**
* Use -Xmx30G -Xms30G for 100,000,000 rows
*/
public class MapUpdater2 {
public static void main(String[] args) throws InterruptedException {