IgorBerman

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;


public class Test {
    private AtomicReference<Config> configHolder = new AtomicReference<Config>();
    private static final class Config {
        private final Map<String, Double> factorsMap;
        Config(Map<String, Double> factorsMap) {

IgorBerman

import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

import javax.inject.Provider;

public class Test {
    //it's safe to read from it concurrently, however we don't want that same bean will be created twice
    private ConcurrentHashMap<String, ConcurrentMap<String, Object>> scopes 
        = new ConcurrentHashMap<String, ConcurrentMap<String, Object>>(50);

IgorBerman

def register_citext_type(conn, con_record):
    from psycopg2.extensions import new_type, register_type
    from contextlib import closing
    def cast_citext(in_str, cursor):
        if in_str == None:
            return None
        return unicode(in_str, cursor.connection.encoding)
    citext_oid = None
    with closing(conn.cursor()) as c:

IgorBerman

download twitter core, stream and spark-streaming-twitter from maven central and place them into spark/lib
run from bin directory:
spark-shell --jars ../lib/twitter4j-core-3.0.3.jar,../lib/twitter4j-stream-3.0.3.jar,../lib/spark-streaming-twitter_2.10-1.2.0.jar 

--------------------------------------------------------------
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.twitter._
import  java.io.FileWriter

val filters = List("postgres", "timeseries")

IgorBerman

package com.myproject.cassandraloading;

import static com.google.common.collect.Iterables.filter;
import static com.google.common.collect.Iterables.transform;

import java.io.File;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;

IgorBerman

package org.apache.flink.streaming.connectors.fs.avro;

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at

IgorBerman

knife search "recipes:xxx*" -a ipaddress | grep ipaddress | awk '{print $2}'

IgorBerman

import jenkins.*
import jenkins.model.*
import hudson.*
import hudson.model.*

def DRY_RUN = false
def DIRS_FILTER = ['.hg', 'production','logs','apache', 'projects', 'qlickview']
def JOBS_WS_ROOT = 'path-to-your-workspaces-root-dir'
def JOBS_IRRELEVANT_SYMBOLS = "[_,-]|\\s"
def allJobs = Jenkins.instance.items.findAll { it instanceof hudson.model.Job }.collect { it.name };

IgorBerman

1. General Background and Overview

• Probabilistic Data Structures for Web Analytics and Data Mining : A great overview of the space of probabilistic data structures and how they are used in approximation algorithm implementation.
• Models and Issues in Data Stream Systems
• Philippe Flajolet’s contribution to streaming algorithms : A presentation by Jérémie Lumbroso that visits some of the hostorical perspectives and how it all began with Flajolet
• Approximate Frequency Counts over Data Streams by Gurmeet Singh Manku & Rajeev Motwani : One of the early papers on the subject.
• [Methods for Finding Frequent Items in Data Streams](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.187.9800&rep

IgorBerman

1. Setup your process java properties
-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=1098 -Dcom.sun.management.jmxremote.rmi.port=1098 -Djava.rmi.server.hostname=127.0.0.1

2. ssh to your server on ec2 with a) port forwarding of 1098(jmx port) and b) with socks server on 10003
ssh -L 1098:127.0.0.1:1098 -D 10003 my.ec2.instance.com

3. Open visualvm with socks proxy on on same proxy port
jvisualvm -J-DsocksProxyHost=localhost -J-DsocksProxyPort=10003

4. create local jmx connection on localhost:1098 in visualvm