bnyeggen

package com.nyeggen.strintmap;

import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;

/** Lockfree quasi-queue of integers.  Conceptually there is a queue of integers
 *  at the head, followed by a stream of integers from n to Integer.MAX_VALUE.
 *  add() places at the end of the queue; get() retrieves first from the queue,
 *  and then from the increasing stream.  The stream is implemented with an
 *  AtomicInteger; continuing beyond Integer.MAX_VALUE will cycle back down to

bnyeggen

<plugin>
	<!-- uberjar via "mvn package shade:shade" -->
    <groupId>org.apache.maven.plugins</groupId>
    <artifactId>maven-shade-plugin</artifactId>
    <version>2.3</version>
    <executions>
        <execution>
            <phase>package</phase>
            <goals>
                <goal>shade</goal>

bnyeggen

#redundancy is the max number of survivable failures, so eg 1 for RAID5
#mtbf_array is an array of either actual mean-time-between-failures, or a nested RAID array

# RAID([100]*7,2) #7 disk RAID 6
# RAID([RAID([100]*3,1),RAID([1000]*3,1)],0) # RAID 50, 2 arrays of 3
# RAID([100,100,50,50],1) #RAID 5 with varying reliabilities

from random import random

class RAID(object):

bnyeggen

(comment You will want just about everything in your hive/lib dir included in your Classpath)

(ns myproj.core
  (:import [org.apache.hadoop.hive.service HiveClient]
           [org.apache.thrift.transport TSocket]
           [org.apache.thrift.protocol TBinaryProtocol]))

(defn send-hive
  "Creates a new socket and Hive client connection, runs the query, pulls the result, and closes the connection.
  Eventually modify to split and parse according to schema of result.

bnyeggen

;dummy.csv is of format "1,2,3\n4,5,6\n"

;fails since everything is lazy (but works if you do something that forces resolution)
(with-open [r (reader "file:///home/brycen/dummy.csv")]
  (for [lines (read-csv r)]
    (zipmap [:x :y :z] lines)))

;Doesn't close "properly" (but works most of the time)
(for [lines (read-csv (reader "file:///home/brycen/dummy.csv"))]
  (zipmap [:x :y :z] lines))

bnyeggen

(defn flexigroup
  "Like group-by, but allows arbitrary calculation of keys and values from 
   source coll, (for instance, to roll up distinct particular elements of a 
   vector by distinct other parts) and arbitrary combinations of old and new
   vectors (for instance, to perform an efficient online count).
   (flexigroup identity identity conj [] coll) == group-by
   (flexigroup identity identity (fn [a b] (inc a)) 0) == online count
   (flexigroup #(subvec % 0 1) #(subvec % 1) conj #{} coll) == all the 
   distinct rests of the vectors, grouped by the first element"
  [key-f val-f combine-f init-v coll]

bnyeggen

(defn switch-refs
  "Must be called in transaction"
  [a b]
  (let [av @a bv @b
        setter (fn [_ v] v)]
    (commute a setter bv)
    (commute b setter av)))

(defn switch!
  "Blocks on both sequentially, nontransactional"

bnyeggen

(defn intern-map
  [m]
  (let [val-set (set (vals m))]
    (into {} 
      (for [[k v] m]
        {k (val-set v)}))))

(defn make-duped-hashmap []
  (into {} 
    (for [k (range 10000000)]

bnyeggen

(defmacro with-interns
  "interns => [intern1 intern2...]
   Evaluates body while making available many functions, bound to the 
   symbols in interns. Each fn, when called, returns a deduped reference to
   its argument. The deduplication is with respect to any previously-called
   arguments to that fn.

   (with-interns [intern]
     (into {} 
       (for [k (range 100000)]

bnyeggen

(defn unique-vec1
  "Straight-up Clojure sequence traversal"
  [s]
  (vec (set s)))

(defn unique-vec2
  "Conj into transient set and vectorize"
  [s]
  (loop [remaining s
         seen (transient #{})]