Skip to content

Instantly share code, notes, and snippets.

Avatar

Anton Parkhomenko chuwy

View GitHub Profile
@chuwy
chuwy / run_shredder.py
Created Jan 10, 2022
No Dataflow Runner
View run_shredder.py
import boto3
connection = boto3.client(
'emr',
region_name='eu-central-1',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
)
cluster_id = connection.run_job_flow(
View topology.scala
def input[F[_]: Timer] =
(Stream(1, 2, -3, 4, -5) ++
Stream.sleep_(1.second) ++
Stream(4, 3, -1, -2, 1) ++
Stream.sleep_(1.second) ++
Stream(0, 2, -5) ++
Stream.sleep_(1.second) ++
Stream(-6, 1, 2, 0, -1, -4, 2, 9)
).covary[F].map(i => if (i % 2 == 0) i.asRight else (i.toString + "b").asLeft)
View test-pg-loader.sh
#!/bin/sh
set -e
if [ -z ${TRAVIS_BUILD_DIR+x} ]; then
echo "TRAVIS_BUILD_DIR is unset";
exit 1
fi
IGLUCTL_ZIP="igluctl_0.6.0.zip"
@chuwy
chuwy / README.md
Last active Jun 17, 2020
Second version of Iglu root metaschema
View README.md

Files

  • metaschema.json - new root metaschema to put in $schema properties
  • example-meta.json - example registry metaschema, i.e. one hosted on Acme Ltd. (Mixpanel, Iteratively, Snowplow) Iglu registry and thus specific to all its schemas. Can have 2-0-0 or 1-0-0 root metaschema
  • example-schema.json - plain (non-meta, usual) schema hosting on Acme Ltd. registry and containing metadata described by above metaschema

Changes

Compared to 1-0-0

@chuwy
chuwy / 0001-Unify.patch
Created Jun 13, 2020
Combine enrich submodules into a single SBT project
View 0001-Unify.patch
From fe136f1c24a2d3510f654348fa4522f48cdc6efb Mon Sep 17 00:00:00 2001
From: Anton Parkhomenko <mailbox@chuwy.me>
Date: Sat, 13 Jun 2020 14:52:47 +0700
Subject: [PATCH] Unify
---
beam.sbt | 88 ---------
build.sbt | 172 ++++++++++++++++++
common.sbt | 68 -------
...mmonSettings.scala => BuildSettings.scala} | 68 ++++++-
@chuwy
chuwy / extract.sh
Created Jun 13, 2020
Extract enrich repositories from snowplow/snowplow monorepo
View extract.sh
#!/bin/sh
cp -R ../snowplow .
cd snowplow/
git filter-repo \
--path 3-enrich/scala-common-enrich \
--path 3-enrich/spark-enrich \
--path 3-enrich/beam-enrich \
View Data.hs
data Option a where
Some : a -> Option a
None : Option a
instance Functor
fmap : (a -> b) -> Option a -> Option b
fmap f (Some a) = Some (f a)
fmap _ None = None
deriving instance Show given Show a
View gist:1854efe7a70a7344f22c102a39266113
trait Mutability[F[_]] {
def init[A](a: A): F[Mutability.Var[F, A]]
}
object Mutability {
trait Var[F[_], A] {
def get: F[A]
def put(a: A): F[Unit]
}
View types.scala
/** Always positive integer, nobody can do `new PositiveInt(-3)` */
class PositiveInt private(value: Int) extends AnyVal
/** Set of constructors for PositiveInt */
object PositiveInt {
/** The only proper constructor. Everything else is workaround */
def fromInt(int: Int): Option[PositiveInt] = if (int > 0) Some(new PositiveInt(int)) else None
/**
* This is a workaround and must be called ONLY when caller is sure the `int` is positive
View WeirdChunks.scala
object WeirdChunks {
import cats.effect.concurrent.Ref
import fs2.concurrent.Queue
import scala.concurrent.duration._
implicit val t = IO.timer(scala.concurrent.ExecutionContext.global)
implicit val cs = IO.contextShift(scala.concurrent.ExecutionContext.global)
def init: IO[(Ref[IO, Int], Queue[IO, String])] =