Skip to content

Instantly share code, notes, and snippets.

@egalpin
egalpin / merge_junit_results.py
Created September 28, 2016 13:41 — forked from cgoldberg/merge_junit_results.py
Merge multiple JUnit XML results files into a single results file.
#!/usr/bin/env python
#
# Corey Goldberg, Dec 2012
#
import os
import sys
import xml.etree.ElementTree as ET
import argparse
from typing import Iterable
from typing import List
from typing import Tuple
from typing import Union
import apache_beam as beam
from apache_beam.transforms.combiners import ToDictCombineFn
from apache_beam.transforms.combiners import ToSetCombineFn
@egalpin
egalpin / DatastoreEntitySerializer.java
Last active June 9, 2022 17:01
Jackson JSON serializer for Google Datastore Entity
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import com.google.cloud.Timestamp;
import com.google.cloud.datastore.Blob;
import com.google.cloud.datastore.Entity;
import com.google.cloud.datastore.Key;
import com.google.cloud.datastore.Value;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
@egalpin
egalpin / RateLimit.java
Last active May 24, 2023 20:25
Apache Beam RateLimit
import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
import com.google.auto.value.AutoValue;
import java.io.Serializable;
import java.util.concurrent.TimeUnit;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
import org.apache.beam.sdk.state.BagState;
import org.apache.beam.sdk.state.CombiningState;
import org.apache.beam.sdk.state.State;
@egalpin
egalpin / JsonMutationFn.java
Last active February 23, 2024 00:51
JsonMutationFn using Beam and Jackson Databind
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.ArrayList;
import java.util.List;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
/**
* This DoFn is mostly used to rename json fields, but is flexible enough, thanks to json pointers,
@egalpin
egalpin / RateLimitTest.java
Created May 24, 2023 21:26
Tests for RateLimit.java
import static org.junit.Assert.assertTrue;
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PeriodicImpulse;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.Values;