Skip to content

Instantly share code, notes, and snippets.

@rdblue
rdblue / flink_iceberg_diff_1.14_to_1.15.diff
Created May 3, 2022 — forked from kbendick/flink_iceberg_diff_1.14_to_1.15.txt
Propsed Iceberg Support for Flink 1.15 Diff from Iceberg Flink 1.14
View flink_iceberg_diff_1.14_to_1.15.diff
View FlinkFilters.java
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
View SchemaWithPartnerVisitor.java
package org.apache.iceberg.types;
import java.util.List;
import org.apache.iceberg.Schema;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
public abstract class SchemaWithPartnerVisitor<P, R> {
public interface PartnerAccessors<P> {
P fieldPartner(P partnerStruct, int fieldId, String name);
P mapKeyPartner(P partnerMap);
View WriteToDataSourceV2.scala
/**
* The base physical plan for writing data into data source v2.
*/
abstract class V2TableWriteExec(
options: Map[String, String],
query: SparkPlan) extends SparkPlan {
import org.apache.spark.sql.sources.v2.DataSourceV2Implicits._
def partitioning: Seq[PartitionTransform]
override def children: Seq[SparkPlan] = Seq(query)
@rdblue
rdblue / DataFrameWriterV2.scala
Created May 22, 2019
Prototype DataFrameWriter for v2 tables
View DataFrameWriterV2.scala
/**
* Interface used to write a [[Dataset]] to external storage using the v2 API.
*
* @since 3.0.0
*/
@Experimental
final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
extends CreateTableWriter[T] with LookupCatalog {
import org.apache.spark.sql.catalog.v2.CatalogV2Implicits._
@rdblue
rdblue / DSv2Analysis.scala
Last active Sep 19, 2018
DataSourceV2 SQL Analysis Rules
View DSv2Analysis.scala
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
View YarnPySparkSuite.scala
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@rdblue
rdblue / Comparators.java
Created Nov 9, 2017
Java Comparators for CharSequence and ByteBuffer
View Comparators.java
import java.nio.ByteBuffer;
import java.util.Comparator;
public class Comparators {
public static Comparator<ByteBuffer> unsignedBytes() {
return UnsignedByteBufComparator.INSTANCE;
}
public static Comparator<ByteBuffer> signedBytes() {
return Comparator.naturalOrder();
@rdblue
rdblue / RefinableView.java
Created Apr 2, 2015
Column projection methods
View RefinableView.java
/**
* Creates a copy of this {@code View} for entities of the given type class.
*
* @param type a Java Class to use for entities
* @return a copy of this view for the given type
* @throws org.kitesdk.data.IncompatibleSchemaException
* If the schema for {@code type} is incompatible with the
* underlying dataset's Schema.
*/
<T> RefinableView<T> asType(Class<T> type);
@rdblue
rdblue / TestProvidedConfiguration.java
Created Sep 12, 2014
Tests that hdfs-site.xml is correctly loaded in the master branch.
View TestProvidedConfiguration.java
/*
* Copyright 2013 Cloudera Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software