Skip to content

Instantly share code, notes, and snippets.

View rest-scan-planning.json
POST /v1/namespaces/ns/tables/t/scans
{ "filter": { "type": "in", "term": "x", "values": [1, 2, 3] }, "select": ["x", "a.b"]}
{ "scan-id": 1, "shards": { "id": 1, "manifests": ["C"] }, { "id": 2, "manifests": ["D"] } }
POST /v1/namespaces/ns/tables/t/scans?scan-id=1
{ "shard": { "id": 1, "manifests": ["C"] }, "filter": {"type": "in", "term": "x", "values": [1, 2, 3] } }
{ "file-scan-tasks": [...] }
@rdblue
rdblue / flink_iceberg_diff_1.14_to_1.15.diff
Created May 3, 2022 15:50 — forked from kbendick/flink_iceberg_diff_1.14_to_1.15.txt
Propsed Iceberg Support for Flink 1.15 Diff from Iceberg Flink 1.14
View flink_iceberg_diff_1.14_to_1.15.diff
View FlinkFilters.java
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
View SchemaWithPartnerVisitor.java
package org.apache.iceberg.types;
import java.util.List;
import org.apache.iceberg.Schema;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
public abstract class SchemaWithPartnerVisitor<P, R> {
public interface PartnerAccessors<P> {
P fieldPartner(P partnerStruct, int fieldId, String name);
P mapKeyPartner(P partnerMap);
View WriteToDataSourceV2.scala
/**
* The base physical plan for writing data into data source v2.
*/
abstract class V2TableWriteExec(
options: Map[String, String],
query: SparkPlan) extends SparkPlan {
import org.apache.spark.sql.sources.v2.DataSourceV2Implicits._
def partitioning: Seq[PartitionTransform]
override def children: Seq[SparkPlan] = Seq(query)
@rdblue
rdblue / DataFrameWriterV2.scala
Created May 22, 2019 22:21
Prototype DataFrameWriter for v2 tables
View DataFrameWriterV2.scala
/**
* Interface used to write a [[Dataset]] to external storage using the v2 API.
*
* @since 3.0.0
*/
@Experimental
final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
extends CreateTableWriter[T] with LookupCatalog {
import org.apache.spark.sql.catalog.v2.CatalogV2Implicits._
@rdblue
rdblue / DSv2Analysis.scala
Last active September 19, 2018 22:23
DataSourceV2 SQL Analysis Rules
View DSv2Analysis.scala
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
View YarnPySparkSuite.scala
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@rdblue
rdblue / Comparators.java
Created November 9, 2017 16:49
Java Comparators for CharSequence and ByteBuffer
View Comparators.java
import java.nio.ByteBuffer;
import java.util.Comparator;
public class Comparators {
public static Comparator<ByteBuffer> unsignedBytes() {
return UnsignedByteBufComparator.INSTANCE;
}
public static Comparator<ByteBuffer> signedBytes() {
return Comparator.naturalOrder();
@rdblue
rdblue / RefinableView.java
Created April 2, 2015 23:31
Column projection methods
View RefinableView.java
/**
* Creates a copy of this {@code View} for entities of the given type class.
*
* @param type a Java Class to use for entities
* @return a copy of this view for the given type
* @throws org.kitesdk.data.IncompatibleSchemaException
* If the schema for {@code type} is incompatible with the
* underlying dataset's Schema.
*/
<T> RefinableView<T> asType(Class<T> type);