Skip to content

Instantly share code, notes, and snippets.

Ryan Blue rdblue

Block or report user

Report or block rdblue

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
View WriteToDataSourceV2.scala
/**
* The base physical plan for writing data into data source v2.
*/
abstract class V2TableWriteExec(
options: Map[String, String],
query: SparkPlan) extends SparkPlan {
import org.apache.spark.sql.sources.v2.DataSourceV2Implicits._
def partitioning: Seq[PartitionTransform]
override def children: Seq[SparkPlan] = Seq(query)
@rdblue
rdblue / DataFrameWriterV2.scala
Created May 22, 2019
Prototype DataFrameWriter for v2 tables
View DataFrameWriterV2.scala
/**
* Interface used to write a [[Dataset]] to external storage using the v2 API.
*
* @since 3.0.0
*/
@Experimental
final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
extends CreateTableWriter[T] with LookupCatalog {
import org.apache.spark.sql.catalog.v2.CatalogV2Implicits._
@rdblue
rdblue / DSv2Analysis.scala
Last active Sep 19, 2018
DataSourceV2 SQL Analysis Rules
View DSv2Analysis.scala
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
View YarnPySparkSuite.scala
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@rdblue
rdblue / Comparators.java
Created Nov 9, 2017
Java Comparators for CharSequence and ByteBuffer
View Comparators.java
import java.nio.ByteBuffer;
import java.util.Comparator;
public class Comparators {
public static Comparator<ByteBuffer> unsignedBytes() {
return UnsignedByteBufComparator.INSTANCE;
}
public static Comparator<ByteBuffer> signedBytes() {
return Comparator.naturalOrder();
@rdblue
rdblue / RefinableView.java
Created Apr 2, 2015
Column projection methods
View RefinableView.java
/**
* Creates a copy of this {@code View} for entities of the given type class.
*
* @param type a Java Class to use for entities
* @return a copy of this view for the given type
* @throws org.kitesdk.data.IncompatibleSchemaException
* If the schema for {@code type} is incompatible with the
* underlying dataset's Schema.
*/
<T> RefinableView<T> asType(Class<T> type);
@rdblue
rdblue / TestProvidedConfiguration.java
Created Sep 12, 2014
Tests that hdfs-site.xml is correctly loaded in the master branch.
View TestProvidedConfiguration.java
/*
* Copyright 2013 Cloudera Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@rdblue
rdblue / gist:ca14f2a469f36d7f012c
Created May 17, 2014
Movie schema with column mappings.
View gist:ca14f2a469f36d7f012c
{
"type" : "record",
"name" : "Movie",
"doc" : "Schema generated by Kite",
"fields" : [ {
"name" : "id",
"type" : "long"
}, {
"name" : "title",
"type" : [ "null", "string" ]
@rdblue
rdblue / TestAvroSchemaConverter.java
Last active Aug 29, 2015
Test to catch parquet-mr #12
View TestAvroSchemaConverter.java
@Test
public void testArrayOfRecords() throws Exception {
Schema innerRecord = Schema.createRecord("InnerRecord", null, null, false);
Schema optionalString = optional(Schema.create(Schema.Type.STRING));
innerRecord.setFields(Lists.newArrayList(
new Schema.Field("s1", optionalString, null, NullNode.getInstance()),
new Schema.Field("s2", optionalString, null, NullNode.getInstance())
));
Schema schema = Schema.createRecord("HasArray", null, null, false);
schema.setFields(Lists.newArrayList(
You can’t perform that action at this time.