Created
August 29, 2021 06:38
-
-
Save amitgupta1202/6266ae38074c35180c8b978162ba0cf7 to your computer and use it in GitHub Desktop.
snippet to write the avro records to big query
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
//SomeRecord is generated via avro code generation, instance of SpecificRecord | |
var tableSchema = AvroToTableSchemaConverter.convert({SomeRecord}.SCHEMA$); | |
BigQueryIO.<SomeRecord>write() | |
.to(tableName.getValue()) | |
.withSchema("some-table-name") | |
.useAvroLogicalTypes() | |
.withAvroSchemaFactory(new SerializableFunction<TableSchema, Schema>() { | |
@Override | |
public Schema apply(TableSchema input) { | |
return SomeRecord.SCHEMA$; | |
} | |
}) | |
.withAvroWriter(new SerializableFunction<Schema, DatumWriter<SomeRecord>>() { | |
@Override | |
public DatumWriter<SomeRecord> apply(Schema input) { | |
return new SpecificDatumWriter<>(input); | |
} | |
}) | |
.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) | |
.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment