Last active
June 11, 2017 20:32
-
-
Save alexvanboxel/902099911d86b6827c8ea07f4e1437d4 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package backup; | |
import com.google.api.services.bigquery.model.TableReference; | |
import com.google.api.services.bigquery.model.TableRow; | |
import org.apache.beam.sdk.io.gcp.bigquery.TableDestination; | |
import org.apache.beam.sdk.transforms.SerializableFunction; | |
import org.apache.beam.sdk.values.ValueInSingleWindow; | |
import org.joda.time.format.DateTimeFormat; | |
import org.joda.time.format.DateTimeFormatter; | |
public class TableRefPartition implements SerializableFunction<ValueInSingleWindow<TableRow>, TableDestination> { | |
private final String projectId; | |
private final String datasetId; | |
private final String pattern; | |
private final String table; | |
public static TableRefPartition perDay(String projectId, String datasetId, String tablePrefix) { | |
return new TableRefPartition(projectId, datasetId, "yyyyMMdd", tablePrefix + "$"); | |
} | |
private TableRefPartition(String projectId, String datasetId, String pattern, String table) { | |
this.projectId = projectId; | |
this.datasetId = datasetId; | |
this.pattern = pattern; | |
this.table = table; | |
} | |
@Override | |
public TableDestination apply(ValueInSingleWindow<TableRow> input) { | |
DateTimeFormatter partition = DateTimeFormat.forPattern(pattern).withZoneUTC(); | |
TableReference reference = new TableReference(); | |
reference.setProjectId(this.projectId); | |
reference.setDatasetId(this.datasetId); | |
reference.setTableId(table + input.getWindow().maxTimestamp().toString(partition)); | |
return new TableDestination(reference, null); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment