Skip to content

Instantly share code, notes, and snippets.

@alexvanboxel
Last active June 11, 2017 20:32
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save alexvanboxel/902099911d86b6827c8ea07f4e1437d4 to your computer and use it in GitHub Desktop.
Save alexvanboxel/902099911d86b6827c8ea07f4e1437d4 to your computer and use it in GitHub Desktop.
package backup;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import org.apache.beam.sdk.io.gcp.bigquery.TableDestination;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
public class TableRefPartition implements SerializableFunction<ValueInSingleWindow<TableRow>, TableDestination> {
private final String projectId;
private final String datasetId;
private final String pattern;
private final String table;
public static TableRefPartition perDay(String projectId, String datasetId, String tablePrefix) {
return new TableRefPartition(projectId, datasetId, "yyyyMMdd", tablePrefix + "$");
}
private TableRefPartition(String projectId, String datasetId, String pattern, String table) {
this.projectId = projectId;
this.datasetId = datasetId;
this.pattern = pattern;
this.table = table;
}
@Override
public TableDestination apply(ValueInSingleWindow<TableRow> input) {
DateTimeFormatter partition = DateTimeFormat.forPattern(pattern).withZoneUTC();
TableReference reference = new TableReference();
reference.setProjectId(this.projectId);
reference.setDatasetId(this.datasetId);
reference.setTableId(table + input.getWindow().maxTimestamp().toString(partition));
return new TableDestination(reference, null);
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment