Skip to content

Instantly share code, notes, and snippets.

@ramesh-lingappan
Last active May 7, 2019 08:00
Show Gist options
  • Save ramesh-lingappan/baeab33f147b39e7d064e0c92447f576 to your computer and use it in GitHub Desktop.
Save ramesh-lingappan/baeab33f147b39e7d064e0c92447f576 to your computer and use it in GitHub Desktop.
Cloud Datastore Backup Helper Service
/*
* Service class to trigger datastore backup request to new Cloud Datastore Export Service
* @see <a href="https://cloud.google.com/datastore/docs/schedule-export">Scheduled Export</a>
* This class dependents on Jackson for serialization
*/
public class DatastoreBackupService {
private static final String BACKUP_ENDPOINT = "https://datastore.googleapis.com/v1/projects/%s:export";
private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore";
private String projId;
private String gcsPath;
private List<String> entityKinds;
public DatastoreBackupService(String projId, String gcsPath, List<String> entityKinds) {
this.projId = projId;
this.gcsPath = gcsPath;
this.entityKinds = entityKinds;
}
public boolean triggerBackup() throws IOException {
if (projId == null) throw new NullPointerException("invalid project id ");
if (gcsPath == null) throw new NullPointerException("invalid gcs path");
if (entityKinds == null || entityKinds.isEmpty())
throw new IllegalArgumentException("invalid entity kinds, cannot be empty");
BackupRequest request = new BackupRequest(projId, gcsPath, new EntityFilter(entityKinds));
return makeUrlRequest(request, projId, getDefaultAccessToken());
}
private String getDefaultAccessToken() {
List<String> scopes = new ArrayList<>();
scopes.add(DATASTORE_SCOPE);
AppIdentityService appIdentity = AppIdentityServiceFactory.getAppIdentityService();
AppIdentityService.GetAccessTokenResult accessToken = appIdentity.getAccessToken(scopes);
return accessToken.getAccessToken();
}
private boolean makeUrlRequest(BackupRequest request, String projId, String accessToken) throws IOException {
URL url = new URL(String.format(BACKUP_ENDPOINT, projId));
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setDoOutput(true);
conn.setRequestMethod("POST");
conn.setConnectTimeout(60);
conn.setRequestProperty("Content-Type", "application/json");
conn.setRequestProperty("Authorization", "Bearer " + accessToken);
String payload = new ObjectMapper().writeValueAsString(request);
OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream());
writer.write(payload);
writer.close();
int respCode = conn.getResponseCode();
String responseContent = readResponse(conn);
System.out.println("response : " + responseContent);
if (respCode == HttpURLConnection.HTTP_OK) {
return true;
}
throw new IOException("trigger backup failed with error: " + responseContent);
}
private String readResponse(HttpURLConnection conn) throws IOException {
StringBuffer responseContent = new StringBuffer();
InputStream is = null;
try {
is = conn.getInputStream();
} catch (IOException e) {
if (conn.getResponseCode() != 200) {
is = conn.getErrorStream();
}
}
if (is != null) {
String line;
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
while ((line = reader.readLine()) != null) {
responseContent.append(line);
}
reader.close();
}
return responseContent.toString();
}
}
class BackupRequest {
@JsonProperty("project_id")
private String projectId;
@JsonProperty("output_url_prefix")
private String gcsPath;
@JsonProperty("entity_filter")
private EntityFilter entityFilter;
public BackupRequest(String projectId, String gcsPath, EntityFilter entityFilter) {
this.projectId = projectId;
this.gcsPath = gcsPath;
this.entityFilter = entityFilter;
}
public String getProjectId() {
return projectId;
}
public String getGcsPath() {
return gcsPath;
}
public EntityFilter getEntityFilter() {
return entityFilter;
}
}
class EntityFilter {
@JsonProperty("kinds")
private List<String> kinds;
@JsonProperty("namespace_ids")
private String nameSpaceIds;
public EntityFilter(List<String> kinds) {
this(kinds, null);
}
public EntityFilter(List<String> kinds, String nameSpaceIds) {
this.kinds = kinds;
this.nameSpaceIds = nameSpaceIds;
}
public List<String> getKinds() {
return kinds;
}
public String getNameSpaceIds() {
return nameSpaceIds;
}
}
public class BackupService {
private final String GCS_BACKUP_PATH = "gs://backup/datastore/%s/%s";
public void triggerBackup(String appId, boolean isLive) throws IOException {
DatastoreBackupService backupService = new DatastoreBackupService(appId, getBackupGcsPath(isLive), getBackupEntityKinds());
backupService.triggerBackup();
}
/*
* Generate GCS Url with date like eg: gs://backup/datastore/(prod/stag)/2018/August/16
*/
private String getBackupGcsPath(boolean isLive) {
String dateString = new SimpleDateFormat("yyyy/MMMMM/dd").format(new Date());
return String.format(GCS_BACKUP_PATH, isLive ? "prod" : "stag", dateString);
}
/*List of Datastore Entity Kinds to backup */
private List<String> getBackupEntityKinds() {
List<String> kinds = new ArrayList<>();
kinds.add("Entity1");
kinds.add("Entity2");
return kinds;
}
/*demo for calling this backup service */
public static void main() {
new BackupService().triggerBackup("gae-proj-id", true);
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment