Skip to content

Instantly share code, notes, and snippets.

@ideodora
Last active August 29, 2015 14:27
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ideodora/ab6e0c4bd5a6edb92706 to your computer and use it in GitHub Desktop.
Save ideodora/ab6e0c4bd5a6edb92706 to your computer and use it in GitHub Desktop.
cloud strageのバックアップからcsvに変換
# -*- coding: utf-8 -*-
# referenced at:
# http://gbayer.com/big-data/app-engine-datastore-how-to-efficiently-export-your-data/
# pre required:
# using cloud console to backup datastore into cloud strage then
# gsutil -m cp -R gs://your_bucket_name/your_path /local_target
# to download cloud strage backupdata
# import sys
# sys.path.append('{where your google cloud sdk path}/google-cloud-sdk/platform/google_appengine/')
from google.appengine.api.files import records
from google.appengine.datastore import entity_pb
from google.appengine.api import datastore
import google.appengine.api.datastore_types
import glob
import csv
for filename in glob.glob('datastore_backup_datastore_backup_YYYY_MM_DD_*'):
if not filename.endswith("csv"):
print filename
with open(filename + ".csv", 'w') as f:
ws = csv.writer(f)
try:
raw = open(filename, 'r')
reader = records.RecordsReader(raw)
for i, record in enumerate(reader):
entity_proto = entity_pb.EntityProto(contents=record)
entity = datastore.Entity.FromPb(entity_proto)
if i == 0:
ws.writerow(entity.keys())
list_data = []
for k, v in entity.items():
if isinstance(v, google.appengine.api.datastore_types.Blob):
list_data.append("binary data...")
elif isinstance(v, basestring):
list_data.append("%s" % v.encode('utf-8', 'replace'))
else:
list_data.append("%s" % v)
ws.writerow(list_data)
print "end of try"
except Exception, e:
print "exception occur?"
print e
print "end file"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment