Skip to content

Instantly share code, notes, and snippets.

@fdemmer
Created October 20, 2017 18:16
Show Gist options
  • Save fdemmer/b5bf8914a1c6b7e47ed1bbf08b517bdb to your computer and use it in GitHub Desktop.
Save fdemmer/b5bf8914a1c6b7e47ed1bbf08b517bdb to your computer and use it in GitHub Desktop.
Django SQLite database backend storing the db file on s3.
# -*- coding: utf-8 -*-
import logging
import os
from tempfile import gettempdir
import boto3
import botocore
from django.db.backends.sqlite3.base import DatabaseWrapper
log = logging.getLogger(__name__)
def abspath_join(*args):
return os.path.abspath(os.path.join(*args))
class DatabaseWrapper(DatabaseWrapper):
"""
Django SQLite database backend storing the db file on s3.
Inspired by:
https://blog.zappa.io/posts/s3sqlite-a-serverless-relational-database
DATABASES = {
'default': {
'ENGINE': '...s3sqlite',
'NAME': 's3-bucket-name/sqlite.db',
}
}
"""
def __init__(self, *args, **kwargs):
super(DatabaseWrapper, self).__init__(*args, **kwargs)
# create lazy handle for remote object
self.s3obj = self.get_s3_object()
# set name of local copy
self.local_name = abspath_join(gettempdir(), self.s3obj.key)
def get_s3_resource(self):
# get s3 client
signature_version = self.settings_dict.get("SIGNATURE_VERSION", "s3v4")
return boto3.resource('s3', config=botocore.client.Config(
signature_version=signature_version
))
def get_s3_object(self):
# get s3 object handle
bucket_name, key = self.settings_dict['NAME'].split('/', maxsplit=1)
return self.get_s3_resource().Object(bucket_name, key)
def get_new_connection(self, conn_params):
log.info('get_new_connection(conn_params=%s)', conn_params)
# download if necessary
self.download_database()
# point the default sqlite client/db wrapper to our local copy
conn_params['database'] = self.local_name
return super(DatabaseWrapper, self).get_new_connection(conn_params)
def download_database(self):
# download database to temp file if necessary
if not os.path.isfile(self.local_name):
log.debug('downloading database')
self.s3obj.download_file(self.local_name)
def close(self, *args, **kwargs):
log.info('close()')
super(DatabaseWrapper, self).close()
self.upload_database()
def upload_database(self):
# upload the local copy on every close
if os.path.isfile(self.local_name):
log.debug('uploading database')
self.s3obj.upload_file(self.local_name)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment