jsonkeeper_migrate.py

""" Migrate JSON documents from JSONkeeper 2017-12 to the most recent version
"""

import configparser
import firebase_admin
import json
import os
import requests
import sys
from firebase_admin import auth as firebase_auth
from sqlalchemy import (Column, Table, Integer, ForeignKey, UniqueConstraint,
                        String, UnicodeText, DateTime, create_engine, desc)
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base

# - - - - - - - - - - - - - - - - - - - -

# S E T T I N G S

upload_url = 'http://127.0.0.1:5000/api'
unlisted = 'false'
firebase_api_key = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'

# U S A G E

# - place file in JSONkeeper root folder (same location as jsonkeeper.py and config.ini)
# - adjust above settings
# - run python3 jsonkeeper_migrate.py

# - - - - - - - - - - - - - - - - - - - -

config = configparser.ConfigParser()
config.read('config.ini')
if 'environment' not in config.sections():
    print('Config file needs a [environment] section.')
    sys.exit(1)
elif 'db_uri' not in config['environment'] or \
     'server_url' not in config['environment']:
    print(('Config section [environment] needs parameters "db_uri" and "server'
           '_url".'))
    sys.exit(1)
if 'firebase' in config.sections() and \
       'service_account_key_file' in config['firebase']:
    key_file_path = config['firebase']['service_account_key_file']
    cred = firebase_admin.credentials.Certificate(key_file_path)
    firebase_admin.initialize_app(cred)
else:
    print(('Config section [firebase] needs a parameter "service_account_key_f'
           'ile".'))
    sys.exit(1)

Base = declarative_base()

class JSON_document(Base):
    __tablename__ = 'JSON_document'
    id = Column(String(64), primary_key=True)
    access_token = Column(String(255))
    json_string = Column(UnicodeText())
    created_at = Column(DateTime())
    updated_at = Column(DateTime())

engine = create_engine(config['environment']['db_uri'])
Base.metadata.create_all(engine)
Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
session = DBSession()

users_pre = firebase_auth.list_users()
users = [u for u in users_pre.users]
firebase_user_ids = [u.uid for u in users]

docs = session.query(JSON_document).all()
for doc in docs:
    headers = {'Accept': 'application/json'}
    # access token
    if doc.access_token == '':
        print('doc w/o access_token, ignoring')
        # skip
        continue
    elif doc.access_token in firebase_user_ids:
        print('doc w/ firebase token')
        # get ID token for user
        ct = firebase_auth.create_custom_token(doc.access_token)
        body = {'token' : ct.decode(), 'returnSecureToken' : True}
        params = {'key' : firebase_api_key}
        _verify_token_url = ('https://www.googleapis.com/identitytoolkit/v3/re'
                             'lyingparty/verifyCustomToken')
        resp = requests.request('post',
                                _verify_token_url,
                                params=params,
                                json=body)
        # set header
        headers['X-Firebase-ID-Token'] = resp.json().get('idToken')
    else:
        print('doc w/ freely chosen token')
        # set header
        headers['X-Access-Token'] = doc.access_token

    # json or json-ld
    json_doc = json.loads(doc.json_string)
    if type(json_doc) == dict and '@context' in json_doc:
        # assume it's JSON-LD
        headers['Content-Type'] = 'application/ld+json'
    else:
        headers['Content-Type'] = 'application/json'

    # unlisted
    headers['X-Unlisted'] = unlisted
    print('uploading ...')
    resp = requests.post(upload_url,
                         headers=headers,
                         data=doc.json_string.encode('utf-8'))
    print('{}'.format(resp.status_code))
    if resp.status_code not in [100, 200, 201]:
        print('{}'.format(resp.text))