Scripts for updating Godot demo assets

create-zips.sh

#!/usr/bin/bash

# License: MIT, 2020 Bojidar Marinov.
# Feel free to modify to suit your needs (e.g. use existing clone, make a worktree, etc.)
# Ensure that the repository does not contain any spurious files, by doing e.g. git clean -fxd 

git clone https://github.com/godotengine/godot-demo-projects --branch=3.1 --depth 1

cd godot-demo-projects
 
find . -name 'project.godot' | sed -E 's|/project.godot|/|' | xargs -n 1 -i bash -c 'cd `dirname {}`; rm -f `basename {}`.zip; zip -r `basename {}`.zip `basename {}`/'

mkdir ../zips

find . -name '*.zip' | sed -E 's|\./(.+)|\1|;p;s|/|_|;s|(.+)|../zips/\1|' | xargs -n 2 mv

cd ../

Update assets

Scripts for updating Godot demo assets

LICENSE:

Copyright (c) 2020 Bojidar Marinov

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

update-assets.py

# License: MIT, 2020 Bojidar Marinov
# Example full usage:
# ./create-zips.sh

# cd zips; sha256sum *.zip > ../demos.checksums; cd ../

# nano demos.remaps
# # Format is: <old name> <new name>, for both .zip-s and paths
# #    visual_script/visual_pong visual_script/pong
# #    visual_script_visual_pong.zip visual_script_pong.zip
# # To mark an asset of removal, use <old path> --
# #    2d/kinematic_collision --

# rm edits.txt creates.txt # Clean up old data (backups don't hurt)

# # Interactively write data for new assets (without submitting them yet):
# python ./update-assets.py --api https://godotengine.org/asset-library/api --tag <new tag> --godot-version <version> --checksums demos.checksums --remaps demos.remaps --create edits.txt --create-storage creates.txt --dry-run
# # Note: To make image urls for icons and previews, make a "New Issue" on GitHub, upload the images, and copy the links from there

# # Submit edits for updates and creations in bulk:
# python ./update-assets.py --api https://godotengine.org/asset-library/api --tag <new tag> --godot-version <version> --checksums demos.checksums --remaps demos.remaps --update edits.txt --create edits.txt --create-storage creates.txt

# # _If_ something's wrong, fixup edits in bulk:
# python ./update-assets.py --api https://godotengine.org/asset-library/api --tag <new tag> --godot-version <version> --checksums demos.checksums --remaps demos.remaps --fixup edits.txt

# # Accept edits in bulk (and remove old):
# python ./update-assets.py --api https://godotengine.org/asset-library/api --tag <new tag> --godot-version <version> --checksums demos.checksums --remaps demos.remaps --accept edits.txt --delete
# # This will output an SQL query which should be run for moving the assets to the "Godot Engine" user and setting them to "Official" directly

# # Check that all assets can be downloaded and verified properly
# python ./update-assets.py --api https://godotengine.org/asset-library/api --tag <new tag> --checksums demos.checksums --remaps demos.remaps --check --check-download

# Note that you can run all those ./update-assets.py commands at once, by passing --update edits.txt --create edits.txt --create-storage creates.txt --accept edits.txt --check --check-download

import argparse
import hashlib
import json
import os
import readline
import requests
import sys
from urllib.parse import unquote
from cachecontrol import CacheControl

#  █████  ██████   ██████  ███████
# ██   ██ ██   ██ ██       ██
# ███████ ██████  ██   ███ ███████
# ██   ██ ██   ██ ██    ██      ██
# ██   ██ ██   ██  ██████  ███████

parser = argparse.ArgumentParser(description='Create demo project assets, update demo project assets, and bulk-accept edits')

parser.add_argument('--api', type=str, metavar='URL', default='https://godotengine.org/asset-library/api', help='the api to use')
parser.add_argument('--tag', type=str, help='the target godot-demo-projects tag, the release for which should contain the zips of all the projects')
parser.add_argument('--godot-version', type=str, help='the target godot-demo-projects version')
parser.add_argument('--repo', type=str, metavar='URL',default='https://github.com/godotengine/godot-demo-projects', help='the godot-demo-projects repository (this script expects GitHub repositories)')
parser.add_argument('--old-repo', type=str, metavar='URL',help='the old godot-demo-projects repository')
parser.add_argument('--filter-user', type=str, default='Godot%20Engine', metavar='USER',help='the asset library username to use for filtering')
parser.add_argument('--checksums', type=argparse.FileType('r'), metavar='FILE', help='file to read checksums from')
parser.add_argument('--remaps', type=argparse.FileType('r'), metavar='FILE', help='file to read remaps from')
parser.add_argument('--update', type=argparse.FileType('a'), metavar='FILE', help='creates edits for existing assets in the asset library, writes their ids in the specified file')
parser.add_argument('--delete', action='store_true', help='delete assets which have become obsolete')
parser.add_argument('--fixup', type=argparse.FileType('r'), metavar='FILE', help='fixes edits listed in the specified file')
parser.add_argument('--create', type=argparse.FileType('a'), metavar='FILE', help='creates edits for new demo projects in the asset library, writes their ids in the specified file')
parser.add_argument('--create-storage', type=argparse.FileType('r+'), metavar='FILE', help='sets a file to which create data is persisted')
parser.add_argument('--accept', type=argparse.FileType('r'), metavar='FILE', help='approve edits listed in the specified file')
parser.add_argument('--check', action='store_true', help='check existing demo project assets (Note: when chaining this with --create, it might not pick up new assets)')
parser.add_argument('--check-download', action='store_true', help='download assets to verify that they download properly')
parser.add_argument('--dry-run', action='store_true', help='run a dry run')
parser.add_argument('--yes', action='store_true', help='do not ask questions')

args = parser.parse_args()

args.old_repo = args.old_repo or args.repo

session = CacheControl(requests.session())

token = '<token>'
if not args.dry_run and (args.update or args.fixup or args.accept or args.create):
    envvar = args.api.replace('http://', '').replace('https://', '').replace('-', '_').replace('.', '_').replace('/', '_').upper().strip('_')
    token = os.environ.get(envvar, None)
    if token is None:
        if args.yes:
            raise RuntimeError('Expected to find a token in ' + envvar)
        else:
            token = input('Please enter an access token for the asset library API at ' + args.api + ':\n\t\033[8m')
            print('\033[m')
    token = unquote(token)

checksums = None
if args.checksums:
    checksums = {}
    for line in args.checksums:
        parts = line.split('  ')
        checksums[parts[1].strip()] = parts[0].strip()

remaps = None
if args.remaps:
    remaps = {}
    back_remaps = {}
    for line in args.remaps:
        parts = line.split(' ')
        remaps[parts[0].strip()] = parts[1].strip()

def compute_data(asset):
    version = asset['version_string']
    if version == args.tag:
        return None
    if not asset['download_commit'].startswith(args.old_repo) or not asset['browse_url'].startswith(args.old_repo):
        return None
    download_commit_suffix = asset['download_commit'].replace(args.old_repo + '/releases/download/' + version + '/', '')
    if remaps:
        download_commit_suffix = remaps.get(download_commit_suffix, download_commit_suffix)
    browse_url_suffix = asset['browse_url'].replace(args.old_repo + '/tree/' + version + '/', '')
    if remaps:
        browse_url_suffix = remaps.get(browse_url_suffix, browse_url_suffix)
    if download_commit_suffix == '--' or browse_url_suffix == '--':
        return False

    return {
        'version_string': args.tag,
        'godot_version': args.godot_version,
        'download_provider': 'Custom',
        'download_commit': args.repo + '/releases/download/' + args.tag + '/' + download_commit_suffix,
        'browse_url': args.repo +  '/tree/' + args.tag + '/' + browse_url_suffix,
    }

assets_to_check = None
if args.delete or args.create or args.update:
    assets_to_check = session.get(args.api + '/asset?type=any&godot_version=any&user=' + args.filter_user + '&max_results=500').json()

    print('Got', assets_to_check['total_items'], 'existing demo project assets')
    if len(assets_to_check['result']) != assets_to_check['total_items']:
        raise NotImplementedError('Did not expect results spanning multiple pages')

# ██████  ███████ ██      ███████ ████████ ███████
# ██   ██ ██      ██      ██         ██    ██
# ██   ██ █████   ██      █████      ██    █████
# ██   ██ ██      ██      ██         ██    ██
# ██████  ███████ ███████ ███████    ██    ███████

if args.delete:
    done = 0

    for asset_bare in assets_to_check['result']:
        asset_id = asset_bare['asset_id']
        asset = session.get(args.api + '/asset/' + asset_id).json()

        if compute_data(asset) != False:
            continue

        print('Deleting asset', asset_id, '-', asset_bare['title'])

        data = {}
        print('\tPOST', args.api + '/asset/' + asset_id + '/delete', data)

        if not args.dry_run:
            data['token'] = token
            response = session.post(args.api + '/asset/' + asset_id + '/delete', data=data, headers={}).json()
            print('\t\t', json.dumps(response))
        done += 1

    print('Deleted', done, 'old assets')

# ██    ██ ██████  ██████   █████  ████████ ███████
# ██    ██ ██   ██ ██   ██ ██   ██    ██    ██
# ██    ██ ██████  ██   ██ ███████    ██    █████
# ██    ██ ██      ██   ██ ██   ██    ██    ██
#  ██████  ██      ██████  ██   ██    ██    ███████

if args.update:
    skipped = 0
    done = 0

    for asset_bare in assets_to_check['result']:
        asset_id = asset_bare['asset_id']
        print('Updating asset', asset_id, '-', asset_bare['title'])
        asset = session.get(args.api + '/asset/' + asset_id).json()

        data = compute_data(asset)
        if data is None:
            print('\tNothing to update')
            skipped += 1
            continue
        if data == False:
            print('\tAsset has to be deleted')
            skipped += 1
            continue

        print('\tPOST', args.api + '/asset/' + asset_id, data)

        if not args.dry_run:
            data['token'] = token
            response = session.post(args.api + '/asset/' + asset_id, data=data, headers={}).json()
            print('\t\t', json.dumps(response))
            print(response['id'], file=args.update)
        done += 1

    print('Updated', done, 'assets, skipping', skipped)
    args.update.close()

# ███████ ██ ██   ██ ██    ██ ██████
# ██      ██  ██ ██  ██    ██ ██   ██
# █████   ██   ███   ██    ██ ██████
# ██      ██  ██ ██  ██    ██ ██
# ██      ██ ██   ██  ██████  ██

if args.fixup:

    done = 0
    skipped = 0
    for edit_id in args.fixup:
        edit_id = edit_id.strip()
        print('Fixing edit', edit_id)
        edit = session.get(args.api + '/asset/edit/' + edit_id).json()
        asset_id = edit['asset_id']
        asset = edit['original']
        modifications = compute_data(asset)
        if modifications is None:
            print('\tNothing to fix')
            skipped += 1
            continue

        data = edit
        for k, v in modifications.items():
            edit[k] = v
        edit.pop('previews')
        edit.pop('original')

        print('\tPOST', args.api + '/asset/edit/' + edit_id, data)

        if not args.dry_run:
            data['token'] = token
            response = session.post(args.api + '/asset/edit/' + edit_id, data=data, headers={}).json()
            print('\t\t', json.dumps(response))
            if response['id'] != edit_id:
                print('\tSomething went wrong...')
        done += 1
    print('Fixed', done, 'edits, skipping', skipped)
    args.fixup.close()



#  ██████ ██████  ███████  █████  ████████ ███████
# ██      ██   ██ ██      ██   ██    ██    ██
# ██      ██████  █████   ███████    ██    █████
# ██      ██   ██ ██      ██   ██    ██    ██
#  ██████ ██   ██ ███████ ██   ██    ██    ███████


if args.create:
    if not checksums:
        raise RuntimeError('--create requires --checksums!')

    found = {}

    for asset_bare in assets_to_check['result']:
        asset_id = asset_bare['asset_id']
        asset = session.get(args.api + '/asset/' + asset_id).json()

        download_commit_suffix = asset['download_commit'].replace(args.old_repo + '/releases/download/' + asset['version_string'] + '/', '')
        download_commit_suffix = remaps.get(download_commit_suffix, download_commit_suffix)

        found[download_commit_suffix] = True

    previously_stored = {}
    if args.create_storage:
        for line in args.create_storage:
            asset = json.loads(line)

            download_commit_suffix = asset['download_commit'].replace(args.old_repo + '/releases/download/' + asset['version_string'] + '/', '')

            previously_stored[download_commit_suffix] = asset

    done = 0
    skipped = 0

    for filename in checksums.keys():
        if not found.get(filename, False):
            print('Found missing asset', filename)
            if not args.yes and input('Create a new edit for it [Y/n]? ').strip().lower().startswith('n'):
                skipped += 1
                continue

            title = filename.replace('.zip', '').replace('misc_', '').replace('networking_', '').replace('viewport_', '').replace('_', ' ').title() + ' Demo'

            data = {
                'title': title,
                'version_string': args.tag,
                'category_id': '10',
                'godot_version': args.godot_version,
                'cost': 'MIT',
                'description': title,
                'support_level': 'official',
                'download_provider': 'Custom',
                'download_commit': args.repo + '/releases/download/' + args.tag + '/' + filename,
                'browse_url': args.repo + '/tree/' + args.tag + '/' + filename.replace('.zip', '').replace('_', '/', 1),
                'issues_url': args.repo + '/issues',
                'icon_url': '',
            }
            edit = True
            existing = previously_stored.get(filename, None)
            if existing:
                if args.yes:
                    data = existing
                    edit = False
                else:
                    prompt = input('Reuse stored data [Y/r/n]? ').strip().lower()
                    if prompt.startswith('n'):
                        pass
                    elif prompt.startswith('r'):
                        data = existing
                    else:
                        data = existing
                        edit = False
            elif args.yes:
                skipped += 1
                continue

            if edit:
                data['title'] = input('\tTitle (' + data['title'] + '): ') or data['title']
                input_description = ''
                while true:
                    description_line = input('\tDescription (' + data['description'] + ') (empty to stop): ')
                    if not description_line:
                        break
                    data['description'] = '..'
                    input_description += description_line + '\n'
                data['description'] = input_description or data['description']
                data['icon_url'] = input('\tIcon (' + (data['icon_url'] or 'empty to cancel') + '): ') or data['icon_url']
                if not data['icon_url']:
                    skipped += 1
                    continue

                preview_i = 0
                while True:
                    preview_i += 1
                    if data.get('previews[%d][enabled]' % preview_i, False):
                        continue
                    preview_image = input('\tPreview %d image (empty to stop): ' % preview_i)
                    if not preview_image:
                        break
                    preview_thumb = input('\tPreview %d thumbnail (empty to stop): ' % preview_i)
                    if not preview_thumb:
                        break
                    data['previews[%d][enabled]' % preview_i] = True
                    data['previews[%d][operation]' % preview_i] = 'insert'
                    data['previews[%d][type]' % preview_i] = 'image'
                    data['previews[%d][link]' % preview_i] = preview_image
                    data['previews[%d][thumbnail]' % preview_i] = preview_thumb

                if args.create_storage:
                    print(json.dumps(data), file=args.create_storage)

            print('\tPOST', args.api + '/asset', json.dumps(data))

            if not args.dry_run:
                data['token'] = token
                response = session.post(args.api + '/asset', data=data, headers={}).json()
                print('\t\t', json.dumps(response))
                print(response['id'], file=args.create)
            done += 1
    print('Created', done, 'assets, skipping', skipped)
    args.create.close()


#  █████   ██████  ██████ ███████ ██████  ████████
# ██   ██ ██      ██      ██      ██   ██    ██
# ███████ ██      ██      █████   ██████     ██
# ██   ██ ██      ██      ██      ██         ██
# ██   ██  ██████  ██████ ███████ ██         ██

created_assets = None

if args.accept:
    if not checksums:
        raise RuntimeError('--accept requires --checksums!')

    done = 0
    created_assets = set()
    for edit_id in args.accept:
        edit_id = edit_id.strip()
        print('Approving edit', edit_id)
        edit = session.get(args.api + '/asset/edit/' + edit_id).json()

        review_data = {}
        print('\tPOST', args.api + '/asset/edit/' + edit_id + '/review', review_data)
        if not args.dry_run:
            review_data['token'] = token
            response = session.post( args.api + '/asset/edit/' + edit_id + '/review', data=review_data, headers={}).json()
            print('\t\t', json.dumps(response))

        download_commit_suffix = (edit['download_commit'] or edit['original']['download_commit']).replace(args.repo + '/releases/download/' + (edit['version_string'] or edit['original']['version_string']) + '/', '')
        accept_data = {
            'hash': checksums[download_commit_suffix],
        }

        print('\tPOST', args.api + '/asset/edit/' + edit_id + '/accept', accept_data)
        if not args.dry_run:
            accept_data['token'] = token
            response = session.post( args.api + '/asset/edit/' + edit_id + '/accept', data=accept_data, headers={}).json()
            print('\t\t', json.dumps(response))

            if edit['asset_id'] == '-1':
                created_assets.add(response['id'])
        done += 1
    print('Accepted', done, 'assets')
    args.accept.close()

#  ██████ ██   ██ ███████  ██████ ██   ██
# ██      ██   ██ ██      ██      ██  ██
# ██      ███████ █████   ██      █████
# ██      ██   ██ ██      ██      ██  ██
#  ██████ ██   ██ ███████  ██████ ██   ██

if created_assets:
    print('Please use the following query to update the user and support level for the newly-created demos:')
    print('\n\tUPDATE `as_assets` SET user_id = ( SELECT user_id FROM `as_users` WHERE username="' + unquote(args.filter_user).replace('\\', '\\\\').replace('"', '\\"') + '" ), support_level=2 WHERE asset_id in ( ' + ', '.join(sorted(created_assets)) + ' )\n')

if args.check:
    if created_assets:
        input('Then, press enter to continue')

    assets_to_check = session.get(args.api + '/asset?type=any&godot_version=any&user=' + args.filter_user + '&max_results=500').json()

    print('Got', assets_to_check['total_items'], 'demo project assets')
    if len(assets_to_check['result']) != assets_to_check['total_items']:
        raise NotImplementedError('Did not expect results spanning multiple pages')

    fixup_queries = []

    skipped = 0
    failed = 0
    passed = 0

    for asset_bare in assets_to_check['result']:
        asset_id = asset_bare['asset_id']
        print('Checking asset', asset_id, '-', asset_bare['title'])

        try:
            asset = session.get(args.api + '/asset/' + asset_id).json()

            if asset['version_string'] != args.tag:
                raise Exception('Invalid version')


            if checksums:
                download_commit_suffix = asset['download_commit'].replace(args.repo + '/releases/download/' + asset['version_string'] + '/', '')
                if asset['download_hash'] != checksums[download_commit_suffix]:
                    fixup_queries += ['UPDATE `as_assets` SET `download_hash` = "' + checksums[download_commit_suffix].replace('\\', '\\\\').replace('"', '\\"') + '" WHERE `asset_id` = ' + asset_id]
                    raise Exception('Invalid checksum')


            if args.check_download:
                if not asset['download_commit'].startswith(args.repo) or not asset['browse_url'].startswith(args.repo) or asset['download_hash'] is None:
                    print('\tNothing to validate')
                    skipped += 1
                    continue

                file_response = session.get(asset['download_url'])
                hash = hashlib.sha256()
                for chunk in file_response.iter_content(chunk_size=1024):
                    hash.update(chunk)
                checksum = hash.hexdigest()

                if checksum != asset['download_hash']:
                    fixup_queries += ['UPDATE `as_assets` SET `download_hash` = "' + checksum.replace('\\', '\\\\').replace('"', '\\"') + '" WHERE `asset_id` = ' + asset_id]
                    raise Exception('Invalid hash')
        except Exception as e:
            failed += 1
            print('\tFailed validating asset (', e, ')')
        else:
            passed += 1
            print('\tSuccessfully validated asset')

    if fixup_queries:
        print('Please use the following query to fix the checksums of failed demos:')
        print('\n\t' + ';\n\t'.join(fixup_queries) + ';\n')

    print('Validated', passed, ', failed validating ', failed, ', and skipped', skipped, 'assets')