dodo.py

import json
import secrets
import string
from pathlib import Path

import doit
from doit import tools
from doit.tools import Interactive, LongRunning
from fabric import Connection

from my_project.website.crawler import run_crawler
from my_project.website.utils import setup_var_directories

BASE_DIR = Path(__file__).absolute().parent
DIST_DIR = "./var/dist/"
BUILD_DIR = "./var/tmp/shiv/"

DOIT_CONFIG = {
    "default_tasks": [""],
    "backend": "sqlite3",
    # note to readers: This allows me to use a better format syntax
    # for params instead  of %(param_name)s
    "action_string_formatting": "new",
}


def test_setup_var_directories():
    return {
        "actions": [
            setup_var_directories,
        ],
    }


def task_lock_deps():
    """Lock dependancies using poetry, and export them as a requirements files"""

    return {
        "file_dep": ["pyproject.toml"],
        "actions": [
            "poetry lock",
            "poetry export --without-hashes -f requirements.txt --output requirements.txt",
            "poetry export --without-hashes --dev -f requirements.txt --output all-requirements.txt",
            "grep -Fvxf requirements.txt all-requirements.txt > dev-requirements.txt",
            "rm all-requirements.txt",
        ],
        "targets": ["requirements.txt", "dev-requirements.txt", "poetry.lock"],
    }


def task_precommit():
    """Lint, format and test before commit"""

    return {
        "file_dep": [*Path("my_project").rglob("**/*.py")],
        "actions": [
            "black my_project tests",
            "pylint my_project tests",
            "mypy my_project tests",
            "pytest tests",
        ],
    }


def task_bundle_static_files():

    bundle = str(BASE_DIR / "var/static/bundle.js")
    static_files = Path("my_project/website/static/")
    deps = [
        *static_files.rglob("**/*.js"),
        *static_files.rglob("**/*.css"),
        *Path("my_project/frontend/src/").rglob("**/*.vue"),
    ]

    def update_manifest_json():
        # Because of a bug in vite, we can't use main.js as an entry point
        # and must use a fake index.html. But a script can't load html files.
        # So we do a little hack, duplicate the manifest index.html entry, but
        # name the copy main.js

        conf = Path("var/static/manifest.json")
        data = json.loads(conf.read_text())
        data["./src/main.js"] = data["index.html"]
        conf.write_text(json.dumps(data))

    return {
        "file_dep": deps,
        "actions": [
            "mkdir -p ./var/static/",
            "rm -fr ./var/static/* ",
            "cd my_project/frontend/; npm run build",
            "python manage.py collectstatic --noinput",
            update_manifest_json,
            "cp -r ./var/static/* ",
        ],
        "targets": ["var/static/manifest.json"],
    }


def task_dump_dependencies():

    return {
        "file_dep": ["pyproject.toml"],
        "actions": [
            f"mkdir -p  {BUILD_DIR}",
            f"rm -fr {BUILD_DIR}*",
            f"pip install -r requirements.txt --target {BUILD_DIR}",
        ],
    }


def task_build_zipapp():

    return {
        "task_dep": ["bundle_static_files", "lock_deps", "dump_dependencies"],
        "actions": [
            # Make sure the directories are there and empty
            f"mkdir -p {DIST_DIR}  ",
            f"rm -fr {DIST_DIR}*",
            f"mkdir -p {BUILD_DIR}var/",
            # Put the python project, deps and static files in there
            f"cp -r var/static/ {BUILD_DIR}var/",
            f"cp -r shiv_entry_point.py manage.py my_project {BUILD_DIR} ",
            # Build the zipapp
            f"shiv --site-packages {BUILD_DIR} --compressed -p '/usr/bin/env python3' -o {DIST_DIR}my_project.pyz -e shiv_entry_point.main",
        ],
        "targets": ["{DIST_DIR}/my_project.pyz"],
        "verbosity": 2,
    }


def upload_setup_and_restart():
    with Connection("contact@domain.tld") as c:
        print("Upload pyz")
        c.put(f"{DIST_DIR}my_project.pyz", "/opt/my_project")
        print("Setup prod")

        c.run("cd /opt/my_project && source .env && python3.9 my_project.pyz migrate_prod")

        print("Restart python process")
        c.run("sudo service my_project restart", pty=True)


def task_deploy():

    return {
        "actions": [
            upload_setup_and_restart,
        ],
        "verbosity": 2,
    }


def task_build_and_deploy():
    return {
        "task_dep": ["build_zipapp"],
        "actions": [upload_setup_and_restart],
        "verbosity": 2,
    }


# note to readers: LongRunning is a less costly version of Interactive
# for processes you don't expect to interact with
def task_serve():
    return {
        "actions": [LongRunning("python manage.py runserver")],
    }


def task_vite():
    return {
        "actions": [LongRunning("cd my_project/frontend; npm run dev")],
    }


def task_crawl():

    return {
        "actions": [LongRunning(run_crawler)],
        "verbosity": 2,
    }


def task_shell():
    return {
        "actions": [Interactive("python manage.py shell_plus")],
    }


def task_load_test_users():

    return {
        "actions": [
            "python ./manage.py loaddata tests/fixtures/users.json",
        ],
    }


def task_generate_password():
    def gen():
        alphabet = string.ascii_letters + string.digits
        password = "".join(secrets.choice(alphabet) for i in range(20))
        return {"password": password}

    return {"actions": [gen]}


def task_create_pg_db():

    return {
        "actions": [
            Interactive('sudo -u postgres psql -c "create database {db}"'),
            Interactive(
                """sudo -u postgres psql -c "create user {db} with encrypted password '{password}'" """
            ),
            Interactive(
                'sudo -u postgres psql -c "grant all privileges on database {db} to {db}"'
            ),
            "echo Database '{db}' created for user '{db}' with password '{password}'",
        ],
        "params": [{"name": "db", "default": "my_project", "long": "db"}],
        "getargs": {
            "password": ("generate_password", "password"),
        },
        "verbosity": 2,
    }