Python project autogen.sh

.editorconfig

# Project editorconfig file.
root = true

# Unix-style newlines with a newline ending every file
[*]
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
insert_final_newline = true
max_line_length = 120

[*.{js,py}]
charset = utf-8

[*.py]
indent_style = space
indent_size = 4

.flake8

[flake8]
exclude = .vscode,.idea,.git,__pycache__,old,build,dist,venv,stubs,venv.*,migrations
max-line-length = 160

ignore = E231,C812,C819,E203,SC200,U101,R504,VNE001,D415,D202,D105

per-file-ignores =
    **/*.pyi:U100,D100,D101,D102,D103,E,Y022,Y027,Y037
    setup.py:D100,D101
    **/*/__init__.py:F401
    **/*_model.py:C408
    tests/**/test_*.py:D100,S101,U100
    tests/**/__init__.py:D104
    app/typeutil.py:ECE001

# flake8-formatter-abspath
format = abspath

# flake8-coding
no-accept-encodings = True

# flake8-docstrings
docstring-convention = google

# flake8-spellcheck
dictionaries=en_US,python,technical,django

# flake8-variables-names
use-varnames-strict-mode = True

# flake8-kw-args
max-pos-args = 3

# flake8-requirements
requirements-max-depth=2

# flake8-sfs
extend-ignore =
    # Ignore f-strings
    SFS301
    # Ignore format method
    SFS201

# flake8-tidy-imports
# This is an example of banned-modules. Use it to enforce that certain modules
# should not be allowed by the linter.
banned-modules =
    logging = "Use the application logutil which provides structlog instead"

01virtualenv.sh

#!/bin/bash
# See: https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself
# Note: you can't refactor this out: its at the top of every script so the scripts can find their includes.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"

# Source common includes
source include.sh

requires=(
    "virtualenv"
)

check_requirements "${requires[@]}"

log "Checking for correct Python version"
if command -v pyenv ; then
  python=$(pyenv which python)
else
  if ! command -v python3.9 >/dev/null; then
    fatal 1 "Failed to find python3.9. Consider installing pyenv to set it up."
  fi
  python=$(command -v python3.9)
fi

log "Checking for virtualenv"
if [ ! -d venv ]; then
    if ! virtualenv -p $python venv ; then
        fatal 1 "Could not setup a python 3 virtual environment."
    fi
fi

log "Activating virtualenv and installing development environment"
source activate

if ! pip install -e . ; then
    fatal 1 "Failed to install the application into the virtualenv."
fi

if ! pip install -r "requirements.dev.txt" ; then
    fatal 1 "Failed to install development requirements (hooks and tests will not work properly)."
fi

exit 0

autogen.sh

#!/bin/bash
# See: https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself
# Note: you can't refactor this out: its at the top of every script so the scripts can find their includes.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
cd "${SCRIPT_DIR}" || exit 1  # This is an unlikely failure, so don't worry too much.

# Source common includes
source include.sh

# Check setup requirements
requires=(
    "git"
)

check_requirements "${requires[@]}"

log "Ensure pull request rebase."
# Note: will break for git < 1.7.9 (looking at you Centos 7 users)
log "Setup pull request rebasing"
if ! git config pull.rebase true ; then
    fatal 1 "Failed to setup git pull request rebease."
fi

log "Ensure pull-request checkout for github is active."
if ! git config --get remote.origin.fetch "\+refs/pull/\*/head:refs/remotes/origin/pr/\*" ; then
    log "Setting up automatic fetch of pull requests"
    if ! git config --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*" ; then
        fatal 1 "Failed to configure pull request fetch for origin."
    fi
fi

log "Linking hook scripts"
while read -r hookscript; do
  if [ "$(readlink -f "$hookscript")" != "$(readlink -f .git/hooks/"$(basename "$hookscript")")" ] ; then
    if ! ln -sf "$(readlink -f "$hookscript")" ".git/hooks/$(basename "$hookscript")"; then
      fatal 1 "Failed to activate repository git hooks."
    fi
  fi
done < <(find .githooks -mindepth 1 -maxdepth 1 -type f)

if [ "$(readlink -f .git/hooks)" != "$(readlink -f .githooks)" ] ; then
    if ! ln -sf "$(readlink -f .githooks)" ".git/hooks"; then
        fatal 1 "Failed to activate repository git hooks."
    fi
fi

while read -r autogen_script; do
    if ! "$autogen_script"; then
        fatal 1 "$autogen_script failed."
    fi
done < <(find autogen.d -mindepth 1 -maxdepth 1 -type f -name '*.sh' | sort)

log "Success."
exit 0

fmt

#!/bin/bash
# See: https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself
# Note: you can't refactor this out: its at the top of every script so the scripts can find their includes.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
cd "${SCRIPT_DIR}" || exit 1  # This is an unlikely failure, so don't worry too much.

source include.sh
source activate

result=0

# Format all by default.
if [ -z "$1" ]; then
    black "." || result=1
    isort "." || result=1
else
    black "$@" || result=1
    isort "." || result=1
fi

exit $result

include.sh

#!/bin/bash
# Common bash functions shared by the build scripts.

export bin_dir="$(pwd)/.bin"

# Color initialization guard due to the use of read-only bash variables.
export _colors_initialized=0
function _do_color_setup() {
    # Bash color includes from
    # https://raw.githubusercontent.com/WolfSoftware/bash-colour-include/master/src/bash-colour-include.sh

    fgBlack=''
    fgRed=''
    fgGreen=''
    fgYellow=''
    fgBlue=''
    fgMagenta=''
    fgCyan=''
    fgWhite=''

    bgBlack=''
    bgRed=''
    bgGreen=''
    bgYellow=''
    bgBlue=''
    bgMagenta=''
    bgCyan=''
    bgWhite=''

    screen_width=''
    bold=''
    dim=''
    underline=''
    stop_underline=''
    reverse=''
    cinvis=''
    cnorm=''
    bell=''
    reset=''
    cls=''

    if test -t 1; then
        # see if it supports colors...
        ncolors=$(tput colors)

        if test -n "${ncolors}" && test "${ncolors}" -ge 8; then
            fgBlack=$(tput setaf 0)
            fgRed=$(tput setaf 1)
            fgGreen=$(tput setaf 2)
            fgYellow=$(tput setaf 3)
            fgBlue=$(tput setaf 4)
            fgMagenta=$(tput setaf 5)
            fgCyan=$(tput setaf 6)
            fgWhite=$(tput setaf 7)

            bgBlack=$(tput setab 0)
            bgRed=$(tput setab 1)
            bgGreen=$(tput setab 2)
            bgYellow=$(tput setab 3)
            bgBlue=$(tput setab 4)
            bgMagenta=$(tput setab 5)
            bgCyan=$(tput setab 6)
            bgWhite=$(tput setab 7)

            screen_width=$(tput cols)
            bold=$(tput bold)
            dim=$(tput dim)
            underline=$(tput smul)
            stop_underline=$(tput rmul)
            reverse=$(tput rev)
            cinvis=$(tput civis)
            cnorm=$(tput cnorm)
            bell=$(tput bel)
            reset=$(tput sgr0)
            cls=$(tput clear)
        fi
    fi

    declare -x -r fgBlack
    declare -x -r fgRed
    declare -x -r fgGreen
    declare -x -r fgYellow
    declare -x -r fgBlue
    declare -x -r fgMagenta
    declare -x -r fgCyan
    declare -x -r fgWhite

    declare -x -r bgBlack
    declare -x -r bgRed
    declare -x -r bgGreen
    declare -x -r bgYellow
    declare -x -r bgBlue
    declare -x -r bgMagenta
    declare -x -r bgCyan
    declare -x -r bgWhite

    declare -x -r screen_width
    declare -x -r bold
    declare -x -r dim
    declare -x -r underline
    declare -x -r stop_underline
    declare -x -r reverse
    declare -x -r cinvis
    declare -x -r cnorm
    declare -x -r bell
    declare -x -r reset
    declare -x -r cls

    _colors_initialized=1
}

if [ $_colors_initialized != 1 ]; then
    _do_color_setup
fi

# Begin our includes.

function log() {
    echo "$*" 1>&2
}

function warn() {
    log "${fgYellow}WARN: $*${reset}"
}

function error() {
    log "${fgRed}ERROR: $*${reset}"
}

function success() {
    log "${fgGreen}SUCCESS: $*${reset}"
    exit 0
}

function fatal() {
    local retcode="$1"
    shift
    log "${fgRed}FATAL: $*${reset}"
    exit "$retcode"
}

# check_requirements and alias the input list to the exact paths.
function check_requirements() {
    # Alias all commands to their full paths.
    local cmd
    for p in "$@" ; do
        # Assign
        cmd="$(command -v "${p}")"
        alias "$p=${cmd}"
        # Check the command works.
        if [ ! -x "${cmd}" ]; then
            fatal 1 "Could not find required program (please install it): $p ($cmd)"
        fi
    done
    return 0
}

function is_ci() {
    if [ -n "${CODEBUILD_BUILD_ID+}" ]; then
        # AWS Codebuild CI
        return 0
    fi

    if [ -n "${CI}" ]; then
        # Generic CI
        return 0
    fi

    return 1
}

lint

#!/bin/bash
# See: https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself
# Note: you can't refactor this out: its at the top of every script so the scripts can find their includes.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
cd "${SCRIPT_DIR}" || exit 1  # This is an unlikely failure, so don't worry too much.

source include.sh
source activate

exec flake8 "$@"

pre-commit

#!/bin/bash
# Repository general commit hook.

# Basic principles: get the list of changed files so users can work with an
# inconsistent repository.

source include.sh

log "Pre-Commit"

# NOTE: None of the Python tools we have apply exclusions to their command line, and
# the bigger logic here is that CI will never make this distinction anyway. So
# we *don't* at the moment use the below list in anyway. A better solution would be
# to callout to a Python script which could parse all the different exclude specs.

files_in_commit=()
while read fileline ; do
    commit_file="$(echo "$fileline" | cut -f2-)"
    files_in_commit+=( "$commit_file" )
done < <(git diff --cached --name-only --diff-filter=ACM)

# ./fmt automatically (it's just annoying not too)
retcode=0

if ! ./fmt ; then
    error "Code-formatting failed."
    retcode=1
fi

if ! git add "${files_in_commit[@]}"; then
    fatal "Error adding modified files back to commit."
fi

if ! ./lint ; then
    error "Lint errors. Use git commit --no-verify to ignore."
    retcode=1
fi

if ! ./typecheck ; then
    error "Typechecking errors. Use git commit --no-verify to ignore."
    retcode=1
fi

if [ $retcode != 0 ]; then
    fatal $retcode "Pre-commit hooks failed."
fi

exit 0

pyproject.toml

[tool.black]
line-length = 100
target-version = ['py39']
include = '\.pyi?$'
exclude = '''

(
  /(
      \.eggs         # exclude a few common directories in the
    | \.git          # root of the project
    | \.hg
    | \.mypy_cache
    | \.tox
    | \.venv
    | venv
    | venv\..*
    | _build
    | buck-out
    | build
    | dist
    | setup\.py
    | \.ipynb
    | migrations
  )/
)

'''

[tool.isort]
profile = "black"
filter_files = true
py_version = 39
skip_gitignore = true
skip_glob = [ "*.ipynb" ]

[tool.mypy]
exclude = '''

(
  /(
      \.eggs         # exclude a few common directories in the
    | \.git          # root of the project
    | \.hg
    | \.mypy_cache
    | \.tox
    | \.venv
    | venv
    | venv\..*
    | _build
    | buck-out
    | build
    | dist
    | setup\.py
    | \.ipynb
    | migrations
  )/
)

'''

mypy_path = "stubs"
python_version = "3.9"

namespace_packages = true

disallow_any_unimported = false
disallow_any_expr = false

# Can't practically disable yet.
disallow_any_decorated = false

# Can't practically disable Any yet.
disallow_any_explicit = false

# Also can't practically disable Any in generics.
disallow_any_generics = false

# This winds up not being practical at the moment
disallow_subclassing_any = false

disallow_untyped_calls = false
disallow_untyped_defs = true
disallow_incomplete_defs = true

check_untyped_defs = true

# Need to allow for click at this time
disallow_untyped_decorators = false

no_implicit_optional = true
strict_optional = true

warn_redundant_casts = true
warn_unused_ignores = true
warn_no_return = true
warn_return_any = true
warn_unreachable = true

show_none_errors = false
ignore_errors = false

allow_untyped_globals = false
allow_redefinition = false
implicit_reexport = true

strict_equality = true

# Important!
show_absolute_path = true

warn_unused_configs = true

plugins = "marshmallow_dataclass.mypy"

requirements.dev.txt

# Linters and test tools
# IMPORTANT: changing black version tends to change formatting, which tends to change diffs.
# It can be done, but should be done carefully and with the relevant commit added to the
# .git-blame-ignore-revs file. See: https://www.moxio.com/blog/43/ignoring-bulk-change-commits-with-git-blame
# for details.
black==22.8.0
flake8==4.0.1
mypy
pytest
pytest-cov
isort

# refurb (doesn't work pre-3.10)
#refurb

# This file should contain development requirements.
dictdiffer
funcsigs
pytest-flask
jinja2
pytest-mock
responses
Faker
freezegun
pytest-localserver
cryptography

aiosmtpd

# alembic migrations checker
#alembic-autogen-check

# documentation building
mkdocs
mkdocstrings[python]
mkdocs-material

# flake8-regex is missing this
pep8

# Flake8 extensions
flake8-formatter-abspath
flake8-sfs
#flake8-strings # SFS does the same thing.
flake8-pyi
flake8-functions
#flake8-bandit
flake8-builtins
flake8-commas
flake8-executable
flake8-return
flake8-snippets
flake8-tuple
flake8-datetimez
flake8-bugbear
flake8-continuation
flake8-spellcheck
flake8-mock
flake8-coding
flake8-debugger
flake8-click
flake8-string-format
flake8-exceptions
flake8-docstrings
flake8-regex
flake8-flask
# flake8-requirements # doesn't work properly - left here for reference.
flake8-print
flake8-mutable
flake8-alfred
#flake8-annotations # throws a lot of warnings and ignores pyi files, which is useless in complicated cases
flake8-requests
flake8-breakpoint
flake8-raise
flake8-comprehensions
flake8-expression-complexity
flake8-os-walk
flake8-super-call
flake8-pytest-style
flake8-cognitive-complexity
flake8-if-statements
flake8-keyword-arguments
flake8-blind-except
flake8-variables-names
flake8-unused-arguments
flake8-no-implicit-concat
flake8-class-attributes-order
flake8-tidy-imports

test

#!/bin/bash
# See: https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself
# Note: you can't refactor this out: its at the top of every script so the scripts can find their includes.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
cd "${SCRIPT_DIR}" || exit 1  # This is an unlikely failure, so don't worry too much.

source include.sh
source activate

exec pytest "--junitxml=.testreports/unit.xml" "--cov=app" "--cov-branch" "$@"

typecheck

#!/bin/bash
# See: https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself
# Note: you can't refactor this out: its at the top of every script so the scripts can find their includes.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
cd "${SCRIPT_DIR}" || exit 1  # This is an unlikely failure, so don't worry too much.

source include.sh
source activate

if [ -z "$1" ]; then
  exec mypy "-p" "app"
else
  exec mypy "$@"
fi

watcher

#!/bin/bash
# See: https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself
# Note: you can't refactor this out: its at the top of every script so the scripts can find their includes.
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
cd "${SCRIPT_DIR}" || exit 1  # This is an unlikely failure, so don't worry too much.

# Execute the handlers in order
exit_code=0
if ! ./fmt "$@" ; then
    exit_code=1
fi

if ! ./typecheck "$@" ; then
    exit_code=1
fi

if ! ./lint "$@" ; then
    exit_code=1
fi

exit $exit_code