Load secrets from .env files

.env

DEBUG=yes

AUTH_SECRET=seekrit
AUTH_EXPIRES=3600

HTTPS_ONLY=no
WEB_CONCURRENCY=4
TRUSTED_PROXY=127.0.0.1
BIND_HOST=127.0.0.1

# heroku defines the PORT variable but we rename it to BIND_PORT
PORT=6543
BIND_PORT=${PORT}

.env.local

# do not version control me
AUTH_EXPIRES=86400

render_config.py

import argparse
import jinja2
import os
from pyramid.settings import asbool
import sys

from myapp.utils.settings import load_dotenv


def multiline(val):
    lines = (l.strip() for l in val.strip().split('\n'))
    return '\n    ' + '\n    '.join(lines)


def parse_args(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('template_file', nargs='?', default='site.ini.in')
    parser.add_argument('output_file', nargs='?', default='site.ini')
    return parser.parse_args(argv)


def main(argv=sys.argv[1:]):
    args = parse_args(argv)
    
    for path in (
        os.environ.get('DOTENV_OVERRIDE'),
        '.env.local',
        '.env',
    ):
        if path and os.path.exists(path):
            load_dotenv(path)

    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(os.getcwd()),
        undefined=jinja2.StrictUndefined,
    )
    env.filters['bool'] = asbool
    env.filters['multiline'] = multiline

    template = env.get_template(args.template_file)
    result = template.render({
        'env': os.environ,
    })

    with open(args.output_file, 'w', encoding='utf8') as fp:
        fp.write(result)
        
       
if __name__ == '__main__':
    main()

settings.py

import codecs
import os
import re
from collections import OrderedDict


# ripped from python-dotenv and improved
_escape_decoder = codecs.getdecoder('unicode_escape')
_posix_variable = re.compile('\$\{[^\}]*\}')
_dotenv_entry = re.compile(
    '^\s*(\w*)\s*=\s*("[^"]*"|[^\s]*)\s*$',
    re.MULTILINE,
)


def load_dotenv(path, override=False):
    for k, v in dotenv_values(path).items():
        if override:
            os.environ[k] = v
        else:
            os.environ.setdefault(k, v)

            
def dotenv_values(path):
    with open(path, 'r', encoding='utf8') as fp:
        values = OrderedDict(parse_dotenv(fp))
    values = resolve_nested_variables(values)
    return values


def _decode_escaped(escaped):
    return _escape_decoder(escaped)[0]


def parse_dotenv(fp):
    for k, v in _dotenv_entry.findall(fp.read()):
        if len(v) > 0:
            quoted = v[0] == v[-1] in ['"', "'"]

            if quoted:
                v = _decode_escaped(v[1:-1])

        yield k, v

        
def resolve_nested_variables(values):
    def _re_sub_callback(match_object):
        name = match_object.group()[2:-1]
        return os.getenv(name, values.get(name, ""))

    for k, v in values.items():
        values[k] = _posix_variable.sub(_re_sub_callback, v)

    return values

site.ini.in

{%- set HTTPS_ONLY = env.HTTPS_ONLY | bool -%}
{%- set DEBUG = env.DEBUG | bool -%}
{%- set HEROKU = env.DYNO | default(False) -%}
[alembic]
script_location = myapp.model:migrations
file_template = %%(year)d%%(month).2d%%(day).2d_%%(rev)s

[app:api]
use = egg:myapp#main
{% if DEBUG %}
pyramid.includes =
    pyramid_debugtoolbar
    
debugtoolbar.hosts = 0.0.0.0/0
debugtoolbar.show_on_exc_only = yes
debugtoolbar.exclude_prefixes =
    /static
debugtoolbar.active_panels = performance
{% endif %}

pyramid.reload_assets = {{ 'true' if DEBUG else 'false' }}
pyramid.reload_templates = {{ 'true' if DEBUG else 'false' }}
pyramid.debug_authorization = false
pyramid.debug_notfound = false
pyramid.debug_routematch = false
pyramid.default_locale_name = en

jinja2.trim_blocks = yes
jinja2.lstrip_blocks = yes

tm.annotate_user = false

retry.attempts = 3

auth.secret = {{ env.AUTH_SECRET }}
auth.expires = {{ env.AUTH_EXPIRES }}
auth.secure = {{ env.HTTPS_ONLY }}

[filter:tracker]
use = egg:request-id
{%- if HEROKU %}
source_header = X-Request-Id
format = "{REQUEST_METHOD} {REQUEST_URI} {HTTP_VERSION}" {status} {bytes} {duration} {REMOTE_ADDR} "{HTTP_REFERER}" "{HTTP_USER_AGENT}" - {REQUEST_ID}
{%- else %}
format = {status} {REQUEST_METHOD:<6} {REQUEST_PATH:<40} {REQUEST_ID}
{%- endif %}
exclude_prefixes =
    /static

[pipeline:main]
pipeline =
    tracker
    {%- if HTTPS_ONLY %}
    egg:myapp#ssl_only
    {%- endif %}
    api

[server:main]
use = egg:waitress#main
host = {{ env.BIND_HOST }}
port = {{ env.BIND_PORT }}
threads = {{ env.WEB_CONCURRENCY }}
backlog = {{ env.WEB_CONCURRENCY }}
trusted_proxy = {{ env.TRUSTED_PROXY }}

[loggers]
keys = root, alembic, sqlalchemy, myapp, translogger

[handlers]
keys = console, translogger

[formatters]
keys = generic, minimal

[logger_root]
level = INFO
handlers = console

[logger_alembic]
level = INFO
handlers =
qualname = alembic

[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy

[logger_myapp]
level = DEBUG
handlers =
qualname = myapp

[logger_translogger]
level = INFO
handlers = translogger
qualname = request_id
propagate = 0

[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic

[handler_translogger]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = minimal

[formatter_generic]
{%- if HEROKU %}
format = %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
{%- else %}
format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
{%- endif %}

[formatter_minimal]
format = %(message)s

usage.sh

$ env/bin/render_config
$ env/bin/pserve site.ini