Skip to content

Instantly share code, notes, and snippets.

@monacoremo
Last active May 22, 2022 06:09
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save monacoremo/864814dbd7865849034562a9ce97015f to your computer and use it in GitHub Desktop.
Save monacoremo/864814dbd7865849034562a9ce97015f to your computer and use it in GitHub Desktop.
(Ab-)using Nix for full stack development environments
let
pkgs =
let
pinnedPkgs =
builtins.fetchGit {
name = "nixos-unstable-2019-12-05";
url = https://github.com/nixos/nixpkgs/;
rev = "cc6cf0a96a627e678ffc996a8f9d1416200d6c81";
};
in
import pinnedPkgs {};
postgresql =
pkgs.postgresql_12.withPackages
(
ps: [
ps.pgtap
]
);
pythonTesting =
pkgs.python38.withPackages
(
ps: [
ps.pytest
ps.requests
]
);
postgrest =
pkgs.stdenv.mkDerivation {
name = "postgrest";
src = pkgs.fetchurl {
url = (
"https://github.com/PostgREST/postgrest/"
+ "releases/download/v6.0.2/postgrest-v6.0.2-linux-x64-static.tar.xz"
);
hash = "sha256:09byg9pvq5f3chh1l4rg83y9ycyk2px0086im4xjjhk98z4sd41f";
};
sourceRoot = ".";
unpackCmd = "tar xf $src";
dontConfigure = true;
dontBuild = true;
installPhase = ''
mkdir -p $out/bin
cp postgrest $out/bin
'';
};
elm =
pkgs.elmPackages.elm;
mkService =
args@{ name, watch, ... }:
rec {
inherit name;
setup =
writers.writeCheckedShellScript "${name}-setup" (args.setup or "");
run =
writers.writeCheckedShellScript "${name}-run" (args.run or "");
watch =
writers.writeCheckedShellScript "${name}-watch" args.watch;
test =
writers.writeCheckedShellScript "${name}-test" (args.test or "");
container =
args.container;
launcher =
writers.writeCheckedShellScriptBin "${name}"
''
case $1 in
setup)
${setup}
;;
run)
${run}
;;
watch)
${watch}
;;
test)
${test}
;;
*)
echo "Usage: $0 {setup|run|watch|test}"
exit 1
esac
'';
};
foundationService =
let
tailLogs =
writers.writeCheckedShellScriptBin "foundation-logs"
''
lockfile="$(mktemp)"
function stop () {
rm -f "$lockfile"
kill 0
};
trap stop exit
while read -r logconfig; do
IFS=' ' read -ra args <<< "$logconfig"
label="''${args[0]}"
color="''${args[1]}"
logfile="''${args[2]}"
stdbuf -oL -eL tail -n +1 -F "$logfile" | while read -r line; do
(
# acquire a write lock to make sure that lines are not mixed up
${pkgs.utillinux}/bin/flock 3
${pkgs.ncurses}/bin/tput setaf "$color"
echo -n "$(date -Iseconds) $label: "
${pkgs.ncurses}/bin/tput sgr0
echo "$line"
) 3>"$lockfile";
done &
done
wait
'';
logsConfig =
pkgs.writeText "logs.conf"
''
db 2 $FOUNDATION_BASEDIR/db.log
api 3 $FOUNDATION_BASEDIR/api.log
web 5 $FOUNDATION_BASEDIR/web.log
docs 6 $FOUNDATION_BASEDIR/docs.log
ingress 10 $FOUNDATION_BASEDIR/ingress/logs/access.log
ingress-error 9 $FOUNDATION_BASEDIR/ingress/logs/error.log
'';
in
mkService rec {
name = "foundation";
watch =
''
trap "kill 0" exit
${setup}
echo "Serving on http://localhost:$FOUNDATION_PORT/ from $FOUNDATION_BASEDIR"
${pkgs.envsubst}/bin/envsubst -i ${logsConfig} | ${tailLogs}/bin/foundation-logs &
${dbService.watch} >> "$FOUNDATION_BASEDIR/db.log" 2>&1 &
${apiService.watch} >> "$FOUNDATION_BASEDIR/api.log" &
${webService.watch} >> "$FOUNDATION_BASEDIR/web.log" &
${docsService.watch} >> "$FOUNDATION_BASEDIR/docs.log" 2>&1 &
${ingressService.watch} &
wait
'';
run =
''
trap "kill 0" exit
${setup}
echo "Serving on http://localhost:$FOUNDATION_PORT/ from $FOUNDATION_BASEDIR"
${pkgs.envsubst}/bin/envsubst -i ${logsConfig} | ${tailLogs}/bin/foundation-logs &
${dbService.run} >> "$FOUNDATION_BASEDIR/db.log" &
${apiService.run} >> "$FOUNDATION_BASEDIR/api.log" &
${ingressService.run} &
wait
'';
setup =
''
${dbService.setup} >> "$FOUNDATION_BASEDIR/db.log" &
${apiService.setup} >> "$FOUNDATION_BASEDIR/api.log" &
${docsService.setup} >> "$FOUNDATION_BASEDIR/docs.log" 2>&1 &
${webService.setup} >> "$FOUNDATION_BASEDIR/web.log" &
${ingressService.setup} &
wait
'';
};
apiService =
let
postgrestConf =
pkgs.writeText "postgrest.conf"
''
db-uri = "$FOUNDATION_DB_APISERVER_URI"
db-schema = "api"
db-anon-role = "anonymous"
pre-request = "auth.authenticate"
server-unix-socket = "$FOUNDATION_API_SOCKET"
'';
in
mkService rec {
name = "foundation-api";
setup =
''
mkdir -p "$FOUNDATION_API_DIR"
touch "$FOUNDATION_BASEDIR"/api.log
${pkgs.envsubst}/bin/envsubst -i "${postgrestConf}" -o "$FOUNDATION_API_CONFIG"
'';
watch =
''
while true; do
find "$FOUNDATION_SRC"/db | ${pkgs.entr}/bin/entr -d -r \
${postgrest}/bin/postgrest "$FOUNDATION_API_CONFIG"
done
'';
run =
''
exec ${postgrest}/bin/postgrest "$FOUNDATION_API_CONFIG"
'';
};
dbService =
let
postgresConf =
pkgs.writeText "postgresql.conf"
''
log_min_messages = warning
log_min_error_statement = error
log_min_duration_statement = 100 # ms
log_connections = on
log_disconnections = on
log_duration = on
log_line_prefix = '[%u] '
#log_statement = 'none'
log_timezone = 'UTC'
'';
in
mkService rec {
name = "foundation-db";
setup =
''
set -e
touch "$FOUNDATION_BASEDIR"/db.log
rm -rf "$FOUNDATION_DB_DIR"
mkdir -p "$FOUNDATION_DB_DIR"
# Initialize the PGDATA directory
pwfile=$(mktemp)
echo "$FOUNDATION_DB_SUPERUSER_PW" > "$pwfile"
TZ=UTC ${postgresql}/bin/initdb -D "$FOUNDATION_DB_DIR" \
--no-locale --encoding=UTF8 \
-U postgres -A password --pwfile="$pwfile"
rm "$pwfile"
mkdir -p "$FOUNDATION_DB_DIR/setupsocket"
${postgresql}/bin/pg_ctl start -D "$FOUNDATION_DB_DIR" \
-o "-F -c listen_addresses=\"\" -k $FOUNDATION_DB_DIR/setupsocket"
pushd "$FOUNDATION_SRC/db" >> /dev/null
${postgresql}/bin/psql "$FOUNDATION_DB_SUPERUSER_SETUP_URI" -f app.sql
popd >> /dev/null
${postgresql}/bin/psql "$FOUNDATION_DB_SUPERUSER_SETUP_URI" << EOF
alter role apiserver with password '$FOUNDATION_DB_APISERVER_PW';
EOF
${postgresql}/bin/pg_ctl stop -D "$FOUNDATION_DB_DIR"
cat ${postgresConf} >> "$FOUNDATION_DB_DIR"/postgresql.conf
'';
watch =
let
watchHelper =
writers.writeCheckedShellScript "${name}-watchhelper"
''
${setup}
${run}
'';
in
''
while true; do
find "$FOUNDATION_SRC"/db | ${pkgs.entr}/bin/entr -d -r ${watchHelper}
done
'';
run =
''
exec ${postgresql}/bin/postgres -D "$FOUNDATION_DB_DIR" \
-F -c listen_addresses="" -k "$FOUNDATION_DB_DIR" 2>&1
'';
test =
''
${setup}
${postgresql}/bin/pg_ctl start -D "$FOUNDATION_DB_DIR" \
-o "-F -c listen_addresses=\"\" -k $FOUNDATION_DB_DIR"
${postgresql}/bin/psql "$FOUNDATION_DB_POSTGRES_URI" << EOF
select * from tests.run();
EOF
${postgresql}/bin/pg_ctl stop -D "$FOUNDATION_DB_DIR"
'';
};
ingressService =
let
nginxConf =
writers.writeNginxConf "nginx.conf"
''
events {}
http {
default_type application/ocet-stream;
types {
text/html html;
text/css css;
application/javascript js;
image/png png;
image/svg+xml svg;
font/woff woff;
font/woff2 woff2;
}
gzip off;
sendfile on;
keepalive_timeout 65s;
server {
listen $FOUNDATION_PORT default_server;
# add_header Content-Security-Policy "default-src 'self'; style-src 'unsafe-inline'";
# add_header X-Content-Type-Options "nosniff";
add_header X-Frame-Options "SAMEORIGIN";
add_header X-XSS-Protection "1; mode=block";
location =/ {
access_by_lua_block {
ngx.header['Location'] = '/app/'
ngx.exit(301)
}
}
location =/favicon.ico {
alias ${favicon};
}
location /api/ {
#limit_except GET {
# deny all;
#}
more_clear_input_headers Accept-Encoding;
access_by_lua_file ${antiCsrf};
proxy_pass $FOUNDATION_API_URI;
}
# location /files/ {
# client_max_body_size 1000M;
# access_by_lua_file lib/anticsrf.lua;
# proxy_pass $FILES_BASEURL;
# }
location /docs/ {
alias $FOUNDATION_DOCS_BUILDDIR/;
}
location /app/ {
alias $FOUNDATION_APP_BUILDDIR/;
try_files $$uri /app/index.html;
}
location /fonts/material-design-icons/ {
alias ${pkgs.material-design-icons}/share/fonts/;
}
}
}
'';
favicon =
pkgs.stdenv.mkDerivation rec {
name = "favicon.ico";
src = ./web/assets/favicon.png;
phases = [ "buildPhase" ];
buildPhase =
''
${pkgs.imagemagick}/bin/convert -flatten -background none -resize 16x16 ${src} $out
'';
};
antiCsrf =
writers.writeLuaScript "anticsrf.lua"
''
-- TODO: enable CSRF protection for production
-- First line of defense: Check that origin or referer is set and that they
-- match the current host (using ngx.var.http_origin and ngx.var.http_referer)
-- Defense in depth: Require a custom header for API requests, which can
-- only be set by requests from the same origin
-- if ngx.req.get_headers()['X-Requested-By'] == nil then
-- ngx.header.content_type = 'text/plain'
-- ngx.say('Missing X-Requested-By header - not allowed to mitigate CSRF')
-- ngx.exit(405)
-- end
'';
openApi =
writers.writeLuaScript "openapi.lua"
''
local cjson = require "cjson"
cjson.decode_array_with_array_mt(true)
local res = ngx.location.capture("/api/")
api = cjson.decode(res.body)
api["basePath"] = "/api/"
api["host"] = ""
ngx.say(cjson.encode(api))
'';
in
mkService rec {
name = "foundation-ingress";
setup =
''
touch "$FOUNDATION_BASEDIR"/ingress.log
mkdir -p "$FOUNDATION_INGRESS_DIR"/{logs,conf}
touch "$FOUNDATION_INGRESS_DIR"/logs/{error.log,access.log}
${pkgs.envsubst}/bin/envsubst -i ${nginxConf} \
-o "$FOUNDATION_INGRESS_DIR/conf/nginx.conf"
'';
watch =
''
${setup}
${run}
'';
run =
''
exec ${pkgs.openresty}/bin/openresty -p "$FOUNDATION_INGRESS_DIR" \
-g "daemon off;"
'';
};
webService =
let
name = "foundation-web";
setup =
writers.writeCheckedShellScript "${name}-setup"
''
touch "$FOUNDATION_BASEDIR"/web.log
mkdir -p "$FOUNDATION_APP_BUILDDIR"
webappdir=$(realpath "$FOUNDATION_SRC"/web)
ln -sf "$webappdir"/{elm.json,src} "$FOUNDATION_APP_DIR"
ln -sf "$webappdir"/index.html "$FOUNDATION_APP_BUILDDIR"
'';
build =
writers.writeCheckedShellScript "${name}-build"
''
cd "$FOUNDATION_APP_DIR" || exit 1
webappdir=$(realpath "$FOUNDATION_SRC"/web)
${elm}/bin/elm make src/Main.elm \
--output="$FOUNDATION_APP_BUILDDIR"/app.js --debug
cat "$webappdir/init.js" >> "$FOUNDATION_APP_BUILDDIR"/app.js
'';
in
mkService {
inherit name;
setup =
''
${setup}
${build}
'';
watch =
''
${setup}
while true; do
find "$FOUNDATION_SRC"/web | ${pkgs.entr}/bin/entr -d -r ${build}
done
'';
};
docsService =
let
name = "foundation-docs";
python =
pkgs.python38.withPackages
(
ps: [
ps.sphinx
]
);
setup =
writers.writeCheckedShellScript "${name}-setup"
''
touch "$FOUNDATION_BASEDIR"/docs.log
mkdir -p "$FOUNDATION_DOCS_DIR"
'';
build =
writers.writeCheckedShellScript "${name}-build"
''
${python}/bin/sphinx-build -b html -d "$FOUNDATION_DOCS_DIR"/cache \
"$FOUNDATION_SRC"/docs "$FOUNDATION_DOCS_BUILDDIR"
# TODO: building a pdf via latex
# ${python}/bin/sphinx-build -b latex -d "$FOUNDATION_SRC" \
# "$FOUNDATION_SRC"/docs "$FOUNDATION_DOCS_DIR"/latex
'';
in
mkService {
inherit name;
setup =
''
${setup}
${build}
'';
watch =
''
${setup}
while true; do
find "$FOUNDATION_SRC"/docs | ${pkgs.entr}/bin/entr -d ${build}
done
'';
};
/*
* Create a development enviroment for the project.
*
*/
mkEnv =
writers.writeCheckedShellScriptBin "foundation-mkenv"
''
sourcedir=$(realpath "$1")
basedir=$(realpath "$2")
envfile="$basedir"/env
mkdir -p "$basedir"
cat << EOF > "$envfile"
#!${pkgs.runtimeShell}
export FOUNDATION_SRC="$sourcedir"
export FOUNDATION_PORT=9000
export FOUNDATION_BASEDIR="$basedir"
export FOUNDATION_DB_DIR="\$FOUNDATION_BASEDIR/db"
export FOUNDATION_DB_NAME=postgres
export FOUNDATION_DB_SUPERUSER=postgres
export FOUNDATION_DB_URI="postgresql:///\$FOUNDATION_DB_NAME?host=\$FOUNDATION_DB_DIR"
export FOUNDATION_DB_SUPERUSER_PW=$(${pkgs.pwgen}/bin/pwgen 32 1)
export FOUNDATION_DB_APISERVER_PW=$(${pkgs.pwgen}/bin/pwgen 32 1)
export FOUNDATION_DB_SUPERUSER_SETUP_URI="\$FOUNDATION_DB_URI/setupsocket&user=\$FOUNDATION_DB_SUPERUSER&password=\$FOUNDATION_DB_SUPERUSER_PW"
export FOUNDATION_DB_SUPERUSER_URI="\$FOUNDATION_DB_URI&user=\$FOUNDATION_DB_SUPERUSER&password=\$FOUNDATION_DB_SUPERUSER_PW"
export FOUNDATION_DB_APISERVER_URI="\$FOUNDATION_DB_URI&user=apiserver&password=\$FOUNDATION_DB_APISERVER_PW"
export FOUNDATION_API_DIR="\$FOUNDATION_BASEDIR/api"
export FOUNDATION_API_SOCKET="\$FOUNDATION_API_DIR/postgrest.sock"
export FOUNDATION_API_CONFIG="\$FOUNDATION_API_DIR/postgrest.conf"
export FOUNDATION_API_URI="http://unix:\$FOUNDATION_API_SOCKET:/"
export FOUNDATION_DOCS_DIR="\$FOUNDATION_BASEDIR/docs"
export FOUNDATION_DOCS_BUILDDIR="\$FOUNDATION_DOCS_DIR/build"
export FOUNDATION_APP_DIR="\$FOUNDATION_BASEDIR/app"
export FOUNDATION_APP_BUILDDIR="\$FOUNDATION_APP_DIR/build"
export FOUNDATION_INGRESS_DIR="\$FOUNDATION_BASEDIR/ingress"
# psql variables for convenience
export PGHOST="\$FOUNDATION_DB_DIR"
export PGDATABASE="\$FOUNDATION_DB_NAME"
export PGUSER="\$FOUNDATION_DB_SUPERUSER"
export PGPASSWORD="\$FOUNDATION_DB_SUPERUSER_PW"
EOF
${pkgs.stdenv.shell} -n "$envfile"
${pkgs.shellcheck}/bin/shellcheck "$envfile"
echo "$envfile"
'';
writers = {
/*
* Writes a shell script to bin/<name> and checks it with shellcheck.
*
*/
writeCheckedShellScriptBin =
name: text:
pkgs.writeTextFile {
inherit name;
executable = true;
destination = "/bin/${name}";
text =
''
#!${pkgs.runtimeShell}
${text}
'';
checkPhase =
''
# check syntax
${pkgs.stdenv.shell} -n $out/bin/${name}
# check for shellcheck recommendations
${pkgs.shellcheck}/bin/shellcheck $out/bin/${name}
'';
};
/*
* Writes a shell script and checks it with shellcheck.
*
*/
writeCheckedShellScript =
name: text:
pkgs.writeTextFile {
inherit name;
executable = true;
text =
''
#!${pkgs.runtimeShell}
${text}
'';
checkPhase =
''
# check syntax
${pkgs.stdenv.shell} -n $out
# check for shellcheck recommendations
${pkgs.shellcheck}/bin/shellcheck $out
'';
};
writeNginxConf =
name: text:
pkgs.writeTextFile {
inherit name text;
checkPhase =
''
${pkgs.gixy}/bin/gixy $out > /dev/null
'';
};
writeLuaScript =
name: text:
pkgs.writeTextFile {
inherit name text;
checkPhase =
''
${pkgs.luajit}/bin/luajit -bl $out > /dev/null
'';
};
};
dbContainer =
pkgs.dockerTools.buildImage {
name = "foundation-db";
tag = "latest";
config = {
Cmd = [ "${dbService.run}" ];
WorkingDir = "/";
Volumes = {};
};
};
formatSource =
writers.writeCheckedShellScriptBin "foundation-format"
''
${pkgs.elmPackages.elm-format}/bin/elm-format --yes "$FOUNDATION_SRC"/web/src > /dev/null
${pkgs.nixpkgs-fmt}/bin/nixpkgs-fmt "$FOUNDATION_SRC"/shell.nix > /dev/null
'';
stats =
writers.writeCheckedShellScriptBin "foundation-stats"
''
${pkgs.cloc}/bin/cloc "$FOUNDATION_SRC"
'';
testWeb =
writers.writeCheckedShellScriptBin "foundation-testweb"
''
${pythonTesting}/bin/py.test "$FOUNDATION_SRC"/tests/web
'';
in
pkgs.stdenv.mkDerivation {
name = "foundation";
buildInputs = [
pkgs.graphviz
pkgs.less # for psql display of large tables
postgresql
pythonTesting
mkEnv
pkgs.nodePackages.uglify-js
foundationService.launcher
dbService.launcher
apiService.launcher
ingressService.launcher
docsService.launcher
webService.launcher
formatSource
stats
testWeb
];
shellHook = ''
tmpdir=$(mktemp -d)
trap "rm -rf $tmpdir" exit
source $(${mkEnv}/bin/foundation-mkenv . "$tmpdir")
cat << EOF
$(${pkgs.ncurses}/bin/tput setaf 2)
Foundation development environment
$(${pkgs.ncurses}/bin/tput sgr0)
Temporary environment: $tmpdir (will be deleted on exit)
To use that environment from another shell, run "source $tmpdir/env".
Use "foundation-mkenv" to create a new environment.
To launch a development server, run "foundation run" or "foundation watch".
EOF
'';
}
@steve-chavez
Copy link

This part:

                types {
                  text/html html;
                  text/css css;
                  application/javascript js;
                  image/png png;
                  image/svg+xml svg;
                  font/woff woff;
                  font/woff2 woff2;
                }

I think it can be replaced by:

	include ${pkgs.nginx}/conf/mime.types;

@monacoremo
Copy link
Author

You are right, that's a much nicer solution! I will include it here: https://github.com/monacoremo/full-stack-experiment

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment