(Ab-)using Nix for full stack development environments

foundation.nix

let
  pkgs =
    let
      pinnedPkgs =
        builtins.fetchGit {
          name = "nixos-unstable-2019-12-05";
          url = https://github.com/nixos/nixpkgs/;
          rev = "cc6cf0a96a627e678ffc996a8f9d1416200d6c81";
        };
    in
      import pinnedPkgs {};

  postgresql =
    pkgs.postgresql_12.withPackages
      (
        ps: [
          ps.pgtap
        ]
      );

  pythonTesting =
    pkgs.python38.withPackages
      (
        ps: [
          ps.pytest
          ps.requests
        ]
      );

  postgrest =
    pkgs.stdenv.mkDerivation {
      name = "postgrest";
      src = pkgs.fetchurl {
        url = (
          "https://github.com/PostgREST/postgrest/"
          + "releases/download/v6.0.2/postgrest-v6.0.2-linux-x64-static.tar.xz"
        );
        hash = "sha256:09byg9pvq5f3chh1l4rg83y9ycyk2px0086im4xjjhk98z4sd41f";
      };

      sourceRoot = ".";
      unpackCmd = "tar xf $src";

      dontConfigure = true;
      dontBuild = true;

      installPhase = ''
        mkdir -p $out/bin
        cp postgrest $out/bin
      '';
    };

  elm =
    pkgs.elmPackages.elm;

  mkService =
    args@{ name, watch, ... }:
      rec {
        inherit name;
        setup =
          writers.writeCheckedShellScript "${name}-setup" (args.setup or "");
        run =
          writers.writeCheckedShellScript "${name}-run" (args.run or "");
        watch =
          writers.writeCheckedShellScript "${name}-watch" args.watch;
        test =
          writers.writeCheckedShellScript "${name}-test" (args.test or "");
        container =
          args.container;
        launcher =
          writers.writeCheckedShellScriptBin "${name}"
            ''
              case $1 in
                setup)
                  ${setup}
                  ;;
                run)
                  ${run}
                  ;;
                watch)
                  ${watch}
                  ;;
                test)
                  ${test}
                  ;;
                *)
                  echo "Usage: $0 {setup|run|watch|test}"
                  exit 1
              esac
            '';
      };

  foundationService =
    let
      tailLogs =
        writers.writeCheckedShellScriptBin "foundation-logs"
          ''
            lockfile="$(mktemp)"

            function stop () {
              rm -f "$lockfile"
              kill 0
            };

            trap stop exit

            while read -r logconfig; do
              IFS=' ' read -ra args <<< "$logconfig"
              label="''${args[0]}"
              color="''${args[1]}"
              logfile="''${args[2]}"
              stdbuf -oL -eL tail -n +1 -F "$logfile" | while read -r line; do
                (
                  # acquire a write lock to make sure that lines are not mixed up
                  ${pkgs.utillinux}/bin/flock 3

                  ${pkgs.ncurses}/bin/tput setaf "$color"
                  echo -n "$(date -Iseconds) $label: "
                  ${pkgs.ncurses}/bin/tput sgr0
                  echo "$line"
                ) 3>"$lockfile";
              done &
            done

            wait
          '';

      logsConfig =
        pkgs.writeText "logs.conf"
          ''
            db 2 $FOUNDATION_BASEDIR/db.log
            api 3 $FOUNDATION_BASEDIR/api.log
            web 5 $FOUNDATION_BASEDIR/web.log
            docs 6 $FOUNDATION_BASEDIR/docs.log
            ingress 10 $FOUNDATION_BASEDIR/ingress/logs/access.log
            ingress-error 9 $FOUNDATION_BASEDIR/ingress/logs/error.log
          '';
    in
      mkService rec {
        name = "foundation";
        watch =
          ''
            trap "kill 0" exit

            ${setup}

            echo "Serving on http://localhost:$FOUNDATION_PORT/ from $FOUNDATION_BASEDIR"

            ${pkgs.envsubst}/bin/envsubst -i ${logsConfig} | ${tailLogs}/bin/foundation-logs &

            ${dbService.watch} >> "$FOUNDATION_BASEDIR/db.log" 2>&1 &
            ${apiService.watch} >> "$FOUNDATION_BASEDIR/api.log" &
            ${webService.watch} >> "$FOUNDATION_BASEDIR/web.log" &
            ${docsService.watch} >> "$FOUNDATION_BASEDIR/docs.log" 2>&1 &
            ${ingressService.watch} &

            wait
          '';

        run =
          ''
            trap "kill 0" exit

            ${setup}

            echo "Serving on http://localhost:$FOUNDATION_PORT/ from $FOUNDATION_BASEDIR"

            ${pkgs.envsubst}/bin/envsubst -i ${logsConfig} | ${tailLogs}/bin/foundation-logs &

            ${dbService.run} >> "$FOUNDATION_BASEDIR/db.log" &
            ${apiService.run} >> "$FOUNDATION_BASEDIR/api.log" &
            ${ingressService.run} &

            wait
          '';

        setup =
          ''
            ${dbService.setup} >> "$FOUNDATION_BASEDIR/db.log" &
            ${apiService.setup} >> "$FOUNDATION_BASEDIR/api.log" &
            ${docsService.setup} >> "$FOUNDATION_BASEDIR/docs.log" 2>&1 &
            ${webService.setup} >> "$FOUNDATION_BASEDIR/web.log" &
            ${ingressService.setup} &

            wait
          '';
      };

  apiService =
    let
      postgrestConf =
        pkgs.writeText "postgrest.conf"
          ''
            db-uri = "$FOUNDATION_DB_APISERVER_URI"
            db-schema = "api"
            db-anon-role = "anonymous"

            pre-request = "auth.authenticate"

            server-unix-socket = "$FOUNDATION_API_SOCKET"
          '';
    in
      mkService rec {
        name = "foundation-api";
        setup =
          ''
            mkdir -p "$FOUNDATION_API_DIR"
            touch "$FOUNDATION_BASEDIR"/api.log
            ${pkgs.envsubst}/bin/envsubst -i "${postgrestConf}" -o "$FOUNDATION_API_CONFIG"
          '';
        watch =
          ''
            while true; do
              find "$FOUNDATION_SRC"/db | ${pkgs.entr}/bin/entr -d -r \
                ${postgrest}/bin/postgrest "$FOUNDATION_API_CONFIG"
            done
          '';
        run =
          ''
            exec ${postgrest}/bin/postgrest "$FOUNDATION_API_CONFIG"
          '';
      };

  dbService =
    let
      postgresConf =
        pkgs.writeText "postgresql.conf"
          ''
            log_min_messages = warning
            log_min_error_statement = error

            log_min_duration_statement = 100  # ms

            log_connections = on
            log_disconnections = on
            log_duration = on
            log_line_prefix = '[%u] '
            #log_statement = 'none'
            log_timezone = 'UTC'
          '';
    in
    mkService rec {
      name = "foundation-db";
      setup =
        ''
          set -e
          touch "$FOUNDATION_BASEDIR"/db.log
          rm -rf "$FOUNDATION_DB_DIR"
          mkdir -p "$FOUNDATION_DB_DIR"

          # Initialize the PGDATA directory
          pwfile=$(mktemp)
          echo "$FOUNDATION_DB_SUPERUSER_PW" > "$pwfile"

          TZ=UTC ${postgresql}/bin/initdb -D "$FOUNDATION_DB_DIR" \
            --no-locale --encoding=UTF8 \
            -U postgres -A password --pwfile="$pwfile"

          rm "$pwfile"

          mkdir -p "$FOUNDATION_DB_DIR/setupsocket"

          ${postgresql}/bin/pg_ctl start -D "$FOUNDATION_DB_DIR" \
            -o "-F -c listen_addresses=\"\" -k $FOUNDATION_DB_DIR/setupsocket"

          pushd "$FOUNDATION_SRC/db" >> /dev/null
          ${postgresql}/bin/psql "$FOUNDATION_DB_SUPERUSER_SETUP_URI" -f app.sql
          popd >> /dev/null

          ${postgresql}/bin/psql "$FOUNDATION_DB_SUPERUSER_SETUP_URI" << EOF
            alter role apiserver with password '$FOUNDATION_DB_APISERVER_PW';
          EOF

          ${postgresql}/bin/pg_ctl stop -D "$FOUNDATION_DB_DIR"

          cat ${postgresConf} >> "$FOUNDATION_DB_DIR"/postgresql.conf
        '';
      watch =
        let
          watchHelper =
            writers.writeCheckedShellScript "${name}-watchhelper"
              ''
                ${setup}
                ${run}
              '';
        in
          ''
            while true; do
              find "$FOUNDATION_SRC"/db | ${pkgs.entr}/bin/entr -d -r ${watchHelper}
            done
          '';
      run =
        ''
          exec ${postgresql}/bin/postgres -D "$FOUNDATION_DB_DIR" \
            -F -c listen_addresses="" -k "$FOUNDATION_DB_DIR" 2>&1
        '';
      test =
        ''
          ${setup}

          ${postgresql}/bin/pg_ctl start -D "$FOUNDATION_DB_DIR" \
            -o "-F -c listen_addresses=\"\" -k $FOUNDATION_DB_DIR"

          ${postgresql}/bin/psql "$FOUNDATION_DB_POSTGRES_URI" << EOF
            select * from tests.run();
          EOF

          ${postgresql}/bin/pg_ctl stop -D "$FOUNDATION_DB_DIR"
        '';
    };

  ingressService =
    let
      nginxConf =
        writers.writeNginxConf "nginx.conf"
          ''
            events {}

            http {
                default_type application/ocet-stream;

                types {
                  text/html html;
                  text/css css;
                  application/javascript js;
                  image/png png;
                  image/svg+xml svg;
                  font/woff woff;
                  font/woff2 woff2;
                }

                gzip off;
                sendfile on;

                keepalive_timeout 65s;

                server {
                    listen $FOUNDATION_PORT default_server;

                    # add_header Content-Security-Policy "default-src 'self'; style-src 'unsafe-inline'";
                    # add_header X-Content-Type-Options "nosniff";
                    add_header X-Frame-Options "SAMEORIGIN";
                    add_header X-XSS-Protection "1; mode=block";

                    location =/ {
                        access_by_lua_block {
                            ngx.header['Location'] = '/app/'
                            ngx.exit(301)
                        }
                    }

                    location =/favicon.ico {
                        alias ${favicon};
                    }

                    location /api/ {
                        #limit_except GET {
                        #    deny all;
                        #}

                        more_clear_input_headers Accept-Encoding;
                        access_by_lua_file ${antiCsrf};
                        proxy_pass $FOUNDATION_API_URI;
                    }

                    # location /files/ {
                    #     client_max_body_size 1000M;
                    #     access_by_lua_file lib/anticsrf.lua;
                    #     proxy_pass $FILES_BASEURL;
                    # }

                    location /docs/ {
                        alias $FOUNDATION_DOCS_BUILDDIR/;
                    }

                    location /app/ {
                        alias $FOUNDATION_APP_BUILDDIR/;
                        try_files $$uri /app/index.html;
                    }

                    location /fonts/material-design-icons/ {
                        alias ${pkgs.material-design-icons}/share/fonts/;
                    }
                }
            }
          '';

      favicon =
        pkgs.stdenv.mkDerivation rec {
          name = "favicon.ico";
          src = ./web/assets/favicon.png;
          phases = [ "buildPhase" ];
          buildPhase =
            ''
              ${pkgs.imagemagick}/bin/convert -flatten -background none -resize 16x16 ${src} $out
            '';
        };

      antiCsrf =
        writers.writeLuaScript "anticsrf.lua"
          ''
            -- TODO: enable CSRF protection for production

            -- First line of defense: Check that origin or referer is set and that they
            -- match the current host (using ngx.var.http_origin and ngx.var.http_referer)

            -- Defense in depth: Require a custom header for API requests, which can
            -- only be set by requests from the same origin

            -- if ngx.req.get_headers()['X-Requested-By'] == nil then
            --     ngx.header.content_type = 'text/plain'
            --     ngx.say('Missing X-Requested-By header - not allowed to mitigate CSRF')
            --     ngx.exit(405)
            -- end
          '';

      openApi =
        writers.writeLuaScript "openapi.lua"
          ''
            local cjson = require "cjson"
            cjson.decode_array_with_array_mt(true)

            local res = ngx.location.capture("/api/")
            api = cjson.decode(res.body)

            api["basePath"] = "/api/"
            api["host"] = ""

            ngx.say(cjson.encode(api))
          '';
    in
      mkService rec {
        name = "foundation-ingress";
        setup =
          ''
            touch "$FOUNDATION_BASEDIR"/ingress.log
            mkdir -p "$FOUNDATION_INGRESS_DIR"/{logs,conf}
            touch "$FOUNDATION_INGRESS_DIR"/logs/{error.log,access.log}
            ${pkgs.envsubst}/bin/envsubst -i ${nginxConf} \
              -o "$FOUNDATION_INGRESS_DIR/conf/nginx.conf"
          '';
        watch =
          ''
            ${setup}
            ${run}
          '';
        run =
          ''
            exec ${pkgs.openresty}/bin/openresty -p "$FOUNDATION_INGRESS_DIR" \
              -g "daemon off;"
          '';
      };

  webService =
    let
      name = "foundation-web";

      setup =
        writers.writeCheckedShellScript "${name}-setup"
          ''
            touch "$FOUNDATION_BASEDIR"/web.log
            mkdir -p "$FOUNDATION_APP_BUILDDIR"
            webappdir=$(realpath "$FOUNDATION_SRC"/web)
            ln -sf "$webappdir"/{elm.json,src} "$FOUNDATION_APP_DIR"
            ln -sf "$webappdir"/index.html "$FOUNDATION_APP_BUILDDIR"
          '';

      build =
        writers.writeCheckedShellScript "${name}-build"
          ''
            cd "$FOUNDATION_APP_DIR" || exit 1
            webappdir=$(realpath "$FOUNDATION_SRC"/web)
            ${elm}/bin/elm make src/Main.elm \
              --output="$FOUNDATION_APP_BUILDDIR"/app.js --debug
            cat "$webappdir/init.js" >> "$FOUNDATION_APP_BUILDDIR"/app.js
          '';
    in
      mkService {
        inherit name;
        setup =
          ''
            ${setup}
            ${build}
          '';
        watch =
          ''
            ${setup}

            while true; do
              find "$FOUNDATION_SRC"/web | ${pkgs.entr}/bin/entr -d -r ${build}
            done
          '';
      };

  docsService =
    let
      name = "foundation-docs";

      python =
        pkgs.python38.withPackages
          (
            ps: [
              ps.sphinx
            ]
          );

      setup =
        writers.writeCheckedShellScript "${name}-setup"
          ''
            touch "$FOUNDATION_BASEDIR"/docs.log
            mkdir -p "$FOUNDATION_DOCS_DIR"
          '';

      build =
        writers.writeCheckedShellScript "${name}-build"
          ''
            ${python}/bin/sphinx-build -b html -d "$FOUNDATION_DOCS_DIR"/cache \
              "$FOUNDATION_SRC"/docs "$FOUNDATION_DOCS_BUILDDIR"

            # TODO: building a pdf via latex
            # ${python}/bin/sphinx-build -b latex -d "$FOUNDATION_SRC" \
            #   "$FOUNDATION_SRC"/docs "$FOUNDATION_DOCS_DIR"/latex
          '';
    in
      mkService {
        inherit name;
        setup =
          ''
            ${setup}
            ${build}
          '';
        watch =
          ''
            ${setup}

            while true; do
              find "$FOUNDATION_SRC"/docs | ${pkgs.entr}/bin/entr -d ${build}
            done
          '';
      };


  /*
   * Create a development enviroment for the project.
   *
   */
  mkEnv =
    writers.writeCheckedShellScriptBin "foundation-mkenv"
      ''
        sourcedir=$(realpath "$1")
        basedir=$(realpath "$2")
        envfile="$basedir"/env

        mkdir -p "$basedir"

        cat << EOF > "$envfile"
        #!${pkgs.runtimeShell}

        export FOUNDATION_SRC="$sourcedir"
        export FOUNDATION_PORT=9000
        export FOUNDATION_BASEDIR="$basedir"
        export FOUNDATION_DB_DIR="\$FOUNDATION_BASEDIR/db"
        export FOUNDATION_DB_NAME=postgres
        export FOUNDATION_DB_SUPERUSER=postgres
        export FOUNDATION_DB_URI="postgresql:///\$FOUNDATION_DB_NAME?host=\$FOUNDATION_DB_DIR"
        export FOUNDATION_DB_SUPERUSER_PW=$(${pkgs.pwgen}/bin/pwgen 32 1)
        export FOUNDATION_DB_APISERVER_PW=$(${pkgs.pwgen}/bin/pwgen 32 1)
        export FOUNDATION_DB_SUPERUSER_SETUP_URI="\$FOUNDATION_DB_URI/setupsocket&user=\$FOUNDATION_DB_SUPERUSER&password=\$FOUNDATION_DB_SUPERUSER_PW"
        export FOUNDATION_DB_SUPERUSER_URI="\$FOUNDATION_DB_URI&user=\$FOUNDATION_DB_SUPERUSER&password=\$FOUNDATION_DB_SUPERUSER_PW"
        export FOUNDATION_DB_APISERVER_URI="\$FOUNDATION_DB_URI&user=apiserver&password=\$FOUNDATION_DB_APISERVER_PW"

        export FOUNDATION_API_DIR="\$FOUNDATION_BASEDIR/api"
        export FOUNDATION_API_SOCKET="\$FOUNDATION_API_DIR/postgrest.sock"
        export FOUNDATION_API_CONFIG="\$FOUNDATION_API_DIR/postgrest.conf"
        export FOUNDATION_API_URI="http://unix:\$FOUNDATION_API_SOCKET:/"

        export FOUNDATION_DOCS_DIR="\$FOUNDATION_BASEDIR/docs"
        export FOUNDATION_DOCS_BUILDDIR="\$FOUNDATION_DOCS_DIR/build"

        export FOUNDATION_APP_DIR="\$FOUNDATION_BASEDIR/app"
        export FOUNDATION_APP_BUILDDIR="\$FOUNDATION_APP_DIR/build"

        export FOUNDATION_INGRESS_DIR="\$FOUNDATION_BASEDIR/ingress"

        # psql variables for convenience
        export PGHOST="\$FOUNDATION_DB_DIR"
        export PGDATABASE="\$FOUNDATION_DB_NAME"
        export PGUSER="\$FOUNDATION_DB_SUPERUSER"
        export PGPASSWORD="\$FOUNDATION_DB_SUPERUSER_PW"

        EOF

        ${pkgs.stdenv.shell} -n "$envfile"
        ${pkgs.shellcheck}/bin/shellcheck "$envfile"

        echo "$envfile"
      '';

  writers = {
    /*
     * Writes a shell script to bin/<name> and checks it with shellcheck.
     *
     */
    writeCheckedShellScriptBin =
      name: text:
        pkgs.writeTextFile {
          inherit name;
          executable = true;
          destination = "/bin/${name}";
          text =
            ''
              #!${pkgs.runtimeShell}
              ${text}
            '';
          checkPhase =
            ''
              # check syntax
              ${pkgs.stdenv.shell} -n $out/bin/${name}

              # check for shellcheck recommendations
              ${pkgs.shellcheck}/bin/shellcheck $out/bin/${name}
            '';
        };

    /*
     * Writes a shell script and checks it with shellcheck.
     *
     */
    writeCheckedShellScript =
      name: text:
        pkgs.writeTextFile {
          inherit name;
          executable = true;
          text =
            ''
              #!${pkgs.runtimeShell}
              ${text}
            '';
          checkPhase =
            ''
              # check syntax
              ${pkgs.stdenv.shell} -n $out

              # check for shellcheck recommendations
              ${pkgs.shellcheck}/bin/shellcheck $out
            '';
        };

    writeNginxConf =
      name: text:
        pkgs.writeTextFile {
          inherit name text;
          checkPhase =
            ''
              ${pkgs.gixy}/bin/gixy $out > /dev/null
            '';
        };

    writeLuaScript =
      name: text:
        pkgs.writeTextFile {
          inherit name text;
          checkPhase =
            ''
              ${pkgs.luajit}/bin/luajit -bl $out > /dev/null
            '';
        };
  };

  dbContainer =
    pkgs.dockerTools.buildImage {
      name = "foundation-db";
      tag = "latest";
      config = {
        Cmd = [ "${dbService.run}" ];
        WorkingDir = "/";
        Volumes = {};
      };
    };

  formatSource =
    writers.writeCheckedShellScriptBin "foundation-format"
      ''
        ${pkgs.elmPackages.elm-format}/bin/elm-format --yes "$FOUNDATION_SRC"/web/src > /dev/null
        ${pkgs.nixpkgs-fmt}/bin/nixpkgs-fmt "$FOUNDATION_SRC"/shell.nix > /dev/null
      '';

  stats =
    writers.writeCheckedShellScriptBin "foundation-stats"
      ''
        ${pkgs.cloc}/bin/cloc "$FOUNDATION_SRC"
      '';

  testWeb =
    writers.writeCheckedShellScriptBin "foundation-testweb"
      ''
        ${pythonTesting}/bin/py.test "$FOUNDATION_SRC"/tests/web
      '';
in
pkgs.stdenv.mkDerivation {
  name = "foundation";

  buildInputs = [
    pkgs.graphviz
    pkgs.less # for psql display of large tables
    postgresql
    pythonTesting
    mkEnv
    pkgs.nodePackages.uglify-js
    foundationService.launcher
    dbService.launcher
    apiService.launcher
    ingressService.launcher
    docsService.launcher
    webService.launcher
    formatSource
    stats
    testWeb
  ];

  shellHook = ''
    tmpdir=$(mktemp -d)

    trap "rm -rf $tmpdir" exit

    source $(${mkEnv}/bin/foundation-mkenv . "$tmpdir")

    cat << EOF
    $(${pkgs.ncurses}/bin/tput setaf 2)
    Foundation development environment
    $(${pkgs.ncurses}/bin/tput sgr0)
    Temporary environment: $tmpdir (will be deleted on exit)
    To use that environment from another shell, run "source $tmpdir/env".
    Use "foundation-mkenv" to create a new environment.

    To launch a development server, run "foundation run" or "foundation watch".
    EOF
  '';
}

This part:

                types {
                  text/html html;
                  text/css css;
                  application/javascript js;
                  image/png png;
                  image/svg+xml svg;
                  font/woff woff;
                  font/woff2 woff2;
                }

I think it can be replaced by:

	include ${pkgs.nginx}/conf/mime.types;

You are right, that's a much nicer solution! I will include it here: https://github.com/monacoremo/full-stack-experiment