Kubernetes nixops deployment

certs.nix

{
  pkgs ? import <nixpkgs> {},
  internalDomain ? "cluster.local",
  externalDomain ? "<domain_name>",
  serviceClusterIp ? "10.0.0.1"
}:
let
  runWithCFSSL = name: cmd:
    builtins.fromJSON (builtins.readFile (
      pkgs.runCommand "${name}-cfss.json" {
        buildInputs = [ pkgs.cfssl ];
      } "cfssl ${cmd} > $out"
    ));

  writeCFSSL = content:
    pkgs.runCommand content.name {
      buildInputs = [ pkgs.cfssl ];
    } ''
      mkdir -p $out
      cd $out
      cat ${writeFile content} | cfssljson -bare ${content.name}
    '';

  noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;
  noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;

  writeFile = content: pkgs.writeText "content" (
    if pkgs.lib.isAttrs content then builtins.toJSON content
    else toString content
  );

  createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:
    noCSR (
      (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=server -config=${writeFile ca.config} ${writeFile {
        CN = cn;
        hosts = hosts;
        key = { algo = "rsa"; inherit size; };
      }}") // { inherit name; }
    );

  createClientCertKey = { ca, cn, groups ? [], size ? 2048, name ? cn }:
    noCSR (
      (runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=client -config=${writeFile ca.config} ${writeFile {
        CN = cn;
        names = map (group: {O = group;}) groups;
        hosts = [""];
        key = { algo = "rsa"; inherit size; };
      }}") // { inherit name; }
    );

  createSigningCertKey = { C ? "xx", ST ? "x", L ? "x", O ? "x", OU ? "x", CN ? "ca", emailAddress ? "x", expiry ? "43800h", size ? 2048, name ? CN }:
    (noCSR (runWithCFSSL CN "genkey -initca ${writeFile {
      key = { algo = "rsa"; inherit size; };
      names = [{ inherit C ST L O OU CN emailAddress; }];
    }}")) // {
      inherit name;
      config.signing = {
        default.expiry = expiry;
        profiles = {
          server = {
            inherit expiry;
            usages = [
              "signing"
              "key encipherment"
              "server auth"
            ];
          };
          client = {
            inherit expiry;
            usages = [
              "signing"
              "key encipherment"
              "client auth"
            ];
          };
          peer = {
            inherit expiry;
            usages = [
              "signing"
              "key encipherment"
              "server auth"
              "client auth"
            ];
          };
        };
      };
    };

  ca = createSigningCertKey {};

  kube-apiserver = createServingCertKey {
    inherit ca;
    cn = "kube-apiserver";
    hosts = ["kubernetes.default" "kubernetes.default.svc" "localhost" "api.${externalDomain}" serviceClusterIp];
  }; 

  kubelet = createServingCertKey {
    inherit ca;
    cn = "kubelet";
    hosts = ["*.${externalDomain}"];
  };

  service-accounts = createServingCertKey {
    inherit ca;
    cn = "kube-service-accounts";
  };

  etcd = createServingCertKey {
    inherit ca;
    cn = "etcd";
    hosts = ["etcd.${externalDomain}"];
  };

  etcd-client = createClientCertKey {
    inherit ca;
    cn = "etcd-client";
  };

  kubelet-client = createClientCertKey {
    inherit ca;
    cn = "kubelet-client";
    groups = ["system:masters"];
  };

  apiserver-client = {
    kubelet = createClientCertKey {
      inherit ca;
      cn = "apiserver-client-kubelet";
      groups = ["system:nodes"];
    };

    kube-proxy = createClientCertKey {
      inherit ca;
      name = "apiserver-client-kube-proxy";
      cn = "system:kube-proxy";
      groups = ["system:kube-proxy" "system:nodes"];
    };

    kube-controller-manager = createClientCertKey {
      inherit ca;
      name = "apiserver-client-kube-controller-manager";
      cn = "system:kube-controller-manager";
      groups = ["system:masters"];
    };

    kube-scheduler = createClientCertKey {
      inherit ca;
      name = "apiserver-client-kube-scheduler";
      cn = "system:kube-scheduler";
      groups = ["system:kube-scheduler"];
    };

    admin = createClientCertKey {
      inherit ca;
      cn = "admin";
      groups = ["system:masters"];
    };
  };
in {
  master = pkgs.buildEnv {
    name = "master-keys";
    paths = [
      (writeCFSSL (noKey ca))
      (writeCFSSL kube-apiserver)
      (writeCFSSL kubelet-client)
      (writeCFSSL apiserver-client.kube-controller-manager)
      (writeCFSSL apiserver-client.kube-scheduler)
      (writeCFSSL service-accounts)
      (writeCFSSL etcd)
    ];
  };

  worker = pkgs.buildEnv {
    name = "worker-keys";
    paths = [
      (writeCFSSL (noKey ca))
      (writeCFSSL kubelet)
      (writeCFSSL apiserver-client.kubelet)
      (writeCFSSL apiserver-client.kube-proxy)
      (writeCFSSL etcd-client)
    ];
  };

  admin = writeCFSSL apiserver-client.admin;
}

logical.nix

let
  base = { config, pkgs, nodes, lib, ... }: {
    networking = {
      domain = "<domain_name>";

      firewall = {
        allowedTCPPorts = [
          10250 # kubelet
          2379 2380  # etcd
          443 # kubernetes apiserver
        ];
        trustedInterfaces = ["docker0"];

        # allow any traffic from all of the nodes in the cluster
        extraCommands = lib.concatMapStrings  (node: ''
          iptables -A INPUT -s ${node.config.networking.publicIPv4} -j ACCEPT
        '') (lib.attrValues nodes);
      };
    };

    services.kubernetes = {
      featureGates = ["AllAlpha"];

      caFile = "/var/lib/kubernetes/certs/ca.pem";
      verbose = true;

      apiserver = {
        runtimeConfig = "authentication.k8s.io/v1beta1=true,batch/v2alpha1=true";
        tlsCertFile = "/var/lib/kubernetes/certs/kube-apiserver.pem";
        tlsKeyFile = "/var/lib/kubernetes/certs/kube-apiserver-key.pem";
        kubeletClientCertFile = "/var/lib/kubernetes/certs/kubelet-client.pem";
        kubeletClientKeyFile = "/var/lib/kubernetes/certs/kubelet-client-key.pem";
        serviceAccountKeyFile = "/var/lib/kubernetes/certs/kube-service-accounts.pem";
      };
      etcd = {
        servers = ["https://etcd.${config.networking.domain}:2379"];
        certFile = "/var/lib/kubernetes/certs/etcd-client.pem";
        keyFile = "/var/lib/kubernetes/certs/etcd-client-key.pem";
      };
      kubeconfig = {
        server = "https://api.${config.networking.domain}";
      };
      kubelet = {
        tlsCertFile = "/var/lib/kubernetes/certs/kubelet.pem";
        tlsKeyFile = "/var/lib/kubernetes/certs/kubelet-key.pem";
        hostname = "${config.networking.hostName}.${config.networking.domain}";
        kubeconfig = {
          certFile = "/var/lib/kubernetes/certs/apiserver-client-kubelet.pem";
          keyFile = "/var/lib/kubernetes/certs/apiserver-client-kubelet-key.pem";
        };
      };
      controllerManager = {
        serviceAccountKeyFile = "/var/lib/kubernetes/certs/kube-service-accounts-key.pem";
        kubeconfig = {
          certFile = "/var/lib/kubernetes/certs/apiserver-client-kube-controller-manager.pem";
          keyFile = "/var/lib/kubernetes/certs/apiserver-client-kube-controller-manager-key.pem";
        };
      };
      scheduler = {
        kubeconfig = {
          certFile = "/var/lib/kubernetes/certs/apiserver-client-kube-scheduler.pem";
          keyFile = "/var/lib/kubernetes/certs/apiserver-client-kube-scheduler-key.pem"; 
        };
      };
      proxy = {
        kubeconfig = {
          certFile = "/var/lib/kubernetes/certs/apiserver-client-kube-proxy.pem";
          keyFile = "/var/lib/kubernetes/certs/apiserver-client-kube-proxy-key.pem"; 
        };
      };
      flannel.enable = true;
      addons.dashboard.enable = true;
    };
  };
in {
  machine1 = { config, ... }: {
    require = [base];

    services.etcd = {
      enable = true;
      certFile = "/var/lib/kubernetes/certs/etcd.pem";
      keyFile = "/var/lib/kubernetes/certs/etcd-key.pem";
      trustedCaFile = "/var/lib/kubernetes/certs/ca.pem";
      peerClientCertAuth = true;
      listenClientUrls = ["https://0.0.0.0:2379"];
      listenPeerUrls = ["https://0.0.0.0:2380"];

      advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"];
      initialCluster = ["machine1=https://etcd.${config.networking.domain}:2380"];
      initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"];
    };

    services.kubernetes = {
      roles = ["node" "master"];
    };
  };

  machine2 = {
    require = [base];

    services.kubernetes.roles = ["node"];
  };

  machine3 = {
    require = [base];

    services.kubernetes.roles = ["node"];
  };

  machine4 = {
    require = [base];

    services.kubernetes.roles = ["node"];
  };

  machine5 = {
    require = [base];

    services.kubernetes.roles = ["node"];
  };
}