initiator.nix

{ config, lib, pkgs, ... }: with lib;
let
  cfg = config.services.openiscsi;
in {
  options.services.openiscsi = with types; {
    enable = mkEnableOption "the openiscsi iscsi daemon";
    enableAutoLoginOut = mkEnableOption ''
      automatic login and logout of all automatic targets.
      You probably do not want this.
    '';
    discoverPortal = mkOption {
      type = nullOr str;
      default = null;
      description = "Portal to discover targets on";
    };
    name = mkOption {
      type = str;
      description = "Name of this iscsi initiator";
      example = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
    };
    package = mkOption {
      type = package;
      description = "openiscsi package to use";
      default = pkgs.openiscsi;
      defaultText = "pkgs.openiscsi";
    };

    extraConfig = mkOption {
      type = str;
      default = "";
      description = "Lines to append to default iscsid.conf";
    };
  };

  config = mkIf cfg.enable {
    environment.etc."iscsi/iscsid.conf".source = pkgs.runCommand "iscsid.conf" {} ''
      cat "${cfg.package}/etc/iscsi/iscsid.conf" > $out
      cat << 'EOF' >> $out
      ${cfg.extraConfig}
      ${optionalString cfg.enableAutoLoginOut "node.startup = automatic"}
      EOF
    '';
    environment.etc."iscsi/initiatorname.iscsi".text = "InitiatorName=${cfg.name}";

    systemd.packages = [ cfg.package ];

    systemd.services."iscsid".wantedBy = [ "multi-user.target" ];
    systemd.sockets."iscsid".wantedBy = [ "sockets.target" ];

    systemd.services."iscsi" = mkIf cfg.enableAutoLoginOut {
      wantedBy = [ "remote-fs.target" ];
      serviceConfig.ExecStartPre = mkIf (cfg.discoverPortal != null) "${cfg.package}/bin/iscsiadm --mode discoverydb --type sendtargets --portal ${cfg.discoverPortal} --discover";
    };

    environment.systemPackages = [ cfg.package ];
    boot.kernelModules = [ "iscsi_tcp" ]; # TODO
  };
}

root-initiator.nix

{ config, lib, pkgs, ... }: with lib;
let
  cfg = config.boot.iscsi-initiator;
in {
  # Note: Theoretically you might want to connect to multiple portals and log in to multiple targets
  # However, we assume that you're only crazy enough to boot from iSCSI but not actually crazy enough to boot from multiple iSCSI targets
  # If you are, feel free to implement it, it shouldn't be _that_ hard.
  options.boot.iscsi-initiator = with types; {
    name = mkOption {
      description = ''
        Name of the iSCSI initiator to boot from.
      '';
      default = null;
      example = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
      type = nullOr str;
    };

    discoverPortal = mkOption {
      description = ''
        iSCSI portal to boot from.
      '';
      default = null;
      example = "192.168.1.1:3260";
      type = nullOr str;
    };

    target = mkOption {
      description = ''
        Name of the iSCSI target to boot from.
      '';
      default = null;
      example = "iqn.2020-08.org.linux-iscsi.targethost:example";
      type = nullOr str;
    };

    loginAll = mkOption {
      description = ''
        Do not log into a specific target on the portal, but to all that we discover.
        This overrides setting target.
      '';
      type = bool;
      default = false;
    };

    extraConfig = mkOption {
      description = "Extra lines to append to /etc/iscsid.conf";
      default = null;
      type = nullOr str;
    };
  };

  config = mkIf (cfg.name != null) {
    boot.initrd = {
      network.enable = true;

      kernelModules = [ "iscsi_tcp" ];

      extraUtilsCommands = ''
        copy_bin_and_libs ${pkgs.openiscsi}/bin/iscsid
        copy_bin_and_libs ${pkgs.openiscsi}/bin/iscsiadm
        ${optionalString (!config.boot.initrd.network.ssh.enable) "cp -pv ${pkgs.glibc.out}/lib/libnss_files.so.* $out/lib"}

        mkdir -p $out/etc
        cp ${config.environment.etc.hosts.source} $out/etc/hosts
        cp ${pkgs.openiscsi}/etc/iscsi/iscsid.conf $out/etc/iscsid.conf
        chmod +w $out/etc/iscsid.conf
        cat << 'EOF' >> $out/etc/iscsid.conf
        ${optionalString (cfg.extraConfig != null) cfg.extraConfig}
        EOF
      '';

      extraUtilsCommandsTest = ''
        $out/bin/iscsiadm --version
      '';

      preLVMCommands = ''
        ${optionalString (!config.boot.initrd.network.ssh.enable) ''
          # stolen from initrd-ssh.nix
          echo 'root:x:0:0:root:/root:/bin/ash' > /etc/passwd
          echo 'passwd: files' > /etc/nsswitch.conf
        ''}

        # oc
        cp -f $extraUtils/etc/hosts /etc/hosts

        mkdir -p /etc/iscsi /run/lock/iscsi
        echo "InitiatorName=${cfg.name}" > /etc/iscsi/initiatorname.iscsi
        cp -f $extraUtils/etc/iscsid.conf /etc/iscsi/iscsid.conf
        ${optionalString cfg.loginAll ''echo "node.startup = automatic" >> /etc/iscsi/iscsid.conf''}
        iscsid -f -n -d 9 &
        iscsiadm --mode discoverydb --type sendtargets --portal ${cfg.discoverPortal} --discover -d 9
        ${if cfg.loginAll then ''
          iscsiadm --mode node -L all
        '' else ''
          iscsiadm --mode node --targetname ${cfg.target} --login
        ''}
        pkill -9 iscsid
      '';

      postMountCommands = ''
        ln -sfn /nix/var/nix/profiles/system/init /mnt-root/init
        stage2Init=/init
      '';
    };

    services.openiscsi = {
      enable = true;
      inherit (cfg) name;
    };

    assertions = [
      {
        assertion = cfg.loginAll -> cfg.target == null;
        message = "iSCSI target name is set while login on all portals is enabled.";
      }
    ];
  };
}

target.nix

{ config, lib, pkgs, ... }:

with lib;

let
  cfg = config.services.target;
in
{
  ###### interface
  options = {
    services.target = with types; {
      enable = mkOption {
        type = bool;
        default = false;
        description = ''
          Whether to enable the kernel's LIO iscsi target
        '';
      };

      config = mkOption {
        type = attrs;
        default = {};
        description = ''
          Content of /etc/target/saveconfig.json
          This file is normally read and written by targetcli
        '';
      };
    };
  };

  ###### implementation
  config = mkIf cfg.enable {
    environment.etc."target/saveconfig.json" = {
      text = builtins.toJSON cfg.config;
      mode = "0600";
    };

    environment.systemPackages = with pkgs; [ targetcli ];

    boot.kernelModules = [ "configfs" "target_core_mod" "iscsi_target_mod" ];

    systemd.services.iscsi-target = {
      enable = true;
      after = [ "network.target" "local-fs.target" ];
      requires = [ "sys-kernel-config.mount" ];
      wantedBy = [ "multi-user.target" ];
      serviceConfig = {
        Type = "oneshot";
        ExecStart = "${pkgs.python3.pkgs.rtslib}/bin/targetctl restore";
        ExecStop = "${pkgs.python3.pkgs.rtslib}/bin/targetctl clear";
        RemainAfterExit = "yes";

        PrivateDevices = false;
        PrivateUsers = false;
        PrivateMounts = false;
      };

      # sandbox = 2;
      apparmor = {
        enable2 = true;
        extraConfig = ''
          network unix stream,
          network inet stream,
          network inet6 stream,
          capability sys_admin,

          @{PROC}@{pid}/mounts r,
          @{sys}/kernel/config/target/** rwklm,

          /etc/target/saveconfig.json rwklm,
          /dev/dm-* rwklm,
        '';
      };
    };

    systemd.tmpfiles.rules = [
      "d /etc/target 0700 root root - -"
    ];
  };
}

test.nix

import ./make-test-python.nix ({ pkgs, lib, flake, system, ... }:
let
  initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
  targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
in {
  name = "iscsi";
  skipLint = true;

  nodes = {
    target = { nodes, config, pkgs, lib, ... }: {
      services.target = {
        enable = true;
        config = {
          fabric_modules = [ ];
          storage_objects = [{
            dev = "/dev/vdb";
            name = "test";
            plugin = "block";
            write_back = true;
            wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
          }];
          targets = [{
            fabric = "iscsi";
            tpgs = [{
              enable = true;
              attributes = {
                authentication = 0;
                generate_node_acls = 1;
              };
              luns = [{
                alias = "94dfe06967";
                alua_tg_pt_gp_name = "default_tg_pt_gp";
                index = 0;
                storage_object = "/backstores/block/test";
              }];
              node_acls = [{
                mapped_luns = [{
                  alias = "d42f5bdf8a";
                  index = 0;
                  tpg_lun = 0;
                  write_protect = false;
                }];
                node_wwn = initiatorName;
              }];
              portals = [{
                ip_address = "0.0.0.0";
                iser = false;
                offload = false;
                port = 3260;
              }];
              tag = 1;
            }];
            wwn = targetName;
          }];
        };
      };

      networking.firewall.allowedTCPPorts = [ 3260 ];
      networking.firewall.allowedUDPPorts = [ 3260 ];

      virtualisation.memorySize = 2048;
      virtualisation.emptyDiskImages = [ 2048 ];

      system.extraDependencies = [ nodes.initiatorRootTargetName.config.system.build.toplevel ];

      nix.binaryCaches = lib.mkForce [ ];
      nix.extraOptions = ''
        hashed-mirrors =
        connect-timeout = 1
      '';
    };

    initiatorAuto = { config, pkgs, ... }: {
      services.openiscsi = {
        enable = true;
        enableAutoLoginOut = true;
        discoverPortal = "target";
        name = initiatorName;
      };

      environment.systemPackages = with pkgs; [
        xfsprogs
      ];
    };

    initiatorRootTargetName = { config, pkgs, modulesPath, lib, ... }: {
      imports = [
        "${modulesPath}/testing/test-instrumentation.nix"
        "${modulesPath}/virtualisation/qemu-vm.nix"
        ../2configs/tests.nix
      ];

      boot.loader.grub.enable = false;
      boot.kernelParams = lib.mkOverride 5 ([
        "boot.shell_on_fail"
        "console=tty1"
      ] ++ lib.optional (config.networking ? "primaryIPAddress") "ip=${config.networking.primaryIPAddress}:::255.255.255.0::ens9:none");
      # extremely useful for debugging
      # virtualisation.qemu.consoles = lib.mkForce [ "tty1" ];
      # defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
      virtualisation.writableStore = false;

      fileSystems = lib.mkOverride 5 {
        "/" = {
          fsType = "xfs";
          device = "/dev/sda";
        };
      };

      services.openiscsi = {
        enable = true;
        name = initiatorName;
      };

      boot.iscsi-initiator = {
        discoverPortal = "target";
        name = initiatorName;
        target = targetName;
      };
    };
  };

  testScript = { nodes, ... }: ''
    target.start()
    initiatorAuto.start()
    target.wait_for_unit("iscsi-target.service")

    initiatorAuto.wait_for_unit("iscsid.service")
    initiatorAuto.wait_for_unit("iscsi.service")
    initiatorAuto.get_unit_info("iscsi")

    initiatorAuto.succeed("mkfs.xfs /dev/sda")
    initiatorAuto.shutdown()
    target.stop_job("iscsi-target")
    target.succeed("mkdir /mnt; mount /dev/vdb /mnt")

    target.succeed('nixos-install --no-bootloader --no-root-passwd --system ${nodes.initiatorRootTargetName.config.system.build.toplevel}')
    target.succeed("umount /mnt; rmdir /mnt")
    target.start_job("iscsi-target")

    initiatorRootTargetName.start()
    initiatorRootTargetName.wait_for_unit("multi-user.target")
    initiatorRootTargetName.wait_for_unit("iscsid")
  '';
})