cleverca22/amazon-image.nix Secret

## amazon-image.nix
{ pkgs, ... }:

{
    # Mount all formatted ephemeral disks and activate all swap devices.
    # We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options
    # because the set of devices is dependent on the instance type
    # (e.g. "m1.large" has one ephemeral filesystem and one swap device,
    # while "m1.large" has two ephemeral filesystems and no swap
    # devices).  Also, put /tmp and /var on /disk0, since it has a lot
    # more space than the root device.  Similarly, "move" /nix to /disk0
    # by layering a unionfs-fuse mount on top of it so we have a lot more space for
    # Nix operations.
    boot.initrd.postMountCommands =
      ''
        metaDir=$targetRoot/etc/ec2-metadata
        mkdir -m 0755 -p "$metaDir"

        echo "getting EC2 instance metadata..."

        if ! [ -e "$metaDir/ami-manifest-path" ]; then
          wget -q -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
        fi

        if ! [ -e "$metaDir/user-data" ]; then
          wget -q -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data"
        fi

        if ! [ -e "$metaDir/hostname" ]; then
          wget -q -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
        fi

        if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then
          wget -q -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
        fi

        diskNr=0
        diskForUnionfs=
        for device in /dev/xvd[abcde]*; do
            if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi
            fsType=$(blkid -o value -s TYPE "$device" || true)
            if [ "$fsType" = swap ]; then
                echo "activating swap device $device..."
                swapon "$device" || true
            elif [ "$fsType" = ext3 ]; then
                mp="/disk$diskNr"
                diskNr=$((diskNr + 1))
                if mountFS "$device" "$mp" "" ext3; then
                    if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi
                fi
            else
                echo "skipping unknown device type $device"
            fi
        done

        if [ -n "$diskForUnionfs" ]; then
            mkdir -m 755 -p $targetRoot/$diskForUnionfs/root

            mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
            mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp

            if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then
                mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var
                mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var

                mkdir -p /unionfs-chroot/ro-nix
                mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix

                mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix
                mkdir -p /unionfs-chroot/rw-nix
                mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix

                unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
            fi
        fi
      '';

    boot.initrd.extraUtilsCommands =
      ''
        # We need swapon in the initrd.
        copy_bin_and_libs ${pkgs.utillinux}/sbin/swapon
      '';
}

## make-zfs-image.nix
with import <nixpkgs> {};

{
  name ? "zfs-image",
  poolName ? "tank",
  bootSize ? 1024,
  diskSize ? 1024 * 2
}:
let
  eval = import <nixpkgs/nixos> { inherit configuration; };
  config = eval.config;
  configuration = { ... }: {
    imports = [
      <nixpkgs/nixos/modules/profiles/headless.nix>
      <nixpkgs/nixos/modules/virtualisation/ec2-data.nix>
      <nixpkgs/nixos/modules/virtualisation/amazon-init.nix>
      ./amazon-image.nix
    ];
    boot = {
      loader.grub.device = "/dev/xvda";
      initrd = {
        availableKernelModules = [ "virtio_pci" "virtio_blk" "xen-blkfront" "xen-netfront" ];
        postDeviceCommands = lib.mkMerge [
          (lib.mkBefore ''
            echo resizing xvda3
            TMPDIR=/run sh $(type -P growpart) "/dev/xvda" "3"
            udevadm settle
          '')
          # zfs mounts within postDeviceCommands so mkBefore and mkAfter must be used
          (lib.mkAfter ''
            zpool online -e ${poolName} /dev/xvda3
          '')
        ];
        network.enable = true;
      };
      zfs.devNodes = "/dev/";
      # growPartition does not support zfs directly, so the above postDeviceCommands use what this puts into PATH
      growPartition = true;
      kernelParams = [ "console=ttyS0" ];
      blacklistedKernelModules = [ "nouveau" "xen_fbfront" ];
    };
    fileSystems = {
      "/"     = { fsType = "zfs"; device = "${poolName}/root"; };
      "/home" = { fsType = "zfs"; device = "${poolName}/home"; };
      "/nix"  = { fsType = "zfs"; device = "${poolName}/nix"; };
    };
    networking = {
      hostId = "9474d585";
      hostName = lib.mkDefault "";
      # xen host on aws
      timeServers = [ "169.254.169.123" ];
    };
    environment.systemPackages = [ pkgs.cryptsetup ];
    services.openssh = {
      enable = true;
      permitRootLogin = "prohibit-password";
    };
  };
  closureInfo = pkgs.closureInfo { rootPaths = [ config.system.build.toplevel channelSources ]; };
  preVM = ''
    PATH=$PATH:${qemu_kvm}/bin
    mkdir $out
    diskImage=nixos.raw
    qemu-img create -f qcow2 $diskImage ${toString diskSize}M
  '';
  postVM = ''
    qemu-img convert -f qcow2 -O vpc $diskImage $out/nixos.vhd
    ls -ltrhs $out/ $diskImage
    time sync $out/nixos.vhd
    ls -ltrhs $out/
  '';
  modulesTree = aggregateModules [ linuxPackages.kernel linuxPackages.zfs linuxPackages.spl ];
  nixpkgs = lib.cleanSource pkgs.path;
  # FIXME: merge with channel.nix / make-channel.nix.
  channelSources = pkgs.runCommand "nixos-${config.system.nixos.version}" {} ''
    mkdir -p $out
    cp -prd ${nixpkgs} $out/nixos
    chmod -R u+w $out/nixos
    if [ ! -e $out/nixos/nixpkgs ]; then
      ln -s . $out/nixos/nixpkgs
    fi
    rm -rf $out/nixos/.git
    echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix
  '';
in (vmTools.override {
  rootModules = [ "zfs" "9p" "9pnet_virtio" "virtio_pci" "virtio_blk" "rtc_cmos" ];
  kernel = modulesTree;
}).runInLinuxVM (runCommand name { inherit preVM postVM; } ''
  export PATH=${lib.makeBinPath [ nix e2fsprogs zfs utillinux config.system.build.nixos-enter config.system.build.nixos-install ]}:$PATH

  cp -sv /dev/vda /dev/xvda

  export NIX_STATE_DIR=$TMPDIR/state
  nix-store --load-db < ${closureInfo}/registration

  sfdisk /dev/vda <<EOF
  label: gpt
  device: /dev/vda
  unit: sectors
  1 : size=2048, type=21686148-6449-6E6F-744E-656564454649
  2 : size=${toString (bootSize*2048)}, type=0FC63DAF-8483-4772-8E79-3D69D8477DE4
  3 : type=CA7D7CCB-63ED-4C53-861C-1742536059CC
  EOF

  mkfs.ext4 /dev/vda2 -L NIXOS_BOOT
  zpool create -o ashift=12 -o altroot=/mnt -o autoexpand=on ${poolName} /dev/vda3
  zfs create -o mountpoint=legacy ${poolName}/root
  zfs create -o mountpoint=legacy ${poolName}/home
  zfs create -o mountpoint=legacy ${poolName}/nix
  mount -t zfs ${poolName}/root /mnt
  mkdir /mnt/{home,nix,boot}
  mount -t zfs ${poolName}/home /mnt/home
  mount -t zfs ${poolName}/nix /mnt/nix
  mount -t ext4 /dev/vda2 /mnt/boot
  zfs set compression=lz4 ${poolName}/nix
  zfs set xattr=off ${poolName}/nix
  zfs set atime=off ${poolName}/nix
  echo copying toplevel
  time nix copy --no-check-sigs --to 'local?root=/mnt/' ${config.system.build.toplevel}
  echo copying channels
  time nix copy --no-check-sigs --to 'local?root=/mnt/' ${channelSources}

  echo installing bootloader
  time nixos-install --root /mnt --no-root-passwd --system ${config.system.build.toplevel} --channel ${channelSources} --substituters ""

  zfs inherit compression ${poolName}/nix
  df -h
  umount /mnt/{home,nix,boot,}
  zpool export ${poolName}
'')

## testit.nix
with import <nixpkgs> {};

writeScript "testit" ''
  #!${stdenv.shell}

  set -e

  export PATH=${lib.makeBinPath [ qemu_kvm ]}:$PATH

  qemu-img create -f qcow2 disk.img -o backing_file=result/nixos.vhd -o size=10g
  qemu-kvm -drive file=disk.img,if=virtio,index=0 -serial stdio -m 2048
''

## upload-ami.nix
with import <nixpkgs> {};

writeScript "upload-amis" ''
  #!${stdenv.shell}

  set -e
  export PATH=${lib.makeBinPath [ ec2_ami_tools jq ec2_api_tools awscli qemu ]}:$PATH

  set -o pipefail

  version=${lib.version}-5
  major=${version:0:5}
  echo "NixOS version is $version ($major)"

  stateDir=$HOME/amis/ec2-image-$version/
  mkdir -p $stateDir

  regions="eu-west-1"
  types="hvm"
  stores="ebs"

  for type in $types; do
    link=${import ./make-zfs-image.nix {}}
    imageFile=$link/nixos.vhd
    system=x86_64-linux
    arch=x86_64
    for store in $stores; do
      bucket=iohk-amis
      bucketDir="$version-$type-$store"

      prevAmi=
      prevRegion=

      for region in $regions; do
        name=nixos-$version-$arch-$type-$store
        description="NixOS $system $version ($type-$store)"

        amiFile=$stateDir/$region.$type.$store.ami-id
        if ! [ -e $amiFile ]; then
          echo "doing $name in $region..."
          if [ -n "$prevAmi" ]; then
            ami=$(aws ec2 copy-image \
              --region "$region" \
              --source-region "$prevRegion" --source-image-id "$prevAmi" \
              --name "$name" --description "$description" | jq -r '.ImageId')
            if [ "$ami" = null ]; then break; fi
          else
            vhdFile=$imageFile
            vhdFileLogicalBytes="$(qemu-img info "$vhdFile" | grep ^virtual\ size: | cut -f 2 -d \(  | cut -f 1 -d \ )"
            vhdFileLogicalGigaBytes=$(((vhdFileLogicalBytes-1)/1024/1024/1024+1)) # Round to the next GB
            echo "Disk size is $vhdFileLogicalBytes bytes. Will be registered as $vhdFileLogicalGigaBytes GB."
            taskId=$(cat $stateDir/$region.$type.task-id 2> /dev/null || true)
            volId=$(cat $stateDir/$region.$type.vol-id 2> /dev/null || true)
            snapId=$(cat $stateDir/$region.$type.snap-id 2> /dev/null || true)

            if [ -z "$snapId" -a -z "$volId" -a -z "$taskId" ]; then
              echo "importing $vhdFile..."
              taskId=$(ec2-import-volume $vhdFile --no-upload -f vhd \
                -O "$AWS_ACCESS_KEY_ID" -W "$AWS_SECRET_ACCESS_KEY" \
                -o "$AWS_ACCESS_KEY_ID" -w "$AWS_SECRET_ACCESS_KEY" \
                --region "$region" -z "''${region}a" \
                --bucket "$bucket" --prefix "$bucketDir/" \
                | tee /dev/stderr \
                | sed 's/.*\(import-vol-[0-9a-z]\+\).*/\1/ ; t ; d')
              echo -n "$taskId" > $stateDir/$region.$type.task-id
            fi

            if [ -z "$snapId" -a -z "$volId" ]; then
              ec2-resume-import  $vhdFile -t "$taskId" --region "$region" \
                -O "$AWS_ACCESS_KEY_ID" -W "$AWS_SECRET_ACCESS_KEY" \
                -o "$AWS_ACCESS_KEY_ID" -w "$AWS_SECRET_ACCESS_KEY"
            fi

            # Wait for the volume creation to finish.
            if [ -z "$snapId" -a -z "$volId" ]; then
              echo "waiting for import to finish..."

              while true; do
                volId=$(aws ec2 describe-conversion-tasks --conversion-task-ids "$taskId" --region "$region" | jq -r .ConversionTasks[0].ImportVolume.Volume.Id)
                done=$(aws ec2 describe-conversion-tasks --conversion-task-ids "$taskId" --region "$region" | jq -r .ConversionTasks[0].State)
                if [ "$done" == completed ]; then break; fi
                echo -n .
                sleep 10
              done
              echo -n "$volId" > $stateDir/$region.$type.vol-id
            fi
            if [ -n "$volId" -a -n "$taskId" ]; then
              echo "removing import task..."
              ec2-delete-disk-image -t "$taskId" --region "$region" \
                -O "$AWS_ACCESS_KEY_ID" -W "$AWS_SECRET_ACCESS_KEY" \
                -o "$AWS_ACCESS_KEY_ID" -w "$AWS_SECRET_ACCESS_KEY" || true
              rm -f $stateDir/$region.$type.task-id
            fi
            if [ -z "$snapId" ]; then
              echo "creating snapshot..."
              snapId=$(aws ec2 create-snapshot --volume-id "$volId" --region "$region" --description "$description" | jq -r .SnapshotId)
              if [ "$snapId" = null ]; then exit 1; fi
              echo -n "$snapId" > $stateDir/$region.$type.snap-id
            fi
            echo "waiting for snapshot to finish..."
            while true; do
              status=$(aws ec2 describe-snapshots --snapshot-ids "$snapId" --region "$region" | jq -r .Snapshots[0].State)
              if [ "$status" = completed ]; then break; fi
              sleep 10
            done
            # Delete the volume
            if [ -n "$volId" ]; then
              echo "deleting volume..."
              aws ec2 delete-volume --volume-id "$volId" --region "$region" || true
              rm -f $stateDir/$region.$type.vol-id
            fi
            blockDeviceMappings="DeviceName=/dev/sda1,Ebs={SnapshotId=$snapId,VolumeSize=$vhdFileLogicalGigaBytes,DeleteOnTermination=true,VolumeType=gp2}"
            extraFlags=""
            extraFlags+=" --root-device-name /dev/sda1"
            extraFlags+=" --sriov-net-support simple"
            extraFlags+=" --ena-support"
            blockDeviceMappings+=" DeviceName=/dev/sdb,VirtualName=ephemeral0"
            blockDeviceMappings+=" DeviceName=/dev/sdc,VirtualName=ephemeral1"
            blockDeviceMappings+=" DeviceName=/dev/sdd,VirtualName=ephemeral2"
            blockDeviceMappings+=" DeviceName=/dev/sde,VirtualName=ephemeral3"
            extraFlags+=" --sriov-net-support simple"
            extraFlags+=" --ena-support"
            extraFlags+=" --virtualization-type hvm"

            ami=$(aws ec2 register-image \
              --name "$name" \
              --description "$description" \
              --region "$region" \
              --architecture "$arch" \
              --block-device-mappings $blockDeviceMappings \
              $extraFlags | jq -r .ImageId)
            if [ "$ami" = null ]; then break; fi
          fi
          echo -n "$ami" > $amiFile
          echo "created AMI $ami of type '$type' in $region..."
        else
          ami=$(cat $amiFile)
        fi
        echo "region = $region, type = $type, store = $store, ami = $ami"
        if [ -z "$prevAmi" ]; then
          prevAmi="$ami"
          prevRegion="$region"
        fi
      done
    done
  done
''
	{ pkgs, ... }:

	{
	# Mount all formatted ephemeral disks and activate all swap devices.
	# We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options
	# because the set of devices is dependent on the instance type
	# (e.g. "m1.large" has one ephemeral filesystem and one swap device,
	# while "m1.large" has two ephemeral filesystems and no swap
	# devices). Also, put /tmp and /var on /disk0, since it has a lot
	# more space than the root device. Similarly, "move" /nix to /disk0
	# by layering a unionfs-fuse mount on top of it so we have a lot more space for
	# Nix operations.
	boot.initrd.postMountCommands =
	''
	metaDir=$targetRoot/etc/ec2-metadata
	mkdir -m 0755 -p "$metaDir"

	echo "getting EC2 instance metadata..."

	if ! [ -e "$metaDir/ami-manifest-path" ]; then
	wget -q -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
	fi

	if ! [ -e "$metaDir/user-data" ]; then
	wget -q -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data"
	fi

	if ! [ -e "$metaDir/hostname" ]; then
	wget -q -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
	fi

	if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then
	wget -q -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
	fi

	diskNr=0
	diskForUnionfs=
	for device in /dev/xvd[abcde]*; do
	if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi
	fsType=$(blkid -o value -s TYPE "$device" \|\| true)
	if [ "$fsType" = swap ]; then
	echo "activating swap device $device..."
	swapon "$device" \|\| true
	elif [ "$fsType" = ext3 ]; then
	mp="/disk$diskNr"
	diskNr=$((diskNr + 1))
	if mountFS "$device" "$mp" "" ext3; then
	if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi
	fi
	else
	echo "skipping unknown device type $device"
	fi
	done

	if [ -n "$diskForUnionfs" ]; then
	mkdir -m 755 -p $targetRoot/$diskForUnionfs/root

	mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
	mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp

	if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then
	mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var
	mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var

	mkdir -p /unionfs-chroot/ro-nix
	mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix

	mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix
	mkdir -p /unionfs-chroot/rw-nix
	mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix

	unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
	fi
	fi
	'';

	boot.initrd.extraUtilsCommands =
	''
	# We need swapon in the initrd.
	copy_bin_and_libs ${pkgs.utillinux}/sbin/swapon
	'';
	}
	with import <nixpkgs> {};

	{
	name ? "zfs-image",
	poolName ? "tank",
	bootSize ? 1024,
	diskSize ? 1024 * 2
	}:
	let
	eval = import <nixpkgs/nixos> { inherit configuration; };
	config = eval.config;
	configuration = { ... }: {
	imports = [
	<nixpkgs/nixos/modules/profiles/headless.nix>
	<nixpkgs/nixos/modules/virtualisation/ec2-data.nix>
	<nixpkgs/nixos/modules/virtualisation/amazon-init.nix>
	./amazon-image.nix
	];
	boot = {
	loader.grub.device = "/dev/xvda";
	initrd = {
	availableKernelModules = [ "virtio_pci" "virtio_blk" "xen-blkfront" "xen-netfront" ];
	postDeviceCommands = lib.mkMerge [
	(lib.mkBefore ''
	echo resizing xvda3
	TMPDIR=/run sh $(type -P growpart) "/dev/xvda" "3"
	udevadm settle
	'')
	# zfs mounts within postDeviceCommands so mkBefore and mkAfter must be used
	(lib.mkAfter ''
	zpool online -e ${poolName} /dev/xvda3
	'')
	];
	network.enable = true;
	};
	zfs.devNodes = "/dev/";
	# growPartition does not support zfs directly, so the above postDeviceCommands use what this puts into PATH
	growPartition = true;
	kernelParams = [ "console=ttyS0" ];
	blacklistedKernelModules = [ "nouveau" "xen_fbfront" ];
	};
	fileSystems = {
	"/" = { fsType = "zfs"; device = "${poolName}/root"; };
	"/home" = { fsType = "zfs"; device = "${poolName}/home"; };
	"/nix" = { fsType = "zfs"; device = "${poolName}/nix"; };
	};
	networking = {
	hostId = "9474d585";
	hostName = lib.mkDefault "";
	# xen host on aws
	timeServers = [ "169.254.169.123" ];
	};
	environment.systemPackages = [ pkgs.cryptsetup ];
	services.openssh = {
	enable = true;
	permitRootLogin = "prohibit-password";
	};
	};
	closureInfo = pkgs.closureInfo { rootPaths = [ config.system.build.toplevel channelSources ]; };
	preVM = ''
	PATH=$PATH:${qemu_kvm}/bin
	mkdir $out
	diskImage=nixos.raw
	qemu-img create -f qcow2 $diskImage ${toString diskSize}M
	'';
	postVM = ''
	qemu-img convert -f qcow2 -O vpc $diskImage $out/nixos.vhd
	ls -ltrhs $out/ $diskImage
	time sync $out/nixos.vhd
	ls -ltrhs $out/
	'';
	modulesTree = aggregateModules [ linuxPackages.kernel linuxPackages.zfs linuxPackages.spl ];
	nixpkgs = lib.cleanSource pkgs.path;
	# FIXME: merge with channel.nix / make-channel.nix.
	channelSources = pkgs.runCommand "nixos-${config.system.nixos.version}" {} ''
	mkdir -p $out
	cp -prd ${nixpkgs} $out/nixos
	chmod -R u+w $out/nixos
	if [ ! -e $out/nixos/nixpkgs ]; then
	ln -s . $out/nixos/nixpkgs
	fi
	rm -rf $out/nixos/.git
	echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix
	'';
	in (vmTools.override {
	rootModules = [ "zfs" "9p" "9pnet_virtio" "virtio_pci" "virtio_blk" "rtc_cmos" ];
	kernel = modulesTree;
	}).runInLinuxVM (runCommand name { inherit preVM postVM; } ''
	export PATH=${lib.makeBinPath [ nix e2fsprogs zfs utillinux config.system.build.nixos-enter config.system.build.nixos-install ]}:$PATH

	cp -sv /dev/vda /dev/xvda

	export NIX_STATE_DIR=$TMPDIR/state
	nix-store --load-db < ${closureInfo}/registration

	sfdisk /dev/vda <<EOF
	label: gpt
	device: /dev/vda
	unit: sectors
	1 : size=2048, type=21686148-6449-6E6F-744E-656564454649
	2 : size=${toString (bootSize*2048)}, type=0FC63DAF-8483-4772-8E79-3D69D8477DE4
	3 : type=CA7D7CCB-63ED-4C53-861C-1742536059CC
	EOF

	mkfs.ext4 /dev/vda2 -L NIXOS_BOOT
	zpool create -o ashift=12 -o altroot=/mnt -o autoexpand=on ${poolName} /dev/vda3
	zfs create -o mountpoint=legacy ${poolName}/root
	zfs create -o mountpoint=legacy ${poolName}/home
	zfs create -o mountpoint=legacy ${poolName}/nix
	mount -t zfs ${poolName}/root /mnt
	mkdir /mnt/{home,nix,boot}
	mount -t zfs ${poolName}/home /mnt/home
	mount -t zfs ${poolName}/nix /mnt/nix
	mount -t ext4 /dev/vda2 /mnt/boot
	zfs set compression=lz4 ${poolName}/nix
	zfs set xattr=off ${poolName}/nix
	zfs set atime=off ${poolName}/nix
	echo copying toplevel
	time nix copy --no-check-sigs --to 'local?root=/mnt/' ${config.system.build.toplevel}
	echo copying channels
	time nix copy --no-check-sigs --to 'local?root=/mnt/' ${channelSources}

	echo installing bootloader
	time nixos-install --root /mnt --no-root-passwd --system ${config.system.build.toplevel} --channel ${channelSources} --substituters ""

	zfs inherit compression ${poolName}/nix
	df -h
	umount /mnt/{home,nix,boot,}
	zpool export ${poolName}
	'')
	with import <nixpkgs> {};

	writeScript "testit" ''
	#!${stdenv.shell}

	set -e

	export PATH=${lib.makeBinPath [ qemu_kvm ]}:$PATH

	qemu-img create -f qcow2 disk.img -o backing_file=result/nixos.vhd -o size=10g
	qemu-kvm -drive file=disk.img,if=virtio,index=0 -serial stdio -m 2048
	''
	with import <nixpkgs> {};

	writeScript "upload-amis" ''
	#!${stdenv.shell}

	set -e
	export PATH=${lib.makeBinPath [ ec2_ami_tools jq ec2_api_tools awscli qemu ]}:$PATH

	set -o pipefail

	version=${lib.version}-5
	major=${version:0:5}
	echo "NixOS version is $version ($major)"

	stateDir=$HOME/amis/ec2-image-$version/
	mkdir -p $stateDir

	regions="eu-west-1"
	types="hvm"
	stores="ebs"

	for type in $types; do
	link=${import ./make-zfs-image.nix {}}
	imageFile=$link/nixos.vhd
	system=x86_64-linux
	arch=x86_64
	for store in $stores; do
	bucket=iohk-amis
	bucketDir="$version-$type-$store"

	prevAmi=
	prevRegion=

	for region in $regions; do
	name=nixos-$version-$arch-$type-$store
	description="NixOS $system $version ($type-$store)"

	amiFile=$stateDir/$region.$type.$store.ami-id
	if ! [ -e $amiFile ]; then
	echo "doing $name in $region..."
	if [ -n "$prevAmi" ]; then
	ami=$(aws ec2 copy-image \
	--region "$region" \
	--source-region "$prevRegion" --source-image-id "$prevAmi" \
	--name "$name" --description "$description" \| jq -r '.ImageId')
	if [ "$ami" = null ]; then break; fi
	else
	vhdFile=$imageFile
	vhdFileLogicalBytes="$(qemu-img info "$vhdFile" \| grep ^virtual\ size: \| cut -f 2 -d \( \| cut -f 1 -d \ )"
	vhdFileLogicalGigaBytes=$(((vhdFileLogicalBytes-1)/1024/1024/1024+1)) # Round to the next GB
	echo "Disk size is $vhdFileLogicalBytes bytes. Will be registered as $vhdFileLogicalGigaBytes GB."
	taskId=$(cat $stateDir/$region.$type.task-id 2> /dev/null \|\| true)
	volId=$(cat $stateDir/$region.$type.vol-id 2> /dev/null \|\| true)
	snapId=$(cat $stateDir/$region.$type.snap-id 2> /dev/null \|\| true)

	if [ -z "$snapId" -a -z "$volId" -a -z "$taskId" ]; then
	echo "importing $vhdFile..."
	taskId=$(ec2-import-volume $vhdFile --no-upload -f vhd \
	-O "$AWS_ACCESS_KEY_ID" -W "$AWS_SECRET_ACCESS_KEY" \
	-o "$AWS_ACCESS_KEY_ID" -w "$AWS_SECRET_ACCESS_KEY" \
	--region "$region" -z "''${region}a" \
	--bucket "$bucket" --prefix "$bucketDir/" \
	\| tee /dev/stderr \
	\| sed 's/.\(import-vol-[0-9a-z]\+\)./\1/ ; t ; d')
	echo -n "$taskId" > $stateDir/$region.$type.task-id
	fi

	if [ -z "$snapId" -a -z "$volId" ]; then
	ec2-resume-import $vhdFile -t "$taskId" --region "$region" \
	-O "$AWS_ACCESS_KEY_ID" -W "$AWS_SECRET_ACCESS_KEY" \
	-o "$AWS_ACCESS_KEY_ID" -w "$AWS_SECRET_ACCESS_KEY"
	fi

	# Wait for the volume creation to finish.
	if [ -z "$snapId" -a -z "$volId" ]; then
	echo "waiting for import to finish..."

	while true; do
	volId=$(aws ec2 describe-conversion-tasks --conversion-task-ids "$taskId" --region "$region" \| jq -r .ConversionTasks[0].ImportVolume.Volume.Id)
	done=$(aws ec2 describe-conversion-tasks --conversion-task-ids "$taskId" --region "$region" \| jq -r .ConversionTasks[0].State)
	if [ "$done" == completed ]; then break; fi
	echo -n .
	sleep 10
	done
	echo -n "$volId" > $stateDir/$region.$type.vol-id
	fi
	if [ -n "$volId" -a -n "$taskId" ]; then
	echo "removing import task..."
	ec2-delete-disk-image -t "$taskId" --region "$region" \
	-O "$AWS_ACCESS_KEY_ID" -W "$AWS_SECRET_ACCESS_KEY" \
	-o "$AWS_ACCESS_KEY_ID" -w "$AWS_SECRET_ACCESS_KEY" \|\| true
	rm -f $stateDir/$region.$type.task-id
	fi
	if [ -z "$snapId" ]; then
	echo "creating snapshot..."
	snapId=$(aws ec2 create-snapshot --volume-id "$volId" --region "$region" --description "$description" \| jq -r .SnapshotId)
	if [ "$snapId" = null ]; then exit 1; fi
	echo -n "$snapId" > $stateDir/$region.$type.snap-id
	fi
	echo "waiting for snapshot to finish..."
	while true; do
	status=$(aws ec2 describe-snapshots --snapshot-ids "$snapId" --region "$region" \| jq -r .Snapshots[0].State)
	if [ "$status" = completed ]; then break; fi
	sleep 10
	done
	# Delete the volume
	if [ -n "$volId" ]; then
	echo "deleting volume..."
	aws ec2 delete-volume --volume-id "$volId" --region "$region" \|\| true
	rm -f $stateDir/$region.$type.vol-id
	fi
	blockDeviceMappings="DeviceName=/dev/sda1,Ebs={SnapshotId=$snapId,VolumeSize=$vhdFileLogicalGigaBytes,DeleteOnTermination=true,VolumeType=gp2}"
	extraFlags=""
	extraFlags+=" --root-device-name /dev/sda1"
	extraFlags+=" --sriov-net-support simple"
	extraFlags+=" --ena-support"
	blockDeviceMappings+=" DeviceName=/dev/sdb,VirtualName=ephemeral0"
	blockDeviceMappings+=" DeviceName=/dev/sdc,VirtualName=ephemeral1"
	blockDeviceMappings+=" DeviceName=/dev/sdd,VirtualName=ephemeral2"
	blockDeviceMappings+=" DeviceName=/dev/sde,VirtualName=ephemeral3"
	extraFlags+=" --sriov-net-support simple"
	extraFlags+=" --ena-support"
	extraFlags+=" --virtualization-type hvm"

	ami=$(aws ec2 register-image \
	--name "$name" \
	--description "$description" \
	--region "$region" \
	--architecture "$arch" \
	--block-device-mappings $blockDeviceMappings \
	$extraFlags \| jq -r .ImageId)
	if [ "$ami" = null ]; then break; fi
	fi
	echo -n "$ami" > $amiFile
	echo "created AMI $ami of type '$type' in $region..."
	else
	ami=$(cat $amiFile)
	fi
	echo "region = $region, type = $type, store = $store, ami = $ami"
	if [ -z "$prevAmi" ]; then
	prevAmi="$ami"
	prevRegion="$region"
	fi
	done
	done
	done
	''