Skip to content

Instantly share code, notes, and snippets.

@cleverca22

cleverca22/make-zfs-image.nix Secret

Last active November 24, 2018 06:12
Show Gist options
  • Save cleverca22/db751e05561e7c74a1f1121242b00985 to your computer and use it in GitHub Desktop.
Save cleverca22/db751e05561e7c74a1f1121242b00985 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
make-zfs-image.nix
{ config, lib, pkgs, ... }:
let
name = "zfs-image";
poolName = "tank";
bootSize = 1024;
diskSize = 1024 * 2;
closureInfo = pkgs.closureInfo { rootPaths = [ config.system.build.toplevel channelSources ]; };
preVM = ''
PATH=$PATH:${pkgs.qemu_kvm}/bin
mkdir $out
diskImage=nixos.raw
qemu-img create -f qcow2 $diskImage ${toString diskSize}M
'';
postVM = ''
qemu-img convert -f qcow2 -O vpc $diskImage $out/nixos.vhd
ls -ltrhs $out/ $diskImage
time sync $out/nixos.vhd
ls -ltrhs $out/
'';
modulesTree = pkgs.aggregateModules (with config.boot.kernelPackages; [ kernel zfs spl ]);
nixpkgs = lib.cleanSource pkgs.path;
# FIXME: merge with channel.nix / make-channel.nix.
channelSources = pkgs.runCommand "nixos-${config.system.nixos.version}" {} ''
mkdir -p $out
cp -prd ${nixpkgs} $out/nixos
chmod -R u+w $out/nixos
if [ ! -e $out/nixos/nixpkgs ]; then
ln -s . $out/nixos/nixpkgs
fi
rm -rf $out/nixos/.git
echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix
'';
image = (pkgs.vmTools.override {
rootModules = [ "zfs" "9p" "9pnet_virtio" "virtio_pci" "virtio_blk" "rtc_cmos" ];
kernel = modulesTree;
}).runInLinuxVM (pkgs.runCommand name { inherit preVM postVM; } ''
export PATH=${lib.makeBinPath (with pkgs; [ nix e2fsprogs zfs utillinux config.system.build.nixos-enter config.system.build.nixos-install ])}:$PATH
cp -sv /dev/vda /dev/xvda
export NIX_STATE_DIR=$TMPDIR/state
nix-store --load-db < ${closureInfo}/registration
sfdisk /dev/vda <<EOF
label: gpt
device: /dev/vda
unit: sectors
1 : size=2048, type=21686148-6449-6E6F-744E-656564454649
2 : size=${toString (bootSize*2048)}, type=0FC63DAF-8483-4772-8E79-3D69D8477DE4
3 : type=CA7D7CCB-63ED-4C53-861C-1742536059CC
EOF
mkfs.ext4 /dev/vda2 -L NIXOS_BOOT
zpool create -o ashift=12 -o altroot=/mnt -o autoexpand=on ${poolName} /dev/vda3
zfs create -o mountpoint=legacy ${poolName}/root
zfs create -o mountpoint=legacy ${poolName}/home
zfs create -o mountpoint=legacy ${poolName}/nix
mount -t zfs ${poolName}/root /mnt
mkdir /mnt/{home,nix,boot}
mount -t zfs ${poolName}/home /mnt/home
mount -t zfs ${poolName}/nix /mnt/nix
mount -t ext4 /dev/vda2 /mnt/boot
zfs set compression=lz4 ${poolName}/nix
zfs set xattr=off ${poolName}/nix
zfs set atime=off ${poolName}/nix
echo copying toplevel
time nix copy --no-check-sigs --to 'local?root=/mnt/' ${config.system.build.toplevel}
echo copying channels
time nix copy --no-check-sigs --to 'local?root=/mnt/' ${channelSources}
echo installing bootloader
time nixos-install --root /mnt --no-root-passwd --system ${config.system.build.toplevel} --channel ${channelSources} --substituters ""
zfs inherit compression ${poolName}/nix
df -h
umount /mnt/{home,nix,boot,}
zpool export ${poolName}
'');
in {
imports = [
<nixpkgs/nixos/modules/profiles/headless.nix>
<nixpkgs/nixos/modules/virtualisation/ec2-data.nix>
<nixpkgs/nixos/modules/virtualisation/amazon-init.nix>
];
boot = {
loader.grub.device = "/dev/xvda";
initrd = {
availableKernelModules = [ "virtio_pci" "virtio_blk" "xen-blkfront" "xen-netfront" ];
postDeviceCommands = lib.mkMerge [
(lib.mkBefore ''
echo resizing xvda3
TMPDIR=/run sh $(type -P growpart) "/dev/xvda" "3"
udevadm settle
'')
# zfs mounts within postDeviceCommands so mkBefore and mkAfter must be used
(lib.mkAfter ''
zpool online -e ${poolName} /dev/xvda3
'')
];
network.enable = true;
postMountCommands = ''
metaDir=$targetRoot/etc/ec2-metadata
mkdir -m 0755 -p "$metaDir"
echo "getting EC2 instance metadata..."
if ! [ -e "$metaDir/ami-manifest-path" ]; then
wget -q -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
fi
if ! [ -e "$metaDir/user-data" ]; then
wget -q -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data"
fi
if ! [ -e "$metaDir/hostname" ]; then
wget -q -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
fi
if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then
wget -q -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
fi
'';
};
zfs.devNodes = "/dev/";
# growPartition does not support zfs directly, so the above postDeviceCommands use what this puts into PATH
growPartition = true;
kernelParams = [ "console=ttyS0" ];
blacklistedKernelModules = [ "nouveau" "xen_fbfront" ];
};
fileSystems = {
"/" = { fsType = "zfs"; device = "${poolName}/root"; };
"/home" = { fsType = "zfs"; device = "${poolName}/home"; };
"/nix" = { fsType = "zfs"; device = "${poolName}/nix"; };
};
networking = {
hostId = "9474d585";
hostName = lib.mkDefault "";
# xen host on aws
timeServers = [ "169.254.169.123" ];
};
environment.systemPackages = [ pkgs.cryptsetup ];
services.openssh = {
enable = true;
permitRootLogin = "prohibit-password";
};
system.build.zfsImage = image;
}
Raw
upload-ami.nix
with import <nixpkgs> {};
let
eval = import <nixpkgs/nixos> { inherit configuration; };
configuration = ./make-zfs-image.nix;
image = "${eval.config.system.build.zfsImage}/nixos.vhd";
in writeScript "upload-amis" ''
#!${stdenv.shell}
set -e
export PATH=${lib.makeBinPath [ ec2_ami_tools jq ec2_api_tools awscli qemu ]}:$PATH
set -o pipefail
version=${lib.version}-6
major=${version:0:5}
echo "NixOS version is $version ($major)"
stateDir=$HOME/amis/ec2-image-$version/
mkdir -p $stateDir
regions="eu-west-1"
types="hvm"
stores="ebs"
for type in $types; do
imageFile=${image}
system=x86_64-linux
arch=x86_64
for store in $stores; do
bucket=iohk-amis
bucketDir="$version-$type-$store"
prevAmi=
prevRegion=
for region in $regions; do
name=nixos-$version-$arch-$type-$store
description="NixOS $system $version ($type-$store)"
amiFile=$stateDir/$region.$type.$store.ami-id
if ! [ -e $amiFile ]; then
echo "doing $name in $region..."
if [ -n "$prevAmi" ]; then
ami=$(aws ec2 copy-image \
--region "$region" \
--source-region "$prevRegion" --source-image-id "$prevAmi" \
--name "$name" --description "$description" | jq -r '.ImageId')
if [ "$ami" = null ]; then break; fi
else
vhdFile=$imageFile
vhdFileLogicalBytes="$(qemu-img info "$vhdFile" | grep ^virtual\ size: | cut -f 2 -d \( | cut -f 1 -d \ )"
vhdFileLogicalGigaBytes=$(((vhdFileLogicalBytes-1)/1024/1024/1024+1)) # Round to the next GB
echo "Disk size is $vhdFileLogicalBytes bytes. Will be registered as $vhdFileLogicalGigaBytes GB."
taskId=$(cat $stateDir/$region.$type.task-id 2> /dev/null || true)
volId=$(cat $stateDir/$region.$type.vol-id 2> /dev/null || true)
snapId=$(cat $stateDir/$region.$type.snap-id 2> /dev/null || true)
if [ -z "$snapId" -a -z "$volId" -a -z "$taskId" ]; then
echo "importing $vhdFile..."
taskId=$(ec2-import-volume $vhdFile --no-upload -f vhd \
-O "$AWS_ACCESS_KEY_ID" -W "$AWS_SECRET_ACCESS_KEY" \
-o "$AWS_ACCESS_KEY_ID" -w "$AWS_SECRET_ACCESS_KEY" \
--region "$region" -z "''${region}a" \
--bucket "$bucket" --prefix "$bucketDir/" \
| tee /dev/stderr \
| sed 's/.*\(import-vol-[0-9a-z]\+\).*/\1/ ; t ; d')
echo -n "$taskId" > $stateDir/$region.$type.task-id
fi
if [ -z "$snapId" -a -z "$volId" ]; then
ec2-resume-import $vhdFile -t "$taskId" --region "$region" \
-O "$AWS_ACCESS_KEY_ID" -W "$AWS_SECRET_ACCESS_KEY" \
-o "$AWS_ACCESS_KEY_ID" -w "$AWS_SECRET_ACCESS_KEY"
fi
# Wait for the volume creation to finish.
if [ -z "$snapId" -a -z "$volId" ]; then
echo "waiting for import to finish..."
while true; do
volId=$(aws ec2 describe-conversion-tasks --conversion-task-ids "$taskId" --region "$region" | jq -r .ConversionTasks[0].ImportVolume.Volume.Id)
done=$(aws ec2 describe-conversion-tasks --conversion-task-ids "$taskId" --region "$region" | jq -r .ConversionTasks[0].State)
if [ "$done" == completed ]; then break; fi
echo -n .
sleep 10
done
echo -n "$volId" > $stateDir/$region.$type.vol-id
fi
if [ -n "$volId" -a -n "$taskId" ]; then
echo "removing import task..."
ec2-delete-disk-image -t "$taskId" --region "$region" \
-O "$AWS_ACCESS_KEY_ID" -W "$AWS_SECRET_ACCESS_KEY" \
-o "$AWS_ACCESS_KEY_ID" -w "$AWS_SECRET_ACCESS_KEY" || true
rm -f $stateDir/$region.$type.task-id
fi
if [ -z "$snapId" ]; then
echo "creating snapshot..."
snapId=$(aws ec2 create-snapshot --volume-id "$volId" --region "$region" --description "$description" | jq -r .SnapshotId)
if [ "$snapId" = null ]; then exit 1; fi
echo -n "$snapId" > $stateDir/$region.$type.snap-id
fi
echo "waiting for snapshot to finish..."
while true; do
status=$(aws ec2 describe-snapshots --snapshot-ids "$snapId" --region "$region" | jq -r .Snapshots[0].State)
if [ "$status" = completed ]; then break; fi
sleep 10
done
# Delete the volume
if [ -n "$volId" ]; then
echo "deleting volume..."
aws ec2 delete-volume --volume-id "$volId" --region "$region" || true
rm -f $stateDir/$region.$type.vol-id
fi
blockDeviceMappings="DeviceName=/dev/sda1,Ebs={SnapshotId=$snapId,VolumeSize=$vhdFileLogicalGigaBytes,DeleteOnTermination=true,VolumeType=gp2}"
extraFlags=""
extraFlags+=" --root-device-name /dev/sda1"
extraFlags+=" --sriov-net-support simple"
extraFlags+=" --ena-support"
blockDeviceMappings+=" DeviceName=/dev/sdb,VirtualName=ephemeral0"
blockDeviceMappings+=" DeviceName=/dev/sdc,VirtualName=ephemeral1"
blockDeviceMappings+=" DeviceName=/dev/sdd,VirtualName=ephemeral2"
blockDeviceMappings+=" DeviceName=/dev/sde,VirtualName=ephemeral3"
extraFlags+=" --sriov-net-support simple"
extraFlags+=" --ena-support"
extraFlags+=" --virtualization-type hvm"
ami=$(aws ec2 register-image \
--name "$name" \
--description "$description" \
--region "$region" \
--architecture "$arch" \
--block-device-mappings $blockDeviceMappings \
$extraFlags | jq -r .ImageId)
if [ "$ami" = null ]; then break; fi
fi
echo -n "$ami" > $amiFile
echo "created AMI $ami of type '$type' in $region..."
else
ami=$(cat $amiFile)
fi
echo "region = $region, type = $type, store = $store, ami = $ami"
if [ -z "$prevAmi" ]; then
prevAmi="$ami"
prevRegion="$region"
fi
done
done
done
''
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment