Ubuntu 20.04.3 AutoInstall

HowTo

# For recent versions of Ubuntu:
 - https://www.pugetsystems.com/labs/hpc/ubuntu-22-04-server-autoinstall-iso/

# Docs:
 - https://wiki.ubuntu.com/FoundationsTeam/AutomatedServerInstalls
 - https://wiki.ubuntu.com/FoundationsTeam/AutomatedServerInstalls/ConfigReference
 - https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html
 - https://discourse.ubuntu.com/t/please-test-autoinstalls-for-20-04/15250/53

# Download ISO Installer:
wget https://ubuntu.volia.net/ubuntu-releases/20.04.3/ubuntu-20.04.3-live-server-amd64.iso

# Create ISO distribution dirrectory:
mkdir -p iso/nocloud/

# Extract ISO using 7z:
7z x ubuntu-20.04.3-live-server-amd64.iso -x'![BOOT]' -oiso
# Or extract ISO using xorriso and fix permissions:
xorriso -osirrox on -indev "ubuntu-20.04.3-live-server-amd64.iso" -extract / iso && chmod -R +w iso

# Create empty meta-data file:
touch iso/nocloud/meta-data

# Copy user-data file:
cp user-data iso/nocloud/user-data

# Update boot flags with cloud-init autoinstall:
## Should look similar to this: initrd=/casper/initrd quiet autoinstall ds=nocloud;s=/cdrom/nocloud/ ---
sed -i 's|---|autoinstall ds=nocloud\\\;s=/cdrom/nocloud/ ---|g' iso/boot/grub/grub.cfg
sed -i 's|---|autoinstall ds=nocloud;s=/cdrom/nocloud/ ---|g' iso/isolinux/txt.cfg

# Disable mandatory md5 checksum on boot:
md5sum iso/.disk/info > iso/md5sum.txt
sed -i 's|iso/|./|g' iso/md5sum.txt

# (Optionally) Regenerate md5:
# The find will warn 'File system loop detected' and return non-zero exit status on the 'ubuntu' symlink to '.'
# To avoid that, temporarily move it out of the way
mv iso/ubuntu .
(cd iso; find '!' -name "md5sum.txt" '!' -path "./isolinux/*" -follow -type f -exec "$(which md5sum)" {} \; > ../md5sum.txt)
mv md5sum.txt iso/
mv ubuntu iso

# Create Install ISO from extracted dir (ArchLinux):
xorriso -as mkisofs -r \
  -V Ubuntu\ custom\ amd64 \
  -o ubuntu-20.04.3-live-server-amd64-autoinstall.iso \
  -J -l -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \
  -boot-load-size 4 -boot-info-table \
  -eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
  -isohybrid-gpt-basdat -isohybrid-apm-hfsplus \
  -isohybrid-mbr /usr/lib/syslinux/bios/isohdpfx.bin  \
  iso/boot iso

# Create Install ISO from extracted dir (Ubuntu):
xorriso -as mkisofs -r \
  -V Ubuntu\ custom\ amd64 \
  -o ubuntu-20.04.3-live-server-amd64-autoinstall.iso \
  -J -l -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \
  -boot-load-size 4 -boot-info-table \
  -eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
  -isohybrid-gpt-basdat -isohybrid-apm-hfsplus \
  -isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin  \
  iso/boot iso

# After install:
 - login with 'root:root' and change root user password
 - set correct hostname with 'hostnamectl'

user-data

#cloud-config
autoinstall:
  version: 1
  interactive-sections:
    - network
    - storage
  locale: en_US.UTF-8
  keyboard:
    layout: us
  ssh:
    allow-pw: true
    install-server: false
  late-commands:
    - curtin in-target --target=/target -- apt-get --purge -y --quiet=2 remove apport bcache-tools btrfs-progs byobu cloud-guest-utils cloud-initramfs-copymods cloud-initramfs-dyn-netconf friendly-recovery fwupd landscape-common lxd-agent-loader ntfs-3g open-vm-tools plymouth plymouth-theme-ubuntu-text popularity-contest rsync screen snapd sosreport tmux ufw
    - curtin in-target --target=/target -- apt-get --purge -y --quiet=2 autoremove
    - curtin in-target --target=/target -- apt-get clean
    - sed -i 's/ENABLED=1/ENABLED=0/' /target/etc/default/motd-news
    - sed -i 's|# en_US.UTF-8 UTF-8|en_US.UTF-8 UTF-8|' /target/etc/locale.gen
    - curtin in-target --target=/target -- locale-gen
    - ln -fs /dev/null /target/etc/systemd/system/connman.service
    - ln -fs /dev/null /target/etc/systemd/system/display-manager.service
    - ln -fs /dev/null /target/etc/systemd/system/motd-news.service
    - ln -fs /dev/null /target/etc/systemd/system/motd-news.timer
    - ln -fs /dev/null /target/etc/systemd/system/plymouth-quit-wait.service
    - ln -fs /dev/null /target/etc/systemd/system/plymouth-start.service
    - ln -fs /dev/null /target/etc/systemd/system/systemd-resolved.service
    - ln -fs /usr/share/zoneinfo/Europe/Kiev /target/etc/localtime
    - rm -f /target/etc/resolv.conf
    - printf 'nameserver 8.8.8.8\nnameserver 1.1.1.1\noptions timeout:1\noptions attempts:1\noptions rotate\n' > /target/etc/resolv.conf
    - rm -f /target/etc/update-motd.d/10-help-text
    - rm -rf /target/root/snap
    - rm -rf /target/snap
    - rm -rf /target/var/lib/snapd
    - rm -rf /target/var/snap
    - curtin in-target --target=/target -- passwd -q -u root
    - curtin in-target --target=/target -- passwd -q -x -1 root
    - curtin in-target --target=/target -- passwd -q -e root
    - sed -i 's|^root:.:|root:$6$3b873df474b55246$GIpSsujar7ihMzG8urUKpzF9/2yZJhR.msyFRa5ouGXOKRCVszsc4aBcE2yi3IuFVxtAGwrPKin2WAzK3qOtB.:|' /target/etc/shadow
  user-data:
    disable_root: false

user-data_with_ssh_and_project_quota

#cloud-config
autoinstall:
  version: 1
  interactive-sections:
    - network
    - storage
  locale: en_US.UTF-8
  keyboard:
    layout: us
  ssh:
    allow-pw: true
    install-server: false
  late-commands:
    - curtin in-target --target=/target -- apt-get --purge -y --quiet=2 remove apport bcache-tools btrfs-progs byobu cloud-guest-utils cloud-initramfs-copymods cloud-initramfs-dyn-netconf friendly-recovery fwupd landscape-common lxd-agent-loader ntfs-3g open-vm-tools plymouth plymouth-theme-ubuntu-text popularity-contest screen snapd sosreport tmux ufw
    - curtin in-target --target=/target -- apt-get -qq update
    - curtin in-target --target=/target -- apt-get -y install wget gnupg python2.7 openssh-server
    - curtin in-target --target=/target -- apt-get -qq update
    - curtin in-target --target=/target -- apt-get -y dist-upgrade
    - curtin in-target --target=/target -- apt-get --purge -y --quiet=2 autoremove
    - sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /target/etc/ssh/sshd_config
    - sed -i 's/ENABLED=1/ENABLED=0/' /target/etc/default/motd-news
    - sed -i 's|# en_US.UTF-8 UTF-8|en_US.UTF-8 UTF-8|' /target/etc/locale.gen
    - curtin in-target --target=/target -- locale-gen
    - ln -fs /dev/null /target/etc/systemd/system/connman.service
    - ln -fs /dev/null /target/etc/systemd/system/display-manager.service
    - ln -fs /dev/null /target/etc/systemd/system/motd-news.service
    - ln -fs /dev/null /target/etc/systemd/system/motd-news.timer
    - ln -fs /dev/null /target/etc/systemd/system/plymouth-quit-wait.service
    - ln -fs /dev/null /target/etc/systemd/system/plymouth-start.service
    - ln -fs /dev/null /target/etc/systemd/system/systemd-resolved.service
    - ln -fs /usr/share/zoneinfo/Europe/Kiev /target/etc/localtime
    - rm -f /target/etc/resolv.conf
    - printf 'nameserver 8.8.8.8\nnameserver 1.1.1.1\noptions timeout:1\noptions attempts:1\noptions rotate\n' > /target/etc/resolv.conf
    - rm -f /target/etc/update-motd.d/10-help-text
    - rm -rf /target/root/snap
    - rm -rf /target/snap
    - rm -rf /target/var/lib/snapd
    - rm -rf /target/var/snap
    - curtin in-target --target=/target -- passwd -q -u root
    - curtin in-target --target=/target -- passwd -q -x -1 root
    - curtin in-target --target=/target -- passwd -q -e root
    - sed -i 's|^root:.:|root:$6$3b873df474b55246$GIpSsujar7ihMzG8urUKpzF9/2yZJhR.msyFRa5ouGXOKRCVszsc4aBcE2yi3IuFVxtAGwrPKin2WAzK3qOtB.:|' /target/etc/shadow
    - sed -i 's/ext4 defaults/ext4 prjquota,lazytime/g' /target/etc/fstab
    - findmnt --real -U -n -t ext4 -o source | awk '{system("umount -l "$1)}'
    - blkid -t TYPE="ext4" -o device | awk '{system("tune2fs -O project,quota -Q prjquota "$1)}'
  user-data:
    disable_root: false

Any idea on how to use this on Ubuntu 22.04? I tried to do this today, but there is no isolinux folder anymore in the iso image. Ubuntu 22.04 current release can be downloaded from here: https://cdimage.ubuntu.com/ubuntu-server/daily-live/current/jammy-live-server-amd64.iso

@iondulgheru needs to be adopted for that release, will do when I'll have some free time.

I am also trying to create a customized iso with 21.10 using the above procedure but there is no /isolinux folder and boot/grub/efi.img file inside the extracted folder and these folder/files even not there in 20.10, 21.04, and 21.10. can anyone help if someone already did?

I tried the following but it fails. I am new to this and I can't figure out what is wrong.
xorriso -as mkisofs -r \

-V Ubuntu\ custom\ amd64
-o ubuntu-20.04.3-live-server-amd64-autoinstall.iso
-J -l -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot
-boot-load-size 4 -boot-info-table
-eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot
-isohybrid-gpt-basdat -isohybrid-apm-hfsplus
-isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin
iso/boot iso
xorriso 1.5.2 : RockRidge filesystem manipulator, libburnia project.

Drive current: -outdev 'stdio:ubuntu-20.04.3-live-server-amd64-autoinstall.iso'
Media current: stdio file, overwriteable
Media status : is blank
Media summary: 0 sessions, 0 data blocks, 0 data, 37.1g free
xorriso : WARNING : -volid text problematic as automatic mount point name
xorriso : WARNING : -volid text does not comply to ISO 9660 / ECMA 119 rules
Added to ISO image: directory '/'='/home/devops/iso/boot'
xorriso : UPDATE : 261 files added in 1 seconds
Added to ISO image: directory '/'='/home/devops/iso'
xorriso : FAILURE : Given path does not exist on disk: -boot_image system_area='/usr/lib/ISOLINUX/isohdpfx.bin'
xorriso : UPDATE : 1005 files added in 1 seconds
xorriso : aborting : -abort_on 'FAILURE' encountered 'FAILURE'

Any idea where to start looking? I can't find isohdpfx.bin anywhere in the iso folder.

Can the network&storage be set up automatically?

@dpflick
I was trying to solve your problem and finally, I found this issue on Github.
tadas-s/custom-ubuntu-install#5 (comment)

Yes, you can set up network and storage automatically @liuyueying-nx. You have to remove network and storage from interactive-sections: in a user-data file. Then configure everything you want in sections storage: and network:.

autoinstall:
  version: 1
  interactive-sections:
    - network
    - storage

Check out the documentation -> https://ubuntu.com/server/docs/install/autoinstall-reference

I did but it doesn't seem to work
network:
version: 2
ethernets:
enpls0:
dhcp6: yes
storage:
layout:
name: direct

@liuyueying-nx
Are the sections still interactive or is there an error?

@pat0s still interactive. I want to modify the root password through the code of cloud-init, is this OK?

@dushyanthkumaryd ,
I've got an USB-key with /nocloud , /iso and required stuff to boot (server iso-> usb-key and remove distro files)

In the user-data file i've got references to the usb-key defined by /isodevice/

is your /boot/grub/grub.cfg configured like this?

menuentry "ISO - Ubuntu Server 21.10 {
	set gfxpayload=keep
    set isofile="/iso/ubuntu-21.10-live-server-amd64.iso"
    loopback loop (hd0,msdos1)$isofile
	linux (loop)/casper/vmlinuz boot=casper iso-scan/filename=$isofile noprompt noeject "ds=nocloud;s=/isodevice/nocloud/"  quiet autoinstall --- 
	initrd (loop)/casper/initrd
}

is there a way to choose arbitrary disk to deploy LVM afters.
I used match: {} but it did not work. Has anyone ever had something like this ?

I wrote a Python script that is a boiled-down simplification of the process here, and it's current for 22.04 LTS.

import io
import pathlib

import pycdlib

ubuntu = pathlib.Path('ubuntu-22.04-live-server-amd64.iso')
new_iso_path = pathlib.Path('ubuntu-22.04-live-server-amd64-auto.iso')

iso = pycdlib.PyCdlib()
iso.open(ubuntu)

extracted = io.BytesIO()
iso.get_file_from_iso_fp(extracted, iso_path='/BOOT/GRUB/GRUB.CFG;1')
extracted.seek(0)
data = extracted.read()
print(data.decode())

new = data.replace(b' ---', b'quiet autoinstall ds=nocloud\;s=/cdrom/nocloud/ ---').replace(b'timeout=30', b'timeout=1')
print(new.decode())

iso.rm_file(iso_path='/BOOT/GRUB/GRUB.CFG;1', rr_name='grub.cfg')
iso.add_fp(io.BytesIO(new), len(new), '/BOOT/GRUB/GRUB.CFG;1', rr_name='grub.cfg')

iso.add_directory('/NOCLOUD', rr_name='nocloud')

user_data = b"""#cloud-config
autoinstall:
  version: 1
  identity:
    hostname: ubuntu-server
    password: "$6$exDY1mhS4KUYCE/2$zmn9ToZwTKLhCw.b4/b.ZRTIZM30JZ4QrOQ2aOXJ8yk96xpcCof0kxKwuX1kqLG/ygbJ1f8wxED22bTL4F46P0"
    username: ubuntu
"""

iso.add_fp(io.BytesIO(user_data), len(user_data), '/NOCLOUD/USER_DATA;1', rr_name='user-data')
iso.add_fp(io.BytesIO(b''), len(b''), '/NOCLOUD/META_DATA;1', rr_name='meta-data')
iso.write(new_iso_path)
iso.close()

Here's the gist:

https://gist.github.com/utkonos/718b150de4f86054c37ac798c02b54c6

The username/password for the above is ubuntu/ubuntu

Also, this creates a really dangerous ISO. It will destroy whatever boots it, if possible. Only boot from this if you want an automatic install with no user interaction.

You can split this into two different ISOs also. One which is the Ubuntu installer ISO which has been edited to just do quiet autoinstall. And the other that contains the user-data file. You then boot from the installer ISO and have the other ISO attached as a second CDROM.

First ISO

import io
import pathlib

import pycdlib

ubuntu = pathlib.Path('ubuntu-22.04-live-server-amd64.iso')
new_iso = pathlib.Path('ubuntu-22.04-live-server-amd64-auto.iso')

iso = pycdlib.PyCdlib()
iso.open(ubuntu)

extracted = io.BytesIO()
iso.get_file_from_iso_fp(extracted, iso_path='/BOOT/GRUB/GRUB.CFG;1')
extracted.seek(0)
data = extracted.read()
print(data.decode())

new = data.replace(b' ---', b'quiet autoinstall ---').replace(b'timeout=30', b'timeout=1')
print(new.decode())

iso.rm_file(iso_path='/BOOT/GRUB/GRUB.CFG;1', rr_name='grub.cfg')
iso.add_fp(io.BytesIO(new), len(new), '/BOOT/GRUB/GRUB.CFG;1', rr_name='grub.cfg')

iso.write(new_iso)
iso.close()

Second ISO

import io
import pathlib

import pycdlib

new_iso = pathlib.Path('ubuntu-22.04-auto.iso')

iso = pycdlib.PyCdlib()
iso.new(rock_ridge='1.09', vol_ident='CIDATA')

user_data = b"""#cloud-config
autoinstall:
  version: 1
  identity:
    hostname: ubuntu-server
    password: "$6$exDY1mhS4KUYCE/2$zmn9ToZwTKLhCw.b4/b.ZRTIZM30JZ4QrOQ2aOXJ8yk96xpcCof0kxKwuX1kqLG/ygbJ1f8wxED22bTL4F46P0"
    username: ubuntu
"""

iso.add_fp(io.BytesIO(user_data), len(user_data), '/USERDATA;1', rr_name='user-data')
iso.add_fp(io.BytesIO(b''), len(b''), '/METADATA;1', rr_name='meta-data')

iso.write(new_iso)
iso.close()

Hi!

Is this URL still alive?

https://ubuntu.volia.net/ubuntu-releases/20.04.3/ubuntu-20.04.3-live-server-amd64.iso

I have downloaded the ISO from the official releases site.

https://releases.ubuntu.com/20.04.3/ubuntu-20.04.3-live-server-amd64.iso

Hi!

Is this URL still alive?

https://ubuntu.volia.net/ubuntu-releases/20.04.3/ubuntu-20.04.3-live-server-amd64.iso

I have downloaded the ISO from the official releases site.

https://releases.ubuntu.com/20.04.3/ubuntu-20.04.3-live-server-amd64.iso

should be ok also

@aconte31 please add curtin in-target --target=/target -- before each command as you are now running the commands in the install environment, not the target environment

Does the /target part in curtin in-target --target=/target -- need to be customized in some way or is it literally /target?

@aconte31 please add curtin in-target --target=/target -- before each command as you are now running the commands in the install environment, not the target environment

Does the /target part in curtin in-target --target=/target -- need to be customized in some way or is it literally /target?

as is....literally target

Hi is it possible to point the install environment apt packages e.g cdroom /pool and /dist to the target environment to install packages such as nano. This will be for the minimal server install. ( no internet connection so no apt server , apt address file:///cdrom or something like that

thanks

Maybe somebody already wrote that. The late-commands do not work on current ubuntu-22.04.1-live-server-arm64

For reference I was just using the first three:

  late-commands:
    - curtin in-target --target=/target -- apt-get --purge -y --quiet=2 remove snapd
    - curtin in-target --target=/target -- apt-get --purge -y --quiet=2 autoremove
    - curtin in-target --target=/target -- apt-get clean

Maybe somebody already wrote that. The late-commands do not work on current ubuntu-22.04.1-live-server-arm64

Hmm, could be a bug, https://ubuntu.com/server/docs/install/autoinstall-reference says that late-commands are supported

@dmuiX ,
that is happening because you are trying to remove snapd, I had same issue with it on ubuntu20.04. You see, cloud-init is installed via snapd, and when you try to remove and purge snapd, all the packages, including cloud-init, are removed, thus due to self-preservation, your command fails.
I by passed it by moving the command to runcmd. that way the package will be removed on the first boot, without failing the system install.

@dmuiX , that is happening because you are trying to remove snapd, I had same issue with it on ubuntu20.04. You see, cloud-init is installed via snapd, and when you try to remove and purge snapd, all the packages, including cloud-init, are removed, thus due to self-preservation, your command fails. I by passed it by moving the command to runcmd. that way the package will be removed on the first boot, without failing the system install.

:D :D. So a bit like canibalism^^. Thanks for sharing your solution.
Although: runcmd is not referenced in the documentation: https://ubuntu.com/server/docs/install/autoinstall-reference. have you just added it like the rest? Or where have you found this? Maybe it was removed at the current version 2204? Is there some other way to add commands like the above?

@dmuiX this is was my guide line: https://gist.github.com/dbkinghorn/c236aea31d76028b2b6ccdf6d3c6f07e

@s3rj1k I have tried [user-data_with_ssh_and_project_quota] with ubuntu 22.04 but it was not accessing or not auto installing can you please ping the guide for ubuntu 22.04 server in auto install method.

Thanks in advance

Can you guys ping the working method for installing ubuntu 22.04 @s3rj1k @davosian @dmuiX @zero-pytagoras @zepx @dbkinghorn

@dmuiX this is was my guide line: https://gist.github.com/dbkinghorn/c236aea31d76028b2b6ccdf6d3c6f07e

Thank you.

Can you guys ping the working method for installing ubuntu 22.04 @s3rj1k @davosian @dmuiX @zero-pytagoras @zepx @dbkinghorn

I will look my one up and post it later

If you are interested i understanding the automated ubuntu installation a bit better this might be a good guide:
https://www.jimangel.io/posts/automate-ubuntu-22-04-lts-bare-metal/

Pretty basic. As I decided to install everything via commands. Might be because it was not working pretty well and I had no time or motivation. Switched now to cloud-init and mulitpass to make everything run as this was easier and works better. I think you can use cloud-init to install a ubuntu instance as well you just have to find a way to integrate it in an iso file or provide it via network.

#cloud-config
autoinstall:
  version: 1
  identity:
    hostname: k8s-cluster
    password: "$6$exDY1mhS4KUYCE/2$zmn9ToZwTKLhCw.b4/b.ZRTIZM30JZ4QrOQ2aOXJ8yk96xpcCof0kxKwuX1kqLG/ygbJ1f8wxED22bTL4F46P0"
    username: ubuntu
  locale: de_DE
  keyboard:
    layout: de
    variant: mac
  ssh:
    install-server: yes
    allow-pw: yes
  packages:
    - nala
  updates: all

I am facing error while installing @s3rj1k @dmuiX