Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Docker in LXD
exit #Not that kind of script, these sections should be run manually
# First get lxd working
# Install lxc/lxd. On ubuntu 18.04 I'm using snap to install lxd https://docs.snapcraft.io/installing-snapd/6735
sudo apt install snapd
sudo snap install lxd # https://mangolassi.it/topic/18073/installing-lxd-on-ubuntu-18-04-the-right-way
# Using ZFS for container storage makes it easy to move things around
sudo apt-get install zfsutils-linux
# Initialize lxd (will setup a new zfs pool too, or you can do that before running init)
sudo lxd init
Would you like to use LXD clustering? (yes/no) [default=no]: no
Do you want to configure a new storage pool? (yes/no) [default=yes]: yes
Name of the new storage pool [default=default]:
Name of the storage backend to use (btrfs, ceph, dir, lvm, zfs) [default=zfs]:
Create a new ZFS pool? (yes/no) [default=yes]:
Would you like to use an existing block device? (yes/no) [default=no]: no
Size in GB of the new loop device (1GB minimum) [default=40GB]: 200GB
Would you like to connect to a MAAS server? (yes/no) [default=no]: no
Would you like to create a new local network bridge? (yes/no) [default=yes]: yes
What should the new bridge be called? [default=lxdbr0]:
What IPv4 address should be used? (CIDR subnet notation, “auto” or “none”) [default=auto]: 192.168.4.1/24
Would you like LXD to NAT IPv4 traffic on your bridge? [default=yes]: yes
What IPv6 address should be used? (CIDR subnet notation, “auto” or “none”) [default=auto]: none
Would you like LXD to be available over the network? (yes/no) [default=no]: yes
Address to bind LXD to (not including port) [default=all]: 192.168.0.4
Port to bind LXD to [default=8443]:
Trust password for new clients:
Again:
Would you like stale cached images to be updated automatically? (yes/no) [default=yes]
Would you like a YAML "lxd init" preseed to be printed? (yes/no) [default=no]:
# Add user to lxd group for access without using sudo (similar to docker group)
sudo usermod -a -G lxd $USER
newgrp lxd # start a new shell session with the new group
# Create some test containers
lxc launch images:ubuntu/xenial/amd64 ubuntu-test
lxc launch images:ubuntu/bionic/amd64 ubuntu-test2
# view details of zfs pool
zpool list
NAME SIZE ALLOC FREE EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
default 186G 372K 186G - 0% 0% 1.00x ONLINE -
zpool status
pool: default
state: ONLINE
scan: none requested
config:
NAME STATE READ WRITE CKSUM
default ONLINE 0 0 0
/var/snap/lxd/common/lxd/disks/default.img ONLINE 0 0 0
zfs list
NAME USED AVAIL REFER MOUNTPOINT
default 366M 180G 24K none
default/containers 5.34M 180G 24K none
default/containers/ubuntu-test 2.58M 180G 168M /var/snap/lxd/common/lxd/storage-pools/default/containers/ubuntu-test
default/containers/ubuntu-test2 2.74M 180G 192M /var/snap/lxd/common/lxd/storage-pools/default/containers/ubuntu-test2
default/custom 24K 180G 24K none
default/custom-snapshots 24K 180G 24K none
default/deleted 24K 180G 24K none
default/images 360M 180G 24K none
default/images/6c08ba0fb413ef92d8ea7b6437ded184246644cd0820936cf8351ad6e93e0a13 168M 180G 168M none
default/images/98fa50a3b311163c682bc596ebb6bf054b22e7d480c85430a6db46205b2d2cdc 192M 180G 192M none
default/snapshots 24K 180G 24K none
# snap run lxd in a separate mount namespace so the above mountpoints don't work directly
# instead if you want to access the container rootfs directly (excluding any extra mounts like /tmp /var ...)
sudo ls /var/snap/lxd/common/mntns/var/snap/lxd/common/lxd/storage-pools/default/containers/ubuntu-test/rootfs/
bin boot dev etc home lib lib64 media mnt opt proc root run sbin srv sys tmp usr var
# Fun but optional networking stuff:
# Use ip_forwarding on the host and a static route on your router for "host bridged" style networking https://serverfault.com/a/899402/33095
# once enabled you can disable nat: lxc network set lxdbr0 ipv4.nat false
#
# Change domain assigned to containers (can use local or router's dnsmasq to forward requests: server=/myhost.lxd/192.168.4.1)
# lxc network set lxdbr0 dns.domain myhost.lxd
# host ubuntu-test.myhost.lxd -> ubuntu-test.myhost.lxd has address 192.168.4.221
# If you want to do X11 or audio stuff, check out https://twitter.com/bketelsen/status/1087448612207017986
# Next setup Docker in an LXD container. This makes it more portable (can move the container to another host) and more, uh, contained
# From https://stgraber.org/2016/04/13/lxd-2-0-docker-in-lxd-712/
NAME=docker
lxc launch images:ubuntu/bionic/amd64 $NAME -c security.nesting=true
lxc exec $NAME -- apt update # The above image should be kept up to date, but can force an update if needed
lxc exec $NAME -- apt dist-upgrade -y
# Add an ssh server to the container if you don't want to use exec commands/sessions
lxc exec $NAME -- bash -c 'apt-get install openssh-server -y;mkdir -m 0700 ~/.ssh'
lxc file push -p --uid=0 --gid=0 ~/.ssh/authorized_keys "$NAME/root/.ssh/authorized_keys"
# Next we'll run commands in the container. Could use lxc exec $NAME bash instead but I like treating these like servers
ssh root@docker.myhost.lxd
# change default username (it is benefitial to keep the same UID/GID you have on your host, which is 1000 in my case)
NEWUSER=greg.bray
usermod -l $NEWUSER ubuntu
groupmod -n $NEWUSER ubuntu
mv /home/ubuntu /home/$NEWUSER
sed -i "s/home\/ubuntu/home\/$NEWUSER/" /etc/passwd
# Copy ssh settings from root to new user
cp -r ~/.ssh "/home/$NEWUSER/.ssh"
chown -R $NEWUSER:$NEWUSER "/home/$NEWUSER/.ssh"
# Then install docker. Lots of ways to do that but I like using their repo
sudo apt-get install apt-transport-https ca-certificates curl software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
sudo apt-get update
sudo apt-get install docker-ce
# Run a test container
docker run -it --rm ubuntu:bionic bash -c 'echo hello from $(hostname)'
hello from d9179cea965a
# Great, now we have docker running in an lxd container, what good is that?
# Well, you can install all the client tools on the host and manage the docker instance "remotely"
# New versions at https://docs.docker.com/install/linux/docker-ce/binaries/
cd ~/Installs/docker
VERSION=18.09.6
wget https://download.docker.com/linux/static/stable/`uname -m`/docker-$VERSION.tgz
sudo tar -zxvf docker-$VERSION.tgz --strip-components=1 --directory /usr/local/bin/ docker/docker
# Also docker-compose https://docs.docker.com/compose/install/
wget https://github.com/docker/compose/releases/download/1.24.0/docker-compose-`uname -s`-`uname -m` -O ./docker-compose
chmod +x ./docker-compose
sudo cp ./docker-compose /usr/local/bin/docker-compose
# Install docker machine (optional, but nice to have for connecting to multiple truely remote instances)
wget https://github.com/docker/machine/releases/download/v0.16.1/docker-machine-$(uname -s)-$(uname -m) -O ./docker-machine
chmod +x ./docker-machine
sudo cp ./docker-machine /usr/local/bin/docker-machine
# Setup remote docker daemon (may want to create a snapshot first https://www.cyberciti.biz/faq/create-snapshots-with-lxc-command-lxd/ )
docker-machine create --driver generic --generic-ip-address=docker.myhost.lxd --generic-ssh-key ~/.ssh/id_rsa --generic-ssh-user=root name_of_remote
# Note for lxd may need to change /etc/systemd/system/docker.service.d/10-machine.conf to use vfs instead of aufs
# It will also change the hostname, so change that back:
eval $(docker-machine env name_of_remote)
# But my preferred way is proxying the docker socket. Insecure option over TCP https://hub.docker.com/r/anthonydahanne/docker-socat/
# Or lxd has a built in proxy as well. Blog post for sharing lxc socket to a container, but same approach works for docker container to host https://blog.simos.info/how-to-manage-lxd-from-within-one-of-its-containers/
lxc config device add docker dockersocket proxy connect=unix:/var/run/docker.sock listen=unix:/tmp/docker.sock bind=host uid=0 gid=999 mode=0660 security.uid=65534 security.gid=999
export DOCKER_HOST=unix:///tmp/docker.sock
docker run -it --rm ubuntu bash
# Note, if you mounted it to /var/run/docker.sock on the host it would be picked up by default, but I think there were some permission issues.
# You can also use docker-compose on the host using local docker files https://docs.docker.com/compose/gettingstarted/
# But keep in mind the references in the docker files will be sourced on the docker container not your local host
# So to fix that you can mount your development folder as another device :-P
# Mount local host folder into docker container
lxc config device add docker scripts disk source=/home/$USER/code/scripts/ path=/home/$USER/scripts
lxc config device add docker dockerfiles disk source=/home/$USER/code/dockerfiles path=/home/$USER/dockerfiles
lxc config device add docker go disk source=/home/$USER/code/go path=/home/$USER/go
# Now those folders are mounted in the container, however they currently have a different UID:GID due to the container's user namespace http://man7.org/linux/man-pages/man7/user_namespaces.7.html
root@docker:/home/greg.bray# ls -hl
total 12K
drwxr-xr-x 4 nobody nogroup 4.0K May 13 01:31 dockerfiles
drwxr-xr-x 5 nobody nogroup 4.0K Apr 23 16:09 go
drwxr-xr-x 7 nobody nogroup 4.0K May 22 21:30 scripts
# To fix nobody:nobody you have to add a custom idmap https://tribaal.io/nicer-mounting-home-in-lxd.html
# This is where assuming the UID:GID in the container matches the host comes into play. Should be possible if they are different but much easier if they are the same
# You may need to sudo vim /etc/sub{u,g}id but on ubuntu 18.04 with latest lxd I was able to skip that step
lxc config set docker raw.idmap "both $UID $UID"
lxc restart docker #Should see a message about it remapping the users on the container filesystem
# Now the files mounted in the container have the correct owner:group
lxc exec docker -- bash -c "ls -hl /home/$USER"
total 12K
drwxr-xr-x 4 greg.bray greg.bray 4.0K May 13 01:31 dockerfiles
drwxr-xr-x 5 greg.bray greg.bray 4.0K Apr 23 16:09 go
drwxr-xr-x 7 greg.bray greg.bray 4.0K May 22 21:30 scripts
# And so you can reference them via dockerfiles or docker-compose or as mounted volumes into containers
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment