Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Docker in LXD
exit #Not that kind of script, these sections should be run manually
# First get lxd working
# Install lxc/lxd. On ubuntu 18.04 I'm using snap to install lxd https://docs.snapcraft.io/installing-snapd/6735
sudo apt install snapd
sudo snap install lxd # https://mangolassi.it/topic/18073/installing-lxd-on-ubuntu-18-04-the-right-way
# Using ZFS for container storage makes it easy to move things around
sudo apt-get install zfsutils-linux
# Initialize lxd (will setup a new zfs pool too, or you can do that before running init)
sudo lxd init
Would you like to use LXD clustering? (yes/no) [default=no]: no
Do you want to configure a new storage pool? (yes/no) [default=yes]: yes
Name of the new storage pool [default=default]:
Name of the storage backend to use (btrfs, ceph, dir, lvm, zfs) [default=zfs]:
Create a new ZFS pool? (yes/no) [default=yes]:
Would you like to use an existing block device? (yes/no) [default=no]: no
Size in GB of the new loop device (1GB minimum) [default=40GB]: 200GB
Would you like to connect to a MAAS server? (yes/no) [default=no]: no
Would you like to create a new local network bridge? (yes/no) [default=yes]: yes
What should the new bridge be called? [default=lxdbr0]:
What IPv4 address should be used? (CIDR subnet notation, “auto” or “none”) [default=auto]: 192.168.4.1/24
Would you like LXD to NAT IPv4 traffic on your bridge? [default=yes]: yes
What IPv6 address should be used? (CIDR subnet notation, “auto” or “none”) [default=auto]: none
Would you like LXD to be available over the network? (yes/no) [default=no]: yes
Address to bind LXD to (not including port) [default=all]: 192.168.0.4
Port to bind LXD to [default=8443]:
Trust password for new clients:
Again:
Would you like stale cached images to be updated automatically? (yes/no) [default=yes]
Would you like a YAML "lxd init" preseed to be printed? (yes/no) [default=no]:
# Add user to lxd group for access without using sudo (similar to docker group)
sudo usermod -a -G lxd $USER
newgrp lxd # start a new shell session with the new group
# Create some test containers
lxc launch images:ubuntu/xenial/amd64 ubuntu-test
lxc launch images:ubuntu/bionic/amd64 ubuntu-test2
# view details of zfs pool
zpool list
NAME SIZE ALLOC FREE EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
default 186G 372K 186G - 0% 0% 1.00x ONLINE -
zpool status
pool: default
state: ONLINE
scan: none requested
config:
NAME STATE READ WRITE CKSUM
default ONLINE 0 0 0
/var/snap/lxd/common/lxd/disks/default.img ONLINE 0 0 0
zfs list
NAME USED AVAIL REFER MOUNTPOINT
default 366M 180G 24K none
default/containers 5.34M 180G 24K none
default/containers/ubuntu-test 2.58M 180G 168M /var/snap/lxd/common/lxd/storage-pools/default/containers/ubuntu-test
default/containers/ubuntu-test2 2.74M 180G 192M /var/snap/lxd/common/lxd/storage-pools/default/containers/ubuntu-test2
default/custom 24K 180G 24K none
default/custom-snapshots 24K 180G 24K none
default/deleted 24K 180G 24K none
default/images 360M 180G 24K none
default/images/6c08ba0fb413ef92d8ea7b6437ded184246644cd0820936cf8351ad6e93e0a13 168M 180G 168M none
default/images/98fa50a3b311163c682bc596ebb6bf054b22e7d480c85430a6db46205b2d2cdc 192M 180G 192M none
default/snapshots 24K 180G 24K none
# snap run lxd in a separate mount namespace so the above mountpoints don't work directly
# instead if you want to access the container rootfs directly (excluding any extra mounts like /tmp /var ...)
sudo ls /var/snap/lxd/common/mntns/var/snap/lxd/common/lxd/storage-pools/default/containers/ubuntu-test/rootfs/
bin boot dev etc home lib lib64 media mnt opt proc root run sbin srv sys tmp usr var
# Fun but optional networking stuff:
# Use ip_forwarding on the host and a static route on your router for "host bridged" style networking https://serverfault.com/a/899402/33095
# once enabled you can disable nat: lxc network set lxdbr0 ipv4.nat false
#
# Change domain assigned to containers (can use local or router's dnsmasq to forward requests: server=/myhost.lxd/192.168.4.1)
# lxc network set lxdbr0 dns.domain myhost.lxd
# host ubuntu-test.myhost.lxd -> ubuntu-test.myhost.lxd has address 192.168.4.221
# If you want to do X11 or audio stuff, check out https://twitter.com/bketelsen/status/1087448612207017986
# Next setup Docker in an LXD container. This makes it more portable (can move the container to another host) and more, uh, contained
# From https://stgraber.org/2016/04/13/lxd-2-0-docker-in-lxd-712/
NAME=docker
lxc launch images:ubuntu/bionic/amd64 $NAME -c security.nesting=true
lxc exec $NAME -- apt update # The above image should be kept up to date, but can force an update if needed
lxc exec $NAME -- apt dist-upgrade -y
# Add an ssh server to the container if you don't want to use exec commands/sessions
lxc exec $NAME -- bash -c 'apt-get install openssh-server -y;mkdir -m 0700 ~/.ssh'
lxc file push -p --uid=0 --gid=0 ~/.ssh/authorized_keys "$NAME/root/.ssh/authorized_keys"
# Next we'll run commands in the container. Could use lxc exec $NAME bash instead but I like treating these like servers
ssh root@docker.myhost.lxd
# change default username (it is benefitial to keep the same UID/GID you have on your host, which is 1000 in my case)
NEWUSER=greg.bray
usermod -l $NEWUSER ubuntu
groupmod -n $NEWUSER ubuntu
mv /home/ubuntu /home/$NEWUSER
sed -i "s/home\/ubuntu/home\/$NEWUSER/" /etc/passwd
# Copy ssh settings from root to new user
cp -r ~/.ssh "/home/$NEWUSER/.ssh"
chown -R $NEWUSER:$NEWUSER "/home/$NEWUSER/.ssh"
# Then install docker. Lots of ways to do that but I like using their repo
sudo apt-get install apt-transport-https ca-certificates curl software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
sudo apt-get update
sudo apt-get install docker-ce
# Run a test container
docker run -it --rm ubuntu:bionic bash -c 'echo hello from $(hostname)'
hello from d9179cea965a
# Great, now we have docker running in an lxd container, what good is that?
# Well, you can install all the client tools on the host and manage the docker instance "remotely"
# New versions at https://docs.docker.com/install/linux/docker-ce/binaries/
cd ~/Installs/docker
VERSION=18.09.6
wget https://download.docker.com/linux/static/stable/`uname -m`/docker-$VERSION.tgz
sudo tar -zxvf docker-$VERSION.tgz --strip-components=1 --directory /usr/local/bin/ docker/docker
# Also docker-compose https://docs.docker.com/compose/install/
wget https://github.com/docker/compose/releases/download/1.24.0/docker-compose-`uname -s`-`uname -m` -O ./docker-compose
chmod +x ./docker-compose
sudo cp ./docker-compose /usr/local/bin/docker-compose
# Install docker machine (optional, but nice to have for connecting to multiple truely remote instances)
wget https://github.com/docker/machine/releases/download/v0.16.1/docker-machine-$(uname -s)-$(uname -m) -O ./docker-machine
chmod +x ./docker-machine
sudo cp ./docker-machine /usr/local/bin/docker-machine
# Setup remote docker daemon (may want to create a snapshot first https://www.cyberciti.biz/faq/create-snapshots-with-lxc-command-lxd/ )
docker-machine create --driver generic --generic-ip-address=docker.myhost.lxd --generic-ssh-key ~/.ssh/id_rsa --generic-ssh-user=root name_of_remote
# Note for lxd may need to change /etc/systemd/system/docker.service.d/10-machine.conf to use vfs instead of aufs
# It will also change the hostname, so change that back:
eval $(docker-machine env name_of_remote)
# But my preferred way is proxying the docker socket. Insecure option over TCP https://hub.docker.com/r/anthonydahanne/docker-socat/
# Or lxd has a built in proxy as well. Blog post for sharing lxc socket to a container, but same approach works for docker container to host https://blog.simos.info/how-to-manage-lxd-from-within-one-of-its-containers/
lxc config device add docker dockersocket proxy connect=unix:/var/run/docker.sock listen=unix:/tmp/docker.sock bind=host uid=0 gid=999 mode=0660 security.uid=65534 security.gid=999
export DOCKER_HOST=unix:///tmp/docker.sock
docker run -it --rm ubuntu bash
# Note, if you mounted it to /var/run/docker.sock on the host it would be picked up by default, but I think there were some permission issues.
# You can also use docker-compose on the host using local docker files https://docs.docker.com/compose/gettingstarted/
# But keep in mind the references in the docker files will be sourced on the docker container not your local host
# So to fix that you can mount your development folder as another device :-P
# Mount local host folder into docker container
lxc config device add docker scripts disk source=/home/$USER/code/scripts/ path=/home/$USER/scripts
lxc config device add docker dockerfiles disk source=/home/$USER/code/dockerfiles path=/home/$USER/dockerfiles
lxc config device add docker go disk source=/home/$USER/code/go path=/home/$USER/go
# Now those folders are mounted in the container, however they currently have a different UID:GID due to the container's user namespace http://man7.org/linux/man-pages/man7/user_namespaces.7.html
root@docker:/home/greg.bray# ls -hl
total 12K
drwxr-xr-x 4 nobody nogroup 4.0K May 13 01:31 dockerfiles
drwxr-xr-x 5 nobody nogroup 4.0K Apr 23 16:09 go
drwxr-xr-x 7 nobody nogroup 4.0K May 22 21:30 scripts
# To fix nobody:nobody you have to add a custom idmap https://tribaal.io/nicer-mounting-home-in-lxd.html
# This is where assuming the UID:GID in the container matches the host comes into play. Should be possible if they are different but much easier if they are the same
# You may need to sudo vim /etc/sub{u,g}id but on ubuntu 18.04 with latest lxd I was able to skip that step
lxc config set docker raw.idmap "both $UID $UID"
lxc restart docker #Should see a message about it remapping the users on the container filesystem
# Now the files mounted in the container have the correct owner:group
lxc exec docker -- bash -c "ls -hl /home/$USER"
total 12K
drwxr-xr-x 4 greg.bray greg.bray 4.0K May 13 01:31 dockerfiles
drwxr-xr-x 5 greg.bray greg.bray 4.0K Apr 23 16:09 go
drwxr-xr-x 7 greg.bray greg.bray 4.0K May 22 21:30 scripts
# And so you can reference them via dockerfiles or docker-compose or as mounted volumes into containers
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.