| #!/bin/sh | |
| # Setup a rootfs of Fedora 15 for libvirt lxc | |
| # | |
| # The rootfs is based on http://download.openvz.org/template/precreated/fedora-15-x86_64.tar.gz | |
| # | |
| # See also | |
| # - http://www.mail-archive.com/lxc-users@lists.sourceforge.net/msg01707/lxc-fedora.in | |
| if [ $# != 1 ]; then | |
| echo "usage: $0 <hostname>" | |
| exit 1 | |
| fi | |
| HOSTNAME=$1 | |
| # | |
| # Tweak systemd settings to be bootable inside a container | |
| # | |
| ETC=/etc/systemd/system | |
| LIB=/lib/systemd/system | |
| # Set default run level to 3 (multi user) | |
| ln -sf $LIB/multi-user.target $ETC/default.target | |
| # sysinit.target seems to stick on boot, so disable it. However, we need | |
| # systemd-tmpfiles-setup.service that was started by the dependency of | |
| # sysinit.target to boot up correctly, so start it instead. | |
| cp $LIB/basic.target $ETC/basic.target | |
| sed -i 's/sysinit.target/systemd-tmpfiles-setup.service/' $ETC/basic.target | |
| # Stop starting sysinit.target. Symlinking one to /dev/null is a standard way | |
| # to disable a target (or a service and others). | |
| ln -s /dev/null $ETC/sysinit.target | |
| # It also a cause of stuck on boot | |
| ln -s /dev/null $ETC/udev-settle.service | |
| # It prevents systemd-tmpfiles-setup.service from starting | |
| ln -s /dev/null $ETC/fedora-readonly.service | |
| # Libvirt lxc provides only tty1 | |
| rm -f $ETC/getty.target.wants/getty\@tty{2,3,4,5,6}.service | |
| # It launches sulogin on console(tty1) but it interferes getty@tty1 | |
| ln -s /dev/null $ETC/console-shell.service | |
| # | |
| # Workarounds for libvirt 0.9.4. Without this, getty@tty1 doen't launch | |
| # because a trigger event on tty1 doesn't happen. | |
| # | |
| cp $LIB/getty\@.service $ETC/getty\@.service | |
| sed -i 's/^BindTo/\#&/' $ETC/getty\@.service | |
| ln -sf $ETC/getty\@.service $ETC/getty.target.wants/getty\@tty1.service | |
| # Allow a user who logins via ssh to sudo | |
| sed -i 's/^Defaults\ *requiretty/\#&/' /etc/sudoers | |
| # | |
| # Allow to login at virsh console. loginuid.so doen't work in the absence of auditd | |
| # which cannot run inside a container. | |
| # | |
| sed -i 's/^.*loginuid.so.*$/\#&/' /etc/pam.d/login | |
| # | |
| # Enable eth0 on bootup | |
| # | |
| cat <<EOF > /etc/sysconfig/network-scripts/ifcfg-eth0 | |
| DEVICE=eth0 | |
| BOOTPROTO=dhcp | |
| ONBOOT=yes | |
| EOF | |
| cat <<EOF > /etc/sysconfig/network | |
| NETWORKING=yes | |
| HOSTNAME=$HOSTNAME | |
| EOF | |
| # | |
| # Tweak sshd configuration | |
| # | |
| sed -i 's/^UsePAM\ *yes/\#&/' /etc/ssh/sshd_config | |
| sed -i 's/^GSSAPIAuthentication\ *yes/\#&/' /etc/ssh/sshd_config | |
| # | |
| # Prevent udevd from creating /dev/ptmx | |
| # This is not needed for libvirt 0.9.5 and above. | |
| # | |
| cat <<EOF > /etc/udev/rules.d/00-symlink-dev-ptmx.rules | |
| KERNEL=="ptmx", NAME="pts/%k", SYMLINK+="%k" | |
| EOF | |
| # | |
| # Prevent overwriting /dev/pts. Get rid of devpts entry if exists. | |
| # | |
| sed -i '/devpts/s/^[^#]/\#&/' /etc/fstab | |
| # | |
| # Allow root to login at virsh console | |
| # | |
| echo "pts/0" >> /etc/securetty | |
| # | |
| # Uninstall unnecessary packages of OpenVZ | |
| # | |
| yum -y remove vzdev vzdummy-init-fc15 | |
| # | |
| # Create a user (if you want) | |
| # | |
| #USER=peo3 | |
| #PASS=peo3pass | |
| #adduser -G wheel $USER | |
| #echo "$USER:$PASS" | chpasswd |
| #!/bin/sh | |
| # Setup host environment for libvirt lxc | |
| # | |
| # Tested environment | |
| # | |
| # Host | |
| # - Ubuntu 11.04 | |
| # - libvirt 0.9.2 and 0.9.4 | |
| # Guest | |
| # - Fedora 15 http://download.openvz.org/template/precreated/fedora-15-x86_64.tar.gz | |
| ROOTFS=/opt/rootfs/fedora15 | |
| NAME=fedora15 | |
| HOSTNAME=$NAME | |
| # | |
| # Setup a rootfs of Fedora 15 | |
| # | |
| wget http://download.openvz.org/template/precreated/fedora-15-x86_64.tar.gz | |
| tar zxvf fedora-15-x86_64.tar.gz -C $ROOTFS | |
| # | |
| # Setup packages. The PPA provides latest libvirt packages. | |
| # | |
| add-apt-repository ppa:dnjl/build | |
| apt-get update | |
| apt-get install libcgroup1 libvirt0 libvirt-bin | |
| # | |
| # Allow containers to communicate with outside | |
| # | |
| echo 1 > /proc/sys/net/ipv4/ip_forward | |
| # | |
| # Allow to run dhclient of Fedora 15 inside a container | |
| # | |
| cat - >> /etc/apparmor.d/local/sbin.dhclient <<EOF | |
| /etc/pki/tls/openssl.cnf r, | |
| /var/lib/dhclient/dhclient-*.leases rw, | |
| /var/lib/dhclient/dhclient.leases rw, | |
| EOF | |
| apparmor_parser -r /etc/apparmor.d/sbin.dhclient | |
| # | |
| # Create a container via libvirt | |
| # | |
| XML=/tmp/fedora15.xml | |
| cat - > $XML <<EOF | |
| <domain type='lxc'> | |
| <name>$NAME</name> | |
| <memory>128000</memory> | |
| <os> | |
| <type>exe</type> | |
| <init>/sbin/init</init> | |
| </os> | |
| <vcpu>1</vcpu> | |
| <clock offset='utc'/> | |
| <on_poweroff>destroy</on_poweroff> | |
| <on_reboot>restart</on_reboot> | |
| <on_crash>destroy</on_crash> | |
| <devices> | |
| <emulator>/usr/lib/libvirt/libvirt_lxc</emulator> | |
| <filesystem type='mount'> | |
| <source dir='$ROOTFS'/> | |
| <target dir='/'/> | |
| </filesystem> | |
| <interface type='network'> | |
| <source network='default'/> | |
| </interface> | |
| <console type='pty' /> | |
| </devices> | |
| </domain> | |
| EOF | |
| virsh -c lxc:/// define $XML | |
| # | |
| # Setup the rootfs of the container | |
| # | |
| cp setup_lxc_rootfs_fedora15.sh $ROOTFS/tmp | |
| chroot $ROOTFS sh /tmp/setup_lxc_rootfs_fedora15.sh $HOSTNAME | |
| # And have a fun! | |
| #virsh -c lxc:/// start $NAME | |
| #virsh -c lxc:/// console $NAME | |
| # Known issues | |
| # - Networking doesn't work <- FIXED | |
| # - dhclient fails due to AppArmor in the host <- FIXED | |
| # - Many systemd warnings about getty are output periodically <- FIXED | |
| # - Fails to start httpd <- FIXED | |
| # - Cannot login as root at console <- FIXED | |
| # - Cannot login via ssh <- FIXED | |
| # - Loginning via ssh is so slow <- FIXED | |
| # - Many udevd-work errors found in /var/log/message | |
| # - disabling udev.service prevents login console from starting | |
| # - Many /dev/kmsg errors on console <- FIXED | |
| # - /var/lock/subsys/ isn't created correctly <- FIXED | |
| # - It causes that daemons fail to create lock files | |
| # | |
| # See also | |
| # - http://www.mail-archive.com/lxc-users@lists.sourceforge.net/msg01707/lxc-fedora.in |
I've also tried Ubuntu 11.04 host + lxc tool with a rootfs which is created by the template and confirmed it doesn't work well; lxc-start doesn't return but processes inside the container seem working. lxc-console doesn't work as well.
This is exactly what I meant. If you have a systemd based "host" that launch a lxc container based on systemd, you will have troubles (very impressives sometimes... xscreensaver is launched, tty are lost, reboot, sometimes X crashes...)
This problem is well known... and I'm seeking a nice way to remove the problem. It seems you have good ideas. So I will continue to try with your script as base, and I'll tell you what's wrong or right :)
Hey I have been trying to also launch a f15 container on a f15 host, just applied the configs suggested in the script, X crashed, and container got stuck somewhere, thus unreleasing the cgroup name
peo3 would be of great help if you give me an idea how you did debug those issues
Once again, the problem comes from "systemd"... There is no solution (for now). I'm contacting systemd and lxc administrators to know what should be the solution in the futur :)
I understand that the problem comes from systemd, and I would like to know how did you debug it.
systemd has systemd-nspawn which would init a rootfs in a similar way like lxc without most of the features, and i was able to init a f15 rootfs on a f15 host without the crashes. so that tells me something about lxc
Now if we are able to debug and actually find out what goes wrong. so my question is how did you find out about those fixes? how do you debug systemd?
You're right. systemd-nspawn can init a rootfs, and some people say that lxc can be obsolete in the futur. For now, I don't find a way to debug. I only asked some #fedora-fr users (irc on freenode) some questions... It seems to be complicated to debug systemd...
peo3 could you hint us on how you debugged systemd?
Hmm. In my case systemd was able to launch without any crashes and it just stuck during service initializations. So debugging is not difficult. I could refer several logs: console log via virsh console, libvirtd error messages from libvirtd log files, syslog, dmesg, and of course systemctl outputs.
Anyway I tried replacing /sbin/init with my fake script /sbin/init.sh which does 'exec /sbin/systemd' at the end of itself. By doing so I could get many information, for example what libvirtd actually deployed, or modify some files, e.g., device files, to see how systemd changes its behavior with the modifications.
I also read the source code of systemd and tried a modified version, although it didn't resolve the problem. (But it's not nothing worth because reading source code corrects my wrong assumption.)
you will have troubles (very impressives sometimes... xscreensaver is launched, tty are lost, reboot, sometimes X crashes...)
I had ever seen the behaviors when I was trying old libvirt. The cause might be that some process was trying to grab /dev/console or /dev/tty*...but I don't remember exactly.
Once again, the problem comes from "systemd"...
Yes. My script may be a workaround and we would need correct fixes in systemd and/or maybe in lxc tool.
BTW, can you try libvirt instead of lxc tool? If using libvirt changes the result, it would give us more hints.
Hey, these days I played with lxc and fedora16. I found the same problem of systemd. More specifically, systemd gets a new group /dev/tty_, not those provided by lxc.
LXC starts replacing the /dev/tty1 with a pts (136, 8) while the systemd get the /dev/ of your host machine. So any getty like operation will interfere with your host tty_, which also explains the crash of X. Updated Fedora 16 also has a broken agetty who refuses to cooperate with pts but can be directly replaced by the agetty from debian 6, so that's not a major problem. IMO, it is just insane that systemd pretends to be cgroup aware and can spawn cgroup, but it refuses to reside in a standard spawned cgroup environment, if debian and ubuntu startup daemons also play the trick, then none of them can host each other in containers!
I simply did this patch, :)
diff -ruNp systemd-37/src/mount-setup.c systemd-37.new/src/mount-setup.c
--- systemd-37/src/mount-setup.c 2011-08-31 01:21:41.804076227 +0800
+++ systemd-37.new/src/mount-setup.c 2012-03-11 00:05:55.194903118 +0800
@@ -55,11 +55,6 @@ typedef struct MountPoint {
#define N_EARLY_MOUNT 3
static const MountPoint mount_table[] = {
-
{ "proc", "/proc", "proc", NULL, MS_NOSUID|MS_NOEXEC|MS_NODEV, true }, -
{ "sysfs", "/sys", "sysfs", NULL, MS_NOSUID|MS_NOEXEC|MS_NODEV, true }, -
{ "devtmpfs", "/dev", "devtmpfs", "mode=755", MS_NOSUID, true }, -
{ "tmpfs", "/dev/shm", "tmpfs", "mode=1777", MS_NOSUID|MS_NODEV, true }, -
{ "devpts", "/dev/pts", "devpts", "mode=620,gid=" STRINGIFY(TTY_GID), MS_NOSUID|MS_NOEXEC, false }, { "tmpfs", "/run", "tmpfs", "mode=755", MS_NOSUID|MS_NODEV, true }, { "tmpfs", "/sys/fs/cgroup", "tmpfs", "mode=755", MS_NOSUID|MS_NOEXEC|MS_NODEV, false }, { "cgroup", "/sys/fs/cgroup/systemd", "cgroup", "none,name=systemd", MS_NOSUID|MS_NOEXEC|MS_NODEV, false },
after which I found some other very sneaky bugs(does not occur very time, but when it does, it's rather annoying ! ) in login also routed in systemd...
So I just lost my temper with systemd and wrote a very simple init shell script of my own (i. e. simply agetty the lxc tty and start a sshd), so far so good! :D
I forgot the template because I've never tried it ;-)
And I confirmed that a container using the rootfs works well. So the differences between Ubuntu 11.04 host + libvirt and Fedor 15 host + lxc tool cause the different results...