Firecracker in containerd and Docker

README.md

Firecracker in containers

How to install firectl and firecracker:

docker run -it -v /usr/local/bin:/install luxas/firectl

How to run firectl in Docker:

curl -fsSL -o hello-vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/hello/kernel/hello-vmlinux.bin
curl -fsSL -o hello-rootfs.ext4 https://s3.amazonaws.com/spec.ccfc.min/img/hello/fsfiles/hello-rootfs.ext4

docker run -itd --device /dev/kvm \
    --entrypoint /bin/sh \
    -v $(pwd):/fc \
    --name fc$(date +%s) \
    luxas/firectl \
    -c "/firectl --root-drive /fc/hello-rootfs.ext4 --kernel /fc/hello-vmlinux.bin --firecracker-binary /firecracker"

How to run firectl in containerd:

./fcd

main.go

package main

import (
	"context"
	"fmt"
	"os"
	"os/signal"
	"strings"
	"encoding/csv"

	"github.com/containerd/containerd/log"
	"golang.org/x/sys/unix"
	"github.com/containerd/console"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/oci"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/namespaces"
	"github.com/containerd/containerd/containers"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

type Config struct {
	Detach, TTY bool
	NetHost bool
	Privileged bool
	Namespace string
	Socket string
	Image string
}
  
func main() {
	if err := run(); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
}

func run() error {
	cfg := &Config{
		Detach: true,
		TTY: true,
		NetHost: true,
		Privileged: false,
		Namespace: "firecracker",
		Socket: "/run/containerd/containerd.sock",
		Image: "docker.io/luxas/firectl:latest",
	}

	ctx := context.Background()
	client, err := containerd.New(cfg.Socket, containerd.WithDefaultNamespace(cfg.Namespace))
	if err != nil {
		return err
	}
	defer client.Close()

	image, err := client.GetImage(ctx, cfg.Image)
	if err != nil {
		return err
	}

	unpacked, err := image.IsUnpacked(ctx, containerd.DefaultSnapshotter)
	if err != nil {
		return err
	}
	if !unpacked {
		if err := image.Unpack(ctx, containerd.DefaultSnapshotter); err != nil {
			return err
		}
	}

	cid := "fc-test"

	curDir, err := os.Getwd()
	if err != nil {
		return err
	}

	var opts []oci.SpecOpts
	opts = append(opts, oci.WithDefaultSpec(), oci.WithDefaultUnixDevices)
	mounts := []string{fmt.Sprintf("type=bind,src=%s,dst=/fc,options=rbind:rw", curDir)}
	opts = append(opts, withMounts(mounts))
	opts = append(opts, withKVM)
	opts = append(opts, oci.WithImageConfig(image))
	if cfg.TTY {
		opts = append(opts, oci.WithTTY)
	}
	if cfg.Privileged {
		opts = append(opts, oci.WithPrivileged)
	}
	if cfg.NetHost {
		opts = append(opts, oci.WithHostNamespace(specs.NetworkNamespace), oci.WithHostHostsFile, oci.WithHostResolvconf)
	}
	args := []string{
		"/bin/sh",
		"-c",
		"/firectl --root-drive /fc/hello-rootfs.ext4 --kernel /fc/hello-vmlinux.bin --firecracker-binary /firecracker",
	}
	opts = append(opts, oci.WithProcessArgs(args...))

	cOpts := []containerd.NewContainerOpts{
		containerd.WithImage(image),
		containerd.WithSnapshotter(containerd.DefaultSnapshotter),
		// Even when "readonly" is set, we don't use KindView snapshot here. (#1495)
		// We pass writable snapshot to the OCI runtime, and the runtime remounts it as read-only,
		// after creating some mount points on demand.
		containerd.WithNewSnapshot(cid, image),
		containerd.WithImageStopSignal(image, "SIGTERM"),
		containerd.WithNewSpec(opts...),
	}

	ctx = namespaces.WithNamespace(ctx, cfg.Namespace)
	container, err := client.NewContainer(ctx, cid, cOpts...)
	if err != nil {
		return err
	}

	stdio := cio.NewCreator(cio.WithStdio)
	var con console.Console
	if cfg.TTY {
		con = console.Current()
		defer con.Reset()
		if err := con.SetRaw(); err != nil {
			return err
		}
		stdio = cio.NewCreator(cio.WithStreams(con, con, nil), cio.WithTerminal)
	}
	task, err := container.NewTask(ctx, stdio)
	if err != nil {
		return err
	}

	var statusC <-chan containerd.ExitStatus
	if !cfg.Detach {
		if statusC, err = task.Wait(ctx); err != nil {
			return err
		}
	}

	if err := task.Start(ctx); err != nil {
		return err
	}
	if cfg.Detach {
		fmt.Printf("ID: %q, Pid: %q\n", task.ID(), task.Pid())
		return nil
	}
	if cfg.TTY {
		if err := HandleConsoleResize(ctx, task, con); err != nil {
			return fmt.Errorf("console resize: %v", err)
		}
	}
	status := <-statusC
	code, _, err := status.Result()
	if err != nil {
		return err
	}
	if _, err := task.Delete(ctx); err != nil {
		return err
	}
	if code != 0 {
		return fmt.Errorf("exit with error code: %d", code)
	}
	return nil
}

func withKVM(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
	if s.Linux == nil {
		s.Linux = &specs.Linux{}
	}
	if s.Linux.Resources == nil {
		s.Linux.Resources = &specs.LinuxResources{}
	}
	int64ptr := func(i int64) *int64 {
		return &i
	}
	uint32ptr := func(i uint32) *uint32 {
		return &i
	}
	s.Linux.Resources.Devices = append(s.Linux.Resources.Devices, []specs.LinuxDeviceCgroup{
		{
			// "/dev/kvm",
			Type:   "c",
			Major:  int64ptr(10),
			Minor:  int64ptr(232),
			Access: "rwm",
			Allow:  true,
		},
	}...)
	fm := os.FileMode(8624)
	s.Linux.Devices = append(s.Linux.Devices, []specs.LinuxDevice{
		{
			// "/dev/kvm",
			Path:   "/dev/kvm",
			Type:   "c",
			Major:  10,
			Minor:  232,
			FileMode: &fm,
			UID:  uint32ptr(0),
			GID:  uint32ptr(116),
		},
	}...)
	return nil
}


func withMounts(mountSlice []string) oci.SpecOpts {
	return func(ctx context.Context, client oci.Client, container *containers.Container, s *specs.Spec) error {
		mounts := make([]specs.Mount, 0)
		for _, mount := range mountSlice {
			m, err := parseMountFlag(mount)
			if err != nil {
				return err
			}
			mounts = append(mounts, m)
		}
		return oci.WithMounts(mounts)(ctx, client, container, s)
	}
}
// parseMountFlag parses a mount string in the form "type=foo,source=/path,destination=/target,options=rbind:rw"
func parseMountFlag(m string) (specs.Mount, error) {
	mount := specs.Mount{}
	r := csv.NewReader(strings.NewReader(m))

	fields, err := r.Read()
	if err != nil {
		return mount, err
	}

	for _, field := range fields {
		v := strings.Split(field, "=")
		if len(v) != 2 {
			return mount, fmt.Errorf("invalid mount specification: expected key=val")
		}

		key := v[0]
		val := v[1]
		switch key {
		case "type":
			mount.Type = val
		case "source", "src":
			mount.Source = val
		case "destination", "dst":
			mount.Destination = val
		case "options":
			mount.Options = strings.Split(val, ":")
		default:
			return mount, fmt.Errorf("mount option %q not supported", key)
		}
	}

	return mount, nil
}

type resizer interface {
	Resize(ctx context.Context, w, h uint32) error
}

// HandleConsoleResize resizes the console
func HandleConsoleResize(ctx context.Context, task resizer, con console.Console) error {
	// do an initial resize of the console
	size, err := con.Size()
	if err != nil {
		return err
	}
	if err := task.Resize(ctx, uint32(size.Width), uint32(size.Height)); err != nil {
		log.G(ctx).WithError(err).Error("resize pty")
	}
	s := make(chan os.Signal, 16)
	signal.Notify(s, unix.SIGWINCH)
	go func() {
		for range s {
			size, err := con.Size()
			if err != nil {
				log.G(ctx).WithError(err).Error("get pty size")
				continue
			}
			if err := task.Resize(ctx, uint32(size.Width), uint32(size.Height)); err != nil {
				log.G(ctx).WithError(err).Error("resize pty")
			}
		}
	}()
	return nil
}

Makefile

all: build-docker
build:
	go build -o fcd -mod=vendor .

shell:
	docker run -it -v $(shell pwd):/build -w /build golang:1.11

build-docker:
	docker run -it -v $(shell pwd):/build -w /build golang:1.11 make build

vendor:
	go mod tidy
	go mod vendor

vendor-docker:
	docker run -it -v $(shell pwd):/build -w /build golang:1.11 make vendor