Playing with foreign layers

.gitignore

go.sum

README.md

Playing with foreign layers

Docker images support "foreign layers", which allow pulling layer contents from locations outside the registry serving the image manifest.

This has historically mainly been used to serve Windows image layers from Microsoft-owned servers, to avoid licensing issues with distributing Windows layers to everybody's registries. Instead of pulling Windows layers from my.registry.com, the manifest tells docker pull to fetch from mcr.microsoft.com/... instead.

This experiment (ab)uses foreign layers to fetch Ubuntu rootfs layer tarballs from upstream sources on canonical.com, instead of packaging them up into tarballs and pushing them to registries, like the official ubuntu does.

This doesn't currently work, and even if it did, it probably wouldn't be a good idea to have every client pulling ubuntu to hit Canonical's servers to get it.

---

$ go run ./
gcr.io/imjasonh/fl@sha256:bdad0434111344074600d2b22fb48f12b872ce13a285b188d167bbbab78f56f2
$ crane manifest $(go run ./)
...
  "layers": [
    {
      "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
      "size": 29539783,
      "digest": "sha256:f7d193700113bc4a44551cda1a509802f85978a6ea11f0f40e733de33a2b121a",
      "urls": [
        "https://partner-images.canonical.com/oci/impish/20211207/ubuntu-impish-oci-amd64-root.tar.gz"
      ]
    }
  ]
...
$ docker pull $(go run ./)
gcr.io/imjasonh/fl@sha256:bdad0434111344074600d2b22fb48f12b872ce13a285b188d167bbbab78f56f2: Pulling from imjasonh/fl
f7d193700113: Downloading 
unknown blob

💥

go.mod

module github.com/imjasonh/foreign-layers

go 1.17

require github.com/google/go-containerregistry v0.7.0

require (
	github.com/docker/cli v20.10.10+incompatible // indirect
	github.com/docker/distribution v2.7.1+incompatible // indirect
	github.com/docker/docker v20.10.10+incompatible // indirect
	github.com/docker/docker-credential-helpers v0.6.4 // indirect
	github.com/pkg/errors v0.9.1 // indirect
	github.com/sirupsen/logrus v1.8.1 // indirect
	golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect
)

main.go

package main

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
	"github.com/google/go-containerregistry/pkg/v1/types"
)

var ref = name.MustParseReference("gcr.io/imjasonh/fl")

func main() {
	ctx := context.Background()

	repo := ref.Context()
	reg := repo.Registry

	authr, err := authn.DefaultKeychain.Resolve(repo)
	if err != nil {
		log.Fatal(err)
	}
	t, err := transport.NewWithContext(ctx, reg, authr, &http.Transport{}, []string{repo.Scope(transport.PushScope)})
	if err != nil {
		log.Fatal(err)
	}

	client := &http.Client{Transport: t}

	f, err := os.Open("manifest.json")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		log.Fatal(err)
	}
	if _, err := f.Seek(0, 0); err != nil {
		log.Fatal(err)
	}

	dig := "sha256:" + hex.EncodeToString(h.Sum(nil))
	url := "https://gcr.io/v2/imjasonh/fl/manifests/" + dig
	req, err := http.NewRequest(http.MethodPut, url, f)
	if err != nil {
		log.Fatal(err)
	}
	req.Header.Set("Content-Type", string(types.DockerManifestSchema2))
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusCreated {
		log.Println("PUT", url)
		log.Println(resp.StatusCode)
		io.Copy(os.Stderr, resp.Body)
		os.Exit(1)
	}

	fmt.Println("gcr.io/imjasonh/fl@" + dig)
}

manifest.json

{
  "schemaVersion": 2,
  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
  "config": {
    "mediaType": "application/vnd.docker.container.image.v1+json",
    "size": 1462,
    "digest": "sha256:ba6acccedd2923aee4c2acc6a23780b14ed4b8a5fa4e14e252a23b846df9b6c1"
  },
  "layers": [
    {
      "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
      "size": 29539783,
      "digest": "sha256:f7d193700113bc4a44551cda1a509802f85978a6ea11f0f40e733de33a2b121a",
      "urls": [
        "https://partner-images.canonical.com/oci/impish/20211207/ubuntu-impish-oci-amd64-root.tar.gz"
      ]
    }
  ]
}