Buildkit local cache issue

go.mod

module buildkit-repro

go 1.17

require github.com/moby/buildkit v0.9.3

require (
	github.com/Microsoft/go-winio v0.4.17 // indirect
	github.com/containerd/containerd v1.5.8 // indirect
	github.com/containerd/continuity v0.1.0 // indirect
	github.com/containerd/typeurl v1.0.2 // indirect
	github.com/docker/cli v20.10.7+incompatible // indirect
	github.com/docker/distribution v2.7.1+incompatible // indirect
	github.com/docker/docker v20.10.7+incompatible // indirect
	github.com/gofrs/flock v0.7.3 // indirect
	github.com/gogo/googleapis v1.4.0 // indirect
	github.com/gogo/protobuf v1.3.2 // indirect
	github.com/golang/protobuf v1.5.2 // indirect
	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
	github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 // indirect
	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
	github.com/opencontainers/go-digest v1.0.0 // indirect
	github.com/opencontainers/image-spec v1.0.1 // indirect
	github.com/pkg/errors v0.9.1 // indirect
	github.com/sirupsen/logrus v1.8.1 // indirect
	github.com/tonistiigi/fsutil v0.0.0-20210609172227-d72af97c0eaf // indirect
	go.opentelemetry.io/contrib v0.21.0 // indirect
	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.21.0 // indirect
	go.opentelemetry.io/otel v1.0.0-RC1 // indirect
	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.0-RC1 // indirect
	go.opentelemetry.io/otel/sdk v1.0.0-RC1 // indirect
	go.opentelemetry.io/otel/trace v1.0.0-RC1 // indirect
	go.opentelemetry.io/proto/otlp v0.9.0 // indirect
	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
	golang.org/x/text v0.3.4 // indirect
	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
	google.golang.org/grpc v1.38.0 // indirect
	google.golang.org/protobuf v1.27.1 // indirect
)

main.go

package main

import (
	"context"
	"fmt"
	"os"
	"sync"

	bk "github.com/moby/buildkit/client"
	_ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver
	"github.com/moby/buildkit/client/llb"
	bkgw "github.com/moby/buildkit/frontend/gateway/client"
	"golang.org/x/sync/errgroup"
)

type Client struct {
	c *bk.Client
}

func New(ctx context.Context) (*Client, error) {
	c, err := bk.New(ctx, os.Getenv("BUILDKIT_HOST"))
	if err != nil {
		return nil, err
	}

	return &Client{c}, nil
}

func (c *Client) Do(ctx context.Context) error {
  	// Fill options
	opts := bk.SolveOpt{
		Exports: []bk.ExportEntry{{Type: "local", OutputDir: "result"}},
		CacheImports: []bk.CacheOptionsEntry{{
			Type: "local",
			Attrs: map[string]string{
				"src": "store",
			},
		}},
		CacheExports: []bk.CacheOptionsEntry{{
			Type: "local",
			Attrs: map[string]string{
				"mode": "max",
				"dest": "store",
			},
		}},
	}

	return c.exec(ctx, opts)
}

func (c *Client) exec(ctx context.Context, opts bk.SolveOpt) error {
	eg, ctx := errgroup.WithContext(ctx)

	eg.Go(func() error {
    
    		// Wait for buildkit to finish
		wg := sync.WaitGroup{}
		defer func() {
			wg.Wait()
		}()

    		// Create status
		status := make(chan *bk.SolveStatus)
		wg.Add(1)

		// Catch status
		go func() {
			for _ = range status {
			}
			wg.Done()
		}()

    		// Build & solve LLB
		_, err := c.c.Build(ctx, opts, "",
			func(ctx context.Context, c bkgw.Client) (*bkgw.Result, error) {
        			// A simple image that sleep && echo test into test
				state := llb.
					Image("busybox").
					User("root").
					Run(llb.Shlex(`sh -c "sleep 5 && echo -n test > /test"`))

				def, err := state.Marshal(ctx, llb.LinuxArm64)
				if err != nil {
					return nil, err
				}

				res, err := c.Solve(ctx, bkgw.SolveRequest{
					Definition: def.ToPB(),
				})
				if err != nil {
					return nil, err
				}
        
				return res, nil
			}, status)
    
		return err
	})

	return eg.Wait()
}

func main() {
	ctx := context.Background()
  
  	// Create client
	c, err := New(ctx)
	if err != nil {
		panic(err)
	}

  	// Execute
	err = c.Do(ctx)
	if err != nil {
		fmt.Println(err)
	}
}

Repro-case.md

Repro case

# See file
ls
go.mod  go.sum  main.go

# Start buildkit daemon
docker run --net=host -d --restart always -v dagger-buildkitd:/var/lib/buildkit --name dagger-buildkitd --privileged moby/buildkit:v0.9.3
d0368480d7f2ee03d55ffae5474eeee7a8c7a44453fa978f90825137d9ca1261

# Override host value
export BUILDKIT_HOST=docker-container://dagger-buildkitd

# Run main.go
time go run ./main.go
WARN[0000] local cache import at store not found due to err: could not read store/index.json: open store/index.json: no such file or directory 
go run ./main.go  0.81s user 1.09s system 18% cpu 10.214 total

# List file to see store appear
ls
go.mod  go.sum  main.go result  store

# Check result
cat result/test 
test%

# Check cache
l store/
total 8
drwxr-xr-x  3 tomchauveau  staff    96B Feb  4 15:34 blobs
-rw-r--r--  1 tomchauveau  staff   244B Feb  4 15:34 index.json
drwxr-xr-x  2 tomchauveau  staff    64B Feb  4 15:34 ingest

# Check blob
l store/blobs/sha256 
total 1648
-r--r--r--  1 tomchauveau  staff   571B Feb  4 15:34 1c294434ba7679ee1f7c9e32c29880b3ad5981058cb54884b4cdc1de37d30e31
-r--r--r--  1 tomchauveau  staff   894B Feb  4 15:34 2a086a3ea822a281c45bc43fecc343ef5ffe721fd58f972ed46a0dcc6c810833
-r--r--r--  1 tomchauveau  staff   150B Feb  4 15:34 82789c089c26861754a6fd11f46dcd053aefe99abbf093d9d10f1a1ec1877b68
-r--r--r--  1 tomchauveau  staff   809K Feb  4 15:34 a01966dde7f8d5ba10b6d87e776c7c8fb5a5f6bfa678874bd28b33b1fc6dba34

# Rerun to verify that ephemeral cache works
time go run ./main.go
go run ./main.go  0.76s user 1.21s system 85% cpu 2.318 total

# Remove container && volume
docker container stop dagger-buildkitd && docker container rm dagger-buildkitd && docker volume rm dagger-buildkitd                      
dagger-buildkitd
dagger-buildkitd
dagger-buildkitd

# Restart container
docker run --net=host -d --restart always -v dagger-buildkitd:/var/lib/buildkit --name dagger-buildkitd --privileged moby/buildkit:v0.9.3
c7352ffec6d3fecbbcfa802d3ac8cb3ecd977125c84fadb328d89b3f584e9fbc

# Verify that cache is stll there
l store/blobs/sha256 
total 1648
-r--r--r--  1 tomchauveau  staff   571B Feb  4 15:34 1c294434ba7679ee1f7c9e32c29880b3ad5981058cb54884b4cdc1de37d30e31
-r--r--r--  1 tomchauveau  staff   894B Feb  4 15:34 2a086a3ea822a281c45bc43fecc343ef5ffe721fd58f972ed46a0dcc6c810833
-r--r--r--  1 tomchauveau  staff   150B Feb  4 15:34 82789c089c26861754a6fd11f46dcd053aefe99abbf093d9d10f1a1ec1877b68
-r--r--r--  1 tomchauveau  staff   809K Feb  4 15:34 a01966dde7f8d5ba10b6d87e776c7c8fb5a5f6bfa678874bd28b33b1fc6dba34

# Rerun
time go run ./main.go
ERRO[0009] (*service).Write failed                       error="rpc error: code = Canceled desc = context canceled" expected="sha256:c4f68cac3b6805a14eef6e8812915bc64f81990df1667d1d97b720bfc05aaff7" ref="sha256:c4f68cac3b6805a14eef6e8812915bc64f81990df1667d1d97b720bfc05aaff7" total=894
go run ./main.go  0.79s user 1.24s system 20% cpu 10.021 total

As you see, cache is found because there is no import warning but it's not used by buildkit. And sometime, the following error is thrown, it can be on first run, second or whatever.

ERRO[0009] (*service).Write failed                       error="rpc error: code = Canceled desc = context canceled" expected="sha256:c4f68cac3b6805a14eef6e8812915bc64f81990df1667d1d97b720bfc05aaff7" ref="sha256:c4f68cac3b6805a14eef6e8812915bc64f81990df1667d1d97b720bfc05aaff7" total=894