SOCI w/ k8s

README.md

• use initialize-nginx-image.sh to push nginx:latest into an ECR repository and build the soci index (see here for details)
• install fuse using sudo yum install fuse -y
• download and extract files from https://github.com/awslabs/soci-snapshotter/releases/ into /usr/local/bin
• run sudo chown root:root /usr/local/bin/soci-snapshotter-grpc
• copy soci-config.toml to /etc/soci-snapshotter-grpc/config.toml
• copy soci-snapshotter.service to /usr/lib/systemd/system/soci-snapshotter.service
• enable service using sudo systemctl enable --now soci-snapshotter
• install sudo yum install -y amazon-ecr-credential-helper (or install from latest release in repo)
• copy config.json to /root/.docker/config.json
• run sudo chown root:root /root/.docker/config.json
• append the following snippet in append-to-containerd-config.toml to /etc/containerd/config.toml
• add disable_snapshot_annotations = false in the [plugins."io.containerd.grpc.v1.cri".containerd] section of /etc/containerd/config.toml
• restart containerd using systemctl restart containerd
• run "kubectl apply -f soci-runtime.yaml"
• deploy soci-pod.yaml using kubectl create -f soci-pod.yaml

Most of the above is from awslabs/soci-snapshotter#984 with detailed notes shared by @ollypom

append-to-containerd-config.toml

# Create a dedicated runtime class for SOCI
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.soci-runtime]
runtime_type = "io.containerd.runc.v2"
snapshotter = "soci"

# Configure the SOCI Snapshotter as a Remote Snapshotter for containerd
[plugins]
[proxy_plugins.soci]
type = "snapshot"
address = "/run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock"

config.json

{ "credsStore":"ecr-login" }

initialize-nginx-image.sh

# Install nerdctl
curl --fail --location https://github.com/containerd/nerdctl/releases/download/v1.7.2/nerdctl-1.7.2-linux-amd64.tar.gz | sudo tar -C /usr/local/bin -zxf -

# Log into ECR repo
aws ecr get-login-password --region us-east-1 | sudo nerdctl login --username AWS --password-stdin 086752300739.dkr.ecr.us-east-1.amazonaws.com

# Fetch the nginx image
sudo nerdctl pull nginx:latest --all-platforms

# Tag the nginx image so we can push to ECR repo
sudo nerdctl tag nginx:latest 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest

# Create the SOCI index
sudo soci create 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest

# Push all the things!
sudo nerdctl push --all-platforms --snapshotter soci 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest

misc.sh

sudo yum install fuse htop tree -y

curl --fail --location "https://github.com/containerd/nerdctl/releases/download/v1.7.2/nerdctl-1.7.2-linux-amd64.tar.gz" | sudo tar -C /usr/local/bin -zxf -
#curl --fail --location "https://github.com/awslabs/soci-snapshotter/releases/download/v0.4.1/soci-snapshotter-0.4.1-linux-amd64.tar.gz" | sudo tar -C /usr/local/bin -zxf -
sudo yum install golang zlib-devel zlib-static -y
git clone https://github.com/awslabs/soci-snapshotter /tmp/soci-snapshotter >/dev/null 2>&1

pushd /tmp/soci-snapshotter/
make soci-snapshotter-grpc soci
sudo cp out/soci /usr/local/bin/soci
sudo cp out/soci-snapshotter-grpc /usr/local/bin/soci-snapshotter-grpc
sudo /usr/local/bin/soci-snapshotter-grpc --version
popd

sudo chown root:root /usr/local/bin/soci*
sudo curl -s -o /etc/soci-snapshotter-grpc/config.toml https://gist.githubusercontent.com/dims/829e3890282602202e116694ff626de8/raw/70191e9ed0249170d5f3c85514f668c75557552a/soci-config.toml
sudo curl -s -o /usr/lib/systemd/system/soci-snapshotter.service https://gist.githubusercontent.com/dims/829e3890282602202e116694ff626de8/raw/33471d035cbbdd7b04ad7050db129beee2d1f4f9/soci-snapshotter.service
sudo systemctl enable --now soci-snapshotter
sudo yum install -y amazon-ecr-credential-helper
sudo mkdir -p /root/.docker/ && sudo curl -s -o /root/.docker/config.json https://gist.githubusercontent.com/dims/829e3890282602202e116694ff626de8/raw/70191e9ed0249170d5f3c85514f668c75557552a/config.json
curl -s -o /tmp/append-to-containerd-config.toml https://gist.githubusercontent.com/dims/829e3890282602202e116694ff626de8/raw/909dd1ab5d973507104190342718927deabaf8d6/append-to-containerd-config.toml
sudo sed -i '/discard_unpacked_layers = true/a disable_snapshot_annotations = false' /etc/containerd/config.toml
cat /tmp/append-to-containerd-config.toml | sudo tee -a /etc/containerd/config.toml
#sudo sed -i s#registry.k8s.io/pause:3.8#086752300739.dkr.ecr.us-east-1.amazonaws.com/pause:3.8#g /etc/containerd/config.toml

sudo systemctl stop kubelet containerd soci-snapshotter
sudo systemctl daemon-reload
ps -ef | grep shim | grep -v grep | awk '{print $2}' | xargs sudo kill -9
sudo rm -rf /var/lib/containerd
sudo find /var/lib/soci-snapshotter-grpc/snapshotter/snapshots/ -name fs -exec sudo umount {} \;
sudo rm -rf /var/lib/soci-snapshotter-grpc

sudo systemctl start containerd soci-snapshotter
sudo chmod 777 /run/containerd/containerd.sock

find /kubernetes/server/bin -name "*.tar" -print | xargs -L 1 sudo ctr -n k8s.io images import
sudo ctr -n k8s.io images ls -q | grep -e amd64 | xargs -L 1 -I '{}' /bin/bash -c 'sudo ctr -n k8s.io images tag "{}" "$(echo "{}" | sed s/-'amd64':/:/)"'

sudo nerdctl --namespace k8s.io pull --snapshotter soci 086752300739.dkr.ecr.us-east-1.amazonaws.com/rabbitmq:latest
sudo nerdctl --namespace k8s.io pull --snapshotter soci 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest
sudo nerdctl --namespace k8s.io pull --snapshotter soci 086752300739.dkr.ecr.us-east-1.amazonaws.com/pause:3.8

sudo systemctl start kubelet

sudo journalctl -xeu containerd -f
sudo journalctl -xeu soci-snapshotter -f

kubectl apply -f https://gist.githubusercontent.com/dims/829e3890282602202e116694ff626de8/raw/4fd952b7b50b0979296e1eb73bd9165b04f63795/soci-runtime.yaml

kubectl create -f https://gist.githubusercontent.com/dims/829e3890282602202e116694ff626de8/raw/4fd952b7b50b0979296e1eb73bd9165b04f63795/soci-pod.yaml

kubectl delete -f https://gist.githubusercontent.com/dims/829e3890282602202e116694ff626de8/raw/4fd952b7b50b0979296e1eb73bd9165b04f63795/soci-pod.yaml

sudo nerdctl --namespace k8s.io pull --snapshotter soci 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest
sudo nerdctl --namespace k8s.io run -it --rm --snapshotter=soci 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest

REGISTRY=086752300739.dkr.ecr.us-east-1.amazonaws.com
sudo nerdctl pull --snapshotter soci $REGISTRY/nginx:latest
sudo nerdctl run -it --rm --snapshotter=soci $REGISTRY/nginx:latest

sudo nerdctl run -it --rm --snapshotter=soci public.ecr.aws/soci-workshop-examples/rabbitmq:latest


sudo nerdctl pull docker.io/library/rabbitmq:latest
sudo nerdctl image tag docker.io/library/rabbitmq:latest $REGISTRY/rabbitmq:latest
sudo nerdctl push --platform linux/amd64 --snapshotter soci $REGISTRY/rabbitmq:latest

sudo nerdctl pull registry.k8s.io/pause:3.8
sudo nerdctl image tag registry.k8s.io/pause:3.8 $REGISTRY/pause:3.8
sudo nerdctl push --platform linux/amd64 --snapshotter soci $REGISTRY/pause:3.8

sudo nerdctl pull docker.io/library/nginx:latest
sudo nerdctl image tag docker.io/library/nginx:latest $REGISTRY/nginx:latest
sudo nerdctl push --platform linux/amd64 --snapshotter soci $REGISTRY/nginx:latest

# Install nerdctl
curl --fail --location https://github.com/containerd/nerdctl/releases/download/v1.7.2/nerdctl-1.7.2-linux-amd64.tar.gz | sudo tar -C /usr/local/bin -zxf -

# Log into ECR repo
aws ecr get-login-password --region us-east-1 | sudo nerdctl login --username AWS --password-stdin 086752300739.dkr.ecr.us-east-1.amazonaws.com

# Fetch the nginx image
sudo nerdctl pull nginx:latest --all-platforms

# Tag the nginx image so we can push to ECR repo
sudo nerdctl tag nginx:latest 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest

# Create the SOCI index
sudo soci create 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest

# Push all the things!
sudo nerdctl push --all-platforms --snapshotter soci 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest

# Test 
sudo nerdctl --namespace k8s.io run -it --rm --snapshotter=soci 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest


sudo nerdctl --namespace k8s.io run -it --rm --snapshotter=soci public.ecr.aws/soci-workshop-examples/rabbitmq:latest

soci-config.toml

# Compare with https://github.com/awslabs/soci-snapshotter/blob/main/config/config.toml
# An example config showing all of the toml variables used.

# Copy to /etc/soci-snapshotter-grpc/config.toml
# to use on your system.

# NOTE: Many variables set to zero are just an indicator
# to use the built-in default. These values may change over time,
# which is why the config uses zeroes.
# Please see docs/config.md for more info on what these variables do.

# TODO: Can we put these inside dedicated TOML vars to refer to them?

# config/fs.go FSConfig
http_cache_type=""
filesystem_cache_type=""
resolve_result_entry=0
debug=false
allow_no_verification=true
# disable_verification=false
# Causes TestRunWithDefaultConfig to break, but
# fine to use in /etc/soci-snapshotter-grpc-config.toml
max_concurrency=0  # Actually zero
no_prometheus=false
mount_timeout_sec=0
fuse_metrics_emit_wait_duration_sec=0

## config/config.go Config

metrics_address=""
metrics_network="" # Uses default metrics network
# no_prometheus=true # Defined above, can't be redeclared
debug_address=""
metadata_store="db"

[http]
MaxRetries=0
MinWaitMsec=0
MaxWaitMsec=0
DialTimeoutMsec=0
ResponseHeaderTimeoutMsec=0
RequestTimeoutMsec=0

#
## config/fs.go
#

[blob]
valid_interval=0
check_always=false
fetching_timeout_sec=0
force_single_range_mode=false
# max_retries=0 # Set by http.
# min_wait_msec=0 # Set by http.
# max_wait_msec=0 # Set by http.
max_span_verification_retries=0 # Actually zero

[directory_cache]
max_lru_cache_entry=0 # Actually zero
max_cache_fds=0 # Actually zero
sync_add=false
direct=true

[fuse]
attr_timeout=0
entry_timeout=0
negative_timeout=0
log_fuse_operations=false

[background_fetch]
disable=false
silence_period_msec=0
fetch_period_msec=0
max_queue_size=0
emit_metric_period_sec=0

[content_store]
type="containerd" # will set to 'soci' by default
# Socket address for containerd. Only applicable using containerd content store.
# Defaults to '/run/containerd/containerd.sock'
containerd_address=""
namespace="k8s.io" # will set to 'default' by default
   
#
## config/resolver.go
#

[resolver]
  [resolver.host]

#
## config/service.go
#

[kubeconfig_keychain]
enable_keychain=false
kubeconfig_path=""

[cri_keychain]
enable_keychain=false
image_service_path="" # Uses default image service address

[snapshotter]
min_layer_size=0 # Actually zero
allow_invalid_mounts_on_restart=false

#
## service/resolver/cri.go
#

[registry]
config_path=""
mirrors={}
configs={}

[Mirror]
endpoint={}

[RegistryConfig]

[auth]
username=""
password=""
auth=""
identitytoken=""

[tls]
insecure_skip_verify=false
ca_file=""
cert_file=""
key_file=""

soci-pod.yaml

apiVersion: v1
kind: Pod
metadata:
  name: nginxsoci
  namespace: default
  annotations:
    io.containerd.cri.runtime-handler: "soci-runtime"
spec:
  runtimeClassName: soci-runtime
  containers:
  - name: nginx
    image: 086752300739.dkr.ecr.us-east-1.amazonaws.com/nginx:latest
    ports:
    - containerPort: 80

soci-runtime.yaml

apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: soci-runtime
handler: soci-runtime

soci-snapshotter.service

# Copyright The Soci Snapshotter Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Compare with https://github.com/awslabs/soci-snapshotter/blob/main/soci-snapshotter.service
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=soci snapshotter containerd plugin
Documentation=https://github.com/awslabs/soci-snapshotter
After=network.target containerd.service

[Service]
Type=notify
ExecStart=/usr/local/bin/soci-snapshotter-grpc --log-level trace
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target