soci_on_eks

Readme.md

To use it,

Tested with AMI amazon-eks-gpu-node-1.25-v20231230

• deploy all these files on a public S3 or on any webserver
• Add in the user data script, before eks bootstrap something like

curl https://myserver/soci/install.sh | bash -s -- https://myserver/soci

To use it,

• Create a runtime class

apiVersion: node.k8s.io/v1
handler: soci
kind: RuntimeClass
metadata:
  name: soci-runtime

• On the target pod
  • annotations containers: io.containerd.cri.runtime-handler: "soci"
  • runtimeClassName: soci-runtime

containerd_soci.toml

[plugins.cri.containerd]
disable_snapshot_annotations = false
discard_unpacked_layers = true
default_runtime_name = "runc"
snapshotter = "overlayfs"

[plugins.cri.containerd.runtimes.soci]
snapshotter = "soci"
runtime_type = "io.containerd.runtime.v1.linux"

[plugins.cri.containerd.runtimes.soci.options]
Runtime = "/etc/docker-runtimes.d/nvidia"
SystemdCgroup = true

[plugins]
[proxy_plugins.soci]
type = "snapshot"
address = "/run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock"

install.sh

#!/bin/sh -e

root=$1
echo "Using root url: $root"

cd /tmp
wget -q https://github.com/awslabs/soci-snapshotter/releases/download/v0.5.0/soci-snapshotter-0.5.0-linux-amd64-static.tar.gz
mkdir soci
cd soci
tar zvxf ../soci-snapshotter-0.5.0-linux-amd64-static.tar.gz
rm soci-snapshotter-0.5.0-linux-amd64-static.tar.gz
mv soci-snapshotter-grpc /usr/bin/soci-snapshotter-grpc
cd ..
rm -rf soci
wget -q $root/soci-config.toml -O /etc/containerd/soci-config.toml
wget -q $root/soci-snapshotter.service -O /etc/systemd/system/soci-snapshotter.service


systemctl daemon-reload
systemctl enable --now soci-snapshotter

wget -q $root/containerd_soci.toml
cat containerd_soci.toml >> /etc/eks/containerd/containerd-config.toml

yum install -y amazon-ecr-credential-helper fuse

mkdir -p /root/.docker
echo '{ "credsStore":"ecr-login" }' > /root/.docker/config.json

soci-config.toml

# Compare with https://github.com/awslabs/soci-snapshotter/blob/main/config/config.toml
# An example config showing all of the toml variables used.

# Copy to /etc/soci-snapshotter-grpc/config.toml
# to use on your system.

# NOTE: Many variables set to zero are just an indicator
# to use the built-in default. These values may change over time,
# which is why the config uses zeroes.
# Please see docs/config.md for more info on what these variables do.

# TODO: Can we put these inside dedicated TOML vars to refer to them?

# config/fs.go FSConfig
http_cache_type=""
filesystem_cache_type=""
resolve_result_entry=0
debug=false
allow_no_verification=true
# disable_verification=false
# Causes TestRunWithDefaultConfig to break, but
# fine to use in /etc/soci-snapshotter-grpc-config.toml
max_concurrency=0  # Actually zero
no_prometheus=false
mount_timeout_sec=0
fuse_metrics_emit_wait_duration_sec=0

## config/config.go Config

metrics_address=""
metrics_network="" # Uses default metrics network
# no_prometheus=true # Defined above, can't be redeclared
debug_address=""
metadata_store="db"

[http]
MaxRetries=0
MinWaitMsec=0
MaxWaitMsec=0
DialTimeoutMsec=0
ResponseHeaderTimeoutMsec=0
RequestTimeoutMsec=0

#
## config/fs.go
#

[blob]
valid_interval=0
check_always=false
fetching_timeout_sec=0
force_single_range_mode=false
# max_retries=0 # Set by http.
# min_wait_msec=0 # Set by http.
# max_wait_msec=0 # Set by http.
max_span_verification_retries=0 # Actually zero

[directory_cache]
max_lru_cache_entry=0 # Actually zero
max_cache_fds=0 # Actually zero
sync_add=false
direct=true

[fuse]
attr_timeout=0
entry_timeout=0
negative_timeout=0
log_fuse_operations=false

[background_fetch]
disable=false
silence_period_msec=0
fetch_period_msec=0
max_queue_size=0
emit_metric_period_sec=0

[content_store]
type="containerd" # will set to 'soci' by default
# Socket address for containerd. Only applicable using containerd content store.
# Defaults to '/run/containerd/containerd.sock'
containerd_address=""
namespace="k8s.io" # will set to 'default' by default

#
## config/resolver.go
#

[resolver]
  [resolver.host]

#
## config/service.go
#

[kubeconfig_keychain]
enable_keychain=false
kubeconfig_path=""

[cri_keychain]
enable_keychain=false
image_service_path="" # Uses default image service address

[snapshotter]
min_layer_size=0 # Actually zero
allow_invalid_mounts_on_restart=false

#
## service/resolver/cri.go
#

[registry]
config_path=""
mirrors={}
configs={}

[Mirror]
endpoint={}

[RegistryConfig]

[auth]
username=""
password=""
auth=""
identitytoken=""

[tls]
insecure_skip_verify=false
ca_file=""
cert_file=""
key_file=""

soci-snapshotter.service

# Copyright The Soci Snapshotter Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Compare with https://github.com/awslabs/soci-snapshotter/blob/main/soci-snapshotter.service
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=soci snapshotter containerd plugin
Documentation=https://github.com/awslabs/soci-snapshotter
After=network.target containerd.service

[Service]
Type=notify
ExecStart=/usr/bin/soci-snapshotter-grpc --log-level trace --config /etc/containerd/soci-config.toml
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target