Skip to content

Instantly share code, notes, and snippets.

@FlorianOtel
Created August 29, 2019 09:52
Show Gist options
  • Save FlorianOtel/8fe21bc3efe737efeb074594f5aa12a0 to your computer and use it in GitHub Desktop.
Save FlorianOtel/8fe21bc3efe737efeb074594f5aa12a0 to your computer and use it in GitHub Desktop.
### Current content of /etc/cni/net.d"
root@ip-172-16-1-8:~# ls -al /etc/cni/net.d/
total 20
drwxr-xr-x 2 root root 4096 Aug 29 08:10 ./
drwxr-xr-x 3 root root 4096 Aug 23 08:08 ../
-rw-r--r-- 1 root root 260 Aug 29 08:10 10-aws.conflist
-rw-r--r-- 1 root root 294 Mar 13 13:25 100-crio-bridge.conf
-rw-r--r-- 1 root root 54 Mar 13 13:25 200-loopback.conf
### The content of the "10-aws.conflist" plugin file
root@ip-172-16-1-8:~# cat /etc/cni/net.d/10-aws.conflist
{
"cniVersion": "0.3.0",
"name": "aws-cni",
"plugins": [
{
"name": "aws-cni",
"type": "aws-cni",
"vethPrefix": "eni"
},
{
"type": "portmap",
"capabilities": {"portMappings": true},
"snat": true
}
]
}
########
######## /etc/crio/crio.conf -- the only thing changed I changed from stock file is the addition of image registries
########
root@ip-172-16-1-8:~# cat /etc/crio/crio.conf
# The CRI-O configuration file specifies all of the available configuration
# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
# daemon, but in a TOML format that can be more easily modified and versioned.
#
# Please refer to crio.conf(5) for details of all configuration options.
# CRI-O reads its storage defaults from the containers-storage.conf(5) file
# located at /etc/containers/storage.conf. Modify this storage configuration if
# you want to change the system's defaults. If you want to modify storage just
# for CRI-O, you can change the storage configuration options here.
[crio]
# Path to the "root directory". CRI-O stores all of its data, including
# containers images, in this directory.
#root = "/home/lsm5/.local/share/containers/storage"
# Path to the "run directory". CRI-O stores all of its state in this directory.
#runroot = "/tmp/1000"
# Storage driver used to manage the storage of images and containers. Please
# refer to containers-storage.conf(5) to see all available storage drivers.
#storage_driver = "vfs"
# List to pass options to the storage driver. Please refer to
# containers-storage.conf(5) to see all available storage options.
#storage_option = [
#]
# If set to false, in-memory locking will be used instead of file-based locking.
file_locking = false
# Path to the lock file.
file_locking_path = "/run/crio.lock"
# The crio.api table contains settings for the kubelet/gRPC interface.
[crio.api]
# Path to AF_LOCAL socket on which CRI-O will listen.
listen = "/var/run/crio/crio.sock"
# IP address on which the stream server will listen.
stream_address = "127.0.0.1"
# The port on which the stream server will listen.
stream_port = "0"
# Enable encrypted TLS transport of the stream server.
stream_enable_tls = false
# Path to the x509 certificate file used to serve the encrypted stream. This
# file can change, and CRI-O will automatically pick up the changes within 5
# minutes.
stream_tls_cert = ""
# Path to the key file used to serve the encrypted stream. This file can
# change, and CRI-O will automatically pick up the changes within 5 minutes.
stream_tls_key = ""
# Path to the x509 CA(s) file used to verify and authenticate client
# communication with the encrypted stream. This file can change, and CRI-O will
# automatically pick up the changes within 5 minutes.
stream_tls_ca = ""
# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
grpc_max_send_msg_size = 16777216
# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
grpc_max_recv_msg_size = 16777216
# The crio.runtime table contains settings pertaining to the OCI runtime used
# and options for how to set up and manage the OCI runtime.
[crio.runtime]
# A list of ulimits to be set in containers by default, specified as
# "<ulimit name>=<soft limit>:<hard limit>", for example:
# "nofile=1024:2048"
# If nothing is set here, settings will be inherited from the CRI-O daemon
#default_ulimits = [
#]
# default_runtime is the _name_ of the OCI runtime to be used as the default.
# The name is matched against the runtimes map below.
default_runtime = "runc"
# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
no_pivot = false
# Path to the conmon binary, used for monitoring the OCI runtime.
conmon = "/usr/libexec/crio/conmon"
# Environment variable list for the conmon process, used for passing necessary
# environment variables to conmon or the runtime.
conmon_env = [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
]
# If true, SELinux will be used for pod separation on the host.
selinux = false
# Path to the seccomp.json profile which is used as the default seccomp profile
# for the runtime.
seccomp_profile = "/usr/share/containers/seccomp.json"
# Used to change the name of the default AppArmor profile of CRI-O. The default
# profile name is "crio-default-" followed by the version string of CRI-O.
apparmor_profile = "crio-default"
# Cgroup management implementation used for the runtime.
cgroup_manager = "cgroupfs"
# List of default capabilities for containers. If it is empty or commented out,
# only the capabilities defined in the containers json file by the user/kube
# will be added.
default_capabilities = [
"CHOWN",
"DAC_OVERRIDE",
"FSETID",
"FOWNER",
"NET_RAW",
"SETGID",
"SETUID",
"SETPCAP",
"NET_BIND_SERVICE",
"SYS_CHROOT",
"KILL",
]
# List of default sysctls. If it is empty or commented out, only the sysctls
# defined in the container json file by the user/kube will be added.
default_sysctls = [
]
# List of additional devices. specified as
# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
#If it is empty or commented out, only the devices
# defined in the container json file by the user/kube will be added.
additional_devices = [
]
# Path to OCI hooks directories for automatically executed hooks.
hooks_dir = [
]
# List of default mounts for each container. **Deprecated:** this option will
# be removed in future versions in favor of default_mounts_file.
default_mounts = [
]
# Path to the file specifying the defaults mounts for each container. The
# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
# its default mounts from the following two files:
#
# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
# override file, where users can either add in their own default mounts, or
# override the default mounts shipped with the package.
#
# 2) /usr/share/containers/mounts.conf: This is the default file read for
# mounts. If you want CRI-O to read from a different, specific mounts file,
# you can change the default_mounts_file. Note, if this is done, CRI-O will
# only add mounts it finds in this file.
#
#default_mounts_file = ""
# Maximum number of processes allowed in a container.
pids_limit = 1024
# Maximum sized allowed for the container log file. Negative numbers indicate
# that no size limit is imposed. If it is positive, it must be >= 8192 to
# match/exceed conmon's read buffer. The file is truncated and re-opened so the
# limit is never exceeded.
log_size_max = -1
# Whether container output should be logged to journald in addition to the kuberentes log file
log_to_journald = false
# Path to directory in which container exit files are written to by conmon.
container_exits_dir = "/var/run/crio/exits"
# Path to directory for container attach sockets.
container_attach_socket_dir = "/var/run/crio"
# If set to true, all containers will run in read-only mode.
read_only = false
# Changes the verbosity of the logs based on the level it is set to. Options
# are fatal, panic, error, warn, info, and debug.
log_level = "error"
# The UID mappings for the user namespace of each container. A range is
# specified in the form containerUID:HostUID:Size. Multiple ranges must be
# separated by comma.
uid_mappings = ""
# The GID mappings for the user namespace of each container. A range is
# specified in the form containerGID:HostGID:Size. Multiple ranges must be
# separated by comma.
gid_mappings = ""
# The minimal amount of time in seconds to wait before issuing a timeout
# regarding the proper termination of the container.
ctr_stop_timeout = 0
# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
# The runtime to use is picked based on the runtime_handler provided by the CRI.
# If no runtime_handler is provided, the runtime will be picked based on the level
# of trust of the workload.
[crio.runtime.runtimes.runc]
runtime_path = "/usr/lib/cri-o-runc/sbin/runc"
runtime_type = ""
# The crio.image table contains settings pertaining to the management of OCI images.
#
# CRI-O reads its configured registries defaults from the system wide
# containers-registries.conf(5) located in /etc/containers/registries.conf. If
# you want to modify just CRI-O, you can change the registries configuration in
# this file. Otherwise, leave insecure_registries and registries commented out to
# use the system's defaults from /etc/containers/registries.conf.
[crio.image]
# Default transport for pulling images from a remote container storage.
default_transport = "docker://"
# The path to a file containing credentials necessary for pulling images from
# secure registries. The file is similar to that of /var/lib/kubelet/config.json
global_auth_file = ""
# The image used to instantiate infra containers.
pause_image = "k8s.gcr.io/pause:3.1"
# The path to a file containing credentials specific for pulling the pause_image from
# above. The file is similar to that of /var/lib/kubelet/config.json
pause_image_auth_file = ""
# The command to run to have a container stay in the paused state.
pause_command = "/pause"
# Path to the file which decides what sort of policy we use when deciding
# whether or not to trust an image that we've pulled. It is not recommended that
# this option be used, as the default behavior of using the system-wide default
# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
# refer to containers-policy.json(5) for more details.
signature_policy = ""
# Controls how image volumes are handled. The valid values are mkdir, bind and
# ignore; the latter will ignore volumes entirely.
image_volumes = "mkdir"
# List of registries to be used when pulling an unqualified image (e.g.,
# "alpine:latest"). By default, registries is set to "docker.io" for
# compatibility reasons. Depending on your workload and usecase you may add more
# registries (e.g., "quay.io", "registry.fedoraproject.org",
# "registry.opensuse.org", etc.).
registries = [
"k8s.gcr.io",
"quay.io",
"docker.io",
]
# The crio.network table containers settings pertaining to the management of
# CNI plugins.
[crio.network]
# Path to the directory where CNI configuration files are located.
network_dir = "/etc/cni/net.d/"
# Paths to directories where CNI plugin binaries are located.
plugin_dirs = [
"/opt/cni/bin",
]
########
######## The offending lines from crio log (starting with "--log-level=debug")
########
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.421489160Z" level=warning msg="CNI monitoring event \"/etc/cni/net.d/.#10-aws.conflist\": CREATE"
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.421898015Z" level=warning msg="Error loading CNI config list file /etc/cni/net.d/.#10-aws.conflist: error reading /etc/cni/net.d/.#10-aws.conflist: open /etc/cni/net.d/.#10-aws.conflist: no such file or directory"
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.422250556Z" level=info msg="Found CNI network aws-cni (type=aws-cni) at /etc/cni/net.d/10-aws.conflist"
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.422549900Z" level=info msg="Found CNI network crio-bridge (type=bridge) at /etc/cni/net.d/100-crio-bridge.conf"
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.422840290Z" level=info msg="Found CNI network 200-loopback.conf (type=loopback) at /etc/cni/net.d/200-loopback.conf"
##### Subsequently rio falls back to "crio-bridge" when creating a test pod...
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.056824390Z" level=debug msg="Received container pid: 4243"
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.070517295Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:e27c2d46c32ab35274486e6903b4c8b430c8a2f06f3d742c6d02c1d22533b179 NetNS:/proc/4243/ns/net Networks:[] RuntimeConfig:map[crio-bridge:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}"
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.070551314Z" level=info msg="About to add CNI network cni-loopback (type=loopback)"
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.088052838Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:e27c2d46c32ab35274486e6903b4c8b430c8a2f06f3d742c6d02c1d22533b179 NetNS:/proc/4243/ns/net Networks:[] RuntimeConfig:map[crio-bridge:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}"
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.088085267Z" level=info msg="About to add CNI network crio-bridge (type=bridge)"
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.196308092Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:e27c2d46c32ab35274486e6903b4c8b430c8a2f06f3d742c6d02c1d22533b179 NetNS:/proc/4243/ns/net Networks:[] RuntimeConfig:map[crio-bridge:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}"
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.196346198Z" level=info msg="About to check CNI network crio-bridge (type=bridge)"
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.196563836Z" level=debug msg="CNI setup result: Interfaces:[{Name:cni0 Mac:4e:22:8e:af:f4:11 Sandbox:} {Name:vethb7cce2db Mac:d2:22:00:17:99:1e Sandbox:} {Name:eth0 Mac:1a:a0:fd:2c:34:b3 Sandbox:/proc/4243/ns/net}], IP:[{Version:4 Interface:0xc420791628 Address:{IP:10.88.0.3 Mask:ffff0000} Gateway:10.88.0.1}], Routes:[{Dst:{IP:0.0.0.0 Mask:00000000} GW:<nil>}], DNS:{Nameservers:[] Domain: Search:[] Options:[]}"
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.196597201Z" level=info msg="Ran pod sandbox with infra container: default/busybox-debug/POD"
@FlorianOtel
Copy link
Author

When removing the default "crio-bridge" plugin and starting to create a a pod,

Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.941459552Z" level=debug msg="Received container pid: 13187"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.954727034Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:16ac1045cf9f4d6f98186aa50b3883312ba9
6f6c0fb4c7a85213c138522be775 NetNS:/proc/13187/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth: IpRanges:[]}]}"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.954759313Z" level=info msg="About to add CNI network cni-loopback (type=loopback)"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.956418835Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:16ac1045cf9f4d6f98186aa50b3883312ba9
6f6c0fb4c7a85213c138522be775 NetNS:/proc/13187/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth: IpRanges:[]}]}"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.956445324Z" level=info msg="About to add CNI network aws-cni (type=aws-cni)"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960636226Z" level=error msg="Error adding network: add cmd: failed to assign an IP address to container"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960660744Z" level=error msg="Error while adding pod to CNI network "aws-cni": add cmd: failed to assign an IP address to
container"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960697415Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:16ac1045cf9f4d6f98186aa50b3883312ba9
6f6c0fb4c7a85213c138522be775 NetNS:/proc/13187/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth: IpRanges:[]}]}"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960712458Z" level=info msg="About to del CNI network aws-cni (type=aws-cni)"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960722232Z" level=error msg="Error deleting network: invalid version "": the version is empty"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960731431Z" level=error msg="Error while removing pod from CNI network "aws-cni": invalid version "": the version is empty"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960741808Z" level=warning msg="failed to destroy network for pod sandbox k8s_busybox-debug_default_5c1c0f42-7cb4-4998-8966-810dcc4fee0a_0(16ac1045cf9f4d6f98186aa50b3883312ba96f6c0fb4c7a85213c138522be775): invalid version "": the version is empty"

@FlorianOtel
Copy link
Author

Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.923872142Z" level=info msg="Got pod network &{Name:busybox-debug-7 Namespace:default ID:50d32ef3
497fbca24ac9975f8d13ddc29407d96dce37555702f8bb9cfb5f8222 NetNS:/proc/29780/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}"
Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.923904790Z" level=info msg="About to add CNI network aws-cni (type=aws-cni)"
Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.958913163Z" level=info msg="Got pod network &{Name:busybox-debug-7 Namespace:default ID:50d32ef3
497fbca24ac9975f8d13ddc29407d96dce37555702f8bb9cfb5f8222 NetNS:/proc/29780/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}"
Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.958950000Z" level=info msg="About to check CNI network aws-cni (type=aws-cni)"
Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.958964057Z" level=error msg="Error while checking pod to CNI network \"aws-cni\": invalid version \"\": the version is empty"

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment