Created
August 29, 2019 09:52
-
-
Save FlorianOtel/8fe21bc3efe737efeb074594f5aa12a0 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
### Current content of /etc/cni/net.d" | |
root@ip-172-16-1-8:~# ls -al /etc/cni/net.d/ | |
total 20 | |
drwxr-xr-x 2 root root 4096 Aug 29 08:10 ./ | |
drwxr-xr-x 3 root root 4096 Aug 23 08:08 ../ | |
-rw-r--r-- 1 root root 260 Aug 29 08:10 10-aws.conflist | |
-rw-r--r-- 1 root root 294 Mar 13 13:25 100-crio-bridge.conf | |
-rw-r--r-- 1 root root 54 Mar 13 13:25 200-loopback.conf | |
### The content of the "10-aws.conflist" plugin file | |
root@ip-172-16-1-8:~# cat /etc/cni/net.d/10-aws.conflist | |
{ | |
"cniVersion": "0.3.0", | |
"name": "aws-cni", | |
"plugins": [ | |
{ | |
"name": "aws-cni", | |
"type": "aws-cni", | |
"vethPrefix": "eni" | |
}, | |
{ | |
"type": "portmap", | |
"capabilities": {"portMappings": true}, | |
"snat": true | |
} | |
] | |
} | |
######## | |
######## /etc/crio/crio.conf -- the only thing changed I changed from stock file is the addition of image registries | |
######## | |
root@ip-172-16-1-8:~# cat /etc/crio/crio.conf | |
# The CRI-O configuration file specifies all of the available configuration | |
# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime | |
# daemon, but in a TOML format that can be more easily modified and versioned. | |
# | |
# Please refer to crio.conf(5) for details of all configuration options. | |
# CRI-O reads its storage defaults from the containers-storage.conf(5) file | |
# located at /etc/containers/storage.conf. Modify this storage configuration if | |
# you want to change the system's defaults. If you want to modify storage just | |
# for CRI-O, you can change the storage configuration options here. | |
[crio] | |
# Path to the "root directory". CRI-O stores all of its data, including | |
# containers images, in this directory. | |
#root = "/home/lsm5/.local/share/containers/storage" | |
# Path to the "run directory". CRI-O stores all of its state in this directory. | |
#runroot = "/tmp/1000" | |
# Storage driver used to manage the storage of images and containers. Please | |
# refer to containers-storage.conf(5) to see all available storage drivers. | |
#storage_driver = "vfs" | |
# List to pass options to the storage driver. Please refer to | |
# containers-storage.conf(5) to see all available storage options. | |
#storage_option = [ | |
#] | |
# If set to false, in-memory locking will be used instead of file-based locking. | |
file_locking = false | |
# Path to the lock file. | |
file_locking_path = "/run/crio.lock" | |
# The crio.api table contains settings for the kubelet/gRPC interface. | |
[crio.api] | |
# Path to AF_LOCAL socket on which CRI-O will listen. | |
listen = "/var/run/crio/crio.sock" | |
# IP address on which the stream server will listen. | |
stream_address = "127.0.0.1" | |
# The port on which the stream server will listen. | |
stream_port = "0" | |
# Enable encrypted TLS transport of the stream server. | |
stream_enable_tls = false | |
# Path to the x509 certificate file used to serve the encrypted stream. This | |
# file can change, and CRI-O will automatically pick up the changes within 5 | |
# minutes. | |
stream_tls_cert = "" | |
# Path to the key file used to serve the encrypted stream. This file can | |
# change, and CRI-O will automatically pick up the changes within 5 minutes. | |
stream_tls_key = "" | |
# Path to the x509 CA(s) file used to verify and authenticate client | |
# communication with the encrypted stream. This file can change, and CRI-O will | |
# automatically pick up the changes within 5 minutes. | |
stream_tls_ca = "" | |
# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. | |
grpc_max_send_msg_size = 16777216 | |
# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. | |
grpc_max_recv_msg_size = 16777216 | |
# The crio.runtime table contains settings pertaining to the OCI runtime used | |
# and options for how to set up and manage the OCI runtime. | |
[crio.runtime] | |
# A list of ulimits to be set in containers by default, specified as | |
# "<ulimit name>=<soft limit>:<hard limit>", for example: | |
# "nofile=1024:2048" | |
# If nothing is set here, settings will be inherited from the CRI-O daemon | |
#default_ulimits = [ | |
#] | |
# default_runtime is the _name_ of the OCI runtime to be used as the default. | |
# The name is matched against the runtimes map below. | |
default_runtime = "runc" | |
# If true, the runtime will not use pivot_root, but instead use MS_MOVE. | |
no_pivot = false | |
# Path to the conmon binary, used for monitoring the OCI runtime. | |
conmon = "/usr/libexec/crio/conmon" | |
# Environment variable list for the conmon process, used for passing necessary | |
# environment variables to conmon or the runtime. | |
conmon_env = [ | |
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", | |
] | |
# If true, SELinux will be used for pod separation on the host. | |
selinux = false | |
# Path to the seccomp.json profile which is used as the default seccomp profile | |
# for the runtime. | |
seccomp_profile = "/usr/share/containers/seccomp.json" | |
# Used to change the name of the default AppArmor profile of CRI-O. The default | |
# profile name is "crio-default-" followed by the version string of CRI-O. | |
apparmor_profile = "crio-default" | |
# Cgroup management implementation used for the runtime. | |
cgroup_manager = "cgroupfs" | |
# List of default capabilities for containers. If it is empty or commented out, | |
# only the capabilities defined in the containers json file by the user/kube | |
# will be added. | |
default_capabilities = [ | |
"CHOWN", | |
"DAC_OVERRIDE", | |
"FSETID", | |
"FOWNER", | |
"NET_RAW", | |
"SETGID", | |
"SETUID", | |
"SETPCAP", | |
"NET_BIND_SERVICE", | |
"SYS_CHROOT", | |
"KILL", | |
] | |
# List of default sysctls. If it is empty or commented out, only the sysctls | |
# defined in the container json file by the user/kube will be added. | |
default_sysctls = [ | |
] | |
# List of additional devices. specified as | |
# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm". | |
#If it is empty or commented out, only the devices | |
# defined in the container json file by the user/kube will be added. | |
additional_devices = [ | |
] | |
# Path to OCI hooks directories for automatically executed hooks. | |
hooks_dir = [ | |
] | |
# List of default mounts for each container. **Deprecated:** this option will | |
# be removed in future versions in favor of default_mounts_file. | |
default_mounts = [ | |
] | |
# Path to the file specifying the defaults mounts for each container. The | |
# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads | |
# its default mounts from the following two files: | |
# | |
# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the | |
# override file, where users can either add in their own default mounts, or | |
# override the default mounts shipped with the package. | |
# | |
# 2) /usr/share/containers/mounts.conf: This is the default file read for | |
# mounts. If you want CRI-O to read from a different, specific mounts file, | |
# you can change the default_mounts_file. Note, if this is done, CRI-O will | |
# only add mounts it finds in this file. | |
# | |
#default_mounts_file = "" | |
# Maximum number of processes allowed in a container. | |
pids_limit = 1024 | |
# Maximum sized allowed for the container log file. Negative numbers indicate | |
# that no size limit is imposed. If it is positive, it must be >= 8192 to | |
# match/exceed conmon's read buffer. The file is truncated and re-opened so the | |
# limit is never exceeded. | |
log_size_max = -1 | |
# Whether container output should be logged to journald in addition to the kuberentes log file | |
log_to_journald = false | |
# Path to directory in which container exit files are written to by conmon. | |
container_exits_dir = "/var/run/crio/exits" | |
# Path to directory for container attach sockets. | |
container_attach_socket_dir = "/var/run/crio" | |
# If set to true, all containers will run in read-only mode. | |
read_only = false | |
# Changes the verbosity of the logs based on the level it is set to. Options | |
# are fatal, panic, error, warn, info, and debug. | |
log_level = "error" | |
# The UID mappings for the user namespace of each container. A range is | |
# specified in the form containerUID:HostUID:Size. Multiple ranges must be | |
# separated by comma. | |
uid_mappings = "" | |
# The GID mappings for the user namespace of each container. A range is | |
# specified in the form containerGID:HostGID:Size. Multiple ranges must be | |
# separated by comma. | |
gid_mappings = "" | |
# The minimal amount of time in seconds to wait before issuing a timeout | |
# regarding the proper termination of the container. | |
ctr_stop_timeout = 0 | |
# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. | |
# The runtime to use is picked based on the runtime_handler provided by the CRI. | |
# If no runtime_handler is provided, the runtime will be picked based on the level | |
# of trust of the workload. | |
[crio.runtime.runtimes.runc] | |
runtime_path = "/usr/lib/cri-o-runc/sbin/runc" | |
runtime_type = "" | |
# The crio.image table contains settings pertaining to the management of OCI images. | |
# | |
# CRI-O reads its configured registries defaults from the system wide | |
# containers-registries.conf(5) located in /etc/containers/registries.conf. If | |
# you want to modify just CRI-O, you can change the registries configuration in | |
# this file. Otherwise, leave insecure_registries and registries commented out to | |
# use the system's defaults from /etc/containers/registries.conf. | |
[crio.image] | |
# Default transport for pulling images from a remote container storage. | |
default_transport = "docker://" | |
# The path to a file containing credentials necessary for pulling images from | |
# secure registries. The file is similar to that of /var/lib/kubelet/config.json | |
global_auth_file = "" | |
# The image used to instantiate infra containers. | |
pause_image = "k8s.gcr.io/pause:3.1" | |
# The path to a file containing credentials specific for pulling the pause_image from | |
# above. The file is similar to that of /var/lib/kubelet/config.json | |
pause_image_auth_file = "" | |
# The command to run to have a container stay in the paused state. | |
pause_command = "/pause" | |
# Path to the file which decides what sort of policy we use when deciding | |
# whether or not to trust an image that we've pulled. It is not recommended that | |
# this option be used, as the default behavior of using the system-wide default | |
# policy (i.e., /etc/containers/policy.json) is most often preferred. Please | |
# refer to containers-policy.json(5) for more details. | |
signature_policy = "" | |
# Controls how image volumes are handled. The valid values are mkdir, bind and | |
# ignore; the latter will ignore volumes entirely. | |
image_volumes = "mkdir" | |
# List of registries to be used when pulling an unqualified image (e.g., | |
# "alpine:latest"). By default, registries is set to "docker.io" for | |
# compatibility reasons. Depending on your workload and usecase you may add more | |
# registries (e.g., "quay.io", "registry.fedoraproject.org", | |
# "registry.opensuse.org", etc.). | |
registries = [ | |
"k8s.gcr.io", | |
"quay.io", | |
"docker.io", | |
] | |
# The crio.network table containers settings pertaining to the management of | |
# CNI plugins. | |
[crio.network] | |
# Path to the directory where CNI configuration files are located. | |
network_dir = "/etc/cni/net.d/" | |
# Paths to directories where CNI plugin binaries are located. | |
plugin_dirs = [ | |
"/opt/cni/bin", | |
] | |
######## | |
######## The offending lines from crio log (starting with "--log-level=debug") | |
######## | |
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.421489160Z" level=warning msg="CNI monitoring event \"/etc/cni/net.d/.#10-aws.conflist\": CREATE" | |
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.421898015Z" level=warning msg="Error loading CNI config list file /etc/cni/net.d/.#10-aws.conflist: error reading /etc/cni/net.d/.#10-aws.conflist: open /etc/cni/net.d/.#10-aws.conflist: no such file or directory" | |
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.422250556Z" level=info msg="Found CNI network aws-cni (type=aws-cni) at /etc/cni/net.d/10-aws.conflist" | |
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.422549900Z" level=info msg="Found CNI network crio-bridge (type=bridge) at /etc/cni/net.d/100-crio-bridge.conf" | |
Aug 29 08:10:34 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:10:34.422840290Z" level=info msg="Found CNI network 200-loopback.conf (type=loopback) at /etc/cni/net.d/200-loopback.conf" | |
##### Subsequently rio falls back to "crio-bridge" when creating a test pod... | |
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.056824390Z" level=debug msg="Received container pid: 4243" | |
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.070517295Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:e27c2d46c32ab35274486e6903b4c8b430c8a2f06f3d742c6d02c1d22533b179 NetNS:/proc/4243/ns/net Networks:[] RuntimeConfig:map[crio-bridge:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}" | |
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.070551314Z" level=info msg="About to add CNI network cni-loopback (type=loopback)" | |
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.088052838Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:e27c2d46c32ab35274486e6903b4c8b430c8a2f06f3d742c6d02c1d22533b179 NetNS:/proc/4243/ns/net Networks:[] RuntimeConfig:map[crio-bridge:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}" | |
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.088085267Z" level=info msg="About to add CNI network crio-bridge (type=bridge)" | |
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.196308092Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:e27c2d46c32ab35274486e6903b4c8b430c8a2f06f3d742c6d02c1d22533b179 NetNS:/proc/4243/ns/net Networks:[] RuntimeConfig:map[crio-bridge:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}" | |
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.196346198Z" level=info msg="About to check CNI network crio-bridge (type=bridge)" | |
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.196563836Z" level=debug msg="CNI setup result: Interfaces:[{Name:cni0 Mac:4e:22:8e:af:f4:11 Sandbox:} {Name:vethb7cce2db Mac:d2:22:00:17:99:1e Sandbox:} {Name:eth0 Mac:1a:a0:fd:2c:34:b3 Sandbox:/proc/4243/ns/net}], IP:[{Version:4 Interface:0xc420791628 Address:{IP:10.88.0.3 Mask:ffff0000} Gateway:10.88.0.1}], Routes:[{Dst:{IP:0.0.0.0 Mask:00000000} GW:<nil>}], DNS:{Nameservers:[] Domain: Search:[] Options:[]}" | |
Aug 29 08:11:54 ip-172-16-1-8.eu-north-1.compute.internal crio[800]: time="2019-08-29 08:11:54.196597201Z" level=info msg="Ran pod sandbox with infra container: default/busybox-debug/POD" | |
Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.923872142Z" level=info msg="Got pod network &{Name:busybox-debug-7 Namespace:default ID:50d32ef3
497fbca24ac9975f8d13ddc29407d96dce37555702f8bb9cfb5f8222 NetNS:/proc/29780/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}"
Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.923904790Z" level=info msg="About to add CNI network aws-cni (type=aws-cni)"
Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.958913163Z" level=info msg="Got pod network &{Name:busybox-debug-7 Namespace:default ID:50d32ef3
497fbca24ac9975f8d13ddc29407d96dce37555702f8bb9cfb5f8222 NetNS:/proc/29780/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth:<nil> IpRanges:[]}]}"
Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.958950000Z" level=info msg="About to check CNI network aws-cni (type=aws-cni)"
Aug 30 13:22:57 ip-172-16-1-240.eu-north-1.compute.internal crio[28916]: time="2019-08-30 13:22:57.958964057Z" level=error msg="Error while checking pod to CNI network \"aws-cni\": invalid version \"\": the version is empty"
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
When removing the default "crio-bridge" plugin and starting to create a a pod,
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.941459552Z" level=debug msg="Received container pid: 13187"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.954727034Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:16ac1045cf9f4d6f98186aa50b3883312ba9
6f6c0fb4c7a85213c138522be775 NetNS:/proc/13187/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth: IpRanges:[]}]}"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.954759313Z" level=info msg="About to add CNI network cni-loopback (type=loopback)"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.956418835Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:16ac1045cf9f4d6f98186aa50b3883312ba9
6f6c0fb4c7a85213c138522be775 NetNS:/proc/13187/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth: IpRanges:[]}]}"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.956445324Z" level=info msg="About to add CNI network aws-cni (type=aws-cni)"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960636226Z" level=error msg="Error adding network: add cmd: failed to assign an IP address to container"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960660744Z" level=error msg="Error while adding pod to CNI network "aws-cni": add cmd: failed to assign an IP address to
container"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960697415Z" level=info msg="Got pod network &{Name:busybox-debug Namespace:default ID:16ac1045cf9f4d6f98186aa50b3883312ba9
6f6c0fb4c7a85213c138522be775 NetNS:/proc/13187/ns/net Networks:[] RuntimeConfig:map[aws-cni:{IP: PortMappings:[] Bandwidth: IpRanges:[]}]}"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960712458Z" level=info msg="About to del CNI network aws-cni (type=aws-cni)"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960722232Z" level=error msg="Error deleting network: invalid version "": the version is empty"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960731431Z" level=error msg="Error while removing pod from CNI network "aws-cni": invalid version "": the version is empty"
Aug 29 10:09:58 ip-172-16-1-8.eu-north-1.compute.internal crio[10364]: time="2019-08-29 10:09:58.960741808Z" level=warning msg="failed to destroy network for pod sandbox k8s_busybox-debug_default_5c1c0f42-7cb4-4998-8966-810dcc4fee0a_0(16ac1045cf9f4d6f98186aa50b3883312ba96f6c0fb4c7a85213c138522be775): invalid version "": the version is empty"