OpenSearch signal handling MVCE

README.md

OpenSearch container termination signal handling

Minimal, reproducible example demonstrating broken signal handling for OpenSearch.

Steps

1. Build and run the vanilla (default entrypoint) and fixed container.

   make run-vanilla
   make run-fixed
   

2. Both containers should be up, check the container IDs with docker ps.

   > docker ps
   CONTAINER ID   IMAGE          COMMAND                  CREATED              STATUS              PORTS                                    NAMES
   536d57b3aa00   724651c0865b   "/tini -- ./opensear…"   About a minute ago   Up About a minute   9200/tcp, 9300/tcp, 9600/tcp, 9650/tcp   eager_spence
   8a4f3e9dc88a   a0cd42027469   "./opensearch-docker…"   About a minute ago   Up About a minute   9200/tcp, 9300/tcp, 9600/tcp, 9650/tcp   heuristic_turing
   

3. Send a SIGTERM signal to either container and observe different behaviour.

   # Send a signal to the vanilla container, it will not terminate.
   make send-sigterm CONTAINER_ID=8a4f
   # Send a signal to the fixed container, it will terminate.
   make send-sigterm CONTAINER_ID=536
   

4. Observe the process trees in both containers (notice bash as PID 1 in the vanilla container).

   Vanilla

   [opensearch@8a4f3e9dc88a ~]$ ps -ejf
   UID        PID  PPID  PGID   SID  C STIME TTY          TIME CMD
   opensea+     1     0     1     1  0 16:33 pts/0    00:00:00 /bin/bash ./opensearch-docker-entrypoint.sh opensearch
   opensea+    33     1     1     1  2 16:33 pts/0    00:00:08 /usr/share/opensearch/jdk/bin/java -Xshare:auto -Xms4m -Xmx64m -XX:+UseSerialGC -Dlog4j.configurationFile=/usr/share/opensearch/config/opensearch-performance-analyzer/log4j2.xml -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation opensea+    34     1     1     1  6 16:33 pts/0    00:00:26 /usr/share/opensearch/jdk/bin/java -Xshare:auto -Dopensearch.networkaddress.cache.ttl=60 -Dopensearch.networkaddress.cache.negative.ttl=10 -XX:+AlwaysPreTouch -Xss1m -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -XX:-OmitStackTraceInFastTopensea+   391     0   391   391  0 16:39 pts/1    00:00:00 /bin/sh -c eval $(grep ^$(id -un): /etc/passwd | cut -d : -f 7-)
   opensea+   402   391   402   391  0 16:39 pts/1    00:00:00 /bin/bash
   opensea+   416   402   416   391  0 16:39 pts/1    00:00:00 ps -ejf
   

   Fixed

   [opensearch@191e42e045f7 ~]$ ps -ejf
   UID        PID  PPID  PGID   SID  C STIME TTY          TIME CMD
   opensea+     1     0     1     1  0 16:38 pts/0    00:00:00 /tini -- ./opensearch-docker-entrypoint.sh opensearch
   opensea+     7     1     7     1 99 16:38 pts/0    00:00:26 /usr/share/opensearch/jdk/bin/java -Xshare:auto -Dopensearch.networkaddress.cache.ttl=60 -Dopensearch.networkaddress.cache.negative.ttl=10 -XX:+AlwaysPreTouch -Xss1m -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -XX:-OmitStackTraceInFastT
   opensea+   269     0   269   269  0 16:38 pts/1    00:00:00 /bin/sh -c eval $(grep ^$(id -un): /etc/passwd | cut -d : -f 7-)
   opensea+   280   269   280   269  0 16:38 pts/1    00:00:00 /bin/bash
   opensea+   294   280   294   269  0 16:38 pts/1    00:00:00 ps -ejf
   

Dockerfile.fixed

FROM opensearchproject/opensearch:2.5.0

# Add a basic config to run OpenSearch container as a single-node cluster.
COPY opensearch.yml /usr/share/opensearch/config/opensearch.yml

USER root

# Install tools to be able to inspect the process tree more easily.
RUN yum install -y procps

# Add Tini to use as init (PID1) process.
ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod -R +rx /tini

# Switch back to the opensearch user.
USER opensearch

# Add custom entrypoint which sets up the right process tree.
COPY ./entrypoint.sh /usr/share/opensearch/opensearch-docker-entrypoint.sh

ENTRYPOINT ["/tini", "--", "./opensearch-docker-entrypoint.sh"]
CMD ["opensearch"]

Dockerfile.vanilla

FROM opensearchproject/opensearch:2.5.0

# Add a basic config to run OpenSearch container as a single-node cluster.
COPY opensearch.yml /usr/share/opensearch/config/opensearch.yml

USER root

# Install tools to be able to inspect the process tree more easily.
RUN yum install -y procps

# Switch back to the opensearch user.
USER opensearch

gistfile1.txt

#!/bin/bash

# Copyright OpenSearch Contributors
# SPDX-License-Identifier: Apache-2.0

# This script specify the entrypoint startup actions for opensearch
# It will start both opensearch and performance analyzer plugin cli
# If either process failed, the entire docker container will be removed
# in favor of a newly started container

# Export OpenSearch Home
export OPENSEARCH_HOME=/usr/share/opensearch
export OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config

# The virtual file /proc/self/cgroup should list the current cgroup
# membership. For each hierarchy, you can follow the cgroup path from
# this file to the cgroup filesystem (usually /sys/fs/cgroup/) and
# introspect the statistics for the cgroup for the given
# hierarchy. Alas, Docker breaks this by mounting the container
# statistics at the root while leaving the cgroup paths as the actual
# paths. Therefore, OpenSearch provides a mechanism to override
# reading the cgroup path from /proc/self/cgroup and instead uses the
# cgroup path defined the JVM system property
# opensearch.cgroups.hierarchy.override. Therefore, we set this value here so
# that cgroup statistics are available for the container this process
# will run in.
export OPENSEARCH_JAVA_OPTS="-Dopensearch.cgroups.hierarchy.override=/ $OPENSEARCH_JAVA_OPTS"

# Security Plugin
function setupSecurityPlugin {
    SECURITY_PLUGIN="opensearch-security"

    if [ -d "$OPENSEARCH_HOME/plugins/$SECURITY_PLUGIN" ]; then
        if [ "$DISABLE_INSTALL_DEMO_CONFIG" = "true" ]; then
            echo "Disabling execution of install_demo_configuration.sh for OpenSearch Security Plugin"
        else
            echo "Enabling execution of install_demo_configuration.sh for OpenSearch Security Plugin"
            bash $OPENSEARCH_HOME/plugins/$SECURITY_PLUGIN/tools/install_demo_configuration.sh -y -i -s
        fi

        if [ "$DISABLE_SECURITY_PLUGIN" = "true" ]; then
            echo "Disabling OpenSearch Security Plugin"
            opensearch_opt="-Eplugins.security.disabled=true"
            opensearch_opts+=("${opensearch_opt}")
        else
            echo "Enabling OpenSearch Security Plugin"
        fi
    else
        echo "OpenSearch Security Plugin does not exist, disable by default"
    fi
}

# Performance Analyzer Plugin
function setupPerformanceAnalyzerPlugin {
    PERFORMANCE_ANALYZER_PLUGIN="opensearch-performance-analyzer"
    if [ -d "$OPENSEARCH_HOME/plugins/$PERFORMANCE_ANALYZER_PLUGIN" ]; then
        if [ "$DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI" = "true" ]; then
            echo "Disabling execution of $OPENSEARCH_HOME/bin/$PERFORMANCE_ANALYZER_PLUGIN/performance-analyzer-agent-cli for OpenSearch Performance Analyzer Plugin"
        else
            echo "Enabling execution of OPENSEARCH_HOME/bin/$PERFORMANCE_ANALYZER_PLUGIN/performance-analyzer-agent-cli for OpenSearch Performance Analyzer Plugin"
            $OPENSEARCH_HOME/bin/opensearch-performance-analyzer/performance-analyzer-agent-cli > $OPENSEARCH_HOME/logs/performance-analyzer.log 2>&1 &
        fi
    else
        echo "OpenSearch Performance Analyzer Plugin does not exist, disable by default"
    fi
}

# Start up the opensearch and performance analyzer agent processes.
# When either of them halts, this script exits, or we receive a SIGTERM or SIGINT signal then we want to kill both these processes.
function runOpensearch {
    # Files created by OpenSearch should always be group writable too
    umask 0002

    if [[ "$(id -u)" == "0" ]]; then
        echo "OpenSearch cannot run as root. Please start your container as another user."
        exit 1
    fi

    # Parse Docker env vars to customize OpenSearch
    #
    # e.g. Setting the env var cluster.name=testcluster
    # will cause OpenSearch to be invoked with -Ecluster.name=testcluster
    opensearch_opts=()
    while IFS='=' read -r envvar_key envvar_value
    do
        # OpenSearch settings need to have at least two dot separated lowercase
        # words, e.g. `cluster.name`, except for `processors` which we handle
        # specially
        if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ || "$envvar_key" == "processors" ]]; then
            if [[ ! -z $envvar_value ]]; then
            opensearch_opt="-E${envvar_key}=${envvar_value}"
            opensearch_opts+=("${opensearch_opt}")
            fi
        fi
    done < <(env)

    #setupSecurityPlugin
    #setupPerformanceAnalyzerPlugin

    # Start opensearch
    exec "$@" "${opensearch_opts[@]}"
}

# Prepend "opensearch" command if no argument was provided or if the first
# argument looks like a flag (i.e. starts with a dash).
if [ $# -eq 0 ] || [ "${1:0:1}" = '-' ]; then
    set -- opensearch "$@"
fi

if [ "$1" = "opensearch" ]; then
    # If the first argument is opensearch, then run the setup script.
    runOpensearch "$@"
else
    # Otherwise, just exec the command.
    exec "$@"
fi

Makefile

run-vanilla:
	docker run -it --rm $$(docker build -qf Dockerfile.vanilla .)

run-fixed:
	docker run -it --rm $$(docker build -qf Dockerfile.fixed .)

send-sigterm:
	docker kill --signal="SIGTERM" $(CONTAINER_ID)

opensearch.yml

cluster.name: "docker-cluster"
network.host: 0.0.0.0
index.store.type: niofs
node.store.allow_mmap: false
discovery.type: single-node
plugins.security.disabled: true