AiStore test

aisnode_config.sh

export AIS_CONF_FILE=/etc/ais/ais.json
export STATSD_CONF_FILE="/etc/ais/statsd.conf"
export COLLECTD_CONF_FILE="/etc/ais/collectd.conf"

cat > $AIS_CONF_FILE <<EOL
{
	"confdir": "${AIS_CONF_DIR}",
  "cloud": {},
	"mirror": {
		"copies":       2,
		"burst_buffer": 512,
		"util_thresh":  ${MIRROR_UTIL_THRESH:-20},
		"optimize_put": false,
		"enabled":      ${MIRROR_ENABLED:-false}
	},
	"ec": {
		"objsize_limit": ${OBJ_SIZE_LIMIT:-262144},
		"data_slices":   ${DATA_SLICES:-1},
		"parity_slices": ${PARITY_SLICES:-1},
		"compression":   "${COMPRESSION:-never}",
		"enabled":       ${EC_ENABLED:-false},
		"batch_size":    ${EC_BATCH_SIZE:-64}
	},
	"log": {
		"dir":       "${AIS_LOG_DIR:-/tmp/ais$NEXT_TIER/log}",
		"level":     "${AIS_LOG_LEVEL:-3}",
		"max_size":  4194304,
		"max_total": 67108864
	},
	"periodic": {
		"stats_time":        "10s",
		"notif_time":        "30s",
		"retry_sync_time":   "2s"
	},
	"timeout": {
		"max_keepalive":        "4s",
		"cplane_operation":     "2s",
		"send_file_time":       "5m",
		"startup_time":         "1m",
		"max_host_busy":        "1m"
	},
	"client": {
		"client_timeout":      "10s",
		"client_long_timeout": "30m",
		"list_timeout":        "3m"
	},
	"proxy": {
		"primary_url":   "${AIS_PRIMARY_URL}",
		"original_url":  "${AIS_PRIMARY_URL}",
		"discovery_url": "${AIS_DISCOVERY_URL}",
		"non_electable": ${NON_ELECTABLE:-false}
	},
	"lru": {
		"lowwm":             75,
		"highwm":            90,
		"out_of_space":      95,
		"dont_evict_time":   "120m",
		"capacity_upd_time": "10m",
		"enabled":           true
	},
	"disk":{
	    "iostat_time_long":  "${IOSTAT_TIME_LONG:-2s}",
	    "iostat_time_short": "${IOSTAT_TIME_SHORT:-100ms}",
	    "disk_util_low_wm":  20,
	    "disk_util_high_wm": 80,
	    "disk_util_max_wm":  95
	},
	"rebalance": {
		"enabled":         true,
		"dont_run_time":   "0m",
		"dest_retry_time": "2m",
		"quiescent":       "10s",
		"compression":     "${COMPRESSION:-never}",
		"multiplier":      ${REBALANCE_MULTIPLIER:-4}
	},
	"checksum": {
		"type":			"xxhash",
		"validate_cold_get":	true,
		"validate_warm_get":	false,
		"validate_obj_move":	false,
		"enable_read_range":	false
	},
	"compression": {
		"block_size": ${BLOCK_SIZE:-262144},
		"checksum":   ${CHECKSUM:-false}
	},
	"versioning": {
		"enabled":           true,
		"validate_warm_get": false
	},
	"fspaths": {
   	"/mnt/ais": " "
	},
	"test_fspaths": {
		"root":     "${TEST_FSPATH_ROOT:-/tmp/ais$NEXT_TIER/}",
		"count":    ${TEST_FSPATH_COUNT:-0},
		"instance": ${INSTANCE:-0}
	},
	"net": {
		"l4": {
			"proto":              "tcp",
			"port":               "${PORT:-5353}",
			"port_intra_control": "${PORT_INTRA_CONTROL:-9080}",
			"port_intra_data":    "${PORT_INTRA_DATA:-10080}",
			"sndrcv_buf_size":    ${SNDRCV_BUF_SIZE:-131072}
		},
		"http": {
			"use_https":         ${AIS_USE_HTTPS:-false},
			"server_crt":        "${AIS_SERVER_CRT:-server.crt}",
			"server_key":        "${AIS_SERVER_KEY:-server.key}",
			"write_buffer_size": ${HTTP_WRITE_BUFFER_SIZE:-0},
			"read_buffer_size":  ${HTTP_READ_BUFFER_SIZE:-0},
			"chunked_transfer":  ${CHUNKED_TRANSFER:-true},
			"skip_verify":       ${AIS_SKIP_VERIFY_CRT:-false}
		}
	},
	"fshc": {
		"enabled":     true,
		"test_files":  4,
		"error_limit": 2
	},
	"auth": {
		"secret":      "$AIS_SECRET_KEY",
		"enabled":     ${AUTH_ENABLED:-false}
	},
	"keepalivetracker": {
		"proxy": {
			"interval": "10s",
			"name":     "heartbeat",
			"factor":   3
		},
		"target": {
			"interval": "10s",
			"name":     "heartbeat",
			"factor":   3
		},
		"retry_factor":   5,
		"timeout_factor": 3
	},
	"downloader": {
		"timeout": "1h"
	},
	"distributed_sort": {
		"duplicated_records":    "ignore",
		"missing_shards":        "ignore",
		"ekm_malformed_line":    "abort",
		"ekm_missing_key":       "abort",
		"default_max_mem_usage": "80%",
		"dsorter_mem_threshold": "100GB",
		"compression":           "${COMPRESSION:-never}",
		"call_timeout":          "10m"
	}
}
EOL

cat > $STATSD_CONF_FILE <<EOL
{
	graphitePort: ${GRAPHITE_PORT:-2003},
	graphiteHost: "${GRAPHITE_SERVER:-localhost}"
}
EOL

cat > $COLLECTD_CONF_FILE <<EOL
LoadPlugin df
LoadPlugin cpu
LoadPlugin disk
LoadPlugin interface
LoadPlugin load
LoadPlugin memory
LoadPlugin processes
LoadPlugin write_graphite

<Plugin syslog>
	LogLevel info
</Plugin>

<Plugin df>
	FSType rootfs
	FSType sysfs
	FSType proc
	FSType devtmpfs
	FSType devpts
	FSType tmpfs
	FSType fusectl
	FSType cgroup
	IgnoreSelected true
	ValuesPercentage True
</Plugin>

<Plugin write_graphite>
	<Node "graphiting">
		Host "${GRAPHITE_SERVER:-localhost}"
		Port "${GRAPHITE_PORT:-2003}"
		Protocol "tcp"
		LogSendErrors true
		StoreRates true
		AlwaysAppendDS false
		EscapeCharacter "_"
	</Node>
</Plugin>

<Include "/etc/collectd/collectd.conf.d">
	Filter "*.conf"
</Include>
EOL

Dockerfile

FROM ubuntu:20.04

RUN apt-get clean \
  && apt-get update \
  && apt-get --no-install-recommends -y install \
      curl \
      git \
      ca-certificates \
      wget \
      vim \
      sysstat \
      attr \
      net-tools \
      iproute2 \
      make \
      iputils-ping \
  && apt-get -y clean all \
  && rm -rf /var/lib/apt/lists/*

# Setting ENV variables
ENV GOLANG_VERSION 1.15
ENV AISTORE_VERSION 3.2.1

# Reassign arguments to environment variables so run.sh can use them
ENV GOPATH /go
ENV GOBIN $GOPATH/bin
ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
ENV WORKDIR $GOPATH/src/github.com/NVIDIA/aistore

RUN mkdir -p /etc/ais /usr/nvidia "$GOPATH/bin" "$GOPATH/src/github.com/NVIDIA"

# Installing go
RUN chmod -R 777 "$GOPATH"
RUN curl -LO https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-amd64.tar.gz &&\
  tar -C /usr/local -xvzf go${GOLANG_VERSION}.linux-amd64.tar.gz > /dev/null 2>&1 &&\
  rm -rf go${GOLANG_VERSION}.linux-amd64.tar.gz

RUN git clone --branch "$AISTORE_VERSION" --depth 1 https://github.com/NVIDIA/aistore $WORKDIR
WORKDIR $WORKDIR

# Build it here so it is not done multiple times on each proxy/target: less
# downloading and less time spent on everything.
# TODO: we can do it more elegantly (build image ?)
RUN make node

COPY /aisnode_config.sh /etc/ais/aisnode_config.sh

EXPOSE 5353

CMD /etc/ais/aisnode_config.sh && aisnode -config=/etc/ais/ais.json -role="$AIS_ROLE" -ntargets=0 -alsologtostderr=true;

start_proxy.sh

#!/bin/bash

docker build -t ops_aistore .

docker container rm -f ais_proxy || echo "Container was not running continue"

docker run --rm --name ais_proxy \
  --env AIS_ROLE=proxy \
  --network host \
  --env AIS_IS_PRIMARY=true \
  -it ops_aistore

start_target.sh

#!/bin/bash

docker build -t ops_aistore .

docker container rm -f ais_target || echo "Container was not running continue"

AIS_PRIMARY_IP=$1

docker run --rm --name ais_target \
  --env AIS_ROLE=target \
  --env AIS_PRIMARY_URL="http://$AIS_PRIMARY_IP:5353" \
  --network host \
  -v /mnt/data/ops/aistore:/mnt/ais \
  -it ops_aistore