Tango benchmark setup instructions

README.md

Tango Controls benchmark suite

https://github.com/tango-controls/sys-tango-benchmark

Quick launch steps

1. setup the server machine (see setup-server.sh),
2. setup the client machine (see setup-client.sh),
3. update server's host in tango-benchmarks.yml,
4. on the server machine:
   • ensure that Starter is running,
   • ensure that device servers are not running,
5. on the client machine:
   • run tg_benchmarkrunner -c tango-benchmarks-attr-read.yml > results-attr-read.rst,
   • run tg_benchmarkrunner -c tango-benchmarks-attr-write.yml > results-attr-write.rst,
   • run tg_benchmarkrunner -c tango-benchmarks-command.yml > results-command.rst,
   • run tg_benchmarkrunner -c tango-benchmarks-pipe-read.yml > results-pipe-read.rst,
   • run tg_benchmarkrunner -c tango-benchmarks-pipe-write.yml > results-pipe-write.rst,
   • run tg_benchmarkrunner -c tango-benchmarks-event-sub.yml > results-event-sub.rst,
   • run tg_benchmarkrunner -c tango-benchmarks-event-push.yml > results-event-push.rst,
6. share the results!

setup-client.sh

# For debian / ubuntu.
# Requires Tango to be installed.
# Tested on docker image: registry.gitlab.com/s2innovation/tangobox-docker/tangobox-base

apt update

apt install -y \
    git \
    g++ \
    make \
    libomniorb4-dev \
    libzmq3-dev \
    libcos4-dev \
    openjdk-8-jdk-headless \
    maven \
    python-pip

pip install 'docutils<0.15'
pip install sphinx

export TANGO_ROOT=/usr/local

git clone https://github.com/tango-controls/sys-tango-benchmark.git /benchmark

cd /benchmark/benchmarks
pip install .

cd /benchmark/cppclient
make
make install

cd /benchmark/javaclient
./javaclient-build.sh
./javaclient-install.sh

setup-server.sh

# For debian / ubuntu.
# Requires Tango to be installed.
# Tested on docker image: registry.gitlab.com/s2innovation/tangobox-docker/tangobox-base

apt update

apt install -y \
    git \
    g++ \
    make \
    libomniorb4-dev \
    libzmq3-dev \
    libcos4-dev \
    openjdk-8-jdk-headless \
    maven \
    python-pip

pip install 'docutils<0.15'
pip install sphinx

export TANGO_ROOT=/usr/local

git clone https://github.com/tango-controls/sys-tango-benchmark.git /benchmark

cd /benchmark/ds/CppBenchmarkTarget
make
make install

cd /benchmark/ds/JavaBenchmarkTarget
mvn package
cp target/JavaBenchmarkTarget-1.0.jar /usr/local/share/java/
cp src/scripts/JavaBenchmarkTarget /usr/local/bin/

cd /benchmark/ds/PyBenchmarkTarget
pip install .

tango_admin --add-server PyBenchmarkTarget/01 PyBenchmarkTarget sys/benchmark/python01
tango_admin --add-server CppBenchmarkTarget/01 CppBenchmarkTarget sys/benchmark/cpp01
tango_admin --add-server JavaBenchmarkTarget/01 JavaBenchmarkTarget sys/benchmark/java01

host=$(hostname -s)
tango_admin --add-server Starter/$host Starter tango/admin/$host
tango_admin --add-property tango/admin/$host StartDsPath /usr/bin,/usr/local/bin

Starter $host

tango-benchmarks-attr-read.yml

- &opts
  clients: 2,4
  period: 1

- &extopts
  <<: *opts
  worker: tangobenchmarks.client.external.Worker

- &devpy
  device: sys/benchmark/python01

- &devcpp
  device: sys/benchmark/cpp01

- &devjava
  device: sys/benchmark/java01


### devices

- server_instance: PyBenchmarkTarget/01
  device_class: PyBenchmarkTarget
  target_device: sys/benchmark/python01
  host: tangobox-bmark-server

- server_instance: CppBenchmarkTarget/01
  device_class: CppBenchmarkTarget
  target_device: sys/benchmark/cpp01
  host: tangobox-bmark-server

- server_instance: JavaBenchmarkTarget/01
  device_class: JavaBenchmarkTarget
  target_device: sys/benchmark/java01
  host: tangobox-bmark-server


### attribute read (python client)

- benchmark: read_benchmark
  title: "python attribute read benchmark (python client)"
  <<: *opts
  <<: *devpy
  csvfile: results-attr-read-python-python.csv

- benchmark: read_benchmark
  title: "cpp attribute read benchmark (python client)"
  <<: *opts
  <<: *devcpp
  csvfile: results-attr-read-cpp-python.csv

- benchmark: read_benchmark
  title: "java attribute read benchmark (python client)"
  <<: *opts
  <<: *devjava
  csvfile: results-attr-read-java-python.csv


### attribute read (cpp client)

- benchmark: read_benchmark
  title: "python attribute read benchmark (cpp client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_read
  csvfile: results-attr-read-python-cpp.csv

- benchmark: read_benchmark
  title: "cpp attribute read benchmark (cpp client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_read
  csvfile: results-attr-read-cpp-cpp.csv

- benchmark: read_benchmark
  title: "java attribute read benchmark (cpp client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_read
  csvfile: results-attr-read-java-cpp.csv


### attribute read (java client)

- benchmark: read_benchmark
  title: "python attribute read benchmark (java client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_java_read
  csvfile: results-attr-read-python-java.csv

- benchmark: read_benchmark
  title: "cpp attribute read benchmark (java client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_java_read
  csvfile: results-attr-read-cpp-java.csv

- benchmark: read_benchmark
  title: "java attribute read benchmark (java client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_java_read
  csvfile: results-attr-read-java-java.csv
  

tango-benchmarks-attr-write.yml

- &opts
  clients: 2,4
  period: 1

- &extopts
  <<: *opts
  worker: tangobenchmarks.client.external.Worker

- &devpy
  device: sys/benchmark/python01

- &devcpp
  device: sys/benchmark/cpp01

- &devjava
  device: sys/benchmark/java01


### devices

- server_instance: PyBenchmarkTarget/01
  device_class: PyBenchmarkTarget
  target_device: sys/benchmark/python01
  host: tangobox-bmark-server

- server_instance: CppBenchmarkTarget/01
  device_class: CppBenchmarkTarget
  target_device: sys/benchmark/cpp01
  host: tangobox-bmark-server

- server_instance: JavaBenchmarkTarget/01
  device_class: JavaBenchmarkTarget
  target_device: sys/benchmark/java01
  host: tangobox-bmark-server


### attribute write (python client)

- benchmark: write_benchmark
  title: "python attribute write benchmark (python client)"
  <<: *opts
  <<: *devpy
  csvfile: results-attr-write-python-python.csv

- benchmark: write_benchmark
  title: "cpp attribute write benchmark (python client)"
  <<: *opts
  <<: *devcpp
  csvfile: results-attr-write-cpp-python.csv

- benchmark: write_benchmark
  title: "java attribute write benchmark (python client)"
  <<: *opts
  <<: *devjava
  csvfile: results-attr-write-java-python.csv


### attribute write (cpp client)

- benchmark: write_benchmark
  title: "python attribute write benchmark (cpp client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_write
  csvfile: results-attr-write-python-cpp.csv

- benchmark: write_benchmark
  title: "cpp attribute write benchmark (cpp client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_write
  csvfile: results-attr-write-cpp-cpp.csv

- benchmark: write_benchmark
  title: "java attribute write benchmark (cpp client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_read
  csvfile: results-attr-write-java-cpp.csv


### attribute write (java client)

- benchmark: write_benchmark
  title: "python attribute write benchmark (java client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_java_write
  csvfile: results-attr-write-python-java.csv

- benchmark: write_benchmark
  title: "cpp attribute write benchmark (java client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_java_write
  csvfile: results-attr-write-cpp-java.csv

- benchmark: write_benchmark
  title: "java attribute write benchmark (java client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_java_write
  csvfile: results-attr-write-java-java.csv
  

tango-benchmarks-command.yml

- &opts
  clients: 2,4
  period: 1

- &extopts
  <<: *opts
  worker: tangobenchmarks.client.external.Worker

- &devpy
  device: sys/benchmark/python01

- &devcpp
  device: sys/benchmark/cpp01

- &devjava
  device: sys/benchmark/java01


### devices

- server_instance: PyBenchmarkTarget/01
  device_class: PyBenchmarkTarget
  target_device: sys/benchmark/python01
  host: tangobox-bmark-server

- server_instance: CppBenchmarkTarget/01
  device_class: CppBenchmarkTarget
  target_device: sys/benchmark/cpp01
  host: tangobox-bmark-server

- server_instance: JavaBenchmarkTarget/01
  device_class: JavaBenchmarkTarget
  target_device: sys/benchmark/java01
  host: tangobox-bmark-server


### command (python client)

- benchmark: cmd_benchmark
  title: "python command benchmark (python client)"
  <<: *opts
  <<: *devpy
  csvfile: results-command-python-python.csv

- benchmark: cmd_benchmark
  title: "cpp command benchmark (python client)"
  <<: *opts
  <<: *devcpp
  csvfile: results-command-cpp-python.csv

- benchmark: cmd_benchmark
  title: "java command benchmark (python client)"
  <<: *opts
  <<: *devjava
  csvfile: results-command-java-python.csv


### command (cpp client)

- benchmark: cmd_benchmark
  title: "python command benchmark (cpp client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_command
  csvfile: results-command-python-cpp.csv

- benchmark: cmd_benchmark
  title: "cpp command benchmark (cpp client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_command
  csvfile: results-command-cpp-cpp.csv

- benchmark: cmd_benchmark
  title: "java command benchmark (cpp client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_command
  csvfile: results-command-java-cpp.csv


### command (java client)

- benchmark: cmd_benchmark
  title: "python command benchmark (java client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_java_command
  csvfile: results-command-python-java.csv

- benchmark: cmd_benchmark
  title: "cpp command benchmark (java client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_java_command
  csvfile: results-command-cpp-java.csv

- benchmark: cmd_benchmark
  title: "java command benchmark (java client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_java_command
  csvfile: results-command-java-java.csv
  

tango-benchmarks-event-push.yml

- &opts
  clients: 2,4
  period: 1

- &extopts
  <<: *opts
  worker: tangobenchmarks.client.external.Worker

- &devpy
  device: sys/benchmark/python01

- &devcpp
  device: sys/benchmark/cpp01

- &devjava
  device: sys/benchmark/java01


### devices

- server_instance: PyBenchmarkTarget/01
  device_class: PyBenchmarkTarget
  target_device: sys/benchmark/python01
  host: tangobox-bmark-server

- server_instance: CppBenchmarkTarget/01
  device_class: CppBenchmarkTarget
  target_device: sys/benchmark/cpp01
  host: tangobox-bmark-server

- server_instance: JavaBenchmarkTarget/01
  device_class: JavaBenchmarkTarget
  target_device: sys/benchmark/java01
  host: tangobox-bmark-server


### event push (python client)

- benchmark: push_event_benchmark
  title: "python event push benchmark (python client)"
  <<: *opts
  <<: *devpy
  csvfile: results-event-push-python-python.csv

- benchmark: push_event_benchmark
  title: "cpp event push benchmark (python client)"
  <<: *opts
  <<: *devcpp
  csvfile: results-event-push-cpp-python.csv

- benchmark: push_event_benchmark
  title: "java event push benchmark (python client)"
  <<: *opts
  <<: *devjava
  csvfile: results-event-push-java-python.csv


### event push (cpp client)

- benchmark: push_event_benchmark
  title: "python event push benchmark (cpp client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_pushevent
  csvfile: results-event-push-python-cpp.csv

- benchmark: push_event_benchmark
  title: "cpp event push benchmark (cpp client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_pushevent
  csvfile: results-event-push-cpp-cpp.csv

- benchmark: push_event_benchmark
  title: "java event push benchmark (cpp client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_pushevent
  csvfile: results-event-push-java-cpp.csv


### event push (java client)

- benchmark: push_event_benchmark
  title: "python event push benchmark (java client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_java_pushevent
  csvfile: results-event-push-python-java.csv

- benchmark: push_event_benchmark
  title: "cpp event push benchmark (java client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_java_pushevent
  csvfile: results-event-push-cpp-java.csv

- benchmark: push_event_benchmark
  title: "java event push benchmark (java client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_java_pushevent
  csvfile: results-event-push-java-java.csv
  

tango-benchmarks-event-sub.yml

- &opts
  clients: 2,4
  period: 1

- &extopts
  <<: *opts
  worker: tangobenchmarks.client.external.Worker

- &devpy
  device: sys/benchmark/python01

- &devcpp
  device: sys/benchmark/cpp01

- &devjava
  device: sys/benchmark/java01


### devices

- server_instance: PyBenchmarkTarget/01
  device_class: PyBenchmarkTarget
  target_device: sys/benchmark/python01
  host: tangobox-bmark-server

- server_instance: CppBenchmarkTarget/01
  device_class: CppBenchmarkTarget
  target_device: sys/benchmark/cpp01
  host: tangobox-bmark-server

- server_instance: JavaBenchmarkTarget/01
  device_class: JavaBenchmarkTarget
  target_device: sys/benchmark/java01
  host: tangobox-bmark-server


### event sub (python client)

- benchmark: event_benchmark
  title: "python event sub benchmark (python client)"
  <<: *opts
  <<: *devpy
  csvfile: results-event-sub-python-python.csv

- benchmark: event_benchmark
  title: "cpp event sub benchmark (python client)"
  <<: *opts
  <<: *devcpp
  csvfile: results-event-sub-cpp-python.csv

- benchmark: event_benchmark
  title: "java event sub benchmark (python client)"
  <<: *opts
  <<: *devjava
  csvfile: results-event-sub-java-python.csv


### event sub (cpp client)

- benchmark: event_benchmark
  title: "python event sub benchmark (cpp client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_event
  csvfile: results-event-sub-python-cpp.csv

- benchmark: event_benchmark
  title: "cpp event sub benchmark (cpp client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_event
  csvfile: results-event-sub-cpp-cpp.csv

- benchmark: event_benchmark
  title: "java event sub benchmark (cpp client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_event
  csvfile: results-event-sub-java-cpp.csv


### event sub (java client)

- benchmark: event_benchmark
  title: "python event sub benchmark (java client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_java_event
  csvfile: results-event-sub-python-java.csv

- benchmark: event_benchmark
  title: "cpp event sub benchmark (java client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_java_event
  csvfile: results-event-sub-cpp-java.csv

- benchmark: event_benchmark
  title: "java event sub benchmark (java client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_java_event
  csvfile: results-event-sub-java-java.csv
  

tango-benchmarks-pipe-read.yml

- &opts
  clients: 2,4
  period: 1

- &extopts
  <<: *opts
  worker: tangobenchmarks.client.external.Worker

- &devpy
  device: sys/benchmark/python01

- &devcpp
  device: sys/benchmark/cpp01

- &devjava
  device: sys/benchmark/java01


### devices

- server_instance: PyBenchmarkTarget/01
  device_class: PyBenchmarkTarget
  target_device: sys/benchmark/python01
  host: tangobox-bmark-server

- server_instance: CppBenchmarkTarget/01
  device_class: CppBenchmarkTarget
  target_device: sys/benchmark/cpp01
  host: tangobox-bmark-server

- server_instance: JavaBenchmarkTarget/01
  device_class: JavaBenchmarkTarget
  target_device: sys/benchmark/java01
  host: tangobox-bmark-server


### pipe read (python client)

- benchmark: pipe_read_benchmark
  title: "python pipe read benchmark (python client)"
  <<: *opts
  <<: *devpy
  csvfile: results-pipe-read-python-python.csv

- benchmark: pipe_read_benchmark
  title: "cpp pipe read benchmark (python client)"
  <<: *opts
  <<: *devcpp
  csvfile: results-pipe-read-cpp-python.csv

- benchmark: pipe_read_benchmark
  title: "java pipe read benchmark (python client)"
  <<: *opts
  <<: *devjava
  csvfile: results-pipe-read-java-python.csv

  
### pipe read (cpp client)

- benchmark: pipe_read_benchmark
  title: "python pipe read benchmark (cpp client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_pipe_read
  csvfile: results-pipe-read-python-cpp.csv

- benchmark: pipe_read_benchmark
  title: "cpp pipe read benchmark (cpp client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_pipe_read
  csvfile: results-pipe-read-cpp-cpp.csv

- benchmark: pipe_read_benchmark
  title: "java pipe read benchmark (cpp client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_pipe_read
  csvfile: results-pipe-read-java-cpp.csv


### pipe read (java client)

- benchmark: pipe_read_benchmark
  title: "python pipe read benchmark (java client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_java_piperead
  csvfile: results-pipe-read-python-java.csv

- benchmark: pipe_read_benchmark
  title: "cpp pipe read benchmark (java client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_java_piperead
  csvfile: results-pipe-read-cpp-java.csv

- benchmark: pipe_read_benchmark
  title: "java pipe read benchmark (java client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_java_piperead
  csvfile: results-pipe-read-java-java.csv
  

tango-benchmarks-pipe-write.yml

- &opts
  clients: 2,4
  period: 1

- &extopts
  <<: *opts
  worker: tangobenchmarks.client.external.Worker

- &devpy
  device: sys/benchmark/python01

- &devcpp
  device: sys/benchmark/cpp01

- &devjava
  device: sys/benchmark/java01


### devices

- server_instance: PyBenchmarkTarget/01
  device_class: PyBenchmarkTarget
  target_device: sys/benchmark/python01
  host: tangobox-bmark-server

- server_instance: CppBenchmarkTarget/01
  device_class: CppBenchmarkTarget
  target_device: sys/benchmark/cpp01
  host: tangobox-bmark-server

- server_instance: JavaBenchmarkTarget/01
  device_class: JavaBenchmarkTarget
  target_device: sys/benchmark/java01
  host: tangobox-bmark-server


### pipe write (python client)

- benchmark: pipe_write_benchmark
  title: "python pipe write benchmark (python client)"
  <<: *opts
  <<: *devpy
  csvfile: results-pipe-write-python-python.csv

- benchmark: pipe_write_benchmark
  title: "cpp pipe write benchmark (python client)"
  <<: *opts
  <<: *devcpp
  csvfile: results-pipe-write-cpp-python.csv

- benchmark: pipe_write_benchmark
  title: "java pipe write benchmark (python client)"
  <<: *opts
  <<: *devjava
  csvfile: results-pipe-write-java-python.csv


### pipe write (cpp client)

- benchmark: pipe_write_benchmark
  title: "python pipe write benchmark (cpp client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_pipe_write
  csvfile: results-pipe-write-python-cpp.csv

- benchmark: pipe_write_benchmark
  title: "cpp pipe write benchmark (cpp client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_pipe_write
  csvfile: results-pipe-write-cpp-cpp.csv

- benchmark: pipe_write_benchmark
  title: "java pipe write benchmark (cpp client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_pipe_write
  csvfile: results-pipe-write-java-cpp.csv


### pipe write (java client)

- benchmark: pipe_write_benchmark
  title: "python pipe write benchmark (java client)"
  <<: *extopts
  <<: *devpy
  worker_program: tg_benchmark_client_java_pipewrite
  csvfile: results-pipe-write-python-java.csv

- benchmark: pipe_write_benchmark
  title: "cpp pipe write benchmark (java client)"
  <<: *extopts
  <<: *devcpp
  worker_program: tg_benchmark_client_java_pipewrite
  csvfile: results-pipe-write-cpp-java.csv

- benchmark: pipe_write_benchmark
  title: "java pipe write benchmark (java client)"
  <<: *extopts
  <<: *devjava
  worker_program: tg_benchmark_client_java_pipewrite
  csvfile: results-pipe-write-java-java.csv