GPU サーバのログを ELK サーバに送信して管理する ref: https://qiita.com/shun_n/items/e15451888a348706c672

01-inputs.conf

# http input from gpu server on port 5141
input {
  http {
    port => 5141
    type => "gpu"
  }
}

10-gpu-filter.conf

filter {
  if ［type］ == "gpu" {
    grok {
      match => ["message", "(?<name>[^,]*),(?<Date>[^,]*),(?<gpu_util>[^,]*),(?<memory_util>[^,]*),(?<memory_total>[^,]*),(?<memory_free>[^,]*),(?<memory_used>[^,]*),(?<temperature>[^,]*)"]
    }
    date{
      match => ["Date" , "yyyy/MM/dd HH:mm:ss.SSS"]
      timezone => "Asia/Tokyo"
    }
    mutate{
      convert => {
        gpu_util => float
        memory_util => float
        memory_total => float
        memory_free => float
        memory_used => float
        temperature => float
      }
      remove_field => ["headers"]
    }
  }
}

30-outputs.conf

output {
    if [type] == "gpu" {
    elasticsearch {
        hosts =>  ["localhost:9200"]
        index => "logstash-gpu-%{+YYYY.MM.dd}"
    }
}

file0.txt

$ cd /etc/logstash/conf.d/
$ sudo vim ./01-inputs.txt

file11.txt

$ sudo chmod +x ./sendlog.sh

file12.txt

$ sudo apt install curl

file13.txt

$ crontab -e

file14.txt

* * * * * bash /home/${ユーザー名}/sendlog.sh

file2.txt

$ sudo vim ./10-gpu-filter.conf

file4.txt

$ sudo vim ./30-outputs.conf

file6.txt

$ sudo ufw allow 5141

file7.txt

$ nvidia-smi
Sat Jun 23 16:24:39 2018
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 396.26                 Driver Version: 396.26                    |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce GTX 1080    Off  | 00000000:01:00.0 Off |                  N/A |
|  0%   35C    P0    35W / 180W |      0MiB /  8119MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+

file8.txt

Failed to initialize NVML: Driver/library version mismatch

file9.txt

$ cd ~
$ sudo vim ./sendlog.sh

sendlog.sh

#!/bin/bash
nvidia-smi --query-gpu=timestamp,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used,temperature.gpu --format=csv,noheader,nounits | (read log ; curl -XPUT ${ELK サーバの IP }:5141 -d "${GPU サーバの名前},${log}" > /dev/null 2>&1)