Growthforecast setting up

memo.md

GrowthForecast

How to setup GrowthForecast.

see: http://kazeburo.github.io/GrowthForecast/

yum

sudo yum update
sudo yum install pkgconfig glib2-devel gettext libxml2-devel pango-devel cairo-devel
sudo yum groupinstall "Development Tools"

localtime

sudo mv /etc/localtime /etc/localtime.org
sudo ln -s /usr/share/zoneinfo/Japan /etc/localtime

useradd

add a user

sudo useradd growthforecast

change user

sudo su - growthforecast

perlbrew

install perlbrew

curl -kL http://install.perlbrew.pl | bash
echo '[[ -s "$HOME/perl5/perlbrew/etc/bashrc" ]] && source "$HOME/perl5/perlbrew/etc/bashrc"' >> .bash_profile
source $HOME/perl5/perlbrew/etc/bashrc

install perl

perlbrew available
perlbrew install perl-5.18.2
perlbrew switch perl-5.18.2

cpanm

install cpanm

perlbrew install-cpanm

growthforecast

install GrowthForecast

cpanm -n GrowthForecast

mkdir /home/growthforecast/data

supervisor

install supervisord

sudo yum --enablerepo=epel install supervisor

/etc/supervisord.conf

[supervisord]
http_port=/var/tmp/supervisor.sock ; (default is to run a UNIX domain socket server)
logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
logfile_maxbytes=50MB       ; (max main logfile bytes b4 rotation;default 50MB)
logfile_backups=10          ; (num of main logfile rotation backups;default 10)
loglevel=info               ; (logging level;default info; others: debug,warn)
pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
nodaemon=false              ; (start in foreground if true;default false)
minfds=1024                 ; (min. avail startup file descriptors;default 1024)
minprocs=200                ; (min. avail process descriptors;default 200)

[supervisorctl]
serverurl=unix:///var/tmp/supervisor.sock ; use a unix:// URL  for a unix socket

[program:growthforecast]
command=/home/growthforecast/perl5/perlbrew/perls/perl-5.18.2/bin/perl 
        /home/growthforecast/perl5/perlbrew/perls/perl-5.18.2/bin/growthforecast.pl 
        --data-dir /home/growthforecast/data
process_name=%(program_name)s
directory=/home/growthforecast
stdout_logfile=/var/log/supervisor/%(program_name)s.log
stderr_logfile=/var/log/supervisor/%(program_name)s.log
user=growthforecast
autostart=true
autorestart=true

make a directory for logging

sudo mkdir /var/log/supervisor

add superversord to upstart /etc/init/supervisord.conf

description "supervisord"
 
start on runlevel [2345]
stop on runlevel [!2345]
respawn
exec /usr/bin/supervisord -n

start supervisord via upstart

initctl start supervisord

td-agent

install

curl -L http://toolbelt.treasuredata.com/sh/install-redhat.sh | sh

LTSV plugin

git clone https://github.com/stanaka/fluent-plugin-tail-labeled-tsv.git
cp fluent-plugin-tail-labeled-tsv/lib/fluent/plugin/in_tail_labeled_tsv.rb /etc/td-agent/plugin

plugins

/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-file-alternative
/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-datacounter
/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-growthforecast

mv /etc/td-agent/td-agent.conf /etc/td-agent/td-agent.conf.org

/etc/td-agent/td-agent.conf

<source>
  type forward
</source>

<match nginx.access>
  type copy
  <store>
    type file
    path /var/log/td-agent/nginx.access
  </store>
  <store>
    type growthforecast
    gfapi_url http://localhost:5125/api/
    service   nginx
    section   response
    name_keys reqtime,apptime
  </store>
  <store>
    type datacounter
    unit minute
    tag nginx.access.status_count
    count_key status
    pattern1 3xx ^3\d\d$
    pattern2 4xx ^4\d\d$
    pattern3 5xx ^5\d\d$
    pattern4 200 ^200$
  </store>
</match>

<match nginx.access.status_count>
  type growthforecast
  gfapi_url http://localhost:5125/api/
  service   nginx
  section   status_codes
  name_keys nginx.access_5xx_count,nginx.access_4xx_count,nginx.access_3xx_count,nginx.access_200_count
</match>

fluent-agent-lite

install

sudo yum install perl-ExtUtils-MakeMaker
git clone https://github.com/tagomoris/fluent-agent-lite
cd fluent-agent-lite
./bin/install.sh

/etc/fluent-agent-lite.conf

TAG_PREFIX=""

LOGS=$(cat <<"EOF"
nginx.access /var/log/nginx/access_log
EOF
)

PRIMARY_SERVER="0.0.0.0:24224"

nginx

/etc/nginx/nginx.conf

server {
    listen       80 default_server;

    access_log   /var/log/nginx/access.log  ltsv;
    error_log    /var/log/nginx/error.log;

    auth_basic             "Restricted";
    auth_basic_user_file   "/etc/nginx/htpasswd";

    location / {
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Host $http_host;
        proxy_pass http://growthforecast;
        proxy_redirect off;
    }
}

upstream growthforecast {
    server 127.0.0.1:5125;
}

htpasswd

htpasswd -c /etc/nginx/htpasswd username