View gist:79f869749bf54d135f7f6fe61e0e99a7
1. Generate new domain name for logstash server. | |
For this tutorial | |
domain name = logstash-prod.xyz.com | |
ip = 1.2.3.4 | |
* Enter to following directory | |
``` | |
$ sudo mkdir /etc/pki | |
$ cd /etc/pki |
View openssl.cnf
# | |
# OpenSSL example configuration file. | |
# This is mostly being used for generation of certificate requests. | |
# | |
# This definition stops the following lines choking if HOME isn't | |
# defined. | |
HOME = . | |
RANDFILE = $ENV::HOME/.rnd |
View elk-server-setup.sh
#!/bin/bash | |
if [ -z "$1" ] | |
then | |
echo "Pass nginx password" | |
exit 1 | |
fi | |
View filebeat-index-template.json
{ | |
"mappings": { | |
"_default_": { | |
"dynamic_templates": [ | |
{ | |
"template1": { | |
"mapping": { | |
"ignore_above": 1024, | |
"index": "true", | |
"type": "{dynamic_type}" |
View gist:ea46ce6e91edfbd214b05398e8fdb788
=================== Destination ============ | |
9483 rshah 20 0 1490m 379m 5024 S 38.8 1.0 0:01.57 transfer2go | |
9483 rshah 20 0 1887m 727m 5032 S 30.4 2.0 0:02.49 transfer2go | |
9483 rshah 20 0 2944m 1.4g 5032 S 43.8 3.9 0:03.81 transfer2go | |
9483 rshah 20 0 2944m 1.6g 5032 S 20.2 4.5 0:04.42 transfer2go | |
9483 rshah 20 0 2944m 1.7g 5032 S 22.2 4.9 0:05.09 transfer2go | |
9483 rshah 20 0 4275m 2.6g 5032 S 63.3 7.2 0:07.00 transfer2go | |
9483 rshah 20 0 4275m 2.6g 5032 S 14.3 7.4 0:07.43 transfer2go | |
9483 rshah 20 0 4275m 2.7g 5032 S 15.3 7.6 0:07.89 transfer2go |
View source.log
[rshah@source-server transfer2go]# ./transfer2go -config test/config/source.json -auth=false -agent http://main-agent.cern.ch:8989 | |
INFO[0000] Agent Auth=false Config="<Config: name=SourceAgent url=http://localhost:8000 port=8000 base= catalog=test/catalog/source.json protocol=http backend=/tmp/transfer2go/source tool= opts= mfile=SourceAgentmetrics.log minterval=60 staticdir=static workders=10 queuesize=100 register=http://main-agent.cern.ch:8989 type=pull router=false>" Model=pull | |
INFO[0000] Register agent as alias on register Agent="http://localhost:8000" Alias=SourceAgent Register="http://main-agent.cern.ch:8989" | |
INFO[0000] Register agent as alias on register Agent="http://localhost:8000" Alias=SourceAgent Register="http://131.154.141.247:9000" | |
INFO[0000] Register agent as alias on register Agent="http://localhost:8000" Alias=SourceAgent Register="http://localhost:8989" | |
INFO[0000] Catalog="{s |
View main-agent.log
[rshah@main-agent transfer2go]# ./transfer2go -config test/config/main.json -auth=false | |
WARN[0000] WARNING this agent is not registered with remote ones, either provide register in your config or invoke register API call | |
INFO[0000] Agent Auth=false Config="<Config: name=MainAgent url=http://localhost:8989 port=8989 base= catalog=test/catalog/main.json protocol=http backend=/tmp/transfer2go/main tool=/bin/cp opts= mfile=MainAgentMetrics.log minterval=60 staticdir=static workders=10 queuesize=100 register= type=pull router=true>" Model=pull | |
INFO[0000] Catalog="{sqlite3 test/catalog/main.db }" | |
INFO[0000] Enabling router model TrainInterval=24h | |
INFO[0000] Router successfully retrained DataFile=test/data/history.csv TrainInterval=24h | |
INFO[0000] Requests restored from db | |
INFO[0000] Start dispatcher QueueSize=100 Transfer Type=pull Workers=10 | |
INFO[0013] RequestH |
View destination.log
-bash-4.1$ ./transfer2go -agent http://localhost:8989 -config test/config/destination.json -auth=false | |
INFO[0000] Agent Auth=false Config="<Config: name=DestinationAgent url=http://131.154.141.247:9000 port=9000 base= catalog=test/catalog/destination.json protocol=http backend=/tmp/transfer2go/destination tool=/bin/cp opts= mfile=DestinationAgentMetrics.log minterval=60 staticdir=static workders=10 queuesize=100 register=http://localhost:8989 type=pull router=false>" Model=pull | |
INFO[0000] Register agent as alias on register Agent="http://131.154.141.247:9000" Alias=DestinationAgent Register="http://localhost:8989" | |
INFO[0000] Catalog="{sqlite3 test/catalog/destination.db }" | |
INFO[0000] Requests restored from db | |
INFO[0000] Start dispatcher QueueSize=100 Transfer Type=pull Workers=10 | |
INFO[0028] ActionHandler, receive new request Job="<Job TransferRequest=<TransferRequest id=9f6 |
View gist:f386106067dc0cdfd8bf039b03430944
=============== Destination ================= | |
5080 rshah 20 0 211m 9076 4792 S 0.7 0.0 0:01.28 transfer2go | |
5080 rshah 20 0 1385m 298m 5128 S 28.2 0.8 0:02.13 transfer2go | |
5080 rshah 20 0 1765m 678m 5128 S 29.1 1.9 0:03.01 transfer2go | |
5080 rshah 20 0 2293m 1.2g 5128 S 38.1 3.3 0:04.16 transfer2go | |
5080 rshah 20 0 2293m 1.2g 5128 S 17.3 3.3 0:04.68 transfer2go | |
5080 rshah 20 0 3757m 2.6g 5128 S 57.4 7.3 0:06.41 transfer2go | |
5080 rshah 20 0 3757m 2.6g 5128 S 16.2 7.3 0:06.90 transfer2go | |
5080 rshah 20 0 3757m 2.6g 5128 S 16.3 7.3 0:07.39 transfer2go | |
5080 rshah 20 0 3757m 2.6g 5128 S 16.9 7.3 0:07.90 transfer2go |
View router-pull-test.txt
wifi:transfer2go Rishi$ bash test/run.sh pull dataset | |
### Perform test of pull model | |
Kill previous transfer2go processes (if any) | |
create agents areas in /tmp/transfer2go | |
Create config and catalogs | |
main agent: mainAgent (http://localhost:8989) | |
source agent: sourceAgent (http://localhost:8000) | |
destination agent: destinationAgent (http://localhost:9000) | |
create large file in source area | |
0+0 records in |
NewerOlder