Last active
June 1, 2018 17:11
-
-
Save ncloudsbconnelly/b223add86a413cbfc0d9cc97a6a4bfa9 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
MonitoringTaskDefinition: | |
Type: "AWS::ECS::TaskDefinition" | |
Properties: | |
NetworkMode: host | |
Volumes: | |
- Name: docker_sock | |
Host: | |
SourcePath: /var/run/docker.sock | |
- Name: proc | |
Host: | |
SourcePath: /proc/ | |
- Name: cgroup | |
Host: | |
SourcePath: /cgroup/ | |
- Name: tmp | |
Host: | |
SourcePath: /tmp/ | |
- Name: containers | |
Host: | |
SourcePath: /var/lib/docker/containers/ | |
Family: logging-agent-task | |
ContainerDefinitions: | |
- Name: datadog-agent | |
Image: datadog/agent:latest | |
Cpu: 10 | |
Memory: 256 | |
Essential: true | |
MountPoints: | |
- ContainerPath: /var/run/docker.sock | |
SourceVolume: docker_sock | |
ReadOnly: true | |
- ContainerPath: /host/sys/fs/cgroup | |
SourceVolume: cgroup | |
ReadOnly: true | |
- ContainerPath: /host/proc | |
SourceVolume: proc | |
ReadOnly: true | |
Environment: | |
- Name: DD_API_KEY | |
Value: !Sub ${DDAPIKEY} | |
- Name: SD_BACKEND | |
Value: docker | |
- Name: DD_HOSTNAME | |
Value: !Sub ${AWS::StackName} | |
LogConfiguration: | |
LogDriver: json-file | |
Options: | |
max-size: 50m | |
- Name: logzio-docker | |
ReadonlyRootFilesystem: false | |
Image: 867872586470.dkr.ecr.us-east-1.amazonaws.com/fluentd_logzio_docker | |
Cpu: 10 | |
Memory: 256 | |
Essential: true | |
MountPoints: | |
- ContainerPath: /var/run/docker.sock | |
SourceVolume: docker_sock | |
ReadOnly: true | |
- ContainerPath: /tmp | |
SourceVolume: tmp | |
- ContainerPath: /var/lib/docker/containers | |
SourceVolume: containers | |
Environment: | |
- Name: LOGZIO_TOKEN | |
Value: !Sub ${LogzioToken} | |
- Name: LOGZ_IO_URL_1 | |
Value: !Sub https://listener.logz.io:8071?token=${LogzioToken} | |
LogConfiguration: | |
LogDriver: json-file | |
Options: | |
max-size: 50m |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES | |
dbb4fbf66acd 867872586470.dkr.ecr.us-east-1.amazonaws.com/logzio:mtp-logzio-web-2.1.24-357-staging-jenkins "/var/www/traverse-p…" 4 days ago Up 4 days 0.0.0.0:32770->80/tcp ecs-mtp-logzio-3-portal-web-c2b4a0f6f8b297d5ba01 | |
6e043a5a9cde 867872586470.dkr.ecr.us-east-1.amazonaws.com/logzio:mtp-logzio-app-2.1.24-357-staging-jenkins "bash -c /var/www/tr…" 4 days ago Up 4 days 3001/tcp, 6379/tcp ecs-mtp-logzio-3-portal-sidekiq-8292d392ede5ef992a00 | |
71a647f68462 867872586470.dkr.ecr.us-east-1.amazonaws.com/logzio:mtp-logzio-app-2.1.24-357-staging-jenkins "bash -c /var/www/tr…" 4 days ago Up 4 days 3001/tcp, 6379/tcp ecs-mtp-logzio-3-portal-app-d6f28a85eef7d6edee01 | |
b34d428e6356 867872586470.dkr.ecr.us-east-1.amazonaws.com/api-staging:logzio-api-0.0.1-75-staging-jenkins "/var/www/traverse-a…" 4 days ago Up 4 days 3000/tcp ecs-logzio-20-core-sidekiq-face99a1e498c69b7e00 | |
fe3e3bf60c57 867872586470.dkr.ecr.us-east-1.amazonaws.com/api-staging:logzio-api-0.0.1-75-staging-jenkins "/var/www/traverse-a…" 4 days ago Up 4 days 0.0.0.0:32768->3000/tcp ecs-logzio-20-core-api-caf6c38cd4b79e81b601 | |
9886c5a12a81 867872586470.dkr.ecr.us-east-1.amazonaws.com/fluentd_logzio_docker "/bin/entrypoint.sh …" 4 days ago Up 4 days ecs-logging-agent-task-5-logzio-docker-9a9adac6b887cebc7300 | |
d5016f2d54cd datadog/agent:latest "/init" 4 days ago Up 4 days (healthy) ecs-logging-agent-task-5-datadog-agent-ac8ad7afa9c8d7df1300 | |
93d946f667f9 amazon/amazon-ecs-agent:latest "/agent" 4 days ago Up 4 days ecs-agent |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
FROM fluent/fluentd:stable | |
RUN apk add --update --virtual .build-deps \ | |
sudo build-base ruby-dev \ | |
&& sudo gem install \ | |
fluent-plugin-concat \ | |
fluent-plugin-logzio \ | |
fluent-plugin-record-reformer \ | |
fluent-plugin-record-modifier \ | |
fluent-plugin-ecs-metadata-filter \ | |
fluent-plugin-detect-exceptions \ | |
fluent-plugin-docker_metadata_filter \ | |
&& sudo gem sources --clear-all \ | |
&& apk del .build-deps \ | |
&& rm -rf /var/cache/apk/* \ | |
/home/fluent/.gem/ruby/2.3.0/cache/*.gem | |
COPY fluent.conf /fluentd/etc/fluent.conf | |
COPY entrypoint.sh /bin/entrypoint.sh | |
CMD fluentd -c /fluentd/etc/${FLUENTD_CONF} $FLUENTD_OPT |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This source tails the logs that are created by the Docker json-file logging driver | |
<source> | |
@type tail | |
tag reemoveme.ecs.* | |
path /var/lib/docker/containers/*/*-json.log | |
pos_file fluentd-docker.pos | |
format json | |
refresh_interval 10s | |
time_key time | |
time_format %Y-%m-%dT%H:%M:%S | |
</source> | |
# This matches the removeme tag to dectect exceptions. | |
<match reemoveme.**> | |
@type detect_exceptions | |
remove_tag_prefix reemoveme | |
message log | |
languages all | |
multiline_flush_interval 0.1 | |
</match> | |
#This makes use of the ECS Metadata plugin to enrich records with ECS metadata | |
<filter **> | |
@type ecs_metadata | |
</filter> | |
#This match updates the tag to include all ECS metadata for routing | |
<match ecs.**> | |
@type record_reformer | |
tag ${record['ecs']} | |
</match> | |
#This copies the FluentD field log over to "message" field for the Kibana Discover page at Logzio | |
<match ecs.**> | |
@type record_reformer | |
renew_record false | |
enable_ruby false | |
tag logcopy | |
<record> | |
message ${log} | |
</record> | |
</match> | |
#This drops unnecessary keys (just log in this example, but could be extended to others) | |
<match logcopy> | |
@type record_reformer | |
remove_keys log | |
renew_record false | |
enable_ruby false | |
tag final | |
</match> | |
#This is for sending to a file for testing prior to sending to Logz.io | |
<match **> | |
@type logzio_buffered | |
endpoint_url "#{ENV['LOGZ_IO_URL_1']}" | |
output_include_time true | |
output_include_tags true | |
buffer_type file | |
buffer_path /tmp/logsz_buffer | |
flush_interval 1s | |
buffer_chunk_limit 1m # Logz.io has bulk limit of 10M. We recommend set this to 1M, to avoid oversized bulks | |
</match> | |
#This is for sending to a file for testing prior to sending to Logz.io | |
#<match **> | |
# @type file | |
# path /tmp/test0 | |
# time_slice_format %Y%m%d | |
# time_slice_wait 10m | |
# time_format %Y%m%dT%H%M%S%z | |
# utc | |
#</match> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{"method":"GET", | |
"path":"/api/customers/me", | |
"format":"json", | |
"controller":"CustomersController", | |
"action":"show", | |
"status":404, | |
"duration":8.640000000000001, | |
"view":0.19, | |
"db":2.41, | |
"login":null, | |
"portal":{"id":1,"name":"MyNorthTickets"}, | |
"org":null, | |
"client":"portal"} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment