s-fujimoto

#!/usr/bin/env python

import twitter
import boto3
import os

keyword = '#reinvent'
lang = 'en'
region = 'us-east-1'
size = 100 * 100

s-fujimoto

#!/usr/bin/env python

import twitter
import boto3
import os

keyword = 'dev.classmethod.jp'
region = 'us-east-1'
size = 10 * 100

s-fujimoto

##################################################
### Elasticsearch host name
ES_HOST = "search-******************.ap-northeast-1.es.amazonaws.com"

### Elasticsearch prefix for index name
INDEX_PREFIX = "awslogs"

#################################################
### ELB access log format keys
ELB_KEYS = ["timestamp", "elb", "client_ip", "client_port", "backend_ip", "backend_port", "request_processing_time", "backend_processing_time", "response_processing_time", "elb_status_code", "backend_status_code", "received_bytes", "sent_bytes", "request_method", "request_url", "request_version", "user_agent"]

s-fujimoto

from elasticsearch.connection.http_requests import RequestsHttpConnection
from botocore.awsrequest import AWSRequest
from botocore.auth import SigV4Auth
from botocore.credentials import Credentials
from botocore.utils import InstanceMetadataFetcher
from elasticsearch.compat import urlencode
import os
try:
    import requests
    REQUESTS_AVAILABLE = True

s-fujimoto

input {
  s3 {
    bucket => "<CLOUDFRONT_LOG_BUCKET>"
    prefix => "<CLOUDFRONT_LOG_KEY_PREFIX>"
    region => "<BUCKET_REGION_NAME>"
  }
}


filter {