api-gateway_pipeline.yaml

id: apigateway
metric_id: amazon-api-gateway
facets:
  -
    name: Status Code
    source: log
    path: http.status_code
    groups:
      - Web Access
  -
    name: Method
    source: log
    path: http.method
    groups:
      - Web Access
  -
    name: Client IP
    source: log
    path: network.client.ip
    groups:
      - Web Access
  -
    name: URL Path
    source: log
    path: http.url_details.path
    groups:
      - Web Access
  -
    name: Log Group
    source: log
    path: aws.awslogs.logGroup
    groups:
      - AWS
  -
    name: Request ID
    source: log
    path: http.request_id
    groups:
      - Web Access
pipeline:
  type: pipeline
  name: AWS Api Gateway
  enabled: true
  filter:
    query: 'source:apigateway'
  processors:
    -
      type: grok-parser
      name: Parsing Api Gateway logs
      enabled: true
      source: message
      samples:
        - '127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 123432135'
        - '127.0.0.1,-,frank,10/Oct/2000:13:55:36 -0700,GET,/apache_pb.gif,HTTP/1.0,200,2326,123432135'
      grok:
        supportRules: |
          _auth %{notSpace:http.auth:nullIf("-")}
          _bytes_written %{integer:network.bytes_written}
          _client_ip %{ipOrHost:network.client.ip}
          _version HTTP\/%{regex("\\d+\\.\\d+"):http.version}
          _url %{notSpace:http.url}
          _ident %{notSpace:http.ident:nullIf("-")}
          _user_agent %{regex("[^\\\"]*"):http.useragent}
          _referer %{notSpace:http.referer}
          _status_code %{integer:http.status_code}
          _method %{word:http.method}
          _date_access %{date("dd/MMM/yyyy:HH:mm:ss Z"):date_access}
          _x_forwarded_for %{regex("[^\\\"]*"):http._x_forwarded_for:nullIf("-")}
        matchRules: |
          access.common %{_client_ip} %{_ident} %{_auth} \[%{_date_access}\] "(?>%{_method} |)%{_url}(?> %{_version}|)" %{_status_code} (?>%{_bytes_written}|-) %{notSpace:http.request_id}
          Csv_format %{_client_ip},%{_ident},%{_auth},%{_date_access},%{_method},%{_url},(?>%{_version}|),%{_status_code},%{_bytes_written},%{notSpace:http.request_id}
    -
       type: attribute-remapper
       name: 'Remap requestId'
       enabled: true
       sources:
         - requestId
       target: 'http.request_id'
       preserveSource: false
       overrideOnConflict: false
    -
      type: attribute-remapper
      name: 'Remap ip'
      enabled: true
      sources:
        - ip
      target: 'network.client.ip'
      preserveSource: false
      overrideOnConflict: false
    -
      type: attribute-remapper
      name: 'Remap caller'
      enabled: true
      sources:
        - caller
      target: 'http.ident'
      preserveSource: false
      overrideOnConflict: false
    -
      type: attribute-remapper
      name: 'Remap user'
      enabled: true
      sources:
        - user
      target: 'http.auth'
      preserveSource: false
      overrideOnConflict: false
    -
      type: attribute-remapper
      name: 'Remap requestTime'
      enabled: true
      sources:
        - requestTime
      target: 'date_access'
      preserveSource: false
      overrideOnConflict: false
    -
      type: attribute-remapper
      name: 'Remap httpMethod'
      enabled: true
      sources:
        - httpMethod
      target: 'http.method'
      preserveSource: false
      overrideOnConflict: false
    -
      type: attribute-remapper
      name: 'Remap resourcePath'
      enabled: true
      sources:
        - resourcePath
      target: 'http.url'
      preserveSource: false
      overrideOnConflict: false
    -
      type: attribute-remapper
      name: 'Remap protocol'
      enabled: true
      sources:
        - protocol
      target: 'http.version'
      preserveSource: false
      overrideOnConflict: false
    -
      type: attribute-remapper
      name: 'Remap status'
      enabled: true
      sources:
        - status
      target: 'http.status_code'
      preserveSource: false
      overrideOnConflict: false
    -
      type: attribute-remapper
      name: 'Remap responseLength'
      enabled: true
      sources:
        - responseLength
      target: 'network.bytes_written'
      preserveSource: false
      overrideOnConflict: false
    -
      type: url-parser
      name: ''
      enabled: true
      sources:
        - http.url
      target: http.url_details
    -
      type: date-remapper
      name: Define Date_access as the official timestamp of the log
      enabled: true
      sources:
        - date_access
    -
      type: category-processor
      name: Categorise status code
      enabled: true
      categories:
        - filter:
            query: '@http.status_code:[200 TO 299]'
          name: OK
        - filter:
            query: '@http.status_code:[300 TO 399]'
          name: notice
        - filter:
            query: '@http.status_code:[400 TO 499]'
          name: warning
        - filter:
            query: '@http.status_code:[500 TO 599]'
          name: error
      target: http.status_category
    -
      type: status-remapper
      name: Set the log status based on the status code value
      enabled: true
      sources:
        - http.status_category