helpful actions to support anomalous behavior by geo area

es-commands.es

###
# transform definition
###
 {
      "id" : "count_by_grid_15",
      "source" : {
        "index" : [
          "gtfs*"
        ],
        "query" : {
          "bool" : {
            "filter" : {
              "geo_bounding_box" : {
                "pos" : {
                  "top_left" : {
                    "lat" : 29.863613,
                    "lon" : -99.214393
                  },
                  "bottom_right" : {
                    "lat" : 29.021844,
                    "lon" : -97.985532
                  }
                }
              }
            }
          }
        }
      },
      "dest" : {
        "index" : "count_by_grid_15"
      },
      "pivot" : {
        "group_by" : {
          "hour" : {
            "date_histogram" : {
              "field" : "@timestamp",
              "calendar_interval" : "1h"
            }
          },
          "tile" : {
            "geotile_grid" : {
              "field" : "pos",
              "precision" : 15
            }
          }
        },
        "aggregations" : {
          "@timestamp" : {
            "max" : {
              "field" : "@timestamp"
            }
          },
          "pos" : {
            "geo_centroid" : { // How can we calculate the actual center of the tile????
              "field" : "pos"  // We need the true center as the centroid moves slightly in the tile (as points move)
            }                  // This is so ML has a clean "by" or "partition" field value
          },
          "count" : {
            "value_count" : { // we need the count of point values to better calculate as this is "pre-aggregated data"
              "field" : "pos"
            }
          }
        }
      },
      "settings" : { },
      "version" : "8.0.0",
      "create_time" : 1593117191529
    }

###
# Need to script out the center/corner of the tile somehow
###

#After running anomaly detection by "corner/tile" here are mapping updates:

PUT .ml-anomalies-.write-grid-count-15-by-tile-corner/_mapping
{
  "properties": {
    "geo_results.influencers.influencer_field_values": {
      "type": "geo_point"
    },
    "geo_results.influencers.influencer_field_name": {
      "type": "keyword"
    },
    "geo_results.tile_corner": {
      "type": "geo_point"
    },
    "geo_results.by_field_value": {
      "type": "by_field_value"
    }
  }
}

POST .ml-anomalies-.write-grid-count-15-by-tile-corner/_update_by_query
{
  "query": {
    "bool": {
      "filter": [
        {"term": {
          "result_type": "record"
        }}
      ]
    }
  },
  "script": {
    "source": """
    if(ctx._source.geo_results == null) {
      ctx._source.geo_results = new HashMap();
    }
    ctx._source.geo_results.tile_corner = ctx._source.tile_corner;
    ctx._source.geo_results.by_field_value = ctx._source.by_field_value;
    ctx._source.geo_results.influencers = ctx._source.influencers;
    """
  }
}