Skip to content

Instantly share code, notes, and snippets.

@rushirajnenuji

rushirajnenuji/ES_Quereies_nov_2021.json

Last active November 16, 2021 22:13
Show Gist options
  • Save rushirajnenuji/f2b972c0b636a67b2b4f9d78a170b97f to your computer and use it in GitHub Desktop.
Save rushirajnenuji/f2b972c0b636a67b2b4f9d78a170b97f to your computer and use it in GitHub Desktop.
Download ZIP
Metrics Service faceted queries
Raw
ES_Quereies_nov_2021.json
# get event based on Entry ID
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"entryId": "669246"}
},
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:ESS_DIVE"}
}
]
}
},
"collapse": {
"field": "beat.hostname.keyword"
}
}
# get read event based on Entry ID
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"entryId": "669246"}
},
{
"term": {"event.key": "read"}
}
]
}
}
}
# get event based on Entry ID
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"entryId": "669246"}
},
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:ESS_DIVE"}
}
]
}
}
}
# get event based on Entry ID
GET /eventlog-*/_search
{
"_source": "entryId",
"query": {
"bool": {
"must": [
{
"term": {"entryId": "669246"}
},
{
"term": {"event.key": "read"}
}
]
}
},
"aggs": {
"pid_list": {
"composite": {
"sources": [
{
"nodeId": {
"terms": {
"field": "nodeId"
}
}
}
]
}
}
},
"collapse": {
"field": "host.keyword"
},
"size": 30
}
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:LTER"}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
],
"must_not": [
{
"terms": {
"tags": [
"ignore_ip",
"machine_ua",
"robot_ua",
"dataone_ip",
"robot_ip"
]
}
}
]
}
},
"aggs": {
"pid_list": {
"composite": {
"size": 100,
"sources": [
{
"event": {
"terms": {
"field": "formatType"
}
}
}
]
}
}
},
"size": 0
}
# repo stats by date
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:ESS_DIVE"}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
},
{
"range": {
"dateLogged": {
"gte": "2020-05-01||/M",
"lt": "2020-06-15||/M"
}
}
}
],
"must_not": [
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"tags": [
"ignore_ip",
"machine_ua",
"robot_ua",
"dataone_ip",
"robot_ip",
"d1_admin_subject"
]
}
}
]
}
},
"size": 0
}
# all events without sessionId
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
],
"must_not": [
{
"exists": {
"field": "sessionId"
}
}
]
}
},
"size": 0
}
### Dataset Landing Page Query
# facet: "month"
GET /eventlog-1/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"event.key": "read"
}
},
{
"terms": {
"pid.key": [
"doi:10.18739/A24746S61",
"doi:10.18739/A24746S61",
"urn:uuid:18bfc3ac-d66f-4eeb-9bd9-5b57d5c9098c",
"urn:uuid:9ef38bc7-581f-4cef-ab2e-b990e8613f34",
"urn:uuid:68ee9f36-13a9-4b4e-8a54-f457125f341a",
"urn:uuid:b641bcb9-ce06-48d1-8f7d-e3e05553e98b",
"urn:uuid:b34bfa41-bef5-4d9c-96b1-ae78821371fa",
"urn:uuid:b68d4391-0167-4349-9a60-13bf066830d8",
"urn:uuid:25e9dd3e-77e0-4460-b9a4-25452147718a",
"urn:uuid:fe975ac5-dc20-4d94-b6ac-583c05f34ac2",
"resource_map_doi:10.18739/A24746S61",
"resource_map_urn:uuid:8ece467c-77ed-42af-85dc-34aae17366d7",
"urn:uuid:8ece467c-77ed-42af-85dc-34aae17366d7"
]
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
]
}
}
}
# get entry type
GET /eventlog-*/_search
{
"_source": "entryId",
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
},
{
"exists": {
"field": "sessionId"
}
}
],
"must_not": [
{
"terms": {
"tags": [
"ignore_ip",
"machine_ua",
"robot_ua",
"dataone_ip",
"robot_ip",
"d1_admin_subject"
]
}
}
]
}
},
"size": 5
}
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"event.key": "read"
}
},
{
"term": {
"nodeId": "urn:node:ESS_DIVE"
}
},
{
"range": {
"dateLogged": {
"gte": "2020-05-01||/M",
"lt": "2020-06-01||/M"
}
}
}
],
"must_not": []
}
}
}
# get event based on Entry ID
GET /eventlog-*/_search
{
"_source": "entryId",
"query": {
"bool": {
"must": [
{
"term": {"entryId": "669246"}
},
{
"term": {"event.key": "read"}
}
]
}
},
"aggs": {
"seriesId": {
"terms": {
"field": "seriesId.keyword"
}
}
},
"size": 30
}
# repo query
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:ESS_DIVE"}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
],
"must_not": [
{
"terms": {
"tags": [
"ignore_ip",
"machine_ua",
"robot_ua",
"dataone_ip",
"robot_ip"
]
}
}
]
}
},
"aggs": {
"pid_list": {
"composite": {
"size": 100,
"sources": [
{
"format": {
"terms": {
"field": "formatType"
}
}
},
{
"month": {
"date_histogram": {
"field": "dateLogged",
"interval": "month"
}
}
}
]
}
}
},
"size": 0
}
# dedeups repo query
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:ESS_DIVE"}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
],
"must_not": [
{
"terms": {
"tags": [
"ignore_ip",
"machine_ua",
"robot_ua",
"dataone_ip",
"robot_ip"
]
}
}
]
}
},
"aggs": {
"pid_list": {
"composite": {
"size": 100,
"sources": [
{
"format": {
"terms": {
"field": "formatType"
}
}
},
{
"month": {
"date_histogram": {
"field": "dateLogged",
"interval": "month"
}
}
}
]
},
"aggs": {
"unique_doc_count": {
"cardinality": {
"field": "nodeId"
}
}
}
}
},
"size": 0
}
GET /eventlog-1/_mapping
GET /eventlog-1/_search
{
"query": {
"bool": {
"must": [
{
"exists": {
"field": "eventId"
}
}
]
}
}
}
GET /eventlog-1/_search
{
"query": {
"match_all": {}
}
}
# dedeups repo query
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:ARCTIC"}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
],
"must_not": [
{
"terms": {
"tags": [
"ignore_ip",
"machine_ua",
"robot_ua",
"dataone_ip",
"robot_ip",
"d1_admin_subject"
]
}
}
]
}
},
"aggs": {
"pid_list": {
"composite": {
"size": 100,
"sources": [
{
"format": {
"terms": {
"field": "formatType"
}
}
},
{
"month": {
"date_histogram": {
"field": "dateLogged",
"interval": "month"
}
}
}
]
},
"aggs": {
"unique_doc_count": {
"cardinality": {
"field": "eventId"
}
}
}
}
},
"size": 0
}
# deduped downloads without repeated entries
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:ARCTIC"}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA"
]
}
},
{
"range": {
"dateLogged": {
"gte": "2020-05-01||/M",
"lt": "2020-06-01||/M"
}
}
}
],
"must_not": [
{
"terms": {
"tags": [
"ignore_ip",
"machine_ua",
"robot_ua",
"dataone_ip",
"robot_ip",
"d1_admin_subject"
]
}
}
]
}
},
"aggs": {
"unique_doc_count": {
"cardinality": {
"field": "eventId"
}
}
},
"size": 0
}
# deduped downloads without repeated entries
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:ARCTIC"}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA"
]
}
},
{
"range": {
"dateLogged": {
"gte": "2020-05-01||/M",
"lt": "2020-06-01||/M"
}
}
}
],
"must_not": [
{
"terms": {
"tags": [
"ignore_ip",
"machine_ua",
"robot_ua",
"dataone_ip",
"robot_ip",
"d1_admin_subject"
]
}
}
]
}
},
"aggs": {
"ipAddress": {
"terms": {
"field": "ipAddress"
}
}
},
"size": 0
}
Raw
faceted_queries.json
GET _search
{
"query": {
"match_all": {}
}
}
# Beat name total counts
GET /eventlog-1/_search
{
"query": {
"bool": {
"must": [
],
"must_not": [
]
}
},
"aggs": {
"beatname": {
"composite": {
"sources": [
{
"beat": {
"terms": {
"field": "beat.name.keyword"
}
}
}
]
}
}
}
}
# aggregated nodeId stats for a time range
GET /eventlog-1/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"event.key": "read"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
},
{
"range": {
"dateLogged": {
"gte": "2020-05-01||/M",
"lt": "2020-06-15||/M"
}
}
},
{
"exists": {
"field": "sessionId"
}
}
],
"must_not": [
]
}
},
"aggs": {
"pid_list": {
"composite": {
"size": 100,
"sources": [
{
"nodeId": {
"terms": {
"field": "nodeId"
}
}
}
]
}
}
}
}
# check if session id exists
GET /eventlog-1/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"event.key": "read"
}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
]
}
}
}
# check events without session id
GET /eventlog-1/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"event.key": "read"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
],
"must_not": [
{
"exists": {
"field": "sessionId"
}
}
]
}
}
}
### Dataset Landing Page Query
# facet: "month"
GET /eventlog-1/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"event.key": "read"
}
},
{
"terms": {
"pid.key": [
"doi:10.5063/F17H1GS8",
"urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0"
]
}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
]
}
},
"aggs": {
"package_pid_list": {
"aggs": {
"urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0"
}
}
}
}
},
"doi:10.5063/F17H1GS8": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "doi:10.5063/F17H1GS8"
}
}
}
}
}
},
"composite": {
"sources": [
{
"format": {
"terms": {
"field": "formatType"
}
}
}
]
}
},
"pid_list": {
"aggs": {
"urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0"
}
}
}
}
},
"doi:10.5063/F17H1GS8": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "doi:10.5063/F17H1GS8"
}
}
}
}
}
},
"composite": {
"sources": [
{
"country": {
"terms": {
"field": "geoip.country_code2.keyword",
"missing_bucket": "true"
}
}
},
{
"format": {
"terms": {
"field": "formatType"
}
}
},
{
"month": {
"date_histogram": {
"interval": "month",
"field": "dateLogged"
}
}
}
],
"size": 100
}
}
},
"size": 0
}
### Dataset Landing Page Query
# facet: year
GET /eventlog-1/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"event.key": "read"
}
},
{
"terms": {
"pid.key": [
"doi:10.5063/F17H1GS8",
"urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0"
]
}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
]
}
},
"aggs": {
"package_pid_list": {
"aggs": {
"urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0"
}
}
}
}
},
"doi:10.5063/F17H1GS8": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "doi:10.5063/F17H1GS8"
}
}
}
}
}
},
"composite": {
"sources": [
{
"format": {
"terms": {
"field": "formatType"
}
}
}
]
}
},
"pid_list": {
"aggs": {
"urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0"
}
}
}
}
},
"doi:10.5063/F17H1GS8": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "doi:10.5063/F17H1GS8"
}
}
}
}
}
},
"composite": {
"sources": [
{
"country": {
"terms": {
"field": "geoip.country_code2.keyword",
"missing_bucket": "true"
}
}
},
{
"format": {
"terms": {
"field": "formatType"
}
}
},
{
"month": {
"date_histogram": {
"interval": "year",
"field": "dateLogged"
}
}
}
],
"size": 100
}
}
},
"size": 0
}
### Dataset Landing Page Query
# facet: year
GET /eventlog-1/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"event.key": "read"
}
},
{
"terms": {
"pid.key": [
"doi:10.5063/F17H1GS8",
"urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0"
]
}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
]
}
},
"aggs": {
"package_pid_list": {
"aggs": {
"urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0"
}
}
}
}
},
"doi:10.5063/F17H1GS8": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "doi:10.5063/F17H1GS8"
}
}
}
}
}
},
"composite": {
"sources": [
{
"format": {
"terms": {
"field": "formatType"
}
}
}
]
}
},
"pid_list": {
"aggs": {
"urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "urn:uuid:dc4f0e09-e085-4e46-bddd-e86ab875f0d0"
}
}
}
}
},
"doi:10.5063/F17H1GS8": {
"filters": {
"filters": {
"pid.key": {
"term": {
"pid.key": "doi:10.5063/F17H1GS8"
}
}
}
}
}
},
"composite": {
"sources": [
{
"country": {
"terms": {
"field": "geoip.country_code2.keyword",
"missing_bucket": "true"
}
}
},
{
"format": {
"terms": {
"field": "formatType"
}
}
},
{
"month": {
"date_histogram": {
"interval": "year",
"field": "dateLogged"
}
}
}
],
"size": 100
}
}
},
"size": 0
}
#### Repository page query
# facet: month
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:LTER"}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
],
"must_not": [
{
"terms": {
"tags": [
"ignore_ip",
"machine_ua",
"robot_ua",
"dataone_ip",
"robot_ip"
]
}
}
]
}
},
"aggs": {
"pid_list": {
"composite": {
"size": 100,
"sources": [
{
"format": {
"terms": {
"field": "formatType"
}
}
},
{
"month": {
"date_histogram": {
"field": "dateLogged",
"interval": "month"
}
}
}
]
}
}
},
"size": 0
}
#### Repository page query
# facet: year
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:LTER"}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
]
}
},
"aggs": {
"pid_list": {
"composite": {
"size": 100,
"sources": [
{
"format": {
"terms": {
"field": "formatType"
}
}
},
{
"month": {
"date_histogram": {
"field": "dateLogged",
"interval": "year"
}
}
}
]
}
}
},
"size": 0
}
#### Repository page query
# facet: location
GET /eventlog-*/_search
{
"query": {
"bool": {
"must": [
{
"term": {"event.key": "read"}
},
{
"term": {"nodeId": "urn:node:LTER"}
},
{
"exists": {
"field": "sessionId"
}
},
{
"terms": {
"formatType": [
"DATA",
"METADATA"
]
}
}
]
}
},
"aggs": {
"pid_list": {
"composite": {
"size": 100,
"sources": [
{
"format": {
"terms": {
"field": "formatType"
}
}
},
{
"country": {
"terms": {
"field": "geoip.country_code2.keyword",
"missing_bucket": "true"
}
}
}
]
}
}
},
"size": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment