This Generic Extractor (https://developers.keboola.com/extend/generic-extractor/) configuration for Elastic (https://www.elastic.co/) enables you to fetch data using Request Body Search (https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html). It also uses Scroll (https://www.elastic.co/guide/en/elasticsearch/re…

config.json

{
    "parameters": {
        "api": {
            "authentication": {
                "type": "basic"
            },
            "baseUrl": "https://yourElastisearchHost.com",
            "pagination": {
                "method": "response.param",
                "responseParam": "_scroll_id",
                "queryParam": "scroll_id",
                "scrollRequest": {
                    "endpoint": "_search/scroll",
                    "method": "GET",
                    "params": {
                        "scroll": "1m"
                    }
                }
            }
        },
        "config": {
            "outputBucket": "sapi-elasticsearch",
            "debug": true,
            "username": "YOUR_ELASTICSEARCH_USERNAME",
            "#password": "YOUR_ELASTICSEARCH_PASSWORD",
            "jobs": [{
                "endpoint": "/kbc-files-*/_search?search_type=scan&scroll=1m",
                "params": {
                    "query": {
                        "filtered": {
                            "query": {
                                "match_all": []
                            },
                            "filter": {
                                "bool": {
                                    "must": [{
                                        "range": {
                                            "created": {
                                                "from": "now-48h"
                                            }
                                        }
                                    }]
                                }
                            }
                        }
                    },
                    "partial_fields": {
                        "file": {
                            "include": [
                                "id",
                                "projectId",
                                "token.id",
                                "token.name",
                                "s3Path",
                                "name",
                                "uploadType",
                                "created",
                                "runId",
                                "isSliced",
                                "sizeBytes",
                                "isPublic",
                                "isEncrypted",
                                "tags",
                                "maxAgeDays",
                                "export.tableId"
                            ]
                        }
                    },
                    "size": 10000
                },
                "dataField": "hits.hits",
                "dataType": "files",
                "method": "POST"
            }]
        }
    }
}