index-slack.yaml

# Run the application: benthos -c index-slack.yaml
# trigger: 
#    curl -X POST -H "Content-Type: application/json" localhost:14195/ping \
#         -d '{"token":"xoxp-xxxxxxxx", "channel":"xxxxx", "limit": 5}'`

input:
  http_server:
    address: "localhost:14195"
    path: /ping

pipeline:
  processors:
    - metadata: { key: "cursor",  operator: "set", value: "" }
    - while:
        at_least_once: true
        condition:
          not:
            metadata: { key: "cursor", operator: equals_cs, arg: "" }
        processors:
          - metadata: { key: "url",  operator: "set", value: "https://slack.com/api/conversations.history?token=${!json_field:token}&channel=${!json_field:channel}&inclusive=1&limit=${!json_field:limit}" }
          - log: { level: INFO, message: "cursor (before): ${!metadata:cursor}" }
          - log: { level: INFO, message: "${!metadata:url}" }
          - process_map:
              processors:
                - text: { operator: set, value: "" }
                - http: { request: { verb: GET, url: "${!metadata:url}" } }
              postmap:
                messages: messages
                nextCursor: response_metadata.next_cursor
          - metadata: { key: "cursor",  operator: "set", value: "${!json_field:nextCursor}" }
          - log: { level: INFO, message: "cursor (after): ${!metadata:cursor}" }

output:
  stdout: {}