AWS Step Function: Download full MP4 mixer.com recordings. Periodically check a queue for new recordings, then multi-part upload three parts in parallel to S3.

mixer-download-step-function.json

{
  "StartAt": "PollQueue",
  "States": {
    "PollQueue": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_poll_queue",
      "Next": "CheckForWork"
    },
    "CheckForWork": {
      "Type": "Choice",
      "Choices": [
        {
          "Variable": "$.id",
          "NumericGreaterThan": -1,
          "Next": "GetRecordingSize"
        }
      ],
      "Default": "EndWithSuccess"
    },
    "GetRecordingSize": {
      "Type": "Task",
      "Resource": "arn:aws:XXX:XXX:function:sfn_get_recording_size",
      "Next": "CheckMaximumFileSize"
    },
    "CheckMaximumFileSize": {
      "Type": "Choice",
      "Choices": [
        {
          "And": [
            {
              "Variable": "$.size",
              "NumericGreaterThan": 0
            },
            {
              "Variable": "$.size",
              "NumericLessThanEquals": 8589934592
            }
          ],
          "Next": "InitiateMultipartUpload"
        },
        {
          "Variable": "$.size",
          "NumericLessThanEquals": -1,
          "Next": "IgnoreRecording"
        }
      ],
      "Default": "EndWithFailure"
    },
    "InitiateMultipartUpload": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_initiate_multipart_upload",
      "Next": "IsUploadFinished"
    },
    "IsUploadFinished": {
      "Type": "Choice",
      "Choices": [
        {
          "Variable": "$.is_completed",
          "BooleanEquals": false,
          "Next": "UploadLoop"
        }
      ],
      "Default": "CompleteMultipartUpload"
    },
    "UploadLoop": {
      "Type": "Parallel",
      "Next": "MergePartialUploads",
      "Branches": [
        {
          "StartAt": "First",
          "States": {
            "First": {
              "Type": "Task",
              "Parameters": {
                "number": 1,
                "url.$": "$.url",
                "id.$": "$.id",
                "part_base_no.$": "$.part_base_no",
                "upload_id.$": "$.upload_id",
                "size.$": "$.size",
                "is_completed.$": "$.is_completed",
                "completed_parts.$": "$.completed_parts",
                "message_id.$": "$.message_id",
                "message_receipt_handle.$": "$.message_receipt_handle",
                "key.$": "$.key"
              },
              "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_upload_part",
              "End": true
            }
          }
        },
        {
          "StartAt": "Second",
          "States": {
            "Second": {
              "Type": "Task",
              "Parameters": {
                "number": 2,
                "url.$": "$.url",
                "id.$": "$.id",
                "part_base_no.$": "$.part_base_no",
                "upload_id.$": "$.upload_id",
                "size.$": "$.size",
                "is_completed.$": "$.is_completed",
                "completed_parts.$": "$.completed_parts",
                "message_id.$": "$.message_id",
                "message_receipt_handle.$": "$.message_receipt_handle",
                "key.$": "$.key"
              },
              "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_upload_part",
              "End": true
            }
          }
        },
        {
          "StartAt": "Third",
          "States": {
            "Third": {
              "Type": "Task",
              "Parameters": {
                "number": 3,
                "url.$": "$.url",
                "id.$": "$.id",
                "part_base_no.$": "$.part_base_no",
                "upload_id.$": "$.upload_id",
                "size.$": "$.size",
                "is_completed.$": "$.is_completed",
                "completed_parts.$": "$.completed_parts",
                "message_id.$": "$.message_id",
                "message_receipt_handle.$": "$.message_receipt_handle",
                "key.$": "$.key"
              },
              "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_upload_part",
              "End": true
            }
          }
        }
      ],
      "Catch": [
        {
          "ErrorEquals": [
            "States.ALL"
          ],
          "Next": "AbortMultipartUpload",
          "ResultPath": "$.error"
        }
      ]
    },
    "MergePartialUploads": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_merge_partial_uploads",
      "Next": "IsUploadFinished"
    },
    "CompleteMultipartUpload": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_complete_upload",
      "Next": "FinalizeRecording"
    },
    "FinalizeRecording": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_finalize_recording",
      "Next": "EndWithSuccess"
    },
    "IgnoreRecording": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_ignore_recording",
      "Next": "EndWithSuccess"
    },
    "AbortMultipartUpload": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:XXX:XXX:function:sfn_abort_upload",
      "Next": "EndWithFailure"
    },
    "EndWithFailure": {
      "Type": "Fail"
    },
    "EndWithSuccess": {
      "Type": "Succeed"
    }
  }
}