sjakthol

import logging

import boto3

def _log_botocore_errors(*_args, **kwargs):
    """Handler for botocore needs-retry events."""
    # Arg parsing as per
    # https://github.com/boto/botocore/blob/6451ae1fad57f4453af97649e7ed9192b0f623be/botocore/retries/standard.py#L106
    attempt_number = kwargs["attempts"]
    operation_model = kwargs["operation"]

sjakthol

2019-05-04 10:20:07,160 INFO node_provider.py:34 -- ClusterState: Loaded cluster state: {'YOUR_HEAD_NODE_HOSTNAME': {'tags': {'ray-node-type': 'head'}, 'state': 'terminated'}}
2019-05-04 10:20:07,161 INFO node_provider.py:59 -- ClusterState: Writing cluster state: {'YOUR_HEAD_NODE_HOSTNAME': {'tags': {'ray-node-type': 'head'}, 'state': 'terminated'}}
2019-05-04 10:20:07,162 INFO autoscaler.py:380 -- StandardAutoscaler: {'cluster_name': 'default', 'min_workers': 0, 'max_workers': 1, 'initial_workers': 0, 'target_utilization_fraction': 0.8, 'idle_timeout_minutes': 5, 'provider': {'type': 'local', 'head_ip': 'YOUR_HEAD_NODE_HOSTNAME', 'worker_ips': []}, 'auth': {'ssh_user': 'YOUR_USERNAME', 'ssh_private_key': '~/.ssh/id_rsa'}, 'head_node': {}, 'worker_nodes': {}, 'file_mounts': {}, 'setup_commands': [], 'head_setup_commands': [], 'worker_setup_commands': [], 'head_start_ray_commands': [], 'worker_start_ray_commands': []}
2019-05-04 10:20:07,172 INFO autoscaler.py:630 -- StandardAutoscaler: 0/0 target nodes (0 pe

sjakthol

#!/usr/bin/env python3
"""List S3 bucket objects in parallel.

This module contains a parallel implementation of S3 list_objects_v2()
API call. The implementation lists objects under every distinct prefix
in parallel. Hence, a speedup is achieved if objects are spread under
multiple distinct prefixes.
"""

import argparse