Skip to content

Instantly share code, notes, and snippets.

@steveloughran
Last active December 16, 2020 18:20
Show Gist options
  • Save steveloughran/50ad1cf3d22d6683d4f2f038453be803 to your computer and use it in GitHub Desktop.
Save steveloughran/50ad1cf3d22d6683d4f2f038453be803 to your computer and use it in GitHub Desktop.
S3A IOStatistics Dump from an ITestS3AOperationCost run (remote)
Aggregate FileSystem Statistics
counters=((action_executor_acquired=68)
(action_executor_acquired.failures=0)
(action_http_get_request=4)
(action_http_get_request.failures=0)
(action_http_head_request=284)
(action_http_head_request.failures=0)
(committer_bytes_committed=0)
(committer_bytes_uploaded=0)
(committer_commit_job=0)
(committer_commit_job.failures=0)
(committer_commits.failures=0)
(committer_commits_aborted=0)
(committer_commits_completed=0)
(committer_commits_created=0)
(committer_commits_reverted=0)
(committer_jobs_completed=0)
(committer_jobs_failed=0)
(committer_magic_files_created=0)
(committer_materialize_file=0)
(committer_materialize_file.failures=0)
(committer_stage_file_upload=0)
(committer_stage_file_upload.failures=0)
(committer_tasks_completed=0)
(committer_tasks_failed=0)
(delegation_token_issued=0)
(delegation_token_issued.failures=0)
(directories_created=136)
(directories_deleted=34)
(fake_directories_created=0)
(fake_directories_deleted=143)
(files_copied=0)
(files_copied_bytes=0)
(files_created=68)
(files_delete_rejected=0)
(files_deleted=64)
(ignored_errors=0)
(multipart_instantiated=0)
(multipart_upload_abort_under_path_invoked=0)
(multipart_upload_aborted=0)
(multipart_upload_completed=0)
(multipart_upload_part_put=0)
(multipart_upload_part_put_bytes=0)
(multipart_upload_started=0)
(object_bulk_delete_request=141)
(object_bulk_delete_request.failures=0)
(object_continue_list_request=0)
(object_continue_list_request.failures=0)
(object_copy_requests=0)
(object_delete_objects=241)
(object_delete_request=34)
(object_delete_request.failures=0)
(object_list_request=496)
(object_list_request.failures=0)
(object_metadata_request=284)
(object_multipart_aborted=0)
(object_multipart_initiated=0)
(object_put_bytes=40960)
(object_put_request=204)
(object_put_request_completed=204)
(object_select_requests=0)
(op_copy_from_local_file=4)
(op_create=74)
(op_create_non_recursive=2)
(op_delete=100)
(op_exists=100)
(op_get_delegation_token=0)
(op_get_file_checksum=0)
(op_get_file_status=186)
(op_glob_status=6)
(op_is_directory=54)
(op_is_file=4)
(op_list_files=80)
(op_list_located_status=12)
(op_list_status=18)
(op_mkdirs=138)
(op_open=4)
(op_rename=0)
(s3guard_metadatastore_authoritative_directories_updated=33)
(s3guard_metadatastore_initialization=50)
(s3guard_metadatastore_put_path_request=99)
(s3guard_metadatastore_record_deletes=25)
(s3guard_metadatastore_record_reads=468)
(s3guard_metadatastore_record_writes=184)
(s3guard_metadatastore_retry=0)
(s3guard_metadatastore_throttled=0)
(store_io_request=0)
(store_io_retry=0)
(store_io_throttled=0)
(stream_aborted=0)
(stream_read_bytes=40960)
(stream_read_bytes_backwards_on_seek=0)
(stream_read_bytes_discarded_in_abort=0)
(stream_read_bytes_discarded_in_close=0)
(stream_read_close_operations=4)
(stream_read_closed=4)
(stream_read_exceptions=0)
(stream_read_fully_operations=0)
(stream_read_opened=4)
(stream_read_operations=4)
(stream_read_operations_incomplete=0)
(stream_read_seek_backward_operations=0)
(stream_read_seek_bytes_discarded=0)
(stream_read_seek_bytes_skipped=0)
(stream_read_seek_forward_operations=0)
(stream_read_seek_operations=0)
(stream_read_seek_policy_changed=4)
(stream_read_total_bytes=40960)
(stream_read_version_mismatches=0)
(stream_write_block_uploads=68)
(stream_write_block_uploads_aborted=0)
(stream_write_block_uploads_committed=0)
(stream_write_bytes=40960)
(stream_write_exceptions=0)
(stream_write_exceptions_completing_upload=0)
(stream_write_queue_duration=0)
(stream_write_queue_duration.failures=0)
(stream_write_total_data=81920)
(stream_write_total_time=0));
gauges=();
minimums=((action_executor_acquired.failures.min=-1)
(action_executor_acquired.min=0)
(action_http_get_request.failures.min=-1)
(action_http_get_request.min=22)
(action_http_head_request.failures.min=-1)
(action_http_head_request.min=15)
(committer_commit_job.failures.min=-1)
(committer_commit_job.min=-1)
(committer_materialize_file.failures.min=-1)
(committer_materialize_file.min=-1)
(committer_stage_file_upload.failures.min=-1)
(committer_stage_file_upload.min=-1)
(delegation_token_issued.failures.min=-1)
(delegation_token_issued.min=-1)
(object_bulk_delete_request.failures.min=-1)
(object_bulk_delete_request.min=32)
(object_continue_list_request.failures.min=-1)
(object_continue_list_request.min=-1)
(object_delete_request.failures.min=-1)
(object_delete_request.min=27)
(object_list_request.failures.min=-1)
(object_list_request.min=19)
(stream_write_queue_duration.failures.min=-1)
(stream_write_queue_duration.min=-1));
maximums=((action_executor_acquired.failures.max=-1)
(action_executor_acquired.max=0)
(action_http_get_request.failures.max=-1)
(action_http_get_request.max=63)
(action_http_head_request.failures.max=-1)
(action_http_head_request.max=528)
(committer_commit_job.failures.max=-1)
(committer_commit_job.max=-1)
(committer_materialize_file.failures.max=-1)
(committer_materialize_file.max=-1)
(committer_stage_file_upload.failures.max=-1)
(committer_stage_file_upload.max=-1)
(delegation_token_issued.failures.max=-1)
(delegation_token_issued.max=-1)
(object_bulk_delete_request.failures.max=-1)
(object_bulk_delete_request.max=194)
(object_continue_list_request.failures.max=-1)
(object_continue_list_request.max=-1)
(object_delete_request.failures.max=-1)
(object_delete_request.max=91)
(object_list_request.failures.max=-1)
(object_list_request.max=934)
(stream_write_queue_duration.failures.max=-1)
(stream_write_queue_duration.max=-1));
means=((action_executor_acquired.failures.mean=(samples=0, sum=0, mean=0.0000))
(action_executor_acquired.mean=(samples=68, sum=0, mean=0.0000))
(action_http_get_request.failures.mean=(samples=0, sum=0, mean=0.0000))
(action_http_get_request.mean=(samples=4, sum=140, mean=35.0000))
(action_http_head_request.failures.mean=(samples=0, sum=0, mean=0.0000))
(action_http_head_request.mean=(samples=284, sum=9782, mean=34.4437))
(committer_commit_job.failures.mean=(samples=0, sum=0, mean=0.0000))
(committer_commit_job.mean=(samples=0, sum=0, mean=0.0000))
(committer_materialize_file.failures.mean=(samples=0, sum=0, mean=0.0000))
(committer_materialize_file.mean=(samples=0, sum=0, mean=0.0000))
(committer_stage_file_upload.failures.mean=(samples=0, sum=0, mean=0.0000))
(committer_stage_file_upload.mean=(samples=0, sum=0, mean=0.0000))
(delegation_token_issued.failures.mean=(samples=0, sum=0, mean=0.0000))
(delegation_token_issued.mean=(samples=0, sum=0, mean=0.0000))
(object_bulk_delete_request.failures.mean=(samples=0, sum=0, mean=0.0000))
(object_bulk_delete_request.mean=(samples=141, sum=7931, mean=56.2482))
(object_continue_list_request.failures.mean=(samples=0, sum=0, mean=0.0000))
(object_continue_list_request.mean=(samples=0, sum=0, mean=0.0000))
(object_delete_request.failures.mean=(samples=0, sum=0, mean=0.0000))
(object_delete_request.mean=(samples=34, sum=1293, mean=38.0294))
(object_list_request.failures.mean=(samples=0, sum=0, mean=0.0000))
(object_list_request.mean=(samples=496, sum=145877, mean=294.1069))
(stream_write_queue_duration.failures.mean=(samples=0, sum=0, mean=0.0000))
(stream_write_queue_duration.mean=(samples=0, sum=0, mean=0.0000)));
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment