Created
February 2, 2017 15:54
-
-
Save RickardCardell/8f7c43b8a002627486892c745682c845 to your computer and use it in GitHub Desktop.
Hadoop counters from Mapreduce Jobhistory ReST API
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "jobCounters": { | |
| "counterGroup": [ | |
| { | |
| "counter": [ | |
| { | |
| "mapCounterValue": 0, | |
| "name": "FILE_BYTES_READ", | |
| "reduceCounterValue": 292032, | |
| "totalCounterValue": 292032 | |
| }, | |
| { | |
| "mapCounterValue": 4115879, | |
| "name": "FILE_BYTES_WRITTEN", | |
| "reduceCounterValue": 1566021, | |
| "totalCounterValue": 5681900 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "FILE_READ_OPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "FILE_LARGE_READ_OPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "FILE_WRITE_OPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 74803737, | |
| "name": "HDFS_BYTES_READ", | |
| "reduceCounterValue": 6318, | |
| "totalCounterValue": 74810055 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "HDFS_BYTES_WRITTEN", | |
| "reduceCounterValue": 426518, | |
| "totalCounterValue": 426518 | |
| }, | |
| { | |
| "mapCounterValue": 36, | |
| "name": "HDFS_READ_OPS", | |
| { | |
| "jobCounters": { | |
| "counterGroup": [ | |
| { | |
| "counter": [ | |
| { | |
| "mapCounterValue": 0, | |
| "name": "FILE_BYTES_READ", | |
| "reduceCounterValue": 292032, | |
| "totalCounterValue": 292032 | |
| }, | |
| { | |
| "mapCounterValue": 4115879, | |
| "name": "FILE_BYTES_WRITTEN", | |
| "reduceCounterValue": 1566021, | |
| "totalCounterValue": 5681900 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "FILE_READ_OPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "FILE_LARGE_READ_OPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "FILE_WRITE_OPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 74803737, | |
| "name": "HDFS_BYTES_READ", | |
| "reduceCounterValue": 6318, | |
| "totalCounterValue": 74810055 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "HDFS_BYTES_WRITTEN", | |
| "reduceCounterValue": 426518, | |
| "totalCounterValue": 426518 | |
| }, | |
| { | |
| "mapCounterValue": 36, | |
| "name": "HDFS_READ_OPS", | |
| "reduceCounterValue": 6, | |
| "totalCounterValue": 42 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "HDFS_LARGE_READ_OPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "HDFS_WRITE_OPS", | |
| "reduceCounterValue": 4, | |
| "totalCounterValue": 4 | |
| } | |
| ], | |
| "counterGroupName": "org.apache.hadoop.mapreduce.FileSystemCounter" | |
| }, | |
| { | |
| "counter": [ | |
| { | |
| "mapCounterValue": 0, | |
| "name": "TOTAL_LAUNCHED_MAPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 6 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "TOTAL_LAUNCHED_REDUCES", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 2 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "RACK_LOCAL_MAPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 6 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "SLOTS_MILLIS_MAPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 48946 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "SLOTS_MILLIS_REDUCES", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 14062 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "MILLIS_MAPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 48946 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "MILLIS_REDUCES", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 14062 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "VCORES_MILLIS_MAPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 48946 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "VCORES_MILLIS_REDUCES", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 14062 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "MB_MILLIS_MAPS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 125301760 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "MB_MILLIS_REDUCES", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 35998720 | |
| } | |
| ], | |
| "counterGroupName": "org.apache.hadoop.mapreduce.JobCounter" | |
| }, | |
| { | |
| "counter": [ | |
| { | |
| "mapCounterValue": 17313, | |
| "name": "MAP_INPUT_RECORDS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 17313 | |
| }, | |
| { | |
| "mapCounterValue": 60, | |
| "name": "MAP_OUTPUT_RECORDS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 60 | |
| }, | |
| { | |
| "mapCounterValue": 1158641, | |
| "name": "MAP_OUTPUT_BYTES", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 1158641 | |
| }, | |
| { | |
| "mapCounterValue": 296586, | |
| "name": "MAP_OUTPUT_MATERIALIZED_BYTES", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 296586 | |
| }, | |
| { | |
| "mapCounterValue": 1482, | |
| "name": "SPLIT_RAW_BYTES", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 1482 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "COMBINE_INPUT_RECORDS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "COMBINE_OUTPUT_RECORDS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "REDUCE_INPUT_GROUPS", | |
| "reduceCounterValue": 60, | |
| "totalCounterValue": 60 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "REDUCE_SHUFFLE_BYTES", | |
| "reduceCounterValue": 296586, | |
| "totalCounterValue": 296586 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "REDUCE_INPUT_RECORDS", | |
| "reduceCounterValue": 26, | |
| "totalCounterValue": 26 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "REDUCE_OUTPUT_RECORDS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 60, | |
| "name": "SPILLED_RECORDS", | |
| "reduceCounterValue": 60, | |
| "totalCounterValue": 120 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "SHUFFLED_MAPS", | |
| "reduceCounterValue": 12, | |
| "totalCounterValue": 12 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "FAILED_SHUFFLE", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "MERGED_MAP_OUTPUTS", | |
| "reduceCounterValue": 12, | |
| "totalCounterValue": 12 | |
| }, | |
| { | |
| "mapCounterValue": 401, | |
| "name": "GC_TIME_MILLIS", | |
| "reduceCounterValue": 131, | |
| "totalCounterValue": 532 | |
| }, | |
| { | |
| "mapCounterValue": 18380, | |
| "name": "CPU_MILLISECONDS", | |
| "reduceCounterValue": 5010, | |
| "totalCounterValue": 23390 | |
| }, | |
| { | |
| "mapCounterValue": 8713932800, | |
| "name": "PHYSICAL_MEMORY_BYTES", | |
| "reduceCounterValue": 683040768, | |
| "totalCounterValue": 9396973568 | |
| }, | |
| { | |
| "mapCounterValue": 17884831744, | |
| "name": "VIRTUAL_MEMORY_BYTES", | |
| "reduceCounterValue": 6056280064, | |
| "totalCounterValue": 23941111808 | |
| }, | |
| { | |
| "mapCounterValue": 12508987392, | |
| "name": "COMMITTED_HEAP_BYTES", | |
| "reduceCounterValue": 4205838336, | |
| "totalCounterValue": 16714825728 | |
| } | |
| ], | |
| "counterGroupName": "org.apache.hadoop.mapreduce.TaskCounter" | |
| }, | |
| { | |
| "counter": [ | |
| { | |
| "mapCounterValue": 0, | |
| "name": "CREATED_FILES", | |
| "reduceCounterValue": 2, | |
| "totalCounterValue": 2 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "DESERIALIZE_ERRORS", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 17313, | |
| "name": "RECORDS_IN", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 17313 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "RECORDS_OUT_0", | |
| "reduceCounterValue": 20, | |
| "totalCounterValue": 20 | |
| }, | |
| { | |
| "mapCounterValue": 60, | |
| "name": "RECORDS_OUT_INTERMEDIATE", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 60 | |
| } | |
| ], | |
| "counterGroupName": "HIVE" | |
| }, | |
| { | |
| "counter": [ | |
| { | |
| "mapCounterValue": 0, | |
| "name": "BAD_ID", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "CONNECTION", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "IO_ERROR", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "WRONG_LENGTH", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "WRONG_MAP", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| }, | |
| { | |
| "mapCounterValue": 0, | |
| "name": "WRONG_REDUCE", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| } | |
| ], | |
| "counterGroupName": "Shuffle Errors" | |
| }, | |
| { | |
| "counter": [ | |
| { | |
| "mapCounterValue": 0, | |
| "name": "BYTES_READ", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| } | |
| ], | |
| "counterGroupName": "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter" | |
| }, | |
| { | |
| "counter": [ | |
| { | |
| "mapCounterValue": 0, | |
| "name": "BYTES_WRITTEN", | |
| "reduceCounterValue": 0, | |
| "totalCounterValue": 0 | |
| } | |
| ], | |
| "counterGroupName": "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter" | |
| } | |
| ], | |
| "id": "job_1485345402719_108769" | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment