Skip to content

Instantly share code, notes, and snippets.

@RickardCardell
Created February 2, 2017 15:54
Show Gist options
  • Save RickardCardell/8f7c43b8a002627486892c745682c845 to your computer and use it in GitHub Desktop.
Save RickardCardell/8f7c43b8a002627486892c745682c845 to your computer and use it in GitHub Desktop.
Hadoop counters from Mapreduce Jobhistory ReST API
{
"jobCounters": {
"counterGroup": [
{
"counter": [
{
"mapCounterValue": 0,
"name": "FILE_BYTES_READ",
"reduceCounterValue": 292032,
"totalCounterValue": 292032
},
{
"mapCounterValue": 4115879,
"name": "FILE_BYTES_WRITTEN",
"reduceCounterValue": 1566021,
"totalCounterValue": 5681900
},
{
"mapCounterValue": 0,
"name": "FILE_READ_OPS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "FILE_LARGE_READ_OPS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "FILE_WRITE_OPS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 74803737,
"name": "HDFS_BYTES_READ",
"reduceCounterValue": 6318,
"totalCounterValue": 74810055
},
{
"mapCounterValue": 0,
"name": "HDFS_BYTES_WRITTEN",
"reduceCounterValue": 426518,
"totalCounterValue": 426518
},
{
"mapCounterValue": 36,
"name": "HDFS_READ_OPS",
{
"jobCounters": {
"counterGroup": [
{
"counter": [
{
"mapCounterValue": 0,
"name": "FILE_BYTES_READ",
"reduceCounterValue": 292032,
"totalCounterValue": 292032
},
{
"mapCounterValue": 4115879,
"name": "FILE_BYTES_WRITTEN",
"reduceCounterValue": 1566021,
"totalCounterValue": 5681900
},
{
"mapCounterValue": 0,
"name": "FILE_READ_OPS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "FILE_LARGE_READ_OPS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "FILE_WRITE_OPS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 74803737,
"name": "HDFS_BYTES_READ",
"reduceCounterValue": 6318,
"totalCounterValue": 74810055
},
{
"mapCounterValue": 0,
"name": "HDFS_BYTES_WRITTEN",
"reduceCounterValue": 426518,
"totalCounterValue": 426518
},
{
"mapCounterValue": 36,
"name": "HDFS_READ_OPS",
"reduceCounterValue": 6,
"totalCounterValue": 42
},
{
"mapCounterValue": 0,
"name": "HDFS_LARGE_READ_OPS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "HDFS_WRITE_OPS",
"reduceCounterValue": 4,
"totalCounterValue": 4
}
],
"counterGroupName": "org.apache.hadoop.mapreduce.FileSystemCounter"
},
{
"counter": [
{
"mapCounterValue": 0,
"name": "TOTAL_LAUNCHED_MAPS",
"reduceCounterValue": 0,
"totalCounterValue": 6
},
{
"mapCounterValue": 0,
"name": "TOTAL_LAUNCHED_REDUCES",
"reduceCounterValue": 0,
"totalCounterValue": 2
},
{
"mapCounterValue": 0,
"name": "RACK_LOCAL_MAPS",
"reduceCounterValue": 0,
"totalCounterValue": 6
},
{
"mapCounterValue": 0,
"name": "SLOTS_MILLIS_MAPS",
"reduceCounterValue": 0,
"totalCounterValue": 48946
},
{
"mapCounterValue": 0,
"name": "SLOTS_MILLIS_REDUCES",
"reduceCounterValue": 0,
"totalCounterValue": 14062
},
{
"mapCounterValue": 0,
"name": "MILLIS_MAPS",
"reduceCounterValue": 0,
"totalCounterValue": 48946
},
{
"mapCounterValue": 0,
"name": "MILLIS_REDUCES",
"reduceCounterValue": 0,
"totalCounterValue": 14062
},
{
"mapCounterValue": 0,
"name": "VCORES_MILLIS_MAPS",
"reduceCounterValue": 0,
"totalCounterValue": 48946
},
{
"mapCounterValue": 0,
"name": "VCORES_MILLIS_REDUCES",
"reduceCounterValue": 0,
"totalCounterValue": 14062
},
{
"mapCounterValue": 0,
"name": "MB_MILLIS_MAPS",
"reduceCounterValue": 0,
"totalCounterValue": 125301760
},
{
"mapCounterValue": 0,
"name": "MB_MILLIS_REDUCES",
"reduceCounterValue": 0,
"totalCounterValue": 35998720
}
],
"counterGroupName": "org.apache.hadoop.mapreduce.JobCounter"
},
{
"counter": [
{
"mapCounterValue": 17313,
"name": "MAP_INPUT_RECORDS",
"reduceCounterValue": 0,
"totalCounterValue": 17313
},
{
"mapCounterValue": 60,
"name": "MAP_OUTPUT_RECORDS",
"reduceCounterValue": 0,
"totalCounterValue": 60
},
{
"mapCounterValue": 1158641,
"name": "MAP_OUTPUT_BYTES",
"reduceCounterValue": 0,
"totalCounterValue": 1158641
},
{
"mapCounterValue": 296586,
"name": "MAP_OUTPUT_MATERIALIZED_BYTES",
"reduceCounterValue": 0,
"totalCounterValue": 296586
},
{
"mapCounterValue": 1482,
"name": "SPLIT_RAW_BYTES",
"reduceCounterValue": 0,
"totalCounterValue": 1482
},
{
"mapCounterValue": 0,
"name": "COMBINE_INPUT_RECORDS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "COMBINE_OUTPUT_RECORDS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "REDUCE_INPUT_GROUPS",
"reduceCounterValue": 60,
"totalCounterValue": 60
},
{
"mapCounterValue": 0,
"name": "REDUCE_SHUFFLE_BYTES",
"reduceCounterValue": 296586,
"totalCounterValue": 296586
},
{
"mapCounterValue": 0,
"name": "REDUCE_INPUT_RECORDS",
"reduceCounterValue": 26,
"totalCounterValue": 26
},
{
"mapCounterValue": 0,
"name": "REDUCE_OUTPUT_RECORDS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 60,
"name": "SPILLED_RECORDS",
"reduceCounterValue": 60,
"totalCounterValue": 120
},
{
"mapCounterValue": 0,
"name": "SHUFFLED_MAPS",
"reduceCounterValue": 12,
"totalCounterValue": 12
},
{
"mapCounterValue": 0,
"name": "FAILED_SHUFFLE",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "MERGED_MAP_OUTPUTS",
"reduceCounterValue": 12,
"totalCounterValue": 12
},
{
"mapCounterValue": 401,
"name": "GC_TIME_MILLIS",
"reduceCounterValue": 131,
"totalCounterValue": 532
},
{
"mapCounterValue": 18380,
"name": "CPU_MILLISECONDS",
"reduceCounterValue": 5010,
"totalCounterValue": 23390
},
{
"mapCounterValue": 8713932800,
"name": "PHYSICAL_MEMORY_BYTES",
"reduceCounterValue": 683040768,
"totalCounterValue": 9396973568
},
{
"mapCounterValue": 17884831744,
"name": "VIRTUAL_MEMORY_BYTES",
"reduceCounterValue": 6056280064,
"totalCounterValue": 23941111808
},
{
"mapCounterValue": 12508987392,
"name": "COMMITTED_HEAP_BYTES",
"reduceCounterValue": 4205838336,
"totalCounterValue": 16714825728
}
],
"counterGroupName": "org.apache.hadoop.mapreduce.TaskCounter"
},
{
"counter": [
{
"mapCounterValue": 0,
"name": "CREATED_FILES",
"reduceCounterValue": 2,
"totalCounterValue": 2
},
{
"mapCounterValue": 0,
"name": "DESERIALIZE_ERRORS",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 17313,
"name": "RECORDS_IN",
"reduceCounterValue": 0,
"totalCounterValue": 17313
},
{
"mapCounterValue": 0,
"name": "RECORDS_OUT_0",
"reduceCounterValue": 20,
"totalCounterValue": 20
},
{
"mapCounterValue": 60,
"name": "RECORDS_OUT_INTERMEDIATE",
"reduceCounterValue": 0,
"totalCounterValue": 60
}
],
"counterGroupName": "HIVE"
},
{
"counter": [
{
"mapCounterValue": 0,
"name": "BAD_ID",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "CONNECTION",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "IO_ERROR",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "WRONG_LENGTH",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "WRONG_MAP",
"reduceCounterValue": 0,
"totalCounterValue": 0
},
{
"mapCounterValue": 0,
"name": "WRONG_REDUCE",
"reduceCounterValue": 0,
"totalCounterValue": 0
}
],
"counterGroupName": "Shuffle Errors"
},
{
"counter": [
{
"mapCounterValue": 0,
"name": "BYTES_READ",
"reduceCounterValue": 0,
"totalCounterValue": 0
}
],
"counterGroupName": "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter"
},
{
"counter": [
{
"mapCounterValue": 0,
"name": "BYTES_WRITTEN",
"reduceCounterValue": 0,
"totalCounterValue": 0
}
],
"counterGroupName": "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter"
}
],
"id": "job_1485345402719_108769"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment