Skip to content

Instantly share code, notes, and snippets.

@dexhunter

dexhunter/10g-nccl-fast.json

Last active July 1, 2021 07:42
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dexhunter/bd1db106ba1c199cfdb65ea61667682b to your computer and use it in GitHub Desktop.
Save dexhunter/bd1db106ba1c199cfdb65ea61667682b to your computer and use it in GitHub Desktop.
Download ZIP
horovod timeline json
Raw
10g-nccl-fast.json
[
{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros":1624937181806323}},
{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}},
{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "allreduce.noname.1"}},
{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 3399382, "pid": 1},{"ph": "X", "name": "8", "ts": 3399384, "pid": 1, "dur": 0},{"ph": "X", "name": "9", "ts": 3399390, "pid": 1, "dur": 0},{"ph": "X", "name": "10", "ts": 3399392, "pid": 1, "dur": 0},{"ph": "X", "name": "11", "ts": 3399393, "pid": 1, "dur": 0},{"ph": "X", "name": "5", "ts": 7133998, "pid": 1, "dur": 0},{"ph": "X", "name": "4", "ts": 7207126, "pid": 1, "dur": 0},{"ph": "X", "name": "15", "ts": 7239969, "pid": 1, "dur": 0},{"ph": "X", "name": "7", "ts": 7248450, "pid": 1, "dur": 0},{"ph": "X", "name": "6", "ts": 7255862, "pid": 1, "dur": 0},{"ph": "X", "name": "13", "ts": 7283415, "pid": 1, "dur": 0},{"ph": "X", "name": "29", "ts": 7359749, "pid": 1, "dur": 0},{"ph": "X", "name": "28", "ts": 7369276, "pid": 1, "dur": 0},{"ph": "X", "name": "12", "ts": 7376683, "pid": 1, "dur": 0},{"ph": "X", "name": "14", "ts": 7388370, "pid": 1, "dur": 0},{"ph": "X", "name": "30", "ts": 7396839, "pid": 1, "dur": 0},{"ph": "X", "name": "31", "ts": 7409564, "pid": 1, "dur": 0},{"ph": "X", "name": "27", "ts": 7507936, "pid": 1, "dur": 0},{"ph": "X", "name": "25", "ts": 7519881, "pid": 1, "dur": 0},{"ph": "X", "name": "24", "ts": 7528334, "pid": 1, "dur": 0},{"ph": "X", "name": "26", "ts": 7636769, "pid": 1, "dur": 0},{"ph": "X", "name": "23", "ts": 7933867, "pid": 1, "dur": 0},{"ph": "X", "name": "21", "ts": 7947659, "pid": 1, "dur": 0},{"ph": "X", "name": "20", "ts": 7990315, "pid": 1, "dur": 0},{"ph": "X", "name": "22", "ts": 7990318, "pid": 1, "dur": 0},{"ph": "X", "name": "0", "ts": 8128459, "pid": 1, "dur": 0},{"ph": "X", "name": "2", "ts": 8136164, "pid": 1, "dur": 0},{"ph": "X", "name": "16", "ts": 8155368, "pid": 1, "dur": 0},{"ph": "X", "name": "3", "ts": 8232767, "pid": 1, "dur": 0},{"ph": "X", "name": "17", "ts": 8257183, "pid": 1, "dur": 0},{"ph": "X", "name": "19", "ts": 8258247, "pid": 1, "dur": 0},{"ph": "X", "name": "18", "ts": 8298579, "pid": 1, "dur": 0},{"ph": "X", "name": "1", "ts": 8345255, "pid": 1, "dur": 0},{"ph": "E", "ts": 8345264, "pid": 1},{"ph": "B", "name": "ALLREDUCE", "ts": 8345471, "pid": 1},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8345479, "pid": 1},{"ph": "E", "ts": 8345666, "pid": 1},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8345669, "pid": 1},{"ph": "E", "ts": 8345730, "pid": 1},{"ph": "B", "name": "INIT_NCCL", "ts": 8345974, "pid": 1},{"ph": "E", "ts": 8521019, "pid": 1},{"ph": "B", "name": "QUEUE", "ts": 8530609, "pid": 1},{"ph": "E", "ts": 8530626, "pid": 1},{"ph": "B", "name": "NCCL_REDUCE", "ts": 8530633, "pid": 1},{"ph": "E", "ts": 8530639, "pid": 1},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8530642, "pid": 1},{"ph": "E", "ts": 8530647, "pid": 1},{"ph": "B", "name": "NCCL_BCAST", "ts": 8530651, "pid": 1},{"ph": "E", "ts": 8530657, "pid": 1},{"ph": "E", "ts": 8530710, "pid": 1, "args": {"dtype": "float32", "shape": "[1]"}},{"name": "process_name", "ph": "M", "pid": 2, "args": {"name": "allreduce.noname.2"}},
{"name": "process_sort_index", "ph": "M", "pid": 2, "args": {"sort_index": 2}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8533119, "pid": 2},{"ph": "X", "name": "0", "ts": 8533120, "pid": 2, "dur": 0},{"ph": "X", "name": "1", "ts": 8534023, "pid": 2, "dur": 0},{"ph": "X", "name": "2", "ts": 8534027, "pid": 2, "dur": 0},{"ph": "X", "name": "3", "ts": 8534031, "pid": 2, "dur": 0},{"ph": "X", "name": "4", "ts": 8534035, "pid": 2, "dur": 0},{"ph": "X", "name": "5", "ts": 8534039, "pid": 2, "dur": 0},{"ph": "X", "name": "6", "ts": 8534041, "pid": 2, "dur": 0},{"ph": "X", "name": "7", "ts": 8534045, "pid": 2, "dur": 0},{"ph": "X", "name": "8", "ts": 8534049, "pid": 2, "dur": 0},{"ph": "X", "name": "9", "ts": 8534053, "pid": 2, "dur": 0},{"ph": "X", "name": "10", "ts": 8534056, "pid": 2, "dur": 0},{"ph": "X", "name": "11", "ts": 8534060, "pid": 2, "dur": 0},{"ph": "X", "name": "12", "ts": 8534064, "pid": 2, "dur": 0},{"ph": "X", "name": "13", "ts": 8534067, "pid": 2, "dur": 0},{"ph": "X", "name": "14", "ts": 8534071, "pid": 2, "dur": 0},{"ph": "X", "name": "15", "ts": 8534074, "pid": 2, "dur": 0},{"ph": "X", "name": "16", "ts": 8534077, "pid": 2, "dur": 0},{"ph": "X", "name": "17", "ts": 8534079, "pid": 2, "dur": 0},{"ph": "X", "name": "18", "ts": 8534082, "pid": 2, "dur": 0},{"ph": "X", "name": "19", "ts": 8534084, "pid": 2, "dur": 0},{"ph": "X", "name": "20", "ts": 8534086, "pid": 2, "dur": 0},{"ph": "X", "name": "21", "ts": 8534089, "pid": 2, "dur": 0},{"ph": "X", "name": "22", "ts": 8534092, "pid": 2, "dur": 0},{"ph": "X", "name": "23", "ts": 8534096, "pid": 2, "dur": 0},{"ph": "X", "name": "24", "ts": 8534099, "pid": 2, "dur": 0},{"ph": "X", "name": "27", "ts": 8534104, "pid": 2, "dur": 0},{"ph": "X", "name": "28", "ts": 8534107, "pid": 2, "dur": 0},{"ph": "X", "name": "29", "ts": 8534110, "pid": 2, "dur": 0},{"ph": "X", "name": "30", "ts": 8534113, "pid": 2, "dur": 0},{"ph": "X", "name": "31", "ts": 8534117, "pid": 2, "dur": 0},{"ph": "X", "name": "25", "ts": 8535075, "pid": 2, "dur": 0},{"ph": "X", "name": "26", "ts": 8535078, "pid": 2, "dur": 0},{"ph": "E", "ts": 8535080, "pid": 2},{"ph": "B", "name": "ALLREDUCE", "ts": 8535151, "pid": 2},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8535155, "pid": 2},{"ph": "E", "ts": 8535164, "pid": 2},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8535167, "pid": 2},{"ph": "E", "ts": 8535228, "pid": 2},{"ph": "B", "name": "QUEUE", "ts": 8535398, "pid": 2},{"ph": "E", "ts": 8535413, "pid": 2},{"ph": "B", "name": "NCCL_REDUCE", "ts": 8535421, "pid": 2},{"ph": "E", "ts": 8535431, "pid": 2},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8535435, "pid": 2},{"ph": "E", "ts": 8535441, "pid": 2},{"ph": "B", "name": "NCCL_BCAST", "ts": 8535446, "pid": 2},{"ph": "E", "ts": 8535451, "pid": 2},{"ph": "E", "ts": 8535482, "pid": 2, "args": {"dtype": "float32", "shape": "[2]"}},{"name": "process_name", "ph": "M", "pid": 3, "args": {"name": "allreduce.noname.3"}},
{"name": "process_sort_index", "ph": "M", "pid": 3, "args": {"sort_index": 3}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8536152, "pid": 3},{"ph": "X", "name": "24", "ts": 8536157, "pid": 3, "dur": 0},{"ph": "X", "name": "0", "ts": 8536380, "pid": 3, "dur": 0},{"ph": "X", "name": "1", "ts": 8537285, "pid": 3, "dur": 0},{"ph": "X", "name": "2", "ts": 8537289, "pid": 3, "dur": 0},{"ph": "X", "name": "3", "ts": 8537292, "pid": 3, "dur": 0},{"ph": "X", "name": "4", "ts": 8537296, "pid": 3, "dur": 0},{"ph": "X", "name": "5", "ts": 8537300, "pid": 3, "dur": 0},{"ph": "X", "name": "6", "ts": 8537303, "pid": 3, "dur": 0},{"ph": "X", "name": "7", "ts": 8537307, "pid": 3, "dur": 0},{"ph": "X", "name": "8", "ts": 8537317, "pid": 3, "dur": 0},{"ph": "X", "name": "9", "ts": 8537320, "pid": 3, "dur": 0},{"ph": "X", "name": "10", "ts": 8537323, "pid": 3, "dur": 0},{"ph": "X", "name": "11", "ts": 8537326, "pid": 3, "dur": 0},{"ph": "X", "name": "12", "ts": 8537330, "pid": 3, "dur": 0},{"ph": "X", "name": "13", "ts": 8537334, "pid": 3, "dur": 0},{"ph": "X", "name": "14", "ts": 8537337, "pid": 3, "dur": 0},{"ph": "X", "name": "15", "ts": 8537341, "pid": 3, "dur": 0},{"ph": "X", "name": "16", "ts": 8537345, "pid": 3, "dur": 0},{"ph": "X", "name": "17", "ts": 8537349, "pid": 3, "dur": 0},{"ph": "X", "name": "18", "ts": 8537353, "pid": 3, "dur": 0},{"ph": "X", "name": "19", "ts": 8537356, "pid": 3, "dur": 0},{"ph": "X", "name": "20", "ts": 8537359, "pid": 3, "dur": 0},{"ph": "X", "name": "21", "ts": 8537363, "pid": 3, "dur": 0},{"ph": "X", "name": "22", "ts": 8537366, "pid": 3, "dur": 0},{"ph": "X", "name": "23", "ts": 8537368, "pid": 3, "dur": 0},{"ph": "X", "name": "25", "ts": 8537372, "pid": 3, "dur": 0},{"ph": "X", "name": "26", "ts": 8537376, "pid": 3, "dur": 0},{"ph": "X", "name": "27", "ts": 8537380, "pid": 3, "dur": 0},{"ph": "X", "name": "28", "ts": 8537384, "pid": 3, "dur": 0},{"ph": "X", "name": "29", "ts": 8537388, "pid": 3, "dur": 0},{"ph": "X", "name": "30", "ts": 8537391, "pid": 3, "dur": 0},{"ph": "X", "name": "31", "ts": 8537395, "pid": 3, "dur": 0},{"ph": "E", "ts": 8537397, "pid": 3},{"ph": "B", "name": "ALLREDUCE", "ts": 8537467, "pid": 3},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8537469, "pid": 3},{"ph": "E", "ts": 8537476, "pid": 3},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8537479, "pid": 3},{"ph": "E", "ts": 8537540, "pid": 3},{"ph": "B", "name": "QUEUE", "ts": 8537652, "pid": 3},{"ph": "E", "ts": 8537660, "pid": 3},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8537666, "pid": 3},{"ph": "E", "ts": 8537672, "pid": 3},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8537678, "pid": 3},{"ph": "E", "ts": 8537683, "pid": 3},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8537689, "pid": 3},{"ph": "E", "ts": 8537695, "pid": 3},{"ph": "E", "ts": 8537720, "pid": 3, "args": {"dtype": "float32", "shape": "[4]"}},{"name": "process_name", "ph": "M", "pid": 4, "args": {"name": "allreduce.noname.4"}},
{"name": "process_sort_index", "ph": "M", "pid": 4, "args": {"sort_index": 4}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8538342, "pid": 4},{"ph": "X", "name": "24", "ts": 8538345, "pid": 4, "dur": 0},{"ph": "X", "name": "0", "ts": 8538672, "pid": 4, "dur": 0},{"ph": "X", "name": "1", "ts": 8539458, "pid": 4, "dur": 0},{"ph": "X", "name": "2", "ts": 8539462, "pid": 4, "dur": 0},{"ph": "X", "name": "3", "ts": 8539465, "pid": 4, "dur": 0},{"ph": "X", "name": "4", "ts": 8539468, "pid": 4, "dur": 0},{"ph": "X", "name": "5", "ts": 8539472, "pid": 4, "dur": 0},{"ph": "X", "name": "6", "ts": 8539475, "pid": 4, "dur": 0},{"ph": "X", "name": "7", "ts": 8539479, "pid": 4, "dur": 0},{"ph": "X", "name": "8", "ts": 8539483, "pid": 4, "dur": 0},{"ph": "X", "name": "9", "ts": 8539486, "pid": 4, "dur": 0},{"ph": "X", "name": "10", "ts": 8539490, "pid": 4, "dur": 0},{"ph": "X", "name": "11", "ts": 8539494, "pid": 4, "dur": 0},{"ph": "X", "name": "12", "ts": 8539499, "pid": 4, "dur": 0},{"ph": "X", "name": "13", "ts": 8539502, "pid": 4, "dur": 0},{"ph": "X", "name": "14", "ts": 8539505, "pid": 4, "dur": 0},{"ph": "X", "name": "15", "ts": 8539509, "pid": 4, "dur": 0},{"ph": "X", "name": "16", "ts": 8539513, "pid": 4, "dur": 0},{"ph": "X", "name": "17", "ts": 8539516, "pid": 4, "dur": 0},{"ph": "X", "name": "18", "ts": 8539520, "pid": 4, "dur": 0},{"ph": "X", "name": "19", "ts": 8539523, "pid": 4, "dur": 0},{"ph": "X", "name": "20", "ts": 8539527, "pid": 4, "dur": 0},{"ph": "X", "name": "21", "ts": 8539530, "pid": 4, "dur": 0},{"ph": "X", "name": "22", "ts": 8539534, "pid": 4, "dur": 0},{"ph": "X", "name": "23", "ts": 8539537, "pid": 4, "dur": 0},{"ph": "X", "name": "25", "ts": 8539541, "pid": 4, "dur": 0},{"ph": "X", "name": "26", "ts": 8539545, "pid": 4, "dur": 0},{"ph": "X", "name": "27", "ts": 8539549, "pid": 4, "dur": 0},{"ph": "X", "name": "28", "ts": 8539552, "pid": 4, "dur": 0},{"ph": "X", "name": "29", "ts": 8539556, "pid": 4, "dur": 0},{"ph": "X", "name": "30", "ts": 8539559, "pid": 4, "dur": 0},{"ph": "X", "name": "31", "ts": 8539562, "pid": 4, "dur": 0},{"ph": "E", "ts": 8539565, "pid": 4},{"ph": "B", "name": "ALLREDUCE", "ts": 8539626, "pid": 4},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8539632, "pid": 4},{"ph": "E", "ts": 8539638, "pid": 4},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8539640, "pid": 4},{"ph": "E", "ts": 8539698, "pid": 4},{"ph": "B", "name": "QUEUE", "ts": 8539813, "pid": 4},{"ph": "E", "ts": 8539821, "pid": 4},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8539828, "pid": 4},{"ph": "E", "ts": 8539835, "pid": 4},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8539839, "pid": 4},{"ph": "E", "ts": 8539845, "pid": 4},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8539849, "pid": 4},{"ph": "E", "ts": 8539858, "pid": 4},{"ph": "E", "ts": 8539884, "pid": 4, "args": {"dtype": "float32", "shape": "[8]"}},{"name": "process_name", "ph": "M", "pid": 5, "args": {"name": "allreduce.noname.5"}},
{"name": "process_sort_index", "ph": "M", "pid": 5, "args": {"sort_index": 5}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8540514, "pid": 5},{"ph": "X", "name": "4", "ts": 8540518, "pid": 5, "dur": 0},{"ph": "X", "name": "24", "ts": 8540538, "pid": 5, "dur": 0},{"ph": "X", "name": "0", "ts": 8540831, "pid": 5, "dur": 0},{"ph": "X", "name": "1", "ts": 8541602, "pid": 5, "dur": 0},{"ph": "X", "name": "2", "ts": 8541605, "pid": 5, "dur": 0},{"ph": "X", "name": "3", "ts": 8541609, "pid": 5, "dur": 0},{"ph": "X", "name": "5", "ts": 8541613, "pid": 5, "dur": 0},{"ph": "X", "name": "6", "ts": 8541617, "pid": 5, "dur": 0},{"ph": "X", "name": "7", "ts": 8541620, "pid": 5, "dur": 0},{"ph": "X", "name": "8", "ts": 8541624, "pid": 5, "dur": 0},{"ph": "X", "name": "9", "ts": 8541627, "pid": 5, "dur": 0},{"ph": "X", "name": "10", "ts": 8541631, "pid": 5, "dur": 0},{"ph": "X", "name": "11", "ts": 8541634, "pid": 5, "dur": 0},{"ph": "X", "name": "12", "ts": 8541638, "pid": 5, "dur": 0},{"ph": "X", "name": "13", "ts": 8541641, "pid": 5, "dur": 0},{"ph": "X", "name": "14", "ts": 8541644, "pid": 5, "dur": 0},{"ph": "X", "name": "15", "ts": 8541648, "pid": 5, "dur": 0},{"ph": "X", "name": "16", "ts": 8541651, "pid": 5, "dur": 0},{"ph": "X", "name": "17", "ts": 8541655, "pid": 5, "dur": 0},{"ph": "X", "name": "18", "ts": 8541659, "pid": 5, "dur": 0},{"ph": "X", "name": "19", "ts": 8541662, "pid": 5, "dur": 0},{"ph": "X", "name": "20", "ts": 8541666, "pid": 5, "dur": 0},{"ph": "X", "name": "21", "ts": 8541670, "pid": 5, "dur": 0},{"ph": "X", "name": "22", "ts": 8541673, "pid": 5, "dur": 0},{"ph": "X", "name": "23", "ts": 8541676, "pid": 5, "dur": 0},{"ph": "X", "name": "25", "ts": 8541681, "pid": 5, "dur": 0},{"ph": "X", "name": "26", "ts": 8541684, "pid": 5, "dur": 0},{"ph": "X", "name": "27", "ts": 8541688, "pid": 5, "dur": 0},{"ph": "X", "name": "28", "ts": 8541691, "pid": 5, "dur": 0},{"ph": "X", "name": "29", "ts": 8541694, "pid": 5, "dur": 0},{"ph": "X", "name": "30", "ts": 8541697, "pid": 5, "dur": 0},{"ph": "X", "name": "31", "ts": 8541700, "pid": 5, "dur": 0},{"ph": "E", "ts": 8541702, "pid": 5},{"ph": "B", "name": "ALLREDUCE", "ts": 8541764, "pid": 5},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8541767, "pid": 5},{"ph": "E", "ts": 8541775, "pid": 5},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8541778, "pid": 5},{"ph": "E", "ts": 8541836, "pid": 5},{"ph": "B", "name": "QUEUE", "ts": 8541949, "pid": 5},{"ph": "E", "ts": 8541957, "pid": 5},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8541961, "pid": 5},{"ph": "E", "ts": 8541967, "pid": 5},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8541973, "pid": 5},{"ph": "E", "ts": 8541980, "pid": 5},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8541984, "pid": 5},{"ph": "E", "ts": 8541990, "pid": 5},{"ph": "E", "ts": 8542008, "pid": 5, "args": {"dtype": "float32", "shape": "[16]"}},{"name": "process_name", "ph": "M", "pid": 6, "args": {"name": "allreduce.noname.6"}},
{"name": "process_sort_index", "ph": "M", "pid": 6, "args": {"sort_index": 6}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8542644, "pid": 6},{"ph": "X", "name": "4", "ts": 8542647, "pid": 6, "dur": 0},{"ph": "X", "name": "24", "ts": 8542666, "pid": 6, "dur": 0},{"ph": "X", "name": "0", "ts": 8542966, "pid": 6, "dur": 0},{"ph": "X", "name": "1", "ts": 8543805, "pid": 6, "dur": 0},{"ph": "X", "name": "2", "ts": 8543808, "pid": 6, "dur": 0},{"ph": "X", "name": "3", "ts": 8543812, "pid": 6, "dur": 0},{"ph": "X", "name": "5", "ts": 8543817, "pid": 6, "dur": 0},{"ph": "X", "name": "6", "ts": 8543821, "pid": 6, "dur": 0},{"ph": "X", "name": "7", "ts": 8543824, "pid": 6, "dur": 0},{"ph": "X", "name": "8", "ts": 8543828, "pid": 6, "dur": 0},{"ph": "X", "name": "9", "ts": 8543831, "pid": 6, "dur": 0},{"ph": "X", "name": "10", "ts": 8543839, "pid": 6, "dur": 0},{"ph": "X", "name": "11", "ts": 8543843, "pid": 6, "dur": 0},{"ph": "X", "name": "12", "ts": 8543846, "pid": 6, "dur": 0},{"ph": "X", "name": "13", "ts": 8543850, "pid": 6, "dur": 0},{"ph": "X", "name": "14", "ts": 8543853, "pid": 6, "dur": 0},{"ph": "X", "name": "15", "ts": 8543857, "pid": 6, "dur": 0},{"ph": "X", "name": "16", "ts": 8543861, "pid": 6, "dur": 0},{"ph": "X", "name": "17", "ts": 8543864, "pid": 6, "dur": 0},{"ph": "X", "name": "18", "ts": 8543868, "pid": 6, "dur": 0},{"ph": "X", "name": "19", "ts": 8543871, "pid": 6, "dur": 0},{"ph": "X", "name": "20", "ts": 8543874, "pid": 6, "dur": 0},{"ph": "X", "name": "21", "ts": 8543878, "pid": 6, "dur": 0},{"ph": "X", "name": "22", "ts": 8543881, "pid": 6, "dur": 0},{"ph": "X", "name": "23", "ts": 8543884, "pid": 6, "dur": 0},{"ph": "X", "name": "25", "ts": 8543889, "pid": 6, "dur": 0},{"ph": "X", "name": "26", "ts": 8543892, "pid": 6, "dur": 0},{"ph": "X", "name": "27", "ts": 8543896, "pid": 6, "dur": 0},{"ph": "X", "name": "28", "ts": 8543899, "pid": 6, "dur": 0},{"ph": "X", "name": "29", "ts": 8543902, "pid": 6, "dur": 0},{"ph": "X", "name": "30", "ts": 8543905, "pid": 6, "dur": 0},{"ph": "X", "name": "31", "ts": 8543908, "pid": 6, "dur": 0},{"ph": "E", "ts": 8543910, "pid": 6},{"ph": "B", "name": "ALLREDUCE", "ts": 8543974, "pid": 6},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8543977, "pid": 6},{"ph": "E", "ts": 8543984, "pid": 6},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8543988, "pid": 6},{"ph": "E", "ts": 8544046, "pid": 6},{"ph": "B", "name": "QUEUE", "ts": 8544182, "pid": 6},{"ph": "E", "ts": 8544195, "pid": 6},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8544202, "pid": 6},{"ph": "E", "ts": 8544208, "pid": 6},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8544215, "pid": 6},{"ph": "E", "ts": 8544220, "pid": 6},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8544225, "pid": 6},{"ph": "E", "ts": 8544232, "pid": 6},{"ph": "E", "ts": 8544259, "pid": 6, "args": {"dtype": "float32", "shape": "[32]"}},{"name": "process_name", "ph": "M", "pid": 7, "args": {"name": "allreduce.noname.7"}},
{"name": "process_sort_index", "ph": "M", "pid": 7, "args": {"sort_index": 7}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8544822, "pid": 7},{"ph": "X", "name": "4", "ts": 8544825, "pid": 7, "dur": 0},{"ph": "X", "name": "24", "ts": 8544843, "pid": 7, "dur": 0},{"ph": "X", "name": "0", "ts": 8545184, "pid": 7, "dur": 0},{"ph": "X", "name": "1", "ts": 8545931, "pid": 7, "dur": 0},{"ph": "X", "name": "2", "ts": 8545934, "pid": 7, "dur": 0},{"ph": "X", "name": "3", "ts": 8545938, "pid": 7, "dur": 0},{"ph": "X", "name": "5", "ts": 8545942, "pid": 7, "dur": 0},{"ph": "X", "name": "6", "ts": 8545946, "pid": 7, "dur": 0},{"ph": "X", "name": "7", "ts": 8545949, "pid": 7, "dur": 0},{"ph": "X", "name": "8", "ts": 8545952, "pid": 7, "dur": 0},{"ph": "X", "name": "9", "ts": 8545956, "pid": 7, "dur": 0},{"ph": "X", "name": "10", "ts": 8545960, "pid": 7, "dur": 0},{"ph": "X", "name": "11", "ts": 8545964, "pid": 7, "dur": 0},{"ph": "X", "name": "12", "ts": 8545967, "pid": 7, "dur": 0},{"ph": "X", "name": "13", "ts": 8545971, "pid": 7, "dur": 0},{"ph": "X", "name": "14", "ts": 8545975, "pid": 7, "dur": 0},{"ph": "X", "name": "15", "ts": 8545978, "pid": 7, "dur": 0},{"ph": "X", "name": "16", "ts": 8545982, "pid": 7, "dur": 0},{"ph": "X", "name": "17", "ts": 8545986, "pid": 7, "dur": 0},{"ph": "X", "name": "18", "ts": 8545988, "pid": 7, "dur": 0},{"ph": "X", "name": "19", "ts": 8545991, "pid": 7, "dur": 0},{"ph": "X", "name": "20", "ts": 8545995, "pid": 7, "dur": 0},{"ph": "X", "name": "21", "ts": 8545997, "pid": 7, "dur": 0},{"ph": "X", "name": "22", "ts": 8546001, "pid": 7, "dur": 0},{"ph": "X", "name": "23", "ts": 8546003, "pid": 7, "dur": 0},{"ph": "X", "name": "25", "ts": 8546006, "pid": 7, "dur": 0},{"ph": "X", "name": "26", "ts": 8546010, "pid": 7, "dur": 0},{"ph": "X", "name": "27", "ts": 8546013, "pid": 7, "dur": 0},{"ph": "X", "name": "28", "ts": 8546016, "pid": 7, "dur": 0},{"ph": "X", "name": "29", "ts": 8546019, "pid": 7, "dur": 0},{"ph": "X", "name": "30", "ts": 8546022, "pid": 7, "dur": 0},{"ph": "X", "name": "31", "ts": 8546025, "pid": 7, "dur": 0},{"ph": "E", "ts": 8546027, "pid": 7},{"ph": "B", "name": "ALLREDUCE", "ts": 8546082, "pid": 7},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8546084, "pid": 7},{"ph": "E", "ts": 8546089, "pid": 7},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8546096, "pid": 7},{"ph": "E", "ts": 8546155, "pid": 7},{"ph": "B", "name": "QUEUE", "ts": 8546287, "pid": 7},{"ph": "E", "ts": 8546301, "pid": 7},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8546308, "pid": 7},{"ph": "E", "ts": 8546315, "pid": 7},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8546319, "pid": 7},{"ph": "E", "ts": 8546324, "pid": 7},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8546328, "pid": 7},{"ph": "E", "ts": 8546333, "pid": 7},{"ph": "E", "ts": 8546365, "pid": 7, "args": {"dtype": "float32", "shape": "[64]"}},{"name": "process_name", "ph": "M", "pid": 8, "args": {"name": "allreduce.noname.8"}},
{"name": "process_sort_index", "ph": "M", "pid": 8, "args": {"sort_index": 8}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8546971, "pid": 8},{"ph": "X", "name": "4", "ts": 8546975, "pid": 8, "dur": 0},{"ph": "X", "name": "24", "ts": 8546992, "pid": 8, "dur": 0},{"ph": "X", "name": "0", "ts": 8547268, "pid": 8, "dur": 0},{"ph": "X", "name": "1", "ts": 8548131, "pid": 8, "dur": 0},{"ph": "X", "name": "2", "ts": 8548134, "pid": 8, "dur": 0},{"ph": "X", "name": "3", "ts": 8548138, "pid": 8, "dur": 0},{"ph": "X", "name": "5", "ts": 8548142, "pid": 8, "dur": 0},{"ph": "X", "name": "6", "ts": 8548146, "pid": 8, "dur": 0},{"ph": "X", "name": "7", "ts": 8548149, "pid": 8, "dur": 0},{"ph": "X", "name": "8", "ts": 8548152, "pid": 8, "dur": 0},{"ph": "X", "name": "9", "ts": 8548155, "pid": 8, "dur": 0},{"ph": "X", "name": "10", "ts": 8548159, "pid": 8, "dur": 0},{"ph": "X", "name": "11", "ts": 8548162, "pid": 8, "dur": 0},{"ph": "X", "name": "12", "ts": 8548166, "pid": 8, "dur": 0},{"ph": "X", "name": "13", "ts": 8548169, "pid": 8, "dur": 0},{"ph": "X", "name": "14", "ts": 8548172, "pid": 8, "dur": 0},{"ph": "X", "name": "15", "ts": 8548174, "pid": 8, "dur": 0},{"ph": "X", "name": "16", "ts": 8548178, "pid": 8, "dur": 0},{"ph": "X", "name": "17", "ts": 8548182, "pid": 8, "dur": 0},{"ph": "X", "name": "18", "ts": 8548185, "pid": 8, "dur": 0},{"ph": "X", "name": "19", "ts": 8548188, "pid": 8, "dur": 0},{"ph": "X", "name": "20", "ts": 8548192, "pid": 8, "dur": 0},{"ph": "X", "name": "21", "ts": 8548195, "pid": 8, "dur": 0},{"ph": "X", "name": "22", "ts": 8548198, "pid": 8, "dur": 0},{"ph": "X", "name": "23", "ts": 8548201, "pid": 8, "dur": 0},{"ph": "X", "name": "25", "ts": 8548205, "pid": 8, "dur": 0},{"ph": "X", "name": "26", "ts": 8548209, "pid": 8, "dur": 0},{"ph": "X", "name": "27", "ts": 8548212, "pid": 8, "dur": 0},{"ph": "X", "name": "28", "ts": 8548215, "pid": 8, "dur": 0},{"ph": "X", "name": "29", "ts": 8548218, "pid": 8, "dur": 0},{"ph": "X", "name": "30", "ts": 8548221, "pid": 8, "dur": 0},{"ph": "X", "name": "31", "ts": 8548224, "pid": 8, "dur": 0},{"ph": "E", "ts": 8548227, "pid": 8},{"ph": "B", "name": "ALLREDUCE", "ts": 8548286, "pid": 8},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8548288, "pid": 8},{"ph": "E", "ts": 8548294, "pid": 8},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8548297, "pid": 8},{"ph": "E", "ts": 8548355, "pid": 8},{"ph": "B", "name": "QUEUE", "ts": 8548467, "pid": 8},{"ph": "E", "ts": 8548482, "pid": 8},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8548489, "pid": 8},{"ph": "E", "ts": 8548496, "pid": 8},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8548501, "pid": 8},{"ph": "E", "ts": 8548507, "pid": 8},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8548511, "pid": 8},{"ph": "E", "ts": 8548517, "pid": 8},{"ph": "E", "ts": 8548548, "pid": 8, "args": {"dtype": "float32", "shape": "[128]"}},{"name": "process_name", "ph": "M", "pid": 9, "args": {"name": "allreduce.noname.9"}},
{"name": "process_sort_index", "ph": "M", "pid": 9, "args": {"sort_index": 9}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8549145, "pid": 9},{"ph": "X", "name": "4", "ts": 8549147, "pid": 9, "dur": 0},{"ph": "X", "name": "24", "ts": 8549164, "pid": 9, "dur": 0},{"ph": "X", "name": "0", "ts": 8549472, "pid": 9, "dur": 0},{"ph": "X", "name": "1", "ts": 8550252, "pid": 9, "dur": 0},{"ph": "X", "name": "2", "ts": 8550255, "pid": 9, "dur": 0},{"ph": "X", "name": "3", "ts": 8550259, "pid": 9, "dur": 0},{"ph": "X", "name": "5", "ts": 8550263, "pid": 9, "dur": 0},{"ph": "X", "name": "6", "ts": 8550267, "pid": 9, "dur": 0},{"ph": "X", "name": "7", "ts": 8550270, "pid": 9, "dur": 0},{"ph": "X", "name": "8", "ts": 8550273, "pid": 9, "dur": 0},{"ph": "X", "name": "9", "ts": 8550276, "pid": 9, "dur": 0},{"ph": "X", "name": "10", "ts": 8550279, "pid": 9, "dur": 0},{"ph": "X", "name": "11", "ts": 8550282, "pid": 9, "dur": 0},{"ph": "X", "name": "12", "ts": 8550289, "pid": 9, "dur": 0},{"ph": "X", "name": "13", "ts": 8550293, "pid": 9, "dur": 0},{"ph": "X", "name": "14", "ts": 8550296, "pid": 9, "dur": 0},{"ph": "X", "name": "15", "ts": 8550299, "pid": 9, "dur": 0},{"ph": "X", "name": "16", "ts": 8550302, "pid": 9, "dur": 0},{"ph": "X", "name": "17", "ts": 8550306, "pid": 9, "dur": 0},{"ph": "X", "name": "18", "ts": 8550309, "pid": 9, "dur": 0},{"ph": "X", "name": "19", "ts": 8550311, "pid": 9, "dur": 0},{"ph": "X", "name": "20", "ts": 8550315, "pid": 9, "dur": 0},{"ph": "X", "name": "21", "ts": 8550317, "pid": 9, "dur": 0},{"ph": "X", "name": "22", "ts": 8550321, "pid": 9, "dur": 0},{"ph": "X", "name": "23", "ts": 8550324, "pid": 9, "dur": 0},{"ph": "X", "name": "25", "ts": 8550327, "pid": 9, "dur": 0},{"ph": "X", "name": "26", "ts": 8550331, "pid": 9, "dur": 0},{"ph": "X", "name": "27", "ts": 8550334, "pid": 9, "dur": 0},{"ph": "X", "name": "28", "ts": 8550337, "pid": 9, "dur": 0},{"ph": "X", "name": "29", "ts": 8550340, "pid": 9, "dur": 0},{"ph": "X", "name": "30", "ts": 8550343, "pid": 9, "dur": 0},{"ph": "X", "name": "31", "ts": 8550346, "pid": 9, "dur": 0},{"ph": "E", "ts": 8550348, "pid": 9},{"ph": "B", "name": "ALLREDUCE", "ts": 8550408, "pid": 9},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8550410, "pid": 9},{"ph": "E", "ts": 8550416, "pid": 9},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8550418, "pid": 9},{"ph": "E", "ts": 8550476, "pid": 9},{"ph": "B", "name": "QUEUE", "ts": 8550614, "pid": 9},{"ph": "E", "ts": 8550631, "pid": 9},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8550639, "pid": 9},{"ph": "E", "ts": 8550647, "pid": 9},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8550652, "pid": 9},{"ph": "E", "ts": 8550658, "pid": 9},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8550665, "pid": 9},{"ph": "E", "ts": 8550671, "pid": 9},{"ph": "E", "ts": 8550700, "pid": 9, "args": {"dtype": "float32", "shape": "[256]"}},{"name": "process_name", "ph": "M", "pid": 10, "args": {"name": "allreduce.noname.10"}},
{"name": "process_sort_index", "ph": "M", "pid": 10, "args": {"sort_index": 10}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8551306, "pid": 10},{"ph": "X", "name": "4", "ts": 8551308, "pid": 10, "dur": 0},{"ph": "X", "name": "24", "ts": 8551326, "pid": 10, "dur": 0},{"ph": "X", "name": "0", "ts": 8551591, "pid": 10, "dur": 0},{"ph": "X", "name": "1", "ts": 8552348, "pid": 10, "dur": 0},{"ph": "X", "name": "2", "ts": 8552351, "pid": 10, "dur": 0},{"ph": "X", "name": "3", "ts": 8552355, "pid": 10, "dur": 0},{"ph": "X", "name": "5", "ts": 8552360, "pid": 10, "dur": 0},{"ph": "X", "name": "6", "ts": 8552363, "pid": 10, "dur": 0},{"ph": "X", "name": "7", "ts": 8552366, "pid": 10, "dur": 0},{"ph": "X", "name": "8", "ts": 8552370, "pid": 10, "dur": 0},{"ph": "X", "name": "9", "ts": 8552374, "pid": 10, "dur": 0},{"ph": "X", "name": "10", "ts": 8552378, "pid": 10, "dur": 0},{"ph": "X", "name": "11", "ts": 8552381, "pid": 10, "dur": 0},{"ph": "X", "name": "12", "ts": 8552384, "pid": 10, "dur": 0},{"ph": "X", "name": "13", "ts": 8552386, "pid": 10, "dur": 0},{"ph": "X", "name": "14", "ts": 8552389, "pid": 10, "dur": 0},{"ph": "X", "name": "15", "ts": 8552393, "pid": 10, "dur": 0},{"ph": "X", "name": "16", "ts": 8552396, "pid": 10, "dur": 0},{"ph": "X", "name": "17", "ts": 8552399, "pid": 10, "dur": 0},{"ph": "X", "name": "18", "ts": 8552402, "pid": 10, "dur": 0},{"ph": "X", "name": "19", "ts": 8552406, "pid": 10, "dur": 0},{"ph": "X", "name": "20", "ts": 8552409, "pid": 10, "dur": 0},{"ph": "X", "name": "21", "ts": 8552412, "pid": 10, "dur": 0},{"ph": "X", "name": "22", "ts": 8552415, "pid": 10, "dur": 0},{"ph": "X", "name": "23", "ts": 8552418, "pid": 10, "dur": 0},{"ph": "X", "name": "25", "ts": 8552422, "pid": 10, "dur": 0},{"ph": "X", "name": "26", "ts": 8552425, "pid": 10, "dur": 0},{"ph": "X", "name": "27", "ts": 8552429, "pid": 10, "dur": 0},{"ph": "X", "name": "28", "ts": 8552432, "pid": 10, "dur": 0},{"ph": "X", "name": "29", "ts": 8552435, "pid": 10, "dur": 0},{"ph": "X", "name": "30", "ts": 8552438, "pid": 10, "dur": 0},{"ph": "X", "name": "31", "ts": 8552442, "pid": 10, "dur": 0},{"ph": "E", "ts": 8552444, "pid": 10},{"ph": "B", "name": "ALLREDUCE", "ts": 8552498, "pid": 10},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8552501, "pid": 10},{"ph": "E", "ts": 8552506, "pid": 10},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8552509, "pid": 10},{"ph": "E", "ts": 8552568, "pid": 10},{"ph": "B", "name": "QUEUE", "ts": 8552651, "pid": 10},{"ph": "E", "ts": 8552660, "pid": 10},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8552666, "pid": 10},{"ph": "E", "ts": 8552672, "pid": 10},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8552675, "pid": 10},{"ph": "E", "ts": 8552681, "pid": 10},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8552684, "pid": 10},{"ph": "E", "ts": 8552689, "pid": 10},{"ph": "E", "ts": 8552708, "pid": 10, "args": {"dtype": "float32", "shape": "[512]"}},{"name": "process_name", "ph": "M", "pid": 11, "args": {"name": "allreduce.noname.11"}},
{"name": "process_sort_index", "ph": "M", "pid": 11, "args": {"sort_index": 11}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8553418, "pid": 11},{"ph": "X", "name": "4", "ts": 8553419, "pid": 11, "dur": 0},{"ph": "X", "name": "24", "ts": 8553438, "pid": 11, "dur": 0},{"ph": "X", "name": "0", "ts": 8553697, "pid": 11, "dur": 0},{"ph": "X", "name": "1", "ts": 8554535, "pid": 11, "dur": 0},{"ph": "X", "name": "2", "ts": 8554538, "pid": 11, "dur": 0},{"ph": "X", "name": "3", "ts": 8554542, "pid": 11, "dur": 0},{"ph": "X", "name": "5", "ts": 8554546, "pid": 11, "dur": 0},{"ph": "X", "name": "6", "ts": 8554550, "pid": 11, "dur": 0},{"ph": "X", "name": "7", "ts": 8554553, "pid": 11, "dur": 0},{"ph": "X", "name": "8", "ts": 8554557, "pid": 11, "dur": 0},{"ph": "X", "name": "9", "ts": 8554560, "pid": 11, "dur": 0},{"ph": "X", "name": "10", "ts": 8554563, "pid": 11, "dur": 0},{"ph": "X", "name": "11", "ts": 8554566, "pid": 11, "dur": 0},{"ph": "X", "name": "12", "ts": 8554569, "pid": 11, "dur": 0},{"ph": "X", "name": "13", "ts": 8554573, "pid": 11, "dur": 0},{"ph": "X", "name": "14", "ts": 8554575, "pid": 11, "dur": 0},{"ph": "X", "name": "15", "ts": 8554578, "pid": 11, "dur": 0},{"ph": "X", "name": "16", "ts": 8554581, "pid": 11, "dur": 0},{"ph": "X", "name": "17", "ts": 8554585, "pid": 11, "dur": 0},{"ph": "X", "name": "18", "ts": 8554588, "pid": 11, "dur": 0},{"ph": "X", "name": "19", "ts": 8554591, "pid": 11, "dur": 0},{"ph": "X", "name": "20", "ts": 8554595, "pid": 11, "dur": 0},{"ph": "X", "name": "21", "ts": 8554598, "pid": 11, "dur": 0},{"ph": "X", "name": "22", "ts": 8554601, "pid": 11, "dur": 0},{"ph": "X", "name": "23", "ts": 8554603, "pid": 11, "dur": 0},{"ph": "X", "name": "25", "ts": 8554607, "pid": 11, "dur": 0},{"ph": "X", "name": "26", "ts": 8554610, "pid": 11, "dur": 0},{"ph": "X", "name": "27", "ts": 8554614, "pid": 11, "dur": 0},{"ph": "X", "name": "28", "ts": 8554617, "pid": 11, "dur": 0},{"ph": "X", "name": "29", "ts": 8554620, "pid": 11, "dur": 0},{"ph": "X", "name": "30", "ts": 8554624, "pid": 11, "dur": 0},{"ph": "X", "name": "31", "ts": 8554627, "pid": 11, "dur": 0},{"ph": "E", "ts": 8554629, "pid": 11},{"ph": "B", "name": "ALLREDUCE", "ts": 8554688, "pid": 11},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8554691, "pid": 11},{"ph": "E", "ts": 8554696, "pid": 11},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8554699, "pid": 11},{"ph": "E", "ts": 8554757, "pid": 11},{"ph": "B", "name": "QUEUE", "ts": 8554927, "pid": 11},{"ph": "E", "ts": 8554939, "pid": 11},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8554944, "pid": 11},{"ph": "E", "ts": 8554949, "pid": 11},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8554952, "pid": 11},{"ph": "E", "ts": 8554956, "pid": 11},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8554960, "pid": 11},{"ph": "E", "ts": 8554964, "pid": 11},{"ph": "E", "ts": 8554989, "pid": 11, "args": {"dtype": "float32", "shape": "[1024]"}},{"name": "process_name", "ph": "M", "pid": 12, "args": {"name": "allreduce.noname.12"}},
{"name": "process_sort_index", "ph": "M", "pid": 12, "args": {"sort_index": 12}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8555600, "pid": 12},{"ph": "X", "name": "4", "ts": 8555602, "pid": 12, "dur": 0},{"ph": "X", "name": "24", "ts": 8555621, "pid": 12, "dur": 0},{"ph": "X", "name": "0", "ts": 8555886, "pid": 12, "dur": 0},{"ph": "X", "name": "1", "ts": 8556703, "pid": 12, "dur": 0},{"ph": "X", "name": "2", "ts": 8556706, "pid": 12, "dur": 0},{"ph": "X", "name": "3", "ts": 8556710, "pid": 12, "dur": 0},{"ph": "X", "name": "5", "ts": 8556714, "pid": 12, "dur": 0},{"ph": "X", "name": "6", "ts": 8556719, "pid": 12, "dur": 0},{"ph": "X", "name": "7", "ts": 8556722, "pid": 12, "dur": 0},{"ph": "X", "name": "8", "ts": 8556725, "pid": 12, "dur": 0},{"ph": "X", "name": "9", "ts": 8556728, "pid": 12, "dur": 0},{"ph": "X", "name": "10", "ts": 8556732, "pid": 12, "dur": 0},{"ph": "X", "name": "11", "ts": 8556735, "pid": 12, "dur": 0},{"ph": "X", "name": "12", "ts": 8556738, "pid": 12, "dur": 0},{"ph": "X", "name": "13", "ts": 8556741, "pid": 12, "dur": 0},{"ph": "X", "name": "14", "ts": 8556744, "pid": 12, "dur": 0},{"ph": "X", "name": "15", "ts": 8556752, "pid": 12, "dur": 0},{"ph": "X", "name": "16", "ts": 8556755, "pid": 12, "dur": 0},{"ph": "X", "name": "17", "ts": 8556758, "pid": 12, "dur": 0},{"ph": "X", "name": "18", "ts": 8556762, "pid": 12, "dur": 0},{"ph": "X", "name": "19", "ts": 8556766, "pid": 12, "dur": 0},{"ph": "X", "name": "20", "ts": 8556769, "pid": 12, "dur": 0},{"ph": "X", "name": "21", "ts": 8556772, "pid": 12, "dur": 0},{"ph": "X", "name": "22", "ts": 8556775, "pid": 12, "dur": 0},{"ph": "X", "name": "23", "ts": 8556778, "pid": 12, "dur": 0},{"ph": "X", "name": "25", "ts": 8556782, "pid": 12, "dur": 0},{"ph": "X", "name": "26", "ts": 8556785, "pid": 12, "dur": 0},{"ph": "X", "name": "27", "ts": 8556789, "pid": 12, "dur": 0},{"ph": "X", "name": "28", "ts": 8556793, "pid": 12, "dur": 0},{"ph": "X", "name": "29", "ts": 8556795, "pid": 12, "dur": 0},{"ph": "X", "name": "30", "ts": 8556798, "pid": 12, "dur": 0},{"ph": "X", "name": "31", "ts": 8556801, "pid": 12, "dur": 0},{"ph": "E", "ts": 8556803, "pid": 12},{"ph": "B", "name": "ALLREDUCE", "ts": 8556863, "pid": 12},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8556866, "pid": 12},{"ph": "E", "ts": 8556871, "pid": 12},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8556874, "pid": 12},{"ph": "E", "ts": 8556933, "pid": 12},{"ph": "B", "name": "QUEUE", "ts": 8557074, "pid": 12},{"ph": "E", "ts": 8557082, "pid": 12},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8557085, "pid": 12},{"ph": "E", "ts": 8557088, "pid": 12},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8557090, "pid": 12},{"ph": "E", "ts": 8557094, "pid": 12},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8557095, "pid": 12},{"ph": "E", "ts": 8557099, "pid": 12},{"ph": "E", "ts": 8557111, "pid": 12, "args": {"dtype": "float32", "shape": "[2048]"}},{"name": "process_name", "ph": "M", "pid": 13, "args": {"name": "allreduce.noname.13"}},
{"name": "process_sort_index", "ph": "M", "pid": 13, "args": {"sort_index": 13}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8557775, "pid": 13},{"ph": "X", "name": "24", "ts": 8557777, "pid": 13, "dur": 0},{"ph": "X", "name": "0", "ts": 8558053, "pid": 13, "dur": 0},{"ph": "X", "name": "1", "ts": 8558867, "pid": 13, "dur": 0},{"ph": "X", "name": "2", "ts": 8558871, "pid": 13, "dur": 0},{"ph": "X", "name": "3", "ts": 8558875, "pid": 13, "dur": 0},{"ph": "X", "name": "4", "ts": 8558878, "pid": 13, "dur": 0},{"ph": "X", "name": "5", "ts": 8558882, "pid": 13, "dur": 0},{"ph": "X", "name": "6", "ts": 8558886, "pid": 13, "dur": 0},{"ph": "X", "name": "7", "ts": 8558889, "pid": 13, "dur": 0},{"ph": "X", "name": "8", "ts": 8558893, "pid": 13, "dur": 0},{"ph": "X", "name": "9", "ts": 8558896, "pid": 13, "dur": 0},{"ph": "X", "name": "10", "ts": 8558900, "pid": 13, "dur": 0},{"ph": "X", "name": "11", "ts": 8558903, "pid": 13, "dur": 0},{"ph": "X", "name": "12", "ts": 8558907, "pid": 13, "dur": 0},{"ph": "X", "name": "13", "ts": 8558910, "pid": 13, "dur": 0},{"ph": "X", "name": "14", "ts": 8558913, "pid": 13, "dur": 0},{"ph": "X", "name": "15", "ts": 8558917, "pid": 13, "dur": 0},{"ph": "X", "name": "16", "ts": 8558919, "pid": 13, "dur": 0},{"ph": "X", "name": "17", "ts": 8558922, "pid": 13, "dur": 0},{"ph": "X", "name": "18", "ts": 8558926, "pid": 13, "dur": 0},{"ph": "X", "name": "19", "ts": 8558929, "pid": 13, "dur": 0},{"ph": "X", "name": "20", "ts": 8558932, "pid": 13, "dur": 0},{"ph": "X", "name": "21", "ts": 8558935, "pid": 13, "dur": 0},{"ph": "X", "name": "22", "ts": 8558938, "pid": 13, "dur": 0},{"ph": "X", "name": "23", "ts": 8558941, "pid": 13, "dur": 0},{"ph": "X", "name": "25", "ts": 8558945, "pid": 13, "dur": 0},{"ph": "X", "name": "26", "ts": 8558948, "pid": 13, "dur": 0},{"ph": "X", "name": "27", "ts": 8558951, "pid": 13, "dur": 0},{"ph": "X", "name": "28", "ts": 8558954, "pid": 13, "dur": 0},{"ph": "X", "name": "29", "ts": 8558957, "pid": 13, "dur": 0},{"ph": "X", "name": "30", "ts": 8558961, "pid": 13, "dur": 0},{"ph": "X", "name": "31", "ts": 8558963, "pid": 13, "dur": 0},{"ph": "E", "ts": 8558965, "pid": 13},{"ph": "B", "name": "ALLREDUCE", "ts": 8559021, "pid": 13},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8559027, "pid": 13},{"ph": "E", "ts": 8559033, "pid": 13},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8559036, "pid": 13},{"ph": "E", "ts": 8559094, "pid": 13},{"ph": "B", "name": "QUEUE", "ts": 8559218, "pid": 13},{"ph": "E", "ts": 8559235, "pid": 13},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8559242, "pid": 13},{"ph": "E", "ts": 8559264, "pid": 13},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8559269, "pid": 13},{"ph": "E", "ts": 8559277, "pid": 13},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8559282, "pid": 13},{"ph": "E", "ts": 8559287, "pid": 13},{"ph": "E", "ts": 8559324, "pid": 13, "args": {"dtype": "float32", "shape": "[4096]"}},{"name": "process_name", "ph": "M", "pid": 14, "args": {"name": "allreduce.noname.14"}},
{"name": "process_sort_index", "ph": "M", "pid": 14, "args": {"sort_index": 14}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8559929, "pid": 14},{"ph": "X", "name": "24", "ts": 8559931, "pid": 14, "dur": 0},{"ph": "X", "name": "0", "ts": 8560212, "pid": 14, "dur": 0},{"ph": "X", "name": "1", "ts": 8561027, "pid": 14, "dur": 0},{"ph": "X", "name": "2", "ts": 8561030, "pid": 14, "dur": 0},{"ph": "X", "name": "3", "ts": 8561033, "pid": 14, "dur": 0},{"ph": "X", "name": "4", "ts": 8561037, "pid": 14, "dur": 0},{"ph": "X", "name": "5", "ts": 8561041, "pid": 14, "dur": 0},{"ph": "X", "name": "6", "ts": 8561044, "pid": 14, "dur": 0},{"ph": "X", "name": "7", "ts": 8561048, "pid": 14, "dur": 0},{"ph": "X", "name": "8", "ts": 8561050, "pid": 14, "dur": 0},{"ph": "X", "name": "9", "ts": 8561054, "pid": 14, "dur": 0},{"ph": "X", "name": "10", "ts": 8561058, "pid": 14, "dur": 0},{"ph": "X", "name": "11", "ts": 8561060, "pid": 14, "dur": 0},{"ph": "X", "name": "12", "ts": 8561064, "pid": 14, "dur": 0},{"ph": "X", "name": "13", "ts": 8561067, "pid": 14, "dur": 0},{"ph": "X", "name": "14", "ts": 8561070, "pid": 14, "dur": 0},{"ph": "X", "name": "15", "ts": 8561074, "pid": 14, "dur": 0},{"ph": "X", "name": "16", "ts": 8561078, "pid": 14, "dur": 0},{"ph": "X", "name": "17", "ts": 8561081, "pid": 14, "dur": 0},{"ph": "X", "name": "18", "ts": 8561084, "pid": 14, "dur": 0},{"ph": "X", "name": "19", "ts": 8561088, "pid": 14, "dur": 0},{"ph": "X", "name": "20", "ts": 8561091, "pid": 14, "dur": 0},{"ph": "X", "name": "21", "ts": 8561095, "pid": 14, "dur": 0},{"ph": "X", "name": "22", "ts": 8561097, "pid": 14, "dur": 0},{"ph": "X", "name": "23", "ts": 8561101, "pid": 14, "dur": 0},{"ph": "X", "name": "25", "ts": 8561104, "pid": 14, "dur": 0},{"ph": "X", "name": "26", "ts": 8561108, "pid": 14, "dur": 0},{"ph": "X", "name": "27", "ts": 8561112, "pid": 14, "dur": 0},{"ph": "X", "name": "28", "ts": 8561115, "pid": 14, "dur": 0},{"ph": "X", "name": "29", "ts": 8561118, "pid": 14, "dur": 0},{"ph": "X", "name": "30", "ts": 8561121, "pid": 14, "dur": 0},{"ph": "X", "name": "31", "ts": 8561124, "pid": 14, "dur": 0},{"ph": "E", "ts": 8561126, "pid": 14},{"ph": "B", "name": "ALLREDUCE", "ts": 8561185, "pid": 14},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8561189, "pid": 14},{"ph": "E", "ts": 8561196, "pid": 14},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8561199, "pid": 14},{"ph": "E", "ts": 8561257, "pid": 14},{"ph": "B", "name": "QUEUE", "ts": 8561387, "pid": 14},{"ph": "E", "ts": 8561401, "pid": 14},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8561411, "pid": 14},{"ph": "E", "ts": 8561418, "pid": 14},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8561423, "pid": 14},{"ph": "E", "ts": 8561431, "pid": 14},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8561436, "pid": 14},{"ph": "E", "ts": 8561442, "pid": 14},{"ph": "E", "ts": 8561472, "pid": 14, "args": {"dtype": "float32", "shape": "[8192]"}},{"name": "process_name", "ph": "M", "pid": 15, "args": {"name": "allreduce.noname.15"}},
{"name": "process_sort_index", "ph": "M", "pid": 15, "args": {"sort_index": 15}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8562078, "pid": 15},{"ph": "X", "name": "24", "ts": 8562080, "pid": 15, "dur": 0},{"ph": "X", "name": "0", "ts": 8562378, "pid": 15, "dur": 0},{"ph": "X", "name": "1", "ts": 8563227, "pid": 15, "dur": 0},{"ph": "X", "name": "2", "ts": 8563231, "pid": 15, "dur": 0},{"ph": "X", "name": "3", "ts": 8563235, "pid": 15, "dur": 0},{"ph": "X", "name": "4", "ts": 8563237, "pid": 15, "dur": 0},{"ph": "X", "name": "5", "ts": 8563240, "pid": 15, "dur": 0},{"ph": "X", "name": "6", "ts": 8563243, "pid": 15, "dur": 0},{"ph": "X", "name": "7", "ts": 8563247, "pid": 15, "dur": 0},{"ph": "X", "name": "8", "ts": 8563250, "pid": 15, "dur": 0},{"ph": "X", "name": "9", "ts": 8563253, "pid": 15, "dur": 0},{"ph": "X", "name": "10", "ts": 8563256, "pid": 15, "dur": 0},{"ph": "X", "name": "11", "ts": 8563259, "pid": 15, "dur": 0},{"ph": "X", "name": "12", "ts": 8563262, "pid": 15, "dur": 0},{"ph": "X", "name": "13", "ts": 8563265, "pid": 15, "dur": 0},{"ph": "X", "name": "14", "ts": 8563268, "pid": 15, "dur": 0},{"ph": "X", "name": "15", "ts": 8563271, "pid": 15, "dur": 0},{"ph": "X", "name": "16", "ts": 8563274, "pid": 15, "dur": 0},{"ph": "X", "name": "17", "ts": 8563281, "pid": 15, "dur": 0},{"ph": "X", "name": "18", "ts": 8563284, "pid": 15, "dur": 0},{"ph": "X", "name": "19", "ts": 8563287, "pid": 15, "dur": 0},{"ph": "X", "name": "20", "ts": 8563290, "pid": 15, "dur": 0},{"ph": "X", "name": "21", "ts": 8563294, "pid": 15, "dur": 0},{"ph": "X", "name": "22", "ts": 8563297, "pid": 15, "dur": 0},{"ph": "X", "name": "23", "ts": 8563300, "pid": 15, "dur": 0},{"ph": "X", "name": "25", "ts": 8563304, "pid": 15, "dur": 0},{"ph": "X", "name": "26", "ts": 8563307, "pid": 15, "dur": 0},{"ph": "X", "name": "27", "ts": 8563310, "pid": 15, "dur": 0},{"ph": "X", "name": "28", "ts": 8563312, "pid": 15, "dur": 0},{"ph": "X", "name": "29", "ts": 8563315, "pid": 15, "dur": 0},{"ph": "X", "name": "30", "ts": 8563318, "pid": 15, "dur": 0},{"ph": "X", "name": "31", "ts": 8563321, "pid": 15, "dur": 0},{"ph": "E", "ts": 8563323, "pid": 15},{"ph": "B", "name": "ALLREDUCE", "ts": 8563382, "pid": 15},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8563387, "pid": 15},{"ph": "E", "ts": 8563392, "pid": 15},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8563394, "pid": 15},{"ph": "E", "ts": 8563453, "pid": 15},{"ph": "B", "name": "QUEUE", "ts": 8563579, "pid": 15},{"ph": "E", "ts": 8563593, "pid": 15},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8563606, "pid": 15},{"ph": "E", "ts": 8563615, "pid": 15},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8563622, "pid": 15},{"ph": "E", "ts": 8563628, "pid": 15},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8563636, "pid": 15},{"ph": "E", "ts": 8563642, "pid": 15},{"ph": "E", "ts": 8563671, "pid": 15, "args": {"dtype": "float32", "shape": "[16384]"}},{"name": "process_name", "ph": "M", "pid": 16, "args": {"name": "allreduce.noname.16"}},
{"name": "process_sort_index", "ph": "M", "pid": 16, "args": {"sort_index": 16}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8564573, "pid": 16},{"ph": "X", "name": "0", "ts": 8564576, "pid": 16, "dur": 0},{"ph": "X", "name": "1", "ts": 8565377, "pid": 16, "dur": 0},{"ph": "X", "name": "2", "ts": 8565380, "pid": 16, "dur": 0},{"ph": "X", "name": "3", "ts": 8565385, "pid": 16, "dur": 0},{"ph": "X", "name": "4", "ts": 8565388, "pid": 16, "dur": 0},{"ph": "X", "name": "5", "ts": 8565391, "pid": 16, "dur": 0},{"ph": "X", "name": "6", "ts": 8565394, "pid": 16, "dur": 0},{"ph": "X", "name": "7", "ts": 8565398, "pid": 16, "dur": 0},{"ph": "X", "name": "8", "ts": 8565402, "pid": 16, "dur": 0},{"ph": "X", "name": "9", "ts": 8565405, "pid": 16, "dur": 0},{"ph": "X", "name": "10", "ts": 8565408, "pid": 16, "dur": 0},{"ph": "X", "name": "12", "ts": 8565412, "pid": 16, "dur": 0},{"ph": "X", "name": "13", "ts": 8565415, "pid": 16, "dur": 0},{"ph": "X", "name": "14", "ts": 8565419, "pid": 16, "dur": 0},{"ph": "X", "name": "15", "ts": 8565421, "pid": 16, "dur": 0},{"ph": "X", "name": "16", "ts": 8565425, "pid": 16, "dur": 0},{"ph": "X", "name": "17", "ts": 8565428, "pid": 16, "dur": 0},{"ph": "X", "name": "18", "ts": 8565431, "pid": 16, "dur": 0},{"ph": "X", "name": "19", "ts": 8565435, "pid": 16, "dur": 0},{"ph": "X", "name": "20", "ts": 8565438, "pid": 16, "dur": 0},{"ph": "X", "name": "21", "ts": 8565441, "pid": 16, "dur": 0},{"ph": "X", "name": "22", "ts": 8565445, "pid": 16, "dur": 0},{"ph": "X", "name": "23", "ts": 8565448, "pid": 16, "dur": 0},{"ph": "X", "name": "24", "ts": 8565451, "pid": 16, "dur": 0},{"ph": "X", "name": "25", "ts": 8565454, "pid": 16, "dur": 0},{"ph": "X", "name": "26", "ts": 8565457, "pid": 16, "dur": 0},{"ph": "X", "name": "27", "ts": 8565461, "pid": 16, "dur": 0},{"ph": "X", "name": "28", "ts": 8565464, "pid": 16, "dur": 0},{"ph": "X", "name": "29", "ts": 8565467, "pid": 16, "dur": 0},{"ph": "X", "name": "30", "ts": 8565471, "pid": 16, "dur": 0},{"ph": "X", "name": "31", "ts": 8565474, "pid": 16, "dur": 0},{"ph": "X", "name": "11", "ts": 8566382, "pid": 16, "dur": 0},{"ph": "E", "ts": 8566385, "pid": 16},{"ph": "B", "name": "ALLREDUCE", "ts": 8566456, "pid": 16},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8566459, "pid": 16},{"ph": "E", "ts": 8566464, "pid": 16},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8566467, "pid": 16},{"ph": "E", "ts": 8566526, "pid": 16},{"ph": "B", "name": "QUEUE", "ts": 8566669, "pid": 16},{"ph": "E", "ts": 8566687, "pid": 16},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8566698, "pid": 16},{"ph": "E", "ts": 8566705, "pid": 16},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8566711, "pid": 16},{"ph": "E", "ts": 8566731, "pid": 16},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8566736, "pid": 16},{"ph": "E", "ts": 8566744, "pid": 16},{"ph": "E", "ts": 8566775, "pid": 16, "args": {"dtype": "float32", "shape": "[32768]"}},{"name": "process_name", "ph": "M", "pid": 17, "args": {"name": "allreduce.noname.17"}},
{"name": "process_sort_index", "ph": "M", "pid": 17, "args": {"sort_index": 17}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8568587, "pid": 17},{"ph": "X", "name": "2", "ts": 8568589, "pid": 17, "dur": 0},{"ph": "X", "name": "10", "ts": 8568601, "pid": 17, "dur": 0},{"ph": "X", "name": "15", "ts": 8568608, "pid": 17, "dur": 0},{"ph": "X", "name": "24", "ts": 8568617, "pid": 17, "dur": 0},{"ph": "X", "name": "31", "ts": 8568625, "pid": 17, "dur": 0},{"ph": "X", "name": "0", "ts": 8568797, "pid": 17, "dur": 0},{"ph": "X", "name": "1", "ts": 8569703, "pid": 17, "dur": 0},{"ph": "X", "name": "3", "ts": 8569708, "pid": 17, "dur": 0},{"ph": "X", "name": "4", "ts": 8569712, "pid": 17, "dur": 0},{"ph": "X", "name": "5", "ts": 8569715, "pid": 17, "dur": 0},{"ph": "X", "name": "6", "ts": 8569719, "pid": 17, "dur": 0},{"ph": "X", "name": "7", "ts": 8569723, "pid": 17, "dur": 0},{"ph": "X", "name": "8", "ts": 8569726, "pid": 17, "dur": 0},{"ph": "X", "name": "9", "ts": 8569730, "pid": 17, "dur": 0},{"ph": "X", "name": "12", "ts": 8569735, "pid": 17, "dur": 0},{"ph": "X", "name": "13", "ts": 8569739, "pid": 17, "dur": 0},{"ph": "X", "name": "14", "ts": 8569743, "pid": 17, "dur": 0},{"ph": "X", "name": "16", "ts": 8569746, "pid": 17, "dur": 0},{"ph": "X", "name": "17", "ts": 8569749, "pid": 17, "dur": 0},{"ph": "X", "name": "18", "ts": 8569752, "pid": 17, "dur": 0},{"ph": "X", "name": "19", "ts": 8569754, "pid": 17, "dur": 0},{"ph": "X", "name": "20", "ts": 8569757, "pid": 17, "dur": 0},{"ph": "X", "name": "21", "ts": 8569761, "pid": 17, "dur": 0},{"ph": "X", "name": "22", "ts": 8569764, "pid": 17, "dur": 0},{"ph": "X", "name": "23", "ts": 8569768, "pid": 17, "dur": 0},{"ph": "X", "name": "25", "ts": 8569771, "pid": 17, "dur": 0},{"ph": "X", "name": "26", "ts": 8569774, "pid": 17, "dur": 0},{"ph": "X", "name": "27", "ts": 8569778, "pid": 17, "dur": 0},{"ph": "X", "name": "28", "ts": 8569781, "pid": 17, "dur": 0},{"ph": "X", "name": "29", "ts": 8569785, "pid": 17, "dur": 0},{"ph": "X", "name": "30", "ts": 8569787, "pid": 17, "dur": 0},{"ph": "X", "name": "11", "ts": 8570716, "pid": 17, "dur": 0},{"ph": "E", "ts": 8570719, "pid": 17},{"ph": "B", "name": "ALLREDUCE", "ts": 8570790, "pid": 17},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8570793, "pid": 17},{"ph": "E", "ts": 8570799, "pid": 17},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8570803, "pid": 17},{"ph": "E", "ts": 8570861, "pid": 17},{"ph": "B", "name": "QUEUE", "ts": 8571005, "pid": 17},{"ph": "E", "ts": 8571024, "pid": 17},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8571032, "pid": 17},{"ph": "E", "ts": 8571039, "pid": 17},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8571047, "pid": 17},{"ph": "E", "ts": 8571054, "pid": 17},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8571061, "pid": 17},{"ph": "E", "ts": 8571091, "pid": 17},{"ph": "E", "ts": 8571128, "pid": 17, "args": {"dtype": "float32", "shape": "[65536]"}},{"name": "process_name", "ph": "M", "pid": 18, "args": {"name": "allreduce.noname.18"}},
{"name": "process_sort_index", "ph": "M", "pid": 18, "args": {"sort_index": 18}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8573043, "pid": 18},{"ph": "X", "name": "0", "ts": 8573045, "pid": 18, "dur": 0},{"ph": "X", "name": "2", "ts": 8574052, "pid": 18, "dur": 0},{"ph": "X", "name": "3", "ts": 8574056, "pid": 18, "dur": 0},{"ph": "X", "name": "4", "ts": 8574059, "pid": 18, "dur": 0},{"ph": "X", "name": "5", "ts": 8574063, "pid": 18, "dur": 0},{"ph": "X", "name": "6", "ts": 8574066, "pid": 18, "dur": 0},{"ph": "X", "name": "10", "ts": 8574072, "pid": 18, "dur": 0},{"ph": "X", "name": "12", "ts": 8574076, "pid": 18, "dur": 0},{"ph": "X", "name": "13", "ts": 8574079, "pid": 18, "dur": 0},{"ph": "X", "name": "15", "ts": 8574083, "pid": 18, "dur": 0},{"ph": "X", "name": "16", "ts": 8574087, "pid": 18, "dur": 0},{"ph": "X", "name": "17", "ts": 8574090, "pid": 18, "dur": 0},{"ph": "X", "name": "20", "ts": 8574095, "pid": 18, "dur": 0},{"ph": "X", "name": "21", "ts": 8574098, "pid": 18, "dur": 0},{"ph": "X", "name": "22", "ts": 8574102, "pid": 18, "dur": 0},{"ph": "X", "name": "23", "ts": 8574106, "pid": 18, "dur": 0},{"ph": "X", "name": "24", "ts": 8574109, "pid": 18, "dur": 0},{"ph": "X", "name": "25", "ts": 8574112, "pid": 18, "dur": 0},{"ph": "X", "name": "26", "ts": 8574115, "pid": 18, "dur": 0},{"ph": "X", "name": "27", "ts": 8574119, "pid": 18, "dur": 0},{"ph": "X", "name": "28", "ts": 8574126, "pid": 18, "dur": 0},{"ph": "X", "name": "29", "ts": 8574129, "pid": 18, "dur": 0},{"ph": "X", "name": "31", "ts": 8574133, "pid": 18, "dur": 0},{"ph": "X", "name": "1", "ts": 8575123, "pid": 18, "dur": 0},{"ph": "X", "name": "7", "ts": 8575130, "pid": 18, "dur": 0},{"ph": "X", "name": "8", "ts": 8575134, "pid": 18, "dur": 0},{"ph": "X", "name": "9", "ts": 8575137, "pid": 18, "dur": 0},{"ph": "X", "name": "14", "ts": 8575143, "pid": 18, "dur": 0},{"ph": "X", "name": "18", "ts": 8575149, "pid": 18, "dur": 0},{"ph": "X", "name": "19", "ts": 8575152, "pid": 18, "dur": 0},{"ph": "X", "name": "11", "ts": 8576251, "pid": 18, "dur": 0},{"ph": "X", "name": "30", "ts": 8577401, "pid": 18, "dur": 0},{"ph": "E", "ts": 8577403, "pid": 18},{"ph": "B", "name": "ALLREDUCE", "ts": 8577472, "pid": 18},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8577475, "pid": 18},{"ph": "E", "ts": 8577481, "pid": 18},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8577483, "pid": 18},{"ph": "E", "ts": 8577541, "pid": 18},{"ph": "B", "name": "QUEUE", "ts": 8577681, "pid": 18},{"ph": "E", "ts": 8577695, "pid": 18},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8577704, "pid": 18},{"ph": "E", "ts": 8577711, "pid": 18},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8577715, "pid": 18},{"ph": "E", "ts": 8577786, "pid": 18},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8577791, "pid": 18},{"ph": "E", "ts": 8577905, "pid": 18},{"ph": "E", "ts": 8577939, "pid": 18, "args": {"dtype": "float32", "shape": "[131072]"}},{"name": "process_name", "ph": "M", "pid": 19, "args": {"name": "allreduce.noname.19"}},
{"name": "process_sort_index", "ph": "M", "pid": 19, "args": {"sort_index": 19}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8582930, "pid": 19},{"ph": "X", "name": "10", "ts": 8582932, "pid": 19, "dur": 0},{"ph": "X", "name": "24", "ts": 8582946, "pid": 19, "dur": 0},{"ph": "X", "name": "0", "ts": 8582986, "pid": 19, "dur": 0},{"ph": "X", "name": "1", "ts": 8584006, "pid": 19, "dur": 0},{"ph": "X", "name": "2", "ts": 8584010, "pid": 19, "dur": 0},{"ph": "X", "name": "3", "ts": 8584013, "pid": 19, "dur": 0},{"ph": "X", "name": "4", "ts": 8584016, "pid": 19, "dur": 0},{"ph": "X", "name": "6", "ts": 8584020, "pid": 19, "dur": 0},{"ph": "X", "name": "7", "ts": 8584023, "pid": 19, "dur": 0},{"ph": "X", "name": "8", "ts": 8584027, "pid": 19, "dur": 0},{"ph": "X", "name": "9", "ts": 8584030, "pid": 19, "dur": 0},{"ph": "X", "name": "12", "ts": 8584035, "pid": 19, "dur": 0},{"ph": "X", "name": "13", "ts": 8584038, "pid": 19, "dur": 0},{"ph": "X", "name": "14", "ts": 8584042, "pid": 19, "dur": 0},{"ph": "X", "name": "15", "ts": 8584045, "pid": 19, "dur": 0},{"ph": "X", "name": "16", "ts": 8584048, "pid": 19, "dur": 0},{"ph": "X", "name": "17", "ts": 8584052, "pid": 19, "dur": 0},{"ph": "X", "name": "18", "ts": 8584055, "pid": 19, "dur": 0},{"ph": "X", "name": "19", "ts": 8584058, "pid": 19, "dur": 0},{"ph": "X", "name": "20", "ts": 8584061, "pid": 19, "dur": 0},{"ph": "X", "name": "21", "ts": 8584065, "pid": 19, "dur": 0},{"ph": "X", "name": "22", "ts": 8584068, "pid": 19, "dur": 0},{"ph": "X", "name": "23", "ts": 8584072, "pid": 19, "dur": 0},{"ph": "X", "name": "25", "ts": 8584076, "pid": 19, "dur": 0},{"ph": "X", "name": "26", "ts": 8584079, "pid": 19, "dur": 0},{"ph": "X", "name": "27", "ts": 8584083, "pid": 19, "dur": 0},{"ph": "X", "name": "28", "ts": 8584085, "pid": 19, "dur": 0},{"ph": "X", "name": "29", "ts": 8584089, "pid": 19, "dur": 0},{"ph": "X", "name": "31", "ts": 8584093, "pid": 19, "dur": 0},{"ph": "X", "name": "5", "ts": 8585118, "pid": 19, "dur": 0},{"ph": "X", "name": "11", "ts": 8586242, "pid": 19, "dur": 0},{"ph": "X", "name": "30", "ts": 8586257, "pid": 19, "dur": 0},{"ph": "E", "ts": 8586259, "pid": 19},{"ph": "B", "name": "ALLREDUCE", "ts": 8586312, "pid": 19},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8586314, "pid": 19},{"ph": "E", "ts": 8586320, "pid": 19},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8586324, "pid": 19},{"ph": "E", "ts": 8586383, "pid": 19},{"ph": "B", "name": "QUEUE", "ts": 8586540, "pid": 19},{"ph": "E", "ts": 8586555, "pid": 19},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8586562, "pid": 19},{"ph": "E", "ts": 8586603, "pid": 19},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8586609, "pid": 19},{"ph": "E", "ts": 8586740, "pid": 19},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8586746, "pid": 19},{"ph": "E", "ts": 8586934, "pid": 19},{"ph": "E", "ts": 8586972, "pid": 19, "args": {"dtype": "float32", "shape": "[262144]"}},{"name": "process_name", "ph": "M", "pid": 20, "args": {"name": "allreduce.noname.20"}},
{"name": "process_sort_index", "ph": "M", "pid": 20, "args": {"sort_index": 20}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8594004, "pid": 20},{"ph": "X", "name": "24", "ts": 8594007, "pid": 20, "dur": 0},{"ph": "X", "name": "2", "ts": 8595075, "pid": 20, "dur": 0},{"ph": "X", "name": "3", "ts": 8595078, "pid": 20, "dur": 0},{"ph": "X", "name": "4", "ts": 8595081, "pid": 20, "dur": 0},{"ph": "X", "name": "7", "ts": 8595086, "pid": 20, "dur": 0},{"ph": "X", "name": "9", "ts": 8595090, "pid": 20, "dur": 0},{"ph": "X", "name": "10", "ts": 8595094, "pid": 20, "dur": 0},{"ph": "X", "name": "17", "ts": 8595102, "pid": 20, "dur": 0},{"ph": "X", "name": "20", "ts": 8595107, "pid": 20, "dur": 0},{"ph": "X", "name": "22", "ts": 8595111, "pid": 20, "dur": 0},{"ph": "X", "name": "5", "ts": 8596208, "pid": 20, "dur": 0},{"ph": "X", "name": "6", "ts": 8596212, "pid": 20, "dur": 0},{"ph": "X", "name": "12", "ts": 8596218, "pid": 20, "dur": 0},{"ph": "X", "name": "13", "ts": 8596221, "pid": 20, "dur": 0},{"ph": "X", "name": "14", "ts": 8596224, "pid": 20, "dur": 0},{"ph": "X", "name": "15", "ts": 8596227, "pid": 20, "dur": 0},{"ph": "X", "name": "16", "ts": 8596230, "pid": 20, "dur": 0},{"ph": "X", "name": "18", "ts": 8596234, "pid": 20, "dur": 0},{"ph": "X", "name": "21", "ts": 8596239, "pid": 20, "dur": 0},{"ph": "X", "name": "23", "ts": 8596243, "pid": 20, "dur": 0},{"ph": "X", "name": "25", "ts": 8596247, "pid": 20, "dur": 0},{"ph": "X", "name": "26", "ts": 8596250, "pid": 20, "dur": 0},{"ph": "X", "name": "27", "ts": 8596254, "pid": 20, "dur": 0},{"ph": "X", "name": "28", "ts": 8596257, "pid": 20, "dur": 0},{"ph": "X", "name": "29", "ts": 8596261, "pid": 20, "dur": 0},{"ph": "X", "name": "31", "ts": 8596265, "pid": 20, "dur": 0},{"ph": "X", "name": "1", "ts": 8597285, "pid": 20, "dur": 0},{"ph": "X", "name": "19", "ts": 8597301, "pid": 20, "dur": 0},{"ph": "X", "name": "30", "ts": 8597311, "pid": 20, "dur": 0},{"ph": "X", "name": "0", "ts": 8599539, "pid": 20, "dur": 0},{"ph": "X", "name": "11", "ts": 8600586, "pid": 20, "dur": 0},{"ph": "X", "name": "8", "ts": 8603947, "pid": 20, "dur": 0},{"ph": "E", "ts": 8603949, "pid": 20},{"ph": "B", "name": "ALLREDUCE", "ts": 8604018, "pid": 20},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8604020, "pid": 20},{"ph": "E", "ts": 8604028, "pid": 20},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8604030, "pid": 20},{"ph": "E", "ts": 8604089, "pid": 20},{"ph": "B", "name": "QUEUE", "ts": 8604258, "pid": 20},{"ph": "E", "ts": 8604275, "pid": 20},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8604286, "pid": 20},{"ph": "E", "ts": 8604473, "pid": 20},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8604479, "pid": 20},{"ph": "E", "ts": 8604697, "pid": 20},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8604703, "pid": 20},{"ph": "E", "ts": 8605064, "pid": 20},{"ph": "E", "ts": 8605102, "pid": 20, "args": {"dtype": "float32", "shape": "[524288]"}},{"name": "process_name", "ph": "M", "pid": 21, "args": {"name": "allreduce.noname.21"}},
{"name": "process_sort_index", "ph": "M", "pid": 21, "args": {"sort_index": 21}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8618376, "pid": 21},{"ph": "X", "name": "2", "ts": 8618379, "pid": 21, "dur": 0},{"ph": "X", "name": "8", "ts": 8618388, "pid": 21, "dur": 0},{"ph": "X", "name": "10", "ts": 8618391, "pid": 21, "dur": 0},{"ph": "X", "name": "11", "ts": 8618395, "pid": 21, "dur": 0},{"ph": "X", "name": "12", "ts": 8618398, "pid": 21, "dur": 0},{"ph": "X", "name": "13", "ts": 8618402, "pid": 21, "dur": 0},{"ph": "X", "name": "15", "ts": 8618405, "pid": 21, "dur": 0},{"ph": "X", "name": "16", "ts": 8618408, "pid": 21, "dur": 0},{"ph": "X", "name": "17", "ts": 8618412, "pid": 21, "dur": 0},{"ph": "X", "name": "20", "ts": 8618416, "pid": 21, "dur": 0},{"ph": "X", "name": "25", "ts": 8618422, "pid": 21, "dur": 0},{"ph": "X", "name": "26", "ts": 8618425, "pid": 21, "dur": 0},{"ph": "X", "name": "27", "ts": 8618429, "pid": 21, "dur": 0},{"ph": "X", "name": "28", "ts": 8618431, "pid": 21, "dur": 0},{"ph": "X", "name": "31", "ts": 8618436, "pid": 21, "dur": 0},{"ph": "X", "name": "4", "ts": 8619487, "pid": 21, "dur": 0},{"ph": "X", "name": "6", "ts": 8619491, "pid": 21, "dur": 0},{"ph": "X", "name": "7", "ts": 8619495, "pid": 21, "dur": 0},{"ph": "X", "name": "9", "ts": 8619499, "pid": 21, "dur": 0},{"ph": "X", "name": "14", "ts": 8619505, "pid": 21, "dur": 0},{"ph": "X", "name": "19", "ts": 8619510, "pid": 21, "dur": 0},{"ph": "X", "name": "21", "ts": 8619514, "pid": 21, "dur": 0},{"ph": "X", "name": "22", "ts": 8619522, "pid": 21, "dur": 0},{"ph": "X", "name": "23", "ts": 8619525, "pid": 21, "dur": 0},{"ph": "X", "name": "24", "ts": 8619529, "pid": 21, "dur": 0},{"ph": "X", "name": "0", "ts": 8619572, "pid": 21, "dur": 0},{"ph": "X", "name": "3", "ts": 8620588, "pid": 21, "dur": 0},{"ph": "X", "name": "5", "ts": 8620592, "pid": 21, "dur": 0},{"ph": "X", "name": "18", "ts": 8620602, "pid": 21, "dur": 0},{"ph": "X", "name": "29", "ts": 8621691, "pid": 21, "dur": 0},{"ph": "X", "name": "30", "ts": 8621694, "pid": 21, "dur": 0},{"ph": "X", "name": "1", "ts": 8622798, "pid": 21, "dur": 0},{"ph": "E", "ts": 8622800, "pid": 21},{"ph": "B", "name": "ALLREDUCE", "ts": 8622873, "pid": 21},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8622875, "pid": 21},{"ph": "E", "ts": 8622882, "pid": 21},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8622885, "pid": 21},{"ph": "E", "ts": 8622943, "pid": 21},{"ph": "B", "name": "QUEUE", "ts": 8623106, "pid": 21},{"ph": "E", "ts": 8623125, "pid": 21},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8623133, "pid": 21},{"ph": "E", "ts": 8623681, "pid": 21},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8623687, "pid": 21},{"ph": "E", "ts": 8624057, "pid": 21},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8624064, "pid": 21},{"ph": "E", "ts": 8624770, "pid": 21},{"ph": "E", "ts": 8624806, "pid": 21, "args": {"dtype": "float32", "shape": "[1048576]"}},{"name": "process_name", "ph": "M", "pid": 22, "args": {"name": "allreduce.noname.22"}},
{"name": "process_sort_index", "ph": "M", "pid": 22, "args": {"sort_index": 22}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8648313, "pid": 22},{"ph": "X", "name": "8", "ts": 8648315, "pid": 22, "dur": 0},{"ph": "X", "name": "10", "ts": 8648321, "pid": 22, "dur": 0},{"ph": "X", "name": "2", "ts": 8649389, "pid": 22, "dur": 0},{"ph": "X", "name": "11", "ts": 8649398, "pid": 22, "dur": 0},{"ph": "X", "name": "17", "ts": 8649405, "pid": 22, "dur": 0},{"ph": "X", "name": "26", "ts": 8649415, "pid": 22, "dur": 0},{"ph": "X", "name": "4", "ts": 8650531, "pid": 22, "dur": 0},{"ph": "X", "name": "14", "ts": 8650540, "pid": 22, "dur": 0},{"ph": "X", "name": "19", "ts": 8650546, "pid": 22, "dur": 0},{"ph": "X", "name": "20", "ts": 8650550, "pid": 22, "dur": 0},{"ph": "X", "name": "24", "ts": 8650555, "pid": 22, "dur": 0},{"ph": "X", "name": "3", "ts": 8651617, "pid": 22, "dur": 0},{"ph": "X", "name": "6", "ts": 8651622, "pid": 22, "dur": 0},{"ph": "X", "name": "7", "ts": 8651627, "pid": 22, "dur": 0},{"ph": "X", "name": "12", "ts": 8651634, "pid": 22, "dur": 0},{"ph": "X", "name": "13", "ts": 8651639, "pid": 22, "dur": 0},{"ph": "X", "name": "15", "ts": 8651643, "pid": 22, "dur": 0},{"ph": "X", "name": "21", "ts": 8651649, "pid": 22, "dur": 0},{"ph": "X", "name": "22", "ts": 8651653, "pid": 22, "dur": 0},{"ph": "X", "name": "23", "ts": 8651656, "pid": 22, "dur": 0},{"ph": "X", "name": "0", "ts": 8651697, "pid": 22, "dur": 0},{"ph": "X", "name": "9", "ts": 8652726, "pid": 22, "dur": 0},{"ph": "X", "name": "16", "ts": 8652733, "pid": 22, "dur": 0},{"ph": "X", "name": "27", "ts": 8652743, "pid": 22, "dur": 0},{"ph": "X", "name": "28", "ts": 8652746, "pid": 22, "dur": 0},{"ph": "X", "name": "31", "ts": 8652751, "pid": 22, "dur": 0},{"ph": "X", "name": "25", "ts": 8653831, "pid": 22, "dur": 0},{"ph": "X", "name": "5", "ts": 8654946, "pid": 22, "dur": 0},{"ph": "X", "name": "18", "ts": 8654957, "pid": 22, "dur": 0},{"ph": "X", "name": "29", "ts": 8654967, "pid": 22, "dur": 0},{"ph": "X", "name": "30", "ts": 8656058, "pid": 22, "dur": 0},{"ph": "X", "name": "1", "ts": 8657167, "pid": 22, "dur": 0},{"ph": "E", "ts": 8657169, "pid": 22},{"ph": "B", "name": "ALLREDUCE", "ts": 8657243, "pid": 22},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8657246, "pid": 22},{"ph": "E", "ts": 8657255, "pid": 22},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8657257, "pid": 22},{"ph": "E", "ts": 8657316, "pid": 22},{"ph": "B", "name": "QUEUE", "ts": 8657462, "pid": 22},{"ph": "E", "ts": 8657480, "pid": 22},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8657488, "pid": 22},{"ph": "E", "ts": 8658783, "pid": 22},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8658790, "pid": 22},{"ph": "E", "ts": 8659457, "pid": 22},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8659471, "pid": 22},{"ph": "E", "ts": 8660867, "pid": 22},{"ph": "E", "ts": 8660908, "pid": 22, "args": {"dtype": "float32", "shape": "[2097152]"}},{"name": "process_name", "ph": "M", "pid": 23, "args": {"name": "allreduce.noname.23"}},
{"name": "process_sort_index", "ph": "M", "pid": 23, "args": {"sort_index": 23}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8707062, "pid": 23},{"ph": "X", "name": "8", "ts": 8707070, "pid": 23, "dur": 0},{"ph": "X", "name": "11", "ts": 8709274, "pid": 23, "dur": 0},{"ph": "X", "name": "17", "ts": 8709281, "pid": 23, "dur": 0},{"ph": "X", "name": "10", "ts": 8710379, "pid": 23, "dur": 0},{"ph": "X", "name": "15", "ts": 8710385, "pid": 23, "dur": 0},{"ph": "X", "name": "12", "ts": 8711485, "pid": 23, "dur": 0},{"ph": "X", "name": "13", "ts": 8711488, "pid": 23, "dur": 0},{"ph": "X", "name": "14", "ts": 8711491, "pid": 23, "dur": 0},{"ph": "X", "name": "19", "ts": 8711497, "pid": 23, "dur": 0},{"ph": "X", "name": "3", "ts": 8712615, "pid": 23, "dur": 0},{"ph": "X", "name": "4", "ts": 8712618, "pid": 23, "dur": 0},{"ph": "X", "name": "9", "ts": 8712624, "pid": 23, "dur": 0},{"ph": "X", "name": "16", "ts": 8712631, "pid": 23, "dur": 0},{"ph": "X", "name": "21", "ts": 8712637, "pid": 23, "dur": 0},{"ph": "X", "name": "22", "ts": 8712640, "pid": 23, "dur": 0},{"ph": "X", "name": "23", "ts": 8712643, "pid": 23, "dur": 0},{"ph": "X", "name": "24", "ts": 8712647, "pid": 23, "dur": 0},{"ph": "X", "name": "27", "ts": 8712652, "pid": 23, "dur": 0},{"ph": "X", "name": "28", "ts": 8712656, "pid": 23, "dur": 0},{"ph": "X", "name": "31", "ts": 8712660, "pid": 23, "dur": 0},{"ph": "X", "name": "6", "ts": 8713721, "pid": 23, "dur": 0},{"ph": "X", "name": "7", "ts": 8713724, "pid": 23, "dur": 0},{"ph": "X", "name": "25", "ts": 8713738, "pid": 23, "dur": 0},{"ph": "X", "name": "2", "ts": 8715966, "pid": 23, "dur": 0},{"ph": "X", "name": "26", "ts": 8715983, "pid": 23, "dur": 0},{"ph": "X", "name": "5", "ts": 8717061, "pid": 23, "dur": 0},{"ph": "X", "name": "29", "ts": 8719299, "pid": 23, "dur": 0},{"ph": "X", "name": "30", "ts": 8721510, "pid": 23, "dur": 0},{"ph": "X", "name": "0", "ts": 8722661, "pid": 23, "dur": 0},{"ph": "X", "name": "1", "ts": 8724850, "pid": 23, "dur": 0},{"ph": "X", "name": "18", "ts": 8725981, "pid": 23, "dur": 0},{"ph": "X", "name": "20", "ts": 8731501, "pid": 23, "dur": 0},{"ph": "E", "ts": 8731502, "pid": 23},{"ph": "B", "name": "ALLREDUCE", "ts": 8731563, "pid": 23},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8731566, "pid": 23},{"ph": "E", "ts": 8731576, "pid": 23},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8731578, "pid": 23},{"ph": "E", "ts": 8731636, "pid": 23},{"ph": "B", "name": "QUEUE", "ts": 8731791, "pid": 23},{"ph": "E", "ts": 8731805, "pid": 23},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8731815, "pid": 23},{"ph": "E", "ts": 8734517, "pid": 23},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8734523, "pid": 23},{"ph": "E", "ts": 8735808, "pid": 23},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8735814, "pid": 23},{"ph": "E", "ts": 8738641, "pid": 23},{"ph": "E", "ts": 8738675, "pid": 23, "args": {"dtype": "float32", "shape": "[4194304]"}},{"name": "process_name", "ph": "M", "pid": 24, "args": {"name": "allreduce.noname.24"}},
{"name": "process_sort_index", "ph": "M", "pid": 24, "args": {"sort_index": 24}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 8831282, "pid": 24},{"ph": "X", "name": "8", "ts": 8831285, "pid": 24, "dur": 0},{"ph": "X", "name": "10", "ts": 8834628, "pid": 24, "dur": 0},{"ph": "X", "name": "11", "ts": 8835739, "pid": 24, "dur": 0},{"ph": "X", "name": "4", "ts": 8841299, "pid": 24, "dur": 0},{"ph": "X", "name": "6", "ts": 8843502, "pid": 24, "dur": 0},{"ph": "X", "name": "7", "ts": 8843505, "pid": 24, "dur": 0},{"ph": "X", "name": "2", "ts": 8846800, "pid": 24, "dur": 0},{"ph": "X", "name": "19", "ts": 8846813, "pid": 24, "dur": 0},{"ph": "X", "name": "3", "ts": 8847928, "pid": 24, "dur": 0},{"ph": "X", "name": "16", "ts": 8847939, "pid": 24, "dur": 0},{"ph": "X", "name": "17", "ts": 8847943, "pid": 24, "dur": 0},{"ph": "X", "name": "24", "ts": 8847951, "pid": 24, "dur": 0},{"ph": "X", "name": "28", "ts": 8847957, "pid": 24, "dur": 0},{"ph": "X", "name": "29", "ts": 8847961, "pid": 24, "dur": 0},{"ph": "X", "name": "31", "ts": 8847964, "pid": 24, "dur": 0},{"ph": "X", "name": "25", "ts": 8849043, "pid": 24, "dur": 0},{"ph": "X", "name": "26", "ts": 8849047, "pid": 24, "dur": 0},{"ph": "X", "name": "27", "ts": 8849049, "pid": 24, "dur": 0},{"ph": "X", "name": "5", "ts": 8850131, "pid": 24, "dur": 0},{"ph": "X", "name": "21", "ts": 8852359, "pid": 24, "dur": 0},{"ph": "X", "name": "22", "ts": 8852362, "pid": 24, "dur": 0},{"ph": "X", "name": "23", "ts": 8852364, "pid": 24, "dur": 0},{"ph": "X", "name": "1", "ts": 8854552, "pid": 24, "dur": 0},{"ph": "X", "name": "0", "ts": 8857948, "pid": 24, "dur": 0},{"ph": "X", "name": "9", "ts": 8859029, "pid": 24, "dur": 0},{"ph": "X", "name": "15", "ts": 8861256, "pid": 24, "dur": 0},{"ph": "X", "name": "12", "ts": 8863498, "pid": 24, "dur": 0},{"ph": "X", "name": "13", "ts": 8863503, "pid": 24, "dur": 0},{"ph": "X", "name": "14", "ts": 8863507, "pid": 24, "dur": 0},{"ph": "X", "name": "18", "ts": 8865732, "pid": 24, "dur": 0},{"ph": "X", "name": "30", "ts": 8865742, "pid": 24, "dur": 0},{"ph": "X", "name": "20", "ts": 8896738, "pid": 24, "dur": 0},{"ph": "E", "ts": 8896740, "pid": 24},{"ph": "B", "name": "ALLREDUCE", "ts": 8896807, "pid": 24},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 8896810, "pid": 24},{"ph": "E", "ts": 8896820, "pid": 24},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 8896823, "pid": 24},{"ph": "E", "ts": 8896905, "pid": 24},{"ph": "B", "name": "QUEUE", "ts": 8897078, "pid": 24},{"ph": "E", "ts": 8897096, "pid": 24},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 8897103, "pid": 24},{"ph": "E", "ts": 8902628, "pid": 24},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 8902635, "pid": 24},{"ph": "E", "ts": 8905189, "pid": 24},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 8905194, "pid": 24},{"ph": "E", "ts": 8910837, "pid": 24},{"ph": "E", "ts": 8910862, "pid": 24, "args": {"dtype": "float32", "shape": "[8388608]"}},{"name": "process_name", "ph": "M", "pid": 25, "args": {"name": "allreduce.noname.25"}},
{"name": "process_sort_index", "ph": "M", "pid": 25, "args": {"sort_index": 25}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 9098783, "pid": 25},{"ph": "X", "name": "11", "ts": 9098785, "pid": 25, "dur": 0},{"ph": "X", "name": "10", "ts": 9099874, "pid": 25, "dur": 0},{"ph": "X", "name": "12", "ts": 9104313, "pid": 25, "dur": 0},{"ph": "X", "name": "13", "ts": 9108703, "pid": 25, "dur": 0},{"ph": "X", "name": "5", "ts": 9112068, "pid": 25, "dur": 0},{"ph": "X", "name": "4", "ts": 9115403, "pid": 25, "dur": 0},{"ph": "X", "name": "9", "ts": 9115410, "pid": 25, "dur": 0},{"ph": "X", "name": "15", "ts": 9115416, "pid": 25, "dur": 0},{"ph": "X", "name": "6", "ts": 9118719, "pid": 25, "dur": 0},{"ph": "X", "name": "7", "ts": 9118722, "pid": 25, "dur": 0},{"ph": "X", "name": "8", "ts": 9122039, "pid": 25, "dur": 0},{"ph": "X", "name": "29", "ts": 9125385, "pid": 25, "dur": 0},{"ph": "X", "name": "31", "ts": 9125389, "pid": 25, "dur": 0},{"ph": "X", "name": "24", "ts": 9126509, "pid": 25, "dur": 0},{"ph": "X", "name": "27", "ts": 9126513, "pid": 25, "dur": 0},{"ph": "X", "name": "28", "ts": 9126517, "pid": 25, "dur": 0},{"ph": "X", "name": "21", "ts": 9127592, "pid": 25, "dur": 0},{"ph": "X", "name": "26", "ts": 9127599, "pid": 25, "dur": 0},{"ph": "X", "name": "22", "ts": 9128715, "pid": 25, "dur": 0},{"ph": "X", "name": "23", "ts": 9128719, "pid": 25, "dur": 0},{"ph": "X", "name": "25", "ts": 9128723, "pid": 25, "dur": 0},{"ph": "X", "name": "20", "ts": 9129806, "pid": 25, "dur": 0},{"ph": "X", "name": "19", "ts": 9133123, "pid": 25, "dur": 0},{"ph": "X", "name": "17", "ts": 9135330, "pid": 25, "dur": 0},{"ph": "X", "name": "14", "ts": 9138652, "pid": 25, "dur": 0},{"ph": "X", "name": "18", "ts": 9143118, "pid": 25, "dur": 0},{"ph": "X", "name": "30", "ts": 9161987, "pid": 25, "dur": 0},{"ph": "X", "name": "16", "ts": 9166420, "pid": 25, "dur": 0},{"ph": "X", "name": "0", "ts": 9168683, "pid": 25, "dur": 0},{"ph": "X", "name": "1", "ts": 9169732, "pid": 25, "dur": 0},{"ph": "X", "name": "2", "ts": 9169735, "pid": 25, "dur": 0},{"ph": "X", "name": "3", "ts": 9169739, "pid": 25, "dur": 0},{"ph": "E", "ts": 9169741, "pid": 25},{"ph": "B", "name": "ALLREDUCE", "ts": 9169816, "pid": 25},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 9169819, "pid": 25},{"ph": "E", "ts": 9169827, "pid": 25},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 9169829, "pid": 25},{"ph": "E", "ts": 9169889, "pid": 25},{"ph": "B", "name": "QUEUE", "ts": 9169999, "pid": 25},{"ph": "E", "ts": 9170029, "pid": 25},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 9170034, "pid": 25},{"ph": "E", "ts": 9181250, "pid": 25},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 9181255, "pid": 25},{"ph": "E", "ts": 9186353, "pid": 25},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 9186358, "pid": 25},{"ph": "E", "ts": 9197629, "pid": 25},{"ph": "E", "ts": 9197657, "pid": 25, "args": {"dtype": "float32", "shape": "[16777216]"}},{"name": "process_name", "ph": "M", "pid": 26, "args": {"name": "allreduce.noname.26"}},
{"name": "process_sort_index", "ph": "M", "pid": 26, "args": {"sort_index": 26}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 9552462, "pid": 26},{"ph": "X", "name": "8", "ts": 9552464, "pid": 26, "dur": 0},{"ph": "X", "name": "11", "ts": 9569439, "pid": 26, "dur": 0},{"ph": "X", "name": "10", "ts": 9572604, "pid": 26, "dur": 0},{"ph": "X", "name": "15", "ts": 9578970, "pid": 26, "dur": 0},{"ph": "X", "name": "12", "ts": 9590641, "pid": 26, "dur": 0},{"ph": "X", "name": "14", "ts": 9590644, "pid": 26, "dur": 0},{"ph": "X", "name": "13", "ts": 9593801, "pid": 26, "dur": 0},{"ph": "X", "name": "9", "ts": 9603346, "pid": 26, "dur": 0},{"ph": "X", "name": "30", "ts": 9616066, "pid": 26, "dur": 0},{"ph": "X", "name": "19", "ts": 9620315, "pid": 26, "dur": 0},{"ph": "X", "name": "17", "ts": 9622428, "pid": 26, "dur": 0},{"ph": "X", "name": "0", "ts": 9623523, "pid": 26, "dur": 0},{"ph": "X", "name": "2", "ts": 9625603, "pid": 26, "dur": 0},{"ph": "X", "name": "27", "ts": 9625620, "pid": 26, "dur": 0},{"ph": "X", "name": "3", "ts": 9626685, "pid": 26, "dur": 0},{"ph": "X", "name": "24", "ts": 9626697, "pid": 26, "dur": 0},{"ph": "X", "name": "26", "ts": 9626699, "pid": 26, "dur": 0},{"ph": "X", "name": "29", "ts": 9626703, "pid": 26, "dur": 0},{"ph": "X", "name": "31", "ts": 9626706, "pid": 26, "dur": 0},{"ph": "X", "name": "21", "ts": 9627730, "pid": 26, "dur": 0},{"ph": "X", "name": "22", "ts": 9629843, "pid": 26, "dur": 0},{"ph": "X", "name": "23", "ts": 9629845, "pid": 26, "dur": 0},{"ph": "X", "name": "20", "ts": 9630917, "pid": 26, "dur": 0},{"ph": "X", "name": "25", "ts": 9630922, "pid": 26, "dur": 0},{"ph": "X", "name": "4", "ts": 9634077, "pid": 26, "dur": 0},{"ph": "X", "name": "5", "ts": 9634080, "pid": 26, "dur": 0},{"ph": "X", "name": "6", "ts": 9634082, "pid": 26, "dur": 0},{"ph": "X", "name": "7", "ts": 9634085, "pid": 26, "dur": 0},{"ph": "X", "name": "16", "ts": 9634092, "pid": 26, "dur": 0},{"ph": "X", "name": "18", "ts": 9638320, "pid": 26, "dur": 0},{"ph": "X", "name": "1", "ts": 9646795, "pid": 26, "dur": 0},{"ph": "X", "name": "28", "ts": 9651041, "pid": 26, "dur": 0},{"ph": "E", "ts": 9651043, "pid": 26},{"ph": "B", "name": "ALLREDUCE", "ts": 9651095, "pid": 26},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 9651097, "pid": 26},{"ph": "E", "ts": 9651106, "pid": 26},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 9651108, "pid": 26},{"ph": "E", "ts": 9651166, "pid": 26},{"ph": "B", "name": "QUEUE", "ts": 9651331, "pid": 26},{"ph": "E", "ts": 9651351, "pid": 26},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 9651360, "pid": 26},{"ph": "E", "ts": 9673771, "pid": 26},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 9673775, "pid": 26},{"ph": "E", "ts": 9683954, "pid": 26},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 9683957, "pid": 26},{"ph": "E", "ts": 9706473, "pid": 26},{"ph": "E", "ts": 9706498, "pid": 26, "args": {"dtype": "float32", "shape": "[33554432]"}},{"name": "process_name", "ph": "M", "pid": 27, "args": {"name": "allreduce.noname.27"}},
{"name": "process_sort_index", "ph": "M", "pid": 27, "args": {"sort_index": 27}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 10418965, "pid": 27},{"ph": "X", "name": "8", "ts": 10418967, "pid": 27, "dur": 0},{"ph": "X", "name": "11", "ts": 10446557, "pid": 27, "dur": 0},{"ph": "X", "name": "5", "ts": 10450807, "pid": 27, "dur": 0},{"ph": "X", "name": "10", "ts": 10453995, "pid": 27, "dur": 0},{"ph": "X", "name": "15", "ts": 10466740, "pid": 27, "dur": 0},{"ph": "X", "name": "4", "ts": 10479480, "pid": 27, "dur": 0},{"ph": "X", "name": "12", "ts": 10489055, "pid": 27, "dur": 0},{"ph": "X", "name": "14", "ts": 10489058, "pid": 27, "dur": 0},{"ph": "X", "name": "7", "ts": 10491185, "pid": 27, "dur": 0},{"ph": "X", "name": "6", "ts": 10494345, "pid": 27, "dur": 0},{"ph": "X", "name": "13", "ts": 10495434, "pid": 27, "dur": 0},{"ph": "X", "name": "30", "ts": 10507103, "pid": 27, "dur": 0},{"ph": "X", "name": "9", "ts": 10512396, "pid": 27, "dur": 0},{"ph": "X", "name": "19", "ts": 10550667, "pid": 27, "dur": 0},{"ph": "X", "name": "17", "ts": 10554885, "pid": 27, "dur": 0},{"ph": "X", "name": "16", "ts": 10557022, "pid": 27, "dur": 0},{"ph": "X", "name": "29", "ts": 10559144, "pid": 27, "dur": 0},{"ph": "X", "name": "28", "ts": 10560208, "pid": 27, "dur": 0},{"ph": "X", "name": "31", "ts": 10560212, "pid": 27, "dur": 0},{"ph": "X", "name": "27", "ts": 10562327, "pid": 27, "dur": 0},{"ph": "X", "name": "0", "ts": 10568539, "pid": 27, "dur": 0},{"ph": "X", "name": "1", "ts": 10568733, "pid": 27, "dur": 0},{"ph": "X", "name": "2", "ts": 10568735, "pid": 27, "dur": 0},{"ph": "X", "name": "21", "ts": 10568747, "pid": 27, "dur": 0},{"ph": "X", "name": "22", "ts": 10568749, "pid": 27, "dur": 0},{"ph": "X", "name": "24", "ts": 10568753, "pid": 27, "dur": 0},{"ph": "X", "name": "25", "ts": 10568756, "pid": 27, "dur": 0},{"ph": "X", "name": "26", "ts": 10568765, "pid": 27, "dur": 0},{"ph": "X", "name": "23", "ts": 10569808, "pid": 27, "dur": 0},{"ph": "X", "name": "20", "ts": 10572053, "pid": 27, "dur": 0},{"ph": "X", "name": "3", "ts": 10578732, "pid": 27, "dur": 0},{"ph": "X", "name": "18", "ts": 10601625, "pid": 27, "dur": 0},{"ph": "E", "ts": 10601627, "pid": 27},{"ph": "B", "name": "ALLREDUCE", "ts": 10601686, "pid": 27},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 10601688, "pid": 27},{"ph": "E", "ts": 10601700, "pid": 27},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 10601703, "pid": 27},{"ph": "E", "ts": 10601759, "pid": 27},{"ph": "B", "name": "QUEUE", "ts": 10601917, "pid": 27},{"ph": "E", "ts": 10601937, "pid": 27},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 10601945, "pid": 27},{"ph": "E", "ts": 10647048, "pid": 27},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 10647052, "pid": 27},{"ph": "E", "ts": 10667407, "pid": 27},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 10667410, "pid": 27},{"ph": "E", "ts": 10712353, "pid": 27},{"ph": "E", "ts": 10712380, "pid": 27, "args": {"dtype": "float32", "shape": "[67108864]"}},{"name": "process_name", "ph": "M", "pid": 28, "args": {"name": "allreduce.noname.28"}},
{"name": "process_sort_index", "ph": "M", "pid": 28, "args": {"sort_index": 28}},
{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 12148853, "pid": 28},{"ph": "X", "name": "8", "ts": 12148856, "pid": 28, "dur": 0},{"ph": "X", "name": "11", "ts": 12208349, "pid": 28, "dur": 0},{"ph": "X", "name": "10", "ts": 12215789, "pid": 28, "dur": 0},{"ph": "X", "name": "5", "ts": 12232755, "pid": 28, "dur": 0},{"ph": "X", "name": "15", "ts": 12238073, "pid": 28, "dur": 0},{"ph": "X", "name": "4", "ts": 12252884, "pid": 28, "dur": 0},{"ph": "X", "name": "7", "ts": 12280525, "pid": 28, "dur": 0},{"ph": "X", "name": "12", "ts": 12281604, "pid": 28, "dur": 0},{"ph": "X", "name": "6", "ts": 12285897, "pid": 28, "dur": 0},{"ph": "X", "name": "14", "ts": 12289051, "pid": 28, "dur": 0},{"ph": "X", "name": "13", "ts": 12293256, "pid": 28, "dur": 0},{"ph": "X", "name": "9", "ts": 12308481, "pid": 28, "dur": 0},{"ph": "X", "name": "0", "ts": 12412311, "pid": 28, "dur": 0},{"ph": "X", "name": "28", "ts": 12418654, "pid": 28, "dur": 0},{"ph": "X", "name": "29", "ts": 12420762, "pid": 28, "dur": 0},{"ph": "X", "name": "1", "ts": 12421803, "pid": 28, "dur": 0},{"ph": "X", "name": "2", "ts": 12421806, "pid": 28, "dur": 0},{"ph": "X", "name": "27", "ts": 12423939, "pid": 28, "dur": 0},{"ph": "X", "name": "31", "ts": 12424999, "pid": 28, "dur": 0},{"ph": "X", "name": "3", "ts": 12430280, "pid": 28, "dur": 0},{"ph": "X", "name": "24", "ts": 12431359, "pid": 28, "dur": 0},{"ph": "X", "name": "25", "ts": 12431362, "pid": 28, "dur": 0},{"ph": "X", "name": "26", "ts": 12431365, "pid": 28, "dur": 0},{"ph": "X", "name": "21", "ts": 12482292, "pid": 28, "dur": 0},{"ph": "X", "name": "19", "ts": 12484400, "pid": 28, "dur": 0},{"ph": "X", "name": "17", "ts": 12486547, "pid": 28, "dur": 0},{"ph": "X", "name": "23", "ts": 12487602, "pid": 28, "dur": 0},{"ph": "X", "name": "30", "ts": 12487609, "pid": 28, "dur": 0},{"ph": "X", "name": "20", "ts": 12489734, "pid": 28, "dur": 0},{"ph": "X", "name": "16", "ts": 12490771, "pid": 28, "dur": 0},{"ph": "X", "name": "22", "ts": 12494998, "pid": 28, "dur": 0},{"ph": "X", "name": "18", "ts": 12525726, "pid": 28, "dur": 0},{"ph": "E", "ts": 12525730, "pid": 28},{"ph": "B", "name": "ALLREDUCE", "ts": 12525790, "pid": 28},{"ph": "B", "name": "WAIT_FOR_DATA", "ts": 12525793, "pid": 28},{"ph": "E", "ts": 12525807, "pid": 28},{"ph": "B", "name": "WAIT_FOR_OTHER_TENSOR_DATA", "ts": 12525810, "pid": 28},{"ph": "E", "ts": 12525867, "pid": 28},{"ph": "B", "name": "QUEUE", "ts": 12526060, "pid": 28},{"ph": "E", "ts": 12526082, "pid": 28},{"ph": "B", "name": "NCCL_REDUCESCATTER", "ts": 12526100, "pid": 28},{"ph": "E", "ts": 12615896, "pid": 28},{"ph": "B", "name": "NCCL_ALLREDUCE", "ts": 12615899, "pid": 28},{"ph": "E", "ts": 12656615, "pid": 28},{"ph": "B", "name": "NCCL_ALLGATHER", "ts": 12656617, "pid": 28},{"ph": "E", "ts": 12746627, "pid": 28},{"ph": "E", "ts": 12746648, "pid": 28, "args": {"dtype": "float32", "shape": "[134217728]"}}]
Raw
extract-timeline.py
import json
f = open('./data/horovod-results/32gpu/logs/10g-nccl-fast.json')
data = json.load(f)
small = 1e9
large = 0
for i in data:
if i['pid'] == 3 and 'ts' in i:
if i['ts'] > large:
large = i['ts']
if i['ts'] < small:
small = i['ts']
print(large)
print(small)
print(large-small)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment