Skip to content

Instantly share code, notes, and snippets.

@c21
Created November 13, 2021 09:34
Show Gist options
  • Save c21/5cab9751f24ae448d77a259d28cb77d7 to your computer and use it in GitHub Desktop.
Save c21/5cab9751f24ae448d77a259d28cb77d7 to your computer and use it in GitHub Desktop.
== Subtree 5 / 5 (maxMethodCodeSize:590; maxConstantPoolSize:189(0.29% used); numInnerClasses:0) ==
*(5) SortMergeJoin [k1#2L], [(k2#6L % 3)], FullOuter, NOT ((k1#2L + 3) = k2#6L)
:- *(2) Sort [k1#2L ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(k1#2L, 5), ENSURE_REQUIREMENTS, [id=#242]
: +- *(1) Project [id#0L AS k1#2L]
: +- *(1) Range (0, 5, step=1, splits=2)
+- *(4) Sort [(k2#6L % 3) ASC NULLS FIRST], false, 0
+- Exchange hashpartitioning((k2#6L % 3), 5), ENSURE_REQUIREMENTS, [id=#248]
+- *(3) Project [id#4L AS k2#6L]
+- *(3) Range (0, 10, step=1, splits=2)
Generated code:
/* 001 */ public Object generate(Object[] references) {
/* 002 */ return new GeneratedIteratorForCodegenStage5(references);
/* 003 */ }
/* 004 */
/* 005 */ // codegenStageId=5
/* 006 */ final class GeneratedIteratorForCodegenStage5 extends org.apache.spark.sql.execution.BufferedRowIterator {
/* 007 */ private Object[] references;
/* 008 */ private scala.collection.Iterator[] inputs;
/* 009 */ private scala.collection.Iterator smj_leftInput_0;
/* 010 */ private scala.collection.Iterator smj_rightInput_0;
/* 011 */ private InternalRow smj_leftInputRow_0;
/* 012 */ private InternalRow smj_rightInputRow_0;
/* 013 */ private long smj_value_4;
/* 014 */ private InternalRow smj_leftOutputRow_0;
/* 015 */ private InternalRow smj_rightOutputRow_0;
/* 016 */ private java.util.ArrayList<InternalRow> smj_leftBuffer_0;
/* 017 */ private java.util.ArrayList<InternalRow> smj_rightBuffer_0;
/* 018 */ private org.apache.spark.util.collection.BitSet smj_leftMatched_0;
/* 019 */ private org.apache.spark.util.collection.BitSet smj_rightMatched_0;
/* 020 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] smj_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1];
/* 021 */
/* 022 */ public GeneratedIteratorForCodegenStage5(Object[] references) {
/* 023 */ this.references = references;
/* 024 */ }
/* 025 */
/* 026 */ public void init(int index, scala.collection.Iterator[] inputs) {
/* 027 */ partitionIndex = index;
/* 028 */ this.inputs = inputs;
/* 029 */ smj_leftInput_0 = inputs[0];
/* 030 */ smj_rightInput_0 = inputs[1];
/* 031 */
/* 032 */ smj_leftBuffer_0 = new java.util.ArrayList<InternalRow>();
/* 033 */ smj_rightBuffer_0 = new java.util.ArrayList<InternalRow>();
/* 034 */ smj_leftMatched_0 = new org.apache.spark.util.collection.BitSet(1);
/* 035 */ smj_rightMatched_0 = new org.apache.spark.util.collection.BitSet(1);
/* 036 */ smj_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0);
/* 037 */
/* 038 */ }
/* 039 */
/* 040 */ private void smj_consumeFullOuterJoinRow_0() {
/* 041 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1);
/* 042 */
/* 043 */ boolean smj_isNull_9 = true;
/* 044 */ long smj_value_10 = -1L;
/* 045 */ if (smj_leftOutputRow_0 != null) {
/* 046 */ long smj_value_9 = smj_leftOutputRow_0.getLong(0);
/* 047 */ smj_isNull_9 = false;
/* 048 */ smj_value_10 = smj_value_9;
/* 049 */ }
/* 050 */ boolean smj_isNull_11 = true;
/* 051 */ long smj_value_12 = -1L;
/* 052 */ if (smj_rightOutputRow_0 != null) {
/* 053 */ long smj_value_11 = smj_rightOutputRow_0.getLong(0);
/* 054 */ smj_isNull_11 = false;
/* 055 */ smj_value_12 = smj_value_11;
/* 056 */ }
/* 057 */ smj_mutableStateArray_0[0].reset();
/* 058 */
/* 059 */ smj_mutableStateArray_0[0].zeroOutNullBytes();
/* 060 */
/* 061 */ if (smj_isNull_9) {
/* 062 */ smj_mutableStateArray_0[0].setNullAt(0);
/* 063 */ } else {
/* 064 */ smj_mutableStateArray_0[0].write(0, smj_value_10);
/* 065 */ }
/* 066 */
/* 067 */ if (smj_isNull_11) {
/* 068 */ smj_mutableStateArray_0[0].setNullAt(1);
/* 069 */ } else {
/* 070 */ smj_mutableStateArray_0[0].write(1, smj_value_12);
/* 071 */ }
/* 072 */ append((smj_mutableStateArray_0[0].getRow()).copy());
/* 073 */
/* 074 */ }
/* 075 */
/* 076 */ private void wholestagecodegen_findNextJoinRows_0(
/* 077 */ scala.collection.Iterator leftIter,
/* 078 */ scala.collection.Iterator rightIter) {
/* 079 */ int comp = 0;
/* 080 */ smj_leftBuffer_0.clear();
/* 081 */ smj_rightBuffer_0.clear();
/* 082 */
/* 083 */ if (smj_leftInputRow_0 == null) {
/* 084 */ smj_leftInputRow_0 = (InternalRow) leftIter.next();
/* 085 */ }
/* 086 */ if (smj_rightInputRow_0 == null) {
/* 087 */ smj_rightInputRow_0 = (InternalRow) rightIter.next();
/* 088 */ }
/* 089 */
/* 090 */ long smj_value_0 = smj_leftInputRow_0.getLong(0);
/* 091 */ if (false) {
/* 092 */ // The left row join key is null, join it with null row
/* 093 */
/* 094 */ smj_leftOutputRow_0 = smj_leftInputRow_0;
/* 095 */ smj_rightOutputRow_0 = null;
/* 096 */ smj_leftInputRow_0 = null;
/* 097 */ smj_consumeFullOuterJoinRow_0();
/* 098 */
/* 099 */ return;
/* 100 */ }
/* 101 */
/* 102 */ boolean smj_isNull_1 = false;
/* 103 */ long smj_value_1 = -1L;
/* 104 */ if (3L == 0) {
/* 105 */ smj_isNull_1 = true;
/* 106 */ } else {
/* 107 */ long smj_value_2 = smj_rightInputRow_0.getLong(0);
/* 108 */
/* 109 */ smj_value_1 = (long)(smj_value_2 % 3L);
/* 110 */ }
/* 111 */ if (smj_isNull_1) {
/* 112 */ // The right row join key is null, join it with null row
/* 113 */
/* 114 */ smj_rightOutputRow_0 = smj_rightInputRow_0;
/* 115 */ smj_leftOutputRow_0 = null;
/* 116 */ smj_rightInputRow_0 = null;
/* 117 */ smj_consumeFullOuterJoinRow_0();
/* 118 */
/* 119 */ return;
/* 120 */ }
/* 121 */
/* 122 */ comp = 0;
/* 123 */ if (comp == 0) {
/* 124 */ comp = (smj_value_0 > smj_value_1 ? 1 : smj_value_0 < smj_value_1 ? -1 : 0);
/* 125 */ }
/* 126 */
/* 127 */ if (comp < 0) {
/* 128 */ // The left row join key is smaller, join it with null row
/* 129 */
/* 130 */ smj_leftOutputRow_0 = smj_leftInputRow_0;
/* 131 */ smj_rightOutputRow_0 = null;
/* 132 */ smj_leftInputRow_0 = null;
/* 133 */ smj_consumeFullOuterJoinRow_0();
/* 134 */
/* 135 */ return;
/* 136 */ } else if (comp > 0) {
/* 137 */ // The right row join key is smaller, join it with null row
/* 138 */
/* 139 */ smj_rightOutputRow_0 = smj_rightInputRow_0;
/* 140 */ smj_leftOutputRow_0 = null;
/* 141 */ smj_rightInputRow_0 = null;
/* 142 */ smj_consumeFullOuterJoinRow_0();
/* 143 */
/* 144 */ return;
/* 145 */ }
/* 146 */
/* 147 */ smj_value_4 = smj_value_0;
/* 148 */ smj_leftBuffer_0.add(smj_leftInputRow_0.copy());
/* 149 */ smj_rightBuffer_0.add(smj_rightInputRow_0.copy());
/* 150 */ smj_leftInputRow_0 = null;
/* 151 */ smj_rightInputRow_0 = null;
/* 152 */
/* 153 */ // Buffer rows from both sides with same join key
/* 154 */ while (leftIter.hasNext()) {
/* 155 */ smj_leftInputRow_0 = (InternalRow) leftIter.next();
/* 156 */ long smj_value_5 = smj_leftInputRow_0.getLong(0);
/* 157 */
/* 158 */ comp = 0;
/* 159 */ if (comp == 0) {
/* 160 */ comp = (smj_value_5 > smj_value_4 ? 1 : smj_value_5 < smj_value_4 ? -1 : 0);
/* 161 */ }
/* 162 */
/* 163 */ if (comp == 0) {
/* 164 */ smj_leftBuffer_0.add(smj_leftInputRow_0.copy());
/* 165 */ smj_leftInputRow_0 = null;
/* 166 */ } else {
/* 167 */ break;
/* 168 */ }
/* 169 */ }
/* 170 */ while (rightIter.hasNext()) {
/* 171 */ smj_rightInputRow_0 = (InternalRow) rightIter.next();
/* 172 */ boolean smj_isNull_5 = false;
/* 173 */ long smj_value_6 = -1L;
/* 174 */ if (3L == 0) {
/* 175 */ smj_isNull_5 = true;
/* 176 */ } else {
/* 177 */ long smj_value_7 = smj_rightInputRow_0.getLong(0);
/* 178 */
/* 179 */ smj_value_6 = (long)(smj_value_7 % 3L);
/* 180 */ }
/* 181 */
/* 182 */ comp = 0;
/* 183 */ if (comp == 0) {
/* 184 */ comp = (smj_value_6 > smj_value_4 ? 1 : smj_value_6 < smj_value_4 ? -1 : 0);
/* 185 */ }
/* 186 */
/* 187 */ if (comp == 0) {
/* 188 */ smj_rightBuffer_0.add(smj_rightInputRow_0.copy());
/* 189 */ smj_rightInputRow_0 = null;
/* 190 */ } else {
/* 191 */ break;
/* 192 */ }
/* 193 */ }
/* 194 */
/* 195 */ // Reset bit sets of buffers accordingly
/* 196 */ if (smj_leftBuffer_0.size() <= smj_leftMatched_0.capacity()) {
/* 197 */ smj_leftMatched_0.clearUntil(smj_leftBuffer_0.size());
/* 198 */ } else {
/* 199 */ smj_leftMatched_0 = new org.apache.spark.util.collection.BitSet(smj_leftBuffer_0.size());
/* 200 */ }
/* 201 */ if (smj_rightBuffer_0.size() <= smj_rightMatched_0.capacity()) {
/* 202 */ smj_rightMatched_0.clearUntil(smj_rightBuffer_0.size());
/* 203 */ } else {
/* 204 */ smj_rightMatched_0 = new org.apache.spark.util.collection.BitSet(smj_rightBuffer_0.size());
/* 205 */ }
/* 206 */ }
/* 207 */
/* 208 */ protected void processNext() throws java.io.IOException {
/* 209 */ while ((smj_leftInputRow_0 != null || smj_leftInput_0.hasNext()) &&
/* 210 */ (smj_rightInputRow_0 != null || smj_rightInput_0.hasNext())) {
/* 211 */ wholestagecodegen_findNextJoinRows_0(smj_leftInput_0, smj_rightInput_0);
/* 212 */
/* 213 */ int smj_leftIndex_0;
/* 214 */ int smj_rightIndex_0;
/* 215 */
/* 216 */ for (smj_leftIndex_0 = 0; smj_leftIndex_0 < smj_leftBuffer_0.size(); smj_leftIndex_0++) {
/* 217 */ smj_leftOutputRow_0 = (InternalRow) smj_leftBuffer_0.get(smj_leftIndex_0);
/* 218 */ for (smj_rightIndex_0 = 0; smj_rightIndex_0 < smj_rightBuffer_0.size(); smj_rightIndex_0++) {
/* 219 */ smj_rightOutputRow_0 = (InternalRow) smj_rightBuffer_0.get(smj_rightIndex_0);
/* 220 */
/* 221 */ long smj_value_13 = smj_rightOutputRow_0.getLong(0);
/* 222 */
/* 223 */ boolean smj_isNull_9 = true;
/* 224 */ long smj_value_10 = -1L;
/* 225 */ if (smj_leftOutputRow_0 != null) {
/* 226 */ long smj_value_9 = smj_leftOutputRow_0.getLong(0);
/* 227 */ smj_isNull_9 = false;
/* 228 */ smj_value_10 = smj_value_9;
/* 229 */ }
/* 230 */
/* 231 */ long smj_value_16 = -1L;
/* 232 */
/* 233 */ smj_value_16 = smj_value_10 + 3L;
/* 234 */
/* 235 */ boolean smj_value_15 = false;
/* 236 */ smj_value_15 = smj_value_16 == smj_value_13;
/* 237 */ boolean smj_value_14 = false;
/* 238 */ smj_value_14 = !(smj_value_15);
/* 239 */ if (!(false || !smj_value_14))
/* 240 */ {
/* 241 */ smj_consumeFullOuterJoinRow_0();
/* 242 */ smj_leftMatched_0.set(smj_leftIndex_0);
/* 243 */ smj_rightMatched_0.set(smj_rightIndex_0);
/* 244 */ }
/* 245 */ }
/* 246 */
/* 247 */ if (!smj_leftMatched_0.get(smj_leftIndex_0)) {
/* 248 */ smj_rightOutputRow_0 = null;
/* 249 */ smj_consumeFullOuterJoinRow_0();
/* 250 */ }
/* 251 */ }
/* 252 */
/* 253 */ smj_leftOutputRow_0 = null;
/* 254 */ for (smj_rightIndex_0 = 0; smj_rightIndex_0 < smj_rightBuffer_0.size(); smj_rightIndex_0++) {
/* 255 */ if (!smj_rightMatched_0.get(smj_rightIndex_0)) {
/* 256 */ // The right row has never matched any left row, join it with null row
/* 257 */ smj_rightOutputRow_0 = (InternalRow) smj_rightBuffer_0.get(smj_rightIndex_0);
/* 258 */ smj_consumeFullOuterJoinRow_0();
/* 259 */ }
/* 260 */ }
/* 261 */
/* 262 */ if (shouldStop()) return;
/* 263 */ }
/* 264 */
/* 265 */ // The right iterator has no more rows, join left row with null
/* 266 */ while (smj_leftInputRow_0 != null || smj_leftInput_0.hasNext()) {
/* 267 */ if (smj_leftInputRow_0 == null) {
/* 268 */ smj_leftInputRow_0 = (InternalRow) smj_leftInput_0.next();
/* 269 */ }
/* 270 */
/* 271 */ smj_leftOutputRow_0 = smj_leftInputRow_0;
/* 272 */ smj_rightOutputRow_0 = null;
/* 273 */ smj_leftInputRow_0 = null;
/* 274 */ smj_consumeFullOuterJoinRow_0();
/* 275 */
/* 276 */ if (shouldStop()) return;
/* 277 */ }
/* 278 */
/* 279 */ // The left iterator has no more rows, join right row with null
/* 280 */ while (smj_rightInputRow_0 != null || smj_rightInput_0.hasNext()) {
/* 281 */ if (smj_rightInputRow_0 == null) {
/* 282 */ smj_rightInputRow_0 = (InternalRow) smj_rightInput_0.next();
/* 283 */ }
/* 284 */
/* 285 */ smj_rightOutputRow_0 = smj_rightInputRow_0;
/* 286 */ smj_leftOutputRow_0 = null;
/* 287 */ smj_rightInputRow_0 = null;
/* 288 */ smj_consumeFullOuterJoinRow_0();
/* 289 */
/* 290 */ if (shouldStop()) return;
/* 291 */ }
/* 292 */ }
/* 293 */
/* 294 */ }
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment