containsNull.scala

  test("save - append - ArrayType.containsNull") {
    withTempPath { file =>
      val df = Seq.empty[Tuple1[Seq[Int]]].toDF("arrayVal")
      val nonNullSchema = StructType(df.schema.map {
        case f @ StructField(_, a: ArrayType, _, _) =>
          f.copy(dataType = a.copy(containsNull = false))
        case f => f
      })

      sqlContext.createDataFrame(df.rdd, nonNullSchema).save(file.getCanonicalPath)

      (Tuple1(Seq(1, 2)) :: Tuple1(null.asInstanceOf[Seq[Int]]) :: Nil)
        .toDF("arrayVal")
        .save(file.getCanonicalPath, SaveMode.Append)

      checkAnswer(
        parquetFile(file.getCanonicalPath),
        Row(ArrayBuffer(1, 2)) :: Row(null) :: Nil)
    }
  }