Main.scala

import java.util

import org.deeplearning4j.datasets.iterator.DataSetIterator
import org.nd4j.linalg.dataset.DataSet
import org.nd4j.linalg.dataset.api.DataSetPreProcessor
import org.nd4j.linalg.factory.Nd4j

import scala.collection.JavaConverters._
import scala.util.Random

object Main extends App {
  println("This runs fine ...")
  val data: List[DataSet] = new MockIterator(1L).asScala.toList
  DataSet.merge(data.asJava)

  println("\nThis will crash ...")
  val data2: List[DataSet] = new MockIterator(2L).asScala.toList
  DataSet.merge(data2.asJava)
}

class MockIterator(seed: Long) extends DataSetIterator {
  val batchSize = 50
  var counter = 0
  val dataSetSize = 698
  val featureSize = 1000
  val labelSize = 3
  val r = new Random(seed)

  override def next(num: Int): DataSet = {
    val maxNumberOfFeatures = r.nextInt(100) + 1
    print(maxNumberOfFeatures + " ")

    // [miniBatchSize, inputSize, timeSeriesLength]
    val features = Nd4j.create(num, featureSize, maxNumberOfFeatures)
    val labels = Nd4j.create(num, labelSize, maxNumberOfFeatures)
    // [miniBatchSize, timeSeriesLength]
    val featureMask = Nd4j.zeros(num, maxNumberOfFeatures)
    val labelsMask = Nd4j.zeros(num, maxNumberOfFeatures)

    counter += num
    new DataSet(features, labels, featureMask, labelsMask)
  }
  override def batch(): Int = Math.min(batchSize, dataSetSize - counter)
  override def cursor(): Int = counter
  override def totalExamples(): Int = dataSetSize
  override def inputColumns(): Int = featureSize
  override def setPreProcessor(preProcessor: DataSetPreProcessor): Unit = throw new UnsupportedOperationException
  override def getLabels: util.List[String] = Seq("a", "b", "c").asJava
  override def totalOutcomes(): Int = getLabels.size
  override def reset(): Unit = counter = 0
  override def numExamples(): Int = totalExamples()
  override def next(): DataSet = next(batch)
  override def hasNext: Boolean = cursor() < totalExamples()
}

Output:

This runs fine ...
86 89 48 14 55 5 35 7 79 49 70 74 18 64 
This will crash ...
9 73 41 68 90 51 7 20 48 69 95 87 35 15 
Exception in thread "main" java.lang.IllegalArgumentException: Illegal index 6282000
    at org.nd4j.linalg.api.buffer.BaseDataBuffer.put(BaseDataBuffer.java:702)
    at org.nd4j.linalg.api.buffer.BaseDataBuffer.put(BaseDataBuffer.java:696)
    at org.nd4j.linalg.api.buffer.BaseDataBuffer.copyAtStride(BaseDataBuffer.java:282)
    at org.nd4j.linalg.api.ndarray.BaseNDArray.assign(BaseNDArray.java:1027)
    at org.nd4j.linalg.dataset.DataSet.mergeTimeSeries(DataSet.java:154)
    at org.nd4j.linalg.dataset.DataSet.merge(DataSet.java:114)
    at org.nd4j.linalg.dataset.DataSet.merge(DataSet.java:172)
    at no.habitats.corpus.dl4j.Main$delayedInit$body.apply(MockIterator.scala:20)
    at scala.Function0$class.apply$mcV$sp(Function0.scala:40)
    at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
    at scala.App$$anonfun$main$1.apply(App.scala:71)
    at scala.App$$anonfun$main$1.apply(App.scala:71)
    at scala.collection.immutable.List.foreach(List.scala:318)
    at scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:32)
    at scala.App$class.main(App.scala:71)
    at no.habitats.corpus.dl4j.Main$.main(MockIterator.scala:13)
    at no.habitats.corpus.dl4j.Main.main(MockIterator.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:497)
    at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144)

Process finished with exit code 1