Created
May 15, 2016 21:36
-
-
Save eraly/f349cb4c65126361ad19c94fa3832081 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
DEBUG] joining on thread Thread[org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample.main(),5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] | |
[DEBUG] Setting accessibility to true in order to invoke main(). | |
o.n.j.c.CudaEnvironment - Please note, CudaEnvironment is already initialized. Configuration changes won't have effect | |
o.d.m.s.SequenceVectors - Starting vocabulary building... | |
o.d.m.w.w.VocabConstructor - Sequences checked: [30], Current vocabulary size: [2291] | |
o.d.m.s.SequenceVectors - Building learning algorithms: | |
o.d.m.s.SequenceVectors - building ElementsLearningAlgorithm: [SkipGram] | |
o.d.m.s.SequenceVectors - building SequenceLearningAlgorithm: [DBOW] | |
o.d.m.s.SequenceVectors - Starting learning process... | |
o.d.m.s.SequenceVectors - Epoch: [1]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [2]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [3]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [4]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [5]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [6]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [7]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [8]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [9]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [10]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
o.d.m.s.SequenceVectors - Epoch: [11]; Words vectorized so far: [7590]; Lines vectorized so far: [30]; learningRate: [0.001] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:259) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:259) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:259) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
o.d.m.s.SequenceVectors - Epoch: [12]; Words vectorized so far: [6545]; Lines vectorized so far: [26]; learningRate: [0.001] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:259) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 moreCUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] allocation buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:238) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
o.d.m.s.SequenceVectors - Epoch: [13]; Words vectorized so far: [0]; Lines vectorized so far: [0]; learningRate: [0.001] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 moreCUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
... 16 more | |
[WARNING] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
... 16 more | |
[WARNING] | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
o.d.m.s.SequenceVectors - Epoch: [14]; Words vectorized so far: [0]; Lines vectorized so far: [0]; learningRate: [0.001] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
o.d.m.s.SequenceVectors - Epoch: [15]; Words vectorized so far: [0]; Lines vectorized so far: [0]; learningRate: [0.001] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
o.d.m.s.SequenceVectors - Epoch: [16]; Words vectorized so far: [0]; Lines vectorized so far: [0]; learningRate: [0.001] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
o.d.m.s.SequenceVectors - Epoch: [17]; Words vectorized so far: [0]; Lines vectorized so far: [0]; learningRate: [0.001] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
o.d.m.s.SequenceVectors - Epoch: [18]; Words vectorized so far: [0]; Lines vectorized so far: [0]; learningRate: [0.001] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory!CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
o.d.m.s.SequenceVectors - Epoch: [19]; Words vectorized so far: [0]; Lines vectorized so far: [0]; learningRate: [0.001] | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921)CUDA error at /home/ec2-user/core/libnd4j/blas/cuda/NativeOps.cu:4015 code=2(cudaErrorMemoryAllocation) "result" | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[WARNING] | |
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] reduction buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:230) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
o.d.m.s.SequenceVectors - Epoch: [20]; Words vectorized so far: [0]; Lines vectorized so far: [0]; learningRate: [0.001] | |
o.n.j.h.i.CudaZeroHandler - Skipping allocation C on [DEVICE] | |
o.n.j.c.CudaConstantHandler - Overflow at constant space, skipping relocation | |
o.d.e.n.p.ParagraphVectorsClassifierExample - Document 'finance' falls into the following categories: | |
o.d.e.n.p.ParagraphVectorsClassifierExample - finance: 0.8738501667976379 | |
o.d.e.n.p.ParagraphVectorsClassifierExample - health: -0.21891160309314728 | |
o.d.e.n.p.ParagraphVectorsClassifierExample - science: -0.1487337201833725 | |
o.n.j.h.i.CudaZeroHandler - Skipping allocation C on [DEVICE] | |
o.n.j.c.CudaConstantHandler - Overflow at constant space, skipping relocation | |
o.d.e.n.p.ParagraphVectorsClassifierExample - Document 'health' falls into the following categories: | |
o.d.e.n.p.ParagraphVectorsClassifierExample - finance: 0.18979552388191223 | |
o.d.e.n.p.ParagraphVectorsClassifierExample - health: 0.6530775427818298 | |
o.d.e.n.p.ParagraphVectorsClassifierExample - science: -0.25349265336990356 | |
[DEBUG] interrupting thread Thread[UniGC thread 0,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] | |
[DEBUG] interrupting thread Thread[UniGC thread 1,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] | |
[DEBUG] interrupting thread Thread[UniGC thread 2,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] | |
[DEBUG] interrupting thread Thread[UniGC thread 3,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] | |
[DEBUG] joining on thread Thread[UniGC thread 0,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] | |
[WARNING] thread Thread[UniGC thread 0,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] was interrupted but is still alive after waiting at least 15000msecs | |
[WARNING] thread Thread[UniGC thread 0,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] will linger despite being asked to die via interruption | |
[WARNING] thread Thread[UniGC thread 1,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] will linger despite being asked to die via interruption | |
[WARNING] thread Thread[UniGC thread 2,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] will linger despite being asked to die via interruption | |
[WARNING] thread Thread[UniGC thread 3,5,org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample] will linger despite being asked to die via interruption | |
[WARNING] NOTE: 4 thread(s) did not finish despite being asked to via interruption. This is not a problem with exec:java, it is a problem with the running code. Although not serious, it should be remedied. | |
[WARNING] Couldn't destroy threadgroup org.codehaus.mojo.exec.ExecJavaMojo$IsolatedThreadGroup[name=org.deeplearning4j.examples.nlp.paragraphvectors.ParagraphVectorsClassifierExample,maxpri=10] | |
java.lang.IllegalThreadStateException | |
at java.lang.ThreadGroup.destroy(ThreadGroup.java:778) | |
at org.codehaus.mojo.exec.ExecJavaMojo.execute(ExecJavaMojo.java:328) | |
at org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:134) | |
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:207) | |
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:153) | |
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:145) | |
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:116) | |
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:80) | |
at org.apache.maven.lifecycle.internal.builder.singlethreaded.SingleThreadedBuilder.build(SingleThreadedBuilder.java:51) | |
at org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:128) | |
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:307) | |
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:193) | |
at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:106) | |
at org.apache.maven.cli.MavenCli.execute(MavenCli.java:863) | |
at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:288) | |
at org.apache.maven.cli.MavenCli.main(MavenCli.java:199) | |
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
at java.lang.reflect.Method.invoke(Method.java:497) | |
at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:289) | |
at org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:229) | |
at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:415) | |
at org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:356) | |
[INFO] ------------------------------------------------------------------------ | |
[INFO] BUILD FAILURE | |
[INFO] ------------------------------------------------------------------------ | |
[INFO] Total time: 01:14 h | |
[INFO] Finished at: 2016-05-15T21:11:58+00:00 | |
[INFO] Final Memory: 18M/981M | |
[INFO] ------------------------------------------------------------------------ | |
[ERROR] Failed to execute goal org.codehaus.mojo:exec-maven-plugin:1.4.0:java (default-cli) on project deeplearning4j-examples: An exception occured while executing the Java class. java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! -> [Help 1] | |
org.apache.maven.lifecycle.LifecycleExecutionException: Failed to execute goal org.codehaus.mojo:exec-maven-plugin:1.4.0:java (default-cli) on project deeplearning4j-examples: An exception occured while executing the Java class. java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:212) | |
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:153) | |
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:145) | |
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:116) | |
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:80) | |
at org.apache.maven.lifecycle.internal.builder.singlethreaded.SingleThreadedBuilder.build(SingleThreadedBuilder.java:51) | |
at org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:128) | |
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:307) | |
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:193) | |
at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:106) | |
at org.apache.maven.cli.MavenCli.execute(MavenCli.java:863) | |
at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:288) | |
at org.apache.maven.cli.MavenCli.main(MavenCli.java:199) | |
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
at java.lang.reflect.Method.invoke(Method.java:497) | |
at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:289) | |
at org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:229) | |
at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:415) | |
at org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:356) | |
Caused by: org.apache.maven.plugin.MojoExecutionException: An exception occured while executing the Java class. java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.codehaus.mojo.exec.ExecJavaMojo.execute(ExecJavaMojo.java:345) | |
at org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:134) | |
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:207) | |
... 20 more | |
Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:939) | |
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:135) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.getCudaContext(CudaZeroHandler.java:1071) | |
at org.nd4j.jita.handler.impl.CudaZeroHandler.alloc(CudaZeroHandler.java:194) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:374) | |
at org.nd4j.jita.allocator.impl.AtomicAllocator.allocateMemory(AtomicAllocator.java:336) | |
at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.<init>(BaseCudaDataBuffer.java:142) | |
at org.nd4j.linalg.jcublas.buffer.CudaFloatDataBuffer.<init>(CudaFloatDataBuffer.java:44) | |
at org.nd4j.linalg.jcublas.buffer.factory.CudaDataBufferFactory.createFloat(CudaDataBufferFactory.java:226) | |
at org.nd4j.linalg.factory.Nd4j.createBuffer(Nd4j.java:1161) | |
at org.nd4j.linalg.factory.BaseNDArrayFactory.create(BaseNDArrayFactory.java:937) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2805) | |
at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:2673) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.iterateSample(SkipGram.java:168) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.skipGram(SkipGram.java:154) | |
at org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram.learnSequence(SkipGram.java:129) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors.trainSequence(SequenceVectors.java:218) | |
at org.deeplearning4j.models.sequencevectors.SequenceVectors$VectorCalculationsThread.run(SequenceVectors.java:921) | |
Caused by: java.lang.IllegalStateException: Can't allocate [DEVICE] special buffer memory! | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.getDeviceBuffers(BasicContextPool.java:259) | |
at org.nd4j.jita.allocator.context.impl.BasicContextPool.acquireContextForDevice(BasicContextPool.java:90) | |
... 16 more | |
[ERROR] | |
[ERROR] | |
[ERROR] For more information about the errors and possible solutions, please read the following articles: | |
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment