output

gistfile1.py

from pybrain.structure           import FeedForwardNetwork
from pybrain.structure           import LinearLayer, SigmoidLayer
from pybrain.structure           import FullConnection
from pybrain.datasets            import ClassificationDataSet
from pybrain.utilities           import percentError
from pybrain.tools.shortcuts     import buildNetwork
from pybrain.supervised.trainers import BackpropTrainer
from pybrain.structure.modules   import SoftmaxLayer
import time

""" Elaborate stuff is done on the read data to make it consumable by classificationdataset """



alldata_list = []
alldata_list.append(ClassificationDataSet(numProducts, 1, nb_classes=numProducts))

counter = 0
listcounter = 0
starttime = time.time()
midtime1 = time.time()
for datum in trainingSet:
    if not counter > 10000:
        alldata_list[-1].addSample(datum.getInputData(), datum.getClassification())
        counter += 1
    else:
        alldata_list.append(ClassificationDataSet(numProducts, 1, nb_classes=numProducts))
        counter = 0
        midtime2 = time.time()
        print "New list created, elapsed time %.2f" % (midtime2 - midtime1)
        midtime1 = midtime2
endtime = time.time()

stdout

python productTest.py 
890
147011
Starting creation of dataset.
Time elapsed: 16.14 seconds
Created 472057 data
New list created, elapsed time 1.10
New list created, elapsed time 1.04
New list created, elapsed time 2.23
New list created, elapsed time 2.72
New list created, elapsed time 2.51
Traceback (most recent call last):
  File "productTest.py", line 115, in <module>
    alldata_list[-1].addSample(datum.getInputData(), datum.getClassification())
  File "/usr/local/lib/python2.7/dist-packages/PyBrain-0.3-py2.7.egg/pybrain/datasets/supervised.py", line 45, in addSample
    self.appendLinked(inp, target)
  File "/usr/local/lib/python2.7/dist-packages/PyBrain-0.3-py2.7.egg/pybrain/datasets/dataset.py", line 215, in appendLinked
    self._appendUnlinked(l, args[i])
  File "/usr/local/lib/python2.7/dist-packages/PyBrain-0.3-py2.7.egg/pybrain/datasets/dataset.py", line 195, in _appendUnlinked
    self._resize(label)
  File "/usr/local/lib/python2.7/dist-packages/PyBrain-0.3-py2.7.egg/pybrain/datasets/dataset.py", line 180, in _resize
    self.data[l] = self._resizeArray(self.data[l])
  File "/usr/local/lib/python2.7/dist-packages/PyBrain-0.3-py2.7.egg/pybrain/datasets/dataset.py", line 187, in _resizeArray
    return resize(a, shape)
  File "/usr/lib/python2.7/dist-packages/numpy/core/fromnumeric.py", line 864, in resize
    a = concatenate( (a,)*n_copies)
MemoryError

Getting same error. My Dataset is 6.2M with 1.5M dimension. I am unable to load even a single datapoint.

Did you find any way out