This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Fellow instructors, | |
I am teaching the DataAnalyst class so I thought I would write up a demo and share it. I tried when possible to make it fit into the labs and demo collections that the DA class has already. I hope you find this useful. | |
A student in class today asked me to provide a demo of pig over hbase. | |
It took me a few tries to get things to work, so my script here got a little muddy. I thought I would include it anyhow, I think I got the important steps recorded in this email. | |
I am teaching the DataAnalyst class so I thought I would write up a demo and share it. I tried when possible to make it fit into the labs and demo collections that the DA class has already. I hope you find this useful. | |
Here are the steps. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.deeplearning4j.examples.dataExamples; | |
import org.datavec.api.io.labels.ParentPathLabelGenerator; | |
import org.datavec.api.records.listener.impl.LogRecordListener; | |
import org.datavec.api.split.FileSplit; | |
import org.datavec.image.loader.NativeImageLoader; | |
import org.datavec.image.recordreader.ImageRecordReader; | |
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; | |
import org.nd4j.linalg.dataset.DataSet; | |
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.deeplearning4j.examples.dataExamples; | |
import org.datavec.api.io.labels.ParentPathLabelGenerator; | |
import org.datavec.api.records.listener.impl.LogRecordListener; | |
import org.datavec.api.split.FileSplit; | |
import org.datavec.image.loader.NativeImageLoader; | |
import org.datavec.image.recordreader.ImageRecordReader; | |
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; | |
import org.deeplearning4j.eval.Evaluation; | |
import org.deeplearning4j.nn.api.OptimizationAlgorithm; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.deeplearning4j.examples.dataExamples; | |
import org.datavec.api.io.labels.ParentPathLabelGenerator; | |
import org.datavec.api.split.FileSplit; | |
import org.datavec.image.loader.NativeImageLoader; | |
import org.datavec.image.recordreader.ImageRecordReader; | |
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; | |
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; | |
import org.deeplearning4j.util.ModelSerializer; | |
import org.nd4j.linalg.api.ndarray.INDArray; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// How do I apply the same functionality that I have here in this code snippet of a DataSetIterator | |
DataNormalization scaler = new ImagePreProcessingScaler(0,1); | |
scaler.fit(dataIter); | |
dataIter.setPreProcessor(scaler); | |
// this was run when I trained the data | |
// took pixels from 0-255 to 0-1 | |
// Now I read the data like this (Thanks Raver.) | |
File mf = new File(filechose); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.deeplearning4j.examples.dataExamples; | |
import org.datavec.api.io.labels.ParentPathLabelGenerator; | |
import org.datavec.api.split.FileSplit; | |
import org.datavec.image.loader.NativeImageLoader; | |
import org.datavec.image.recordreader.ImageRecordReader; | |
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; | |
import org.deeplearning4j.eval.Evaluation; | |
import org.deeplearning4j.nn.api.OptimizationAlgorithm; | |
import org.deeplearning4j.nn.conf.MultiLayerConfiguration; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.deeplearning4j.examples.dataExamples; | |
import org.datavec.api.io.labels.ParentPathLabelGenerator; | |
import org.datavec.api.split.FileSplit; | |
import org.datavec.image.loader.NativeImageLoader; | |
import org.datavec.image.recordreader.ImageRecordReader; | |
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; | |
import org.deeplearning4j.eval.Evaluation; | |
import org.deeplearning4j.nn.api.OptimizationAlgorithm; | |
import org.deeplearning4j.nn.conf.MultiLayerConfiguration; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.deeplearning4j.examples.dataExamples; | |
import org.datavec.image.loader.NativeImageLoader; | |
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; | |
import org.deeplearning4j.util.ModelSerializer; | |
import org.nd4j.linalg.api.ndarray.INDArray; | |
import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization; | |
import org.nd4j.linalg.dataset.api.preprocessor.ImagePreProcessingScaler; | |
import org.slf4j.Logger; | |
import org.slf4j.LoggerFactory; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
'''Trains a simple deep NN on the MNIST dataset. | |
Gets to 98.40% test accuracy after 20 epochs | |
(there is *a lot* of margin for parameter tuning). | |
2 seconds per epoch on a K520 GPU. | |
''' | |
from __future__ import print_function | |
import numpy as np | |
import tensorflow as tf | |
tf.python.control_flow_ops = tf |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import numpy | |
import pandas | |
from keras.models import Sequential | |
from keras.layers import Dense | |
from keras.wrappers.scikit_learn import KerasClassifier | |
from keras.utils import np_utils | |
from sklearn.cross_validation import cross_val_score, KFold | |
from sklearn.preprocessing import LabelEncoder | |
from sklearn.pipeline import Pipeline |
OlderNewer