SystemML.ipynb

{
    "nbformat_minor": 1, 
    "cells": [
        {
            "execution_count": 1, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "Collecting https://github.com/niketanpansare/future_of_data/raw/master/systemml-1.1.0-SNAPSHOT-python.tar.gz\n  Using cached https://github.com/niketanpansare/future_of_data/raw/master/systemml-1.1.0-SNAPSHOT-python.tar.gz\nRequirement not upgraded as not directly required: numpy>=1.8.2 in /usr/local/src/bluemix_jupyter_bundle.v106/notebook/lib/python2.7/site-packages (from systemml==1.1.0) (1.13.1)\nRequirement not upgraded as not directly required: scipy>=0.15.1 in /usr/local/src/bluemix_jupyter_bundle.v106/notebook/lib/python2.7/site-packages (from systemml==1.1.0) (0.17.0)\nRequirement not upgraded as not directly required: pandas in /usr/local/src/bluemix_jupyter_bundle.v106/notebook/lib/python2.7/site-packages (from systemml==1.1.0) (0.17.1)\nRequirement not upgraded as not directly required: scikit-learn in /usr/local/src/bluemix_jupyter_bundle.v106/notebook/lib/python2.7/site-packages (from systemml==1.1.0) (0.17)\nRequirement not upgraded as not directly required: Pillow>=2.0.0 in /usr/local/src/bluemix_jupyter_bundle.v106/notebook/lib/python2.7/site-packages (from systemml==1.1.0) (3.0.0)\nRequirement not upgraded as not directly required: python-dateutil in /usr/local/src/bluemix_jupyter_bundle.v106/notebook/lib/python2.7/site-packages (from pandas->systemml==1.1.0) (2.4.2)\nRequirement not upgraded as not directly required: pytz>=2011k in /usr/local/src/bluemix_jupyter_bundle.v106/notebook/lib/python2.7/site-packages (from pandas->systemml==1.1.0) (2018.4)\nRequirement not upgraded as not directly required: six>=1.5 in /usr/local/src/bluemix_jupyter_bundle.v106/notebook/lib/python2.7/site-packages (from python-dateutil->pandas->systemml==1.1.0) (1.10.0)\nBuilding wheels for collected packages: systemml\n  Running setup.py bdist_wheel for systemml ... \u001b[?25ldone\n\u001b[?25h  Stored in directory: /gpfs/fs01/user/sd72-5c78ef3376bcb4-2ccd90524418/.cache/pip/wheels/75/ca/50/f14a0f1cec72222376f5ced7749629ca44194d8245c2e4939d\nSuccessfully built systemml\n\u001b[31mtensorflow 1.2.1 has requirement bleach==1.5.0, but you'll have bleach 2.0.0 which is incompatible.\u001b[0m\n\u001b[31mtensorflow 1.2.1 has requirement html5lib==0.9999999, but you'll have html5lib 0.999999999 which is incompatible.\u001b[0m\nInstalling collected packages: systemml\n  Found existing installation: systemml 1.1.0\n    Uninstalling systemml-1.1.0:\n      Successfully uninstalled systemml-1.1.0\nSuccessfully installed systemml-1.1.0\n"
                }
            ], 
            "source": "#!pip install --upgrade systemml\n!pip install --upgrade https://github.com/niketanpansare/future_of_data/raw/master/systemml-1.1.0-SNAPSHOT-python.tar.gz\n!ln -s -f ~/.local/lib/python2.7/site-packages/systemml/systemml-java/*.jar ~/data/libs/"
        }, 
        {
            "execution_count": 2, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "Name: systemml\r\nVersion: 1.1.0\r\nSummary: Apache SystemML is a distributed and declarative machine learning platform.\r\nHome-page: http://systemml.apache.org/\r\nAuthor: Apache SystemML\r\nAuthor-email: dev@systemml.apache.org\r\nLicense: Apache 2.0\r\nLocation: /gpfs/global_fs01/sym_shared/YPProdSpark/user/sd72-5c78ef3376bcb4-2ccd90524418/.local/lib/python2.7/site-packages\r\nRequires: pandas, scipy, Pillow, numpy, scikit-learn\r\nRequired-by: \r\n"
                }
            ], 
            "source": "!pip show systemml"
        }, 
        {
            "execution_count": 3, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 3, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "u'2.1.3'"
                    }, 
                    "output_type": "execute_result"
                }
            ], 
            "source": "sc.version"
        }, 
        {
            "execution_count": 4, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "Archiver-Version: Plexus Archiver\nArtifact-Id: systemml\nBuild-Jdk: 1.8.0_111\nBuild-Time: 2018-01-11 16:39:06 CST\nBuilt-By: biuser\nCreated-By: Apache Maven 3.0.5\nGroup-Id: org.apache.systemml\nMain-Class: org.apache.sysml.api.DMLScript\nManifest-Version: 1.0\nMinimum-Recommended-Spark-Version: 2.1.0\nVersion: 1.1.0-SNAPSHOT\n\n"
                }
            ], 
            "source": "from systemml import MLContext, dml\n# Create a MLContext object\nml = MLContext(sc)\n# And print the information of SystemML version\nprint(ml.info())"
        }, 
        {
            "execution_count": 5, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 5, 
                    "metadata": {}, 
                    "data": {
                        "text/html": "<div>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>fixed acidity</th>\n      <th>volatile acidity</th>\n      <th>citric acid</th>\n      <th>residual sugar</th>\n      <th>chlorides</th>\n      <th>free sulfur dioxide</th>\n      <th>total sulfur dioxide</th>\n      <th>density</th>\n      <th>pH</th>\n      <th>sulphates</th>\n      <th>alcohol</th>\n      <th>quality</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7.4</td>\n      <td>0.70</td>\n      <td>0.00</td>\n      <td>1.9</td>\n      <td>0.076</td>\n      <td>11</td>\n      <td>34</td>\n      <td>0.9978</td>\n      <td>3.51</td>\n      <td>0.56</td>\n      <td>9.4</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7.8</td>\n      <td>0.88</td>\n      <td>0.00</td>\n      <td>2.6</td>\n      <td>0.098</td>\n      <td>25</td>\n      <td>67</td>\n      <td>0.9968</td>\n      <td>3.20</td>\n      <td>0.68</td>\n      <td>9.8</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>7.8</td>\n      <td>0.76</td>\n      <td>0.04</td>\n      <td>2.3</td>\n      <td>0.092</td>\n      <td>15</td>\n      <td>54</td>\n      <td>0.9970</td>\n      <td>3.26</td>\n      <td>0.65</td>\n      <td>9.8</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>11.2</td>\n      <td>0.28</td>\n      <td>0.56</td>\n      <td>1.9</td>\n      <td>0.075</td>\n      <td>17</td>\n      <td>60</td>\n      <td>0.9980</td>\n      <td>3.16</td>\n      <td>0.58</td>\n      <td>9.8</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>7.4</td>\n      <td>0.70</td>\n      <td>0.00</td>\n      <td>1.9</td>\n      <td>0.076</td>\n      <td>11</td>\n      <td>34</td>\n      <td>0.9978</td>\n      <td>3.51</td>\n      <td>0.56</td>\n      <td>9.4</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n</div>", 
                        "text/plain": "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n0            7.4              0.70         0.00             1.9      0.076   \n1            7.8              0.88         0.00             2.6      0.098   \n2            7.8              0.76         0.04             2.3      0.092   \n3           11.2              0.28         0.56             1.9      0.075   \n4            7.4              0.70         0.00             1.9      0.076   \n\n   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \\\n0                   11                    34   0.9978  3.51       0.56   \n1                   25                    67   0.9968  3.20       0.68   \n2                   15                    54   0.9970  3.26       0.65   \n3                   17                    60   0.9980  3.16       0.58   \n4                   11                    34   0.9978  3.51       0.56   \n\n   alcohol  quality  \n0      9.4        5  \n1      9.8        5  \n2      9.8        5  \n3      9.8        6  \n4      9.4        5  "
                    }, 
                    "output_type": "execute_result"
                }
            ], 
            "source": "# The code was removed by Watson Studio for sharing."
        }, 
        {
            "execution_count": 6, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "def add_label (row):\n   if row['quality'] >= 7 :\n      return 1\n   return 0\n\ndf['label'] = df.apply (lambda row: add_label (row), axis=1)"
        }, 
        {
            "execution_count": 7, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 7, 
                    "metadata": {}, 
                    "data": {
                        "text/html": "<div>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>fixed acidity</th>\n      <th>volatile acidity</th>\n      <th>citric acid</th>\n      <th>residual sugar</th>\n      <th>chlorides</th>\n      <th>free sulfur dioxide</th>\n      <th>total sulfur dioxide</th>\n      <th>sulphates</th>\n      <th>alcohol</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7.4</td>\n      <td>0.700</td>\n      <td>0.00</td>\n      <td>1.9</td>\n      <td>0.076</td>\n      <td>11</td>\n      <td>34</td>\n      <td>0.56</td>\n      <td>9.4</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7.8</td>\n      <td>0.880</td>\n      <td>0.00</td>\n      <td>2.6</td>\n      <td>0.098</td>\n      <td>25</td>\n      <td>67</td>\n      <td>0.68</td>\n      <td>9.8</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>7.8</td>\n      <td>0.760</td>\n      <td>0.04</td>\n      <td>2.3</td>\n      <td>0.092</td>\n      <td>15</td>\n      <td>54</td>\n      <td>0.65</td>\n      <td>9.8</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>11.2</td>\n      <td>0.280</td>\n      <td>0.56</td>\n      <td>1.9</td>\n      <td>0.075</td>\n      <td>17</td>\n      <td>60</td>\n      <td>0.58</td>\n      <td>9.8</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>7.4</td>\n      <td>0.700</td>\n      <td>0.00</td>\n      <td>1.9</td>\n      <td>0.076</td>\n      <td>11</td>\n      <td>34</td>\n      <td>0.56</td>\n      <td>9.4</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>7.4</td>\n      <td>0.660</td>\n      <td>0.00</td>\n      <td>1.8</td>\n      <td>0.075</td>\n      <td>13</td>\n      <td>40</td>\n      <td>0.56</td>\n      <td>9.4</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>7.9</td>\n      <td>0.600</td>\n      <td>0.06</td>\n      <td>1.6</td>\n      <td>0.069</td>\n      <td>15</td>\n      <td>59</td>\n      <td>0.46</td>\n      <td>9.4</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>7.3</td>\n      <td>0.650</td>\n      <td>0.00</td>\n      <td>1.2</td>\n      <td>0.065</td>\n      <td>15</td>\n      <td>21</td>\n      <td>0.47</td>\n      <td>10.0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>7.8</td>\n      <td>0.580</td>\n      <td>0.02</td>\n      <td>2.0</td>\n      <td>0.073</td>\n      <td>9</td>\n      <td>18</td>\n      <td>0.57</td>\n      <td>9.5</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>7.5</td>\n      <td>0.500</td>\n      <td>0.36</td>\n      <td>6.1</td>\n      <td>0.071</td>\n      <td>17</td>\n      <td>102</td>\n      <td>0.80</td>\n      <td>10.5</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>6.7</td>\n      <td>0.580</td>\n      <td>0.08</td>\n      <td>1.8</td>\n      <td>0.097</td>\n      <td>15</td>\n      <td>65</td>\n      <td>0.54</td>\n      <td>9.2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>7.5</td>\n      <td>0.500</td>\n      <td>0.36</td>\n      <td>6.1</td>\n      <td>0.071</td>\n      <td>17</td>\n      <td>102</td>\n      <td>0.80</td>\n      <td>10.5</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>5.6</td>\n      <td>0.615</td>\n      <td>0.00</td>\n      <td>1.6</td>\n      <td>0.089</td>\n      <td>16</td>\n      <td>59</td>\n      <td>0.52</td>\n      <td>9.9</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>7.8</td>\n      <td>0.610</td>\n      <td>0.29</td>\n      <td>1.6</td>\n      <td>0.114</td>\n      <td>9</td>\n      <td>29</td>\n      <td>1.56</td>\n      <td>9.1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>8.9</td>\n      <td>0.620</td>\n      <td>0.18</td>\n      <td>3.8</td>\n      <td>0.176</td>\n      <td>52</td>\n      <td>145</td>\n      <td>0.88</td>\n      <td>9.2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>8.9</td>\n      <td>0.620</td>\n      <td>0.19</td>\n      <td>3.9</td>\n      <td>0.170</td>\n      <td>51</td>\n      <td>148</td>\n      <td>0.93</td>\n      <td>9.2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>8.5</td>\n      <td>0.280</td>\n      <td>0.56</td>\n      <td>1.8</td>\n      <td>0.092</td>\n      <td>35</td>\n      <td>103</td>\n      <td>0.75</td>\n      <td>10.5</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>8.1</td>\n      <td>0.560</td>\n      <td>0.28</td>\n      <td>1.7</td>\n      <td>0.368</td>\n      <td>16</td>\n      <td>56</td>\n      <td>1.28</td>\n      <td>9.3</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>7.4</td>\n      <td>0.590</td>\n      <td>0.08</td>\n      <td>4.4</td>\n      <td>0.086</td>\n      <td>6</td>\n      <td>29</td>\n      <td>0.50</td>\n      <td>9.0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>7.9</td>\n      <td>0.320</td>\n      <td>0.51</td>\n      <td>1.8</td>\n      <td>0.341</td>\n      <td>17</td>\n      <td>56</td>\n      <td>1.08</td>\n      <td>9.2</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>", 
                        "text/plain": "    fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n0             7.4             0.700         0.00             1.9      0.076   \n1             7.8             0.880         0.00             2.6      0.098   \n2             7.8             0.760         0.04             2.3      0.092   \n3            11.2             0.280         0.56             1.9      0.075   \n4             7.4             0.700         0.00             1.9      0.076   \n5             7.4             0.660         0.00             1.8      0.075   \n6             7.9             0.600         0.06             1.6      0.069   \n7             7.3             0.650         0.00             1.2      0.065   \n8             7.8             0.580         0.02             2.0      0.073   \n9             7.5             0.500         0.36             6.1      0.071   \n10            6.7             0.580         0.08             1.8      0.097   \n11            7.5             0.500         0.36             6.1      0.071   \n12            5.6             0.615         0.00             1.6      0.089   \n13            7.8             0.610         0.29             1.6      0.114   \n14            8.9             0.620         0.18             3.8      0.176   \n15            8.9             0.620         0.19             3.9      0.170   \n16            8.5             0.280         0.56             1.8      0.092   \n17            8.1             0.560         0.28             1.7      0.368   \n18            7.4             0.590         0.08             4.4      0.086   \n19            7.9             0.320         0.51             1.8      0.341   \n\n    free sulfur dioxide  total sulfur dioxide  sulphates  alcohol  label  \n0                    11                    34       0.56      9.4      0  \n1                    25                    67       0.68      9.8      0  \n2                    15                    54       0.65      9.8      0  \n3                    17                    60       0.58      9.8      0  \n4                    11                    34       0.56      9.4      0  \n5                    13                    40       0.56      9.4      0  \n6                    15                    59       0.46      9.4      0  \n7                    15                    21       0.47     10.0      1  \n8                     9                    18       0.57      9.5      1  \n9                    17                   102       0.80     10.5      0  \n10                   15                    65       0.54      9.2      0  \n11                   17                   102       0.80     10.5      0  \n12                   16                    59       0.52      9.9      0  \n13                    9                    29       1.56      9.1      0  \n14                   52                   145       0.88      9.2      0  \n15                   51                   148       0.93      9.2      0  \n16                   35                   103       0.75     10.5      1  \n17                   16                    56       1.28      9.3      0  \n18                    6                    29       0.50      9.0      0  \n19                   17                    56       1.08      9.2      0  "
                    }, 
                    "output_type": "execute_result"
                }
            ], 
            "source": "df = df.drop(['quality', 'density', 'pH'], axis=1)\ndf.head(20)"
        }, 
        {
            "execution_count": 8, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "from sklearn.cross_validation import train_test_split\n\ndf_x_train, df_x_test, df_y_train, df_y_test = train_test_split(df, df['label'], test_size=0.2)"
        }, 
        {
            "execution_count": 9, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "from systemml.mllearn import LogisticRegression\nlogistic = LogisticRegression(spark)"
        }, 
        {
            "execution_count": 10, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "BEGIN MULTINOMIAL LOGISTIC REGRESSION SCRIPT\nReading X...\nReading Y...\n-- Initially:  Objective = 886.53524393617,  Gradient Norm = 25411.93692503367,  Trust Delta = 0.005680862770971049\n-- Outer Iteration 1: Had 1 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 125.92748666017587,  Predicted = 125.6613187282504  (A/P: 1.0021),  Trust Delta = 0.022243910462940473\n   -- New Objective = 760.6077572759941,  Beta Change Norm = 0.005680862770971048,  Gradient Norm = 19014.33742328914\n \n-- Outer Iteration 2: Had 1 CG iterations\n   -- Obj.Reduction:  Actual = 201.38602395390944,  Predicted = 169.79336943773797  (A/P: 1.1861),  Trust Delta = 0.022243910462940473\n   -- New Objective = 559.2217333220847,  Beta Change Norm = 0.017859509448882673,  Gradient Norm = 5317.0784271281855\n \n-- Outer Iteration 3: Had 2 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 41.65333705623482,  Predicted = 38.38813648629335  (A/P: 1.0851),  Trust Delta = 0.027900032770952595\n   -- New Objective = 517.5683962658499,  Beta Change Norm = 0.02224391046294047,  Gradient Norm = 980.9289662506095\n \n-- Outer Iteration 4: Had 2 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 8.873480514389087,  Predicted = 8.700000631889218  (A/P: 1.0199),  Trust Delta = 0.043520556402896295\n   -- New Objective = 508.69491575146077,  Beta Change Norm = 0.0279000327709526,  Gradient Norm = 201.9412923152867\n \n-- Outer Iteration 5: Had 1 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 5.599881616549112,  Predicted = 5.780445322456403  (A/P: 0.9688),  Trust Delta = 0.05997471625720839\n   -- New Objective = 503.09503413491166,  Beta Change Norm = 0.043520556402896295,  Gradient Norm = 622.927223770613\n \n-- Outer Iteration 6: Had 2 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 7.64358559517035,  Predicted = 7.399335228861549  (A/P: 1.033),  Trust Delta = 0.09146383761840256\n   -- New Objective = 495.4514485397413,  Beta Change Norm = 0.059974716257208384,  Gradient Norm = 283.5102786223076\n \n-- Outer Iteration 7: Had 3 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 12.85697942878005,  Predicted = 12.94450202578477  (A/P: 0.9932),  Trust Delta = 0.1537454036538291\n   -- New Objective = 482.59446911096126,  Beta Change Norm = 0.09146383761840254,  Gradient Norm = 301.4654259995004\n \n-- Outer Iteration 8: Had 3 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 22.037134182637374,  Predicted = 22.033423802897687  (A/P: 1.0002),  Trust Delta = 0.4093391057959914\n   -- New Objective = 460.5573349283239,  Beta Change Norm = 0.15374540365382908,  Gradient Norm = 274.1284793996171\n \n-- Outer Iteration 9: Had 3 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 53.399629607838165,  Predicted = 53.28640853581296  (A/P: 1.0021),  Trust Delta = 1.337665136823052\n   -- New Objective = 407.1577053204857,  Beta Change Norm = 0.4093391057959914,  Gradient Norm = 381.6091200225985\n \n-- Outer Iteration 10: Had 4 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 129.7520498998589,  Predicted = 139.49199989938177  (A/P: 0.9302),  Trust Delta = 2.214458365930755\n   -- New Objective = 277.40565542062683,  Beta Change Norm = 1.3376651368230523,  Gradient Norm = 2953.8022160211594\n \n-- Outer Iteration 11: Had 2 CG iterations\n   -- Obj.Reduction:  Actual = 23.26350462457492,  Predicted = 21.198661299412525  (A/P: 1.0974),  Trust Delta = 2.214458365930755\n   -- New Objective = 254.1421507960519,  Beta Change Norm = 0.045531247574098745,  Gradient Norm = 836.4392759596549\n \n-- Outer Iteration 12: Had 4 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 142.35947571804314,  Predicted = 132.88211760289428  (A/P: 1.0713),  Trust Delta = 3.3689876452392467\n   -- New Objective = 111.78267507800878,  Beta Change Norm = 2.2144583659307555,  Gradient Norm = 779.7756021432907\n \n-- Outer Iteration 13: Had 1 CG iterations\n   -- Obj.Reduction:  Actual = 1.2704846825436817,  Predicted = 1.2086819920083829  (A/P: 1.0511),  Trust Delta = 3.3689876452392467\n   -- New Objective = 110.5121903954651,  Beta Change Norm = 0.0031000764545240983,  Gradient Norm = 76.45635077484538\n \n-- Outer Iteration 14: Had 6 CG iterations\n   -- Obj.Reduction:  Actual = 52.0413869703997,  Predicted = 43.83269698561232  (A/P: 1.1873),  Trust Delta = 3.3689876452392467\n   -- New Objective = 58.4708034250654,  Beta Change Norm = 2.144334665934133,  Gradient Norm = 232.65413978099696\n \n-- Outer Iteration 15: Had 1 CG iterations\n   -- Obj.Reduction:  Actual = 0.2252797771509023,  Predicted = 0.22632059952129874  (A/P: 0.9954),  Trust Delta = 3.3689876452392467\n   -- New Objective = 58.245523647914496,  Beta Change Norm = 0.0019455540291209933,  Gradient Norm = 14.462519119783435\n \n-- Outer Iteration 16: Had 6 CG iterations\n   -- Obj.Reduction:  Actual = 8.353810588805231,  Predicted = 7.308864176690613  (A/P: 1.143),  Trust Delta = 3.3689876452392467\n   -- New Objective = 49.891713059109264,  Beta Change Norm = 1.2785739258802455,  Gradient Norm = 12.93630926150778\n \n-- Outer Iteration 17: Had 6 CG iterations\n   -- Obj.Reduction:  Actual = 0.7983619878183106,  Predicted = 0.7507722690054874  (A/P: 1.0634),  Trust Delta = 3.3689876452392467\n   -- New Objective = 49.093351071290954,  Beta Change Norm = 0.6017329611214042,  Gradient Norm = 3.24120811553839\n \n-- Outer Iteration 18: Had 8 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 2.073106432311832,  Predicted = 2.2149466238534763  (A/P: 0.936),  Trust Delta = 3.9933935949816224\n   -- New Objective = 47.02024463897912,  Beta Change Norm = 3.368987645239246,  Gradient Norm = 12.233444279342025\n \n-- Outer Iteration 19: Had 6 CG iterations\n   -- Obj.Reduction:  Actual = 0.7976444220426941,  Predicted = 0.745360988252125  (A/P: 1.0701),  Trust Delta = 3.9933935949816224\n   -- New Objective = 46.22260021693643,  Beta Change Norm = 0.5358251976052949,  Gradient Norm = 1.6586761231347218\n \n-- Outer Iteration 20: Had 8 CG iterations, trust bound REACHED\n   -- Obj.Reduction:  Actual = 1.502251073892971,  Predicted = 1.426047381146908  (A/P: 1.0534),  Trust Delta = 4.24034461110513\n   -- New Objective = 44.72034914304346,  Beta Change Norm = 3.993393594981622,  Gradient Norm = 11.089098207455248\n \n-- Outer Iteration 21: Had 5 CG iterations\n   -- Obj.Reduction:  Actual = 0.3259645710807675,  Predicted = 0.34153710785484753  (A/P: 0.9544),  Trust Delta = 4.24034461110513\n   -- New Objective = 44.39438457196269,  Beta Change Norm = 0.5224155068302326,  Gradient Norm = 1.379028164679143\n \n-- Outer Iteration 22: Had 11 CG iterations\n   -- Obj.Reduction:  Actual = 0.40792904317866885,  Predicted = 0.39636030874268946  (A/P: 1.0292),  Trust Delta = 4.24034461110513\n   -- New Objective = 43.98645552878402,  Beta Change Norm = 2.4143792307018406,  Gradient Norm = 4.57542701017853\n \n-- Outer Iteration 23: Had 2 CG iterations\n   -- Obj.Reduction:  Actual = 3.5418243957963114E-4,  Predicted = 3.541066293346005E-4  (A/P: 1.0002),  Trust Delta = 4.24034461110513\n   -- New Objective = 43.98610134634444,  Beta Change Norm = 5.576355913218783E-4,  Gradient Norm = 0.3946294027758169\n \n-- Outer Iteration 24: Had 6 CG iterations\n   -- Obj.Reduction:  Actual = 0.0026711548840623323,  Predicted = 0.002655988055070485  (A/P: 1.0057),  Trust Delta = 4.24034461110513\n   -- New Objective = 43.98343019146038,  Beta Change Norm = 0.03159971049178642,  Gradient Norm = 0.025163127711154733\nTermination / Convergence condition satisfied.\nSystemML Statistics:\nTotal execution time:\t\t1.138 sec.\nNumber of executed Spark inst:\t2.\n\n\n[Stage 0:>                                                          (0 + 0) / 2]\n[Stage 0:=============================>                             (1 + 1) / 2]\nSystemML Statistics:\nTotal execution time:\t\t0.002 sec.\nNumber of executed Spark inst:\t0.\n\nSystemML Statistics:\nTotal execution time:\t\t0.006 sec.\nNumber of executed Spark inst:\t0.\n\n                                                                                \n\nLogisticRegression score: 1.000000\n"
                }
            ], 
            "source": "print('LogisticRegression score: %f' % logistic.fit(df_x_train.values, df_y_train.values).score(df_x_test.values, df_y_test.values))"
        }, 
        {
            "execution_count": null, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": ""
        }
    ], 
    "metadata": {
        "kernelspec": {
            "display_name": "Python 2 with Spark 2.1", 
            "name": "python2-spark21", 
            "language": "python"
        }, 
        "language_info": {
            "mimetype": "text/x-python", 
            "nbconvert_exporter": "python", 
            "version": "2.7.14", 
            "name": "python", 
            "file_extension": ".py", 
            "pygments_lexer": "ipython2", 
            "codemirror_mode": {
                "version": 2, 
                "name": "ipython"
            }
        }
    }, 
    "nbformat": 4
}