PDAL_EM1000.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "st_time = dt.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**to speed up the loading and minimize the size of the data I used the `feather format`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_data = pd.read_feather('em_df.ft')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Z</th>\n",
       "      <th>GpsTime</th>\n",
       "      <th>Amplitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.226761e+07</td>\n",
       "      <td>1.226761e+07</td>\n",
       "      <td>1.226761e+07</td>\n",
       "      <td>1.226761e+07</td>\n",
       "      <td>1.226761e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-6.890380e+01</td>\n",
       "      <td>4.100966e+01</td>\n",
       "      <td>-7.718564e+01</td>\n",
       "      <td>5.955593e+10</td>\n",
       "      <td>-2.601706e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8.014719e-01</td>\n",
       "      <td>4.787278e-01</td>\n",
       "      <td>9.581520e+00</td>\n",
       "      <td>3.322059e+05</td>\n",
       "      <td>7.345970e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-6.909418e+01</td>\n",
       "      <td>-1.320920e-03</td>\n",
       "      <td>-1.473200e+02</td>\n",
       "      <td>5.955550e+10</td>\n",
       "      <td>-6.400000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-6.899785e+01</td>\n",
       "      <td>4.096217e+01</td>\n",
       "      <td>-8.258000e+01</td>\n",
       "      <td>5.955559e+10</td>\n",
       "      <td>-3.100000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-6.891094e+01</td>\n",
       "      <td>4.101923e+01</td>\n",
       "      <td>-7.606000e+01</td>\n",
       "      <td>5.955588e+10</td>\n",
       "      <td>-2.475000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-6.883640e+01</td>\n",
       "      <td>4.107753e+01</td>\n",
       "      <td>-7.054000e+01</td>\n",
       "      <td>5.955639e+10</td>\n",
       "      <td>-2.075000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.546906e-03</td>\n",
       "      <td>4.114448e+01</td>\n",
       "      <td>-3.808000e+01</td>\n",
       "      <td>5.955648e+10</td>\n",
       "      <td>3.750000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  X             Y             Z       GpsTime     Amplitude\n",
       "count  1.226761e+07  1.226761e+07  1.226761e+07  1.226761e+07  1.226761e+07\n",
       "mean  -6.890380e+01  4.100966e+01 -7.718564e+01  5.955593e+10 -2.601706e+01\n",
       "std    8.014719e-01  4.787278e-01  9.581520e+00  3.322059e+05  7.345970e+00\n",
       "min   -6.909418e+01 -1.320920e-03 -1.473200e+02  5.955550e+10 -6.400000e+01\n",
       "25%   -6.899785e+01  4.096217e+01 -8.258000e+01  5.955559e+10 -3.100000e+01\n",
       "50%   -6.891094e+01  4.101923e+01 -7.606000e+01  5.955588e+10 -2.475000e+01\n",
       "75%   -6.883640e+01  4.107753e+01 -7.054000e+01  5.955639e+10 -2.075000e+01\n",
       "max    2.546906e-03  4.114448e+01 -3.808000e+01  5.955648e+10  3.750000e+00"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "em_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Z</th>\n",
       "      <th>GpsTime</th>\n",
       "      <th>Amplitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-34.527571</td>\n",
       "      <td>20.512344</td>\n",
       "      <td>-80.39</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-40.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-34.527742</td>\n",
       "      <td>20.512322</td>\n",
       "      <td>-80.92</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-43.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-34.527914</td>\n",
       "      <td>20.512300</td>\n",
       "      <td>-79.79</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-43.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-34.528039</td>\n",
       "      <td>20.512284</td>\n",
       "      <td>-80.02</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-39.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-34.528134</td>\n",
       "      <td>20.512273</td>\n",
       "      <td>-81.11</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-37.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           X          Y      Z       GpsTime  Amplitude\n",
       "0 -34.527571  20.512344 -80.39  5.955646e+10     -40.75\n",
       "1 -34.527742  20.512322 -80.92  5.955646e+10     -43.75\n",
       "2 -34.527914  20.512300 -79.79  5.955646e+10     -43.75\n",
       "3 -34.528039  20.512284 -80.02  5.955646e+10     -39.50\n",
       "4 -34.528134  20.512273 -81.11  5.955646e+10     -37.00"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "em_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "**Group by `GpsTime`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "gtimer = em_data.groupby(['GpsTime'], sort=False)['X'].count().reset_index()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GpsTime</th>\n",
       "      <th>X</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        GpsTime   X\n",
       "0  5.955646e+10  54\n",
       "1  5.955646e+10  56\n",
       "2  5.955646e+10  59\n",
       "3  5.955646e+10  59\n",
       "4  5.955646e+10  58"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gtimer.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Compute the index for the nadir beam of each ping**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Beam Count | Nadir Index   | integer resulting from: |\n",
    "|------|------|------------------------------------------------|\n",
    "|   54  | 27| $\\frac{54}{2}$ |\n",
    "|   56  | 82| $54+\\frac{56}{2}$ |\n",
    "|   59  | 139| $54+56+\\frac{59}{2}$ |\n",
    "|   59  | 198| $54+56+59+\\frac{59}{2}$ |\n",
    "|   58  | 257| $54+56+59+59+\\frac{58}{2}$ |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "beam_list = list(gtimer['X'].values)\n",
    "#beam_list[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "beam_indexes = [[i,v] for i,v in enumerate(beam_list)]\n",
    "#beam_indexes[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# consider using np.cumsum(beamlist) \n",
    "# and then add the alf part to get the middlepoint\n",
    "def getNadirIndex(beam_index, beam_list):\n",
    "    if beam_index[0] == 0:\n",
    "        nadir_index = int( beam_index[1]/2)\n",
    "    if  beam_index[0] != 0:\n",
    "        nadir_index = int(sum(beam_list[0:beam_index[0]])+(beam_index[1]/2))\n",
    "    return nadir_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#beam_indexes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Test getNadirIndex**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for i in range(5):\n",
    "#    mdp = getNadirIndex(beam_indexes[i], beam_list)\n",
    "#    print(mdp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Parallelize `getNadirIndex` to loop over `beamlist`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import parmap\n",
    "#nadir_indexs = parmap.map(getNadirIndex, beam_indexes, beam_list, pm_processes=48, pm_pbar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nadir_indexs = parmap.map(getNadirIndex, beam_indexes, beam_list, pm_processes=48, pm_pbar=True, pm_chunksize=4000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nadirIndex_pickle = open(\"nadirIndex.pickle\",\"wb\")\n",
    "#pickle.dump(nadirIndex, nadirIndex_pickle)\n",
    "#nadirIndex_pickle.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "**Transform the Longitude and Latitude to UTM**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyproj import Proj, transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial import distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wgs84 = Proj(\"+init=EPSG:4326\")\n",
    "UTM19N = Proj(\"+init=EPSG:32619\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xy = em_data[['X','Y']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_data['XUTM'], em_data['YUTM'] = transform(wgs84, UTM19N,xy[:,0], xy[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Z</th>\n",
       "      <th>GpsTime</th>\n",
       "      <th>Amplitude</th>\n",
       "      <th>XUTM</th>\n",
       "      <th>YUTM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-34.527571</td>\n",
       "      <td>20.512344</td>\n",
       "      <td>-80.39</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-40.75</td>\n",
       "      <td>4.266364e+06</td>\n",
       "      <td>2.702726e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-34.527742</td>\n",
       "      <td>20.512322</td>\n",
       "      <td>-80.92</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-43.75</td>\n",
       "      <td>4.266344e+06</td>\n",
       "      <td>2.702718e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-34.527914</td>\n",
       "      <td>20.512300</td>\n",
       "      <td>-79.79</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-43.75</td>\n",
       "      <td>4.266325e+06</td>\n",
       "      <td>2.702711e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-34.528039</td>\n",
       "      <td>20.512284</td>\n",
       "      <td>-80.02</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-39.50</td>\n",
       "      <td>4.266310e+06</td>\n",
       "      <td>2.702705e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-34.528134</td>\n",
       "      <td>20.512273</td>\n",
       "      <td>-81.11</td>\n",
       "      <td>5.955646e+10</td>\n",
       "      <td>-37.00</td>\n",
       "      <td>4.266299e+06</td>\n",
       "      <td>2.702701e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           X          Y      Z       GpsTime  Amplitude          XUTM  \\\n",
       "0 -34.527571  20.512344 -80.39  5.955646e+10     -40.75  4.266364e+06   \n",
       "1 -34.527742  20.512322 -80.92  5.955646e+10     -43.75  4.266344e+06   \n",
       "2 -34.527914  20.512300 -79.79  5.955646e+10     -43.75  4.266325e+06   \n",
       "3 -34.528039  20.512284 -80.02  5.955646e+10     -39.50  4.266310e+06   \n",
       "4 -34.528134  20.512273 -81.11  5.955646e+10     -37.00  4.266299e+06   \n",
       "\n",
       "           YUTM  \n",
       "0  2.702726e+06  \n",
       "1  2.702718e+06  \n",
       "2  2.702711e+06  \n",
       "3  2.702705e+06  \n",
       "4  2.702701e+06  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "em_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gtimer['X'].iloc[0], nadir_indexs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#midpoint = em_data[['XUTM','YUTM']].iloc[nadir_indexs[0]].values\n",
    "#midpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#coords = em_data[['XUTM','YUTM']].values\n",
    "#coords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mid_points = []\n",
    "#for i in range(len(nadir_indexs)):\n",
    "#    mid_points.append(coords[nadir_indexs[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Z = em_data['Z'].values\n",
    "#Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xyz = em_data[['XUTM','YUTM', 'Z']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ping_indexes = np.cumsum(beam_list)\n",
    "#ping_indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAngle(ping_index, xyz, ping_indexes, beam_indexes):\n",
    "    if ping_index == 0:\n",
    "        start=0\n",
    "        stop=ping_indexes[ping_index]\n",
    "        shape=ping_indexes[ping_index]\n",
    "        nadir_index = int( beam_indexes[ping_index][1]/2)\n",
    "    else:\n",
    "        start=ping_indexes[ping_index-1]\n",
    "        stop=ping_indexes[ping_index]\n",
    "        shape=ping_indexes[ping_index] - ping_indexes[ping_index-1]\n",
    "        nadir_index = int(sum(beam_list[0: beam_indexes[ping_index][0]])+( beam_indexes[ping_index][1]/2))\n",
    "    #coords = xyz[:, 0:2]\n",
    "    Z = xyz[:, 2]\n",
    "    #midpoint = coords[nadir_index]\n",
    "    midpoint = xyz[:, 0:2][nadir_index] \n",
    "    \n",
    "    pings = coords[start:stop]\n",
    "    Zi = Z[start:stop]        \n",
    "    XY = [tuple(midpoint)]\n",
    "    coord = [tuple(j) for j in tuple(pings)]\n",
    "    d = distance.cdist(XY, coord, 'euclidean')\n",
    "    angle = np.rad2deg(np.arctan2(np.abs(Zi),d)).reshape(shape,)\n",
    "    return angle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ping_indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#beam_indexes"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# consider using np.cumsum(beamlist) \n",
    "# and then add the alf part to get the middlepoint\n",
    "def getNadirIndex(beam_index, beam_list):\n",
    "    if beam_index[0] == 0:\n",
    "        nadir_index = int( beam_index[1]/2)\n",
    "    if  beam_index[0] != 0:\n",
    "        nadir_index = int(sum(beam_list[0:beam_index[0]])+(beam_index[1]/2))\n",
    "    #print(beam_list[0:beam_index[0]], beam_index[1]/2, '\\n')\n",
    "    #print(beam_index[1])\n",
    "    return nadir_index\n",
    "\n",
    "for i in range(5):\n",
    "    mdp = getNadirIndex(beam_indexes[i], beam_list)\n",
    "    print(mdp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#getAngle(2, xyz, ping_indexes, beam_indexes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_indexes = list(range(len(ping_indexes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 157905/209062 [03:44<01:13, 699.78it/s]"
     ]
    }
   ],
   "source": [
    "angles = parmap.map(getAngle, input_indexes, xyz, ping_indexes, beam_indexes, pm_processes=48, pm_pbar=True)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "em_data['angle']=\"\"\n",
    "angles=[]\n",
    "\n",
    "for i in range(len(ping_indexes)):\n",
    "    if i == 0:\n",
    "        start=0\n",
    "        stop=ping_indexes[i]\n",
    "        shape=ping_indexes[i]\n",
    "    else:\n",
    "        start=ping_indexes[i-1]\n",
    "        stop=ping_indexes[i]\n",
    "        shape=ping_indexes[i] - ping_indexes[i-1]\n",
    "        \n",
    "    midpoint = coords[nadir_indexs[i]]\n",
    "    pings = coords[start:stop]\n",
    "    Zi = Z[start:stop]        \n",
    "    XY = [tuple(midpoint)]\n",
    "    coord = [tuple(j) for j in tuple(pings)]\n",
    "    d = distance.cdist(XY, coord, 'euclidean')\n",
    "    angle = np.rad2deg(np.arctan2(np.abs(Zi),d)).reshape(shape,)\n",
    "    angles.append(angle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ping_indexes\n",
    "#nadir_indexs\n",
    "#coord\n",
    "#Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "angle_values = [beam for ping in angles for beam in ping]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_data['angle'] = angle_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end_time = dt.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end_time-st_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_data.head(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(em_data['angle'].iloc[0:54].values, em_data['Amplitude'].iloc[0:54].values, '-');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!gist PDAL_EM1000.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#em_data.to_csv('em_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "em_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# need to start to discard some bad data ...\n",
    "em_data['Amplitude'][em_data['Amplitude']>=0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!gist PDAL_EM1000.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}