Skip to content

Instantly share code, notes, and snippets.

@cchwala

cchwala/test overflow handling in decode_cf_datetime (clean version).ipynb

Last active July 16, 2017 21:19
Show Gist options
  • Save cchwala/25efac7857c9b53f6f81d6fa44135a45 to your computer and use it in GitHub Desktop.
Save cchwala/25efac7857c9b53f6f81d6fa44135a45 to your computer and use it in GitHub Desktop.
Download ZIP
test overflow handling in decode_cf_datetime (clean version)
Display the source blob
Display the rendered blob
Raw
test overflow handling in decode_cf_datetime (clean version).ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import numpy as np\n",
"import xarray as xr\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"\n",
"from collections import OrderedDict"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Copy-paste `decode_cf_datetime()` from xarray master (`old`) and from my [PR](https://github.com/pydata/xarray/pull/1414) (`new`)\n",
"some minor changes to refere to other functions from same file where needed"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from pandas.errors import OutOfBoundsDatetime\n",
"\n",
"def decode_cf_datetime_new(num_dates, units, calendar=None):\n",
" num_dates = np.asarray(num_dates)\n",
" flat_num_dates = num_dates.ravel()\n",
" if calendar is None:\n",
" calendar = 'standard'\n",
"\n",
" delta, ref_date = xr.conventions._unpack_netcdf_time_units(units)\n",
"\n",
" try:\n",
" if calendar not in xr.conventions._STANDARD_CALENDARS:\n",
" raise OutOfBoundsDatetime\n",
"\n",
" delta = xr.conventions._netcdf_to_numpy_timeunit(delta)\n",
" try:\n",
" ref_date = pd.Timestamp(ref_date)\n",
" except ValueError:\n",
" # ValueError is raised by pd.Timestamp for non-ISO timestamp\n",
" # strings, in which case we fall back to using netCDF4\n",
" raise OutOfBoundsDatetime\n",
"\n",
" # fixes: https://github.com/pydata/pandas/issues/14068\n",
" # these lines check if the the lowest or the highest value in dates\n",
" # cause an OutOfBoundsDatetime (Overflow) error\n",
" pd.to_timedelta(flat_num_dates.min(), delta) + ref_date\n",
" pd.to_timedelta(flat_num_dates.max(), delta) + ref_date\n",
"\n",
" # Cast input dates to integers of nanoseconds because `pd.to_datetime`\n",
" # works much faster when dealing with integers\n",
" flat_num_dates_ns_int = (flat_num_dates *\n",
" _NS_PER_TIME_DELTA[delta]).astype(np.int64)\n",
"\n",
" dates = (pd.to_timedelta(flat_num_dates_ns_int, 'ns') +\n",
" ref_date).values\n",
"\n",
" except (OutOfBoundsDatetime, OverflowError):\n",
" dates = xr.conventions._decode_datetime_with_netcdf4(flat_num_dates.astype(np.float),\n",
" units,\n",
" calendar)\n",
"\n",
" return dates.reshape(num_dates.shape)\n",
"\n",
"def decode_cf_datetime_old(num_dates, units, calendar=None):\n",
" num_dates = np.asarray(num_dates, dtype=float)\n",
" flat_num_dates = num_dates.ravel()\n",
" if calendar is None:\n",
" calendar = 'standard'\n",
"\n",
" delta, ref_date = xr.conventions._unpack_netcdf_time_units(units)\n",
"\n",
" try:\n",
" if calendar not in xr.conventions._STANDARD_CALENDARS:\n",
" raise OutOfBoundsDatetime\n",
"\n",
" delta = xr.conventions._netcdf_to_numpy_timeunit(delta)\n",
" try:\n",
" ref_date = pd.Timestamp(ref_date)\n",
" except ValueError:\n",
" # ValueError is raised by pd.Timestamp for non-ISO timestamp\n",
" # strings, in which case we fall back to using netCDF4\n",
" raise OutOfBoundsDatetime\n",
"\n",
" # fixes: https://github.com/pydata/pandas/issues/14068\n",
" # these lines check if the the lowest or the highest value in dates\n",
" # cause an OutOfBoundsDatetime (Overflow) error\n",
" pd.to_timedelta(flat_num_dates.min(), delta) + ref_date\n",
" pd.to_timedelta(flat_num_dates.max(), delta) + ref_date\n",
"\n",
" dates = (pd.to_timedelta(flat_num_dates, delta) + ref_date).values\n",
"\n",
" except (OutOfBoundsDatetime, OverflowError):\n",
" dates = xr.conventions._decode_datetime_with_netcdf4(flat_num_dates, units, calendar)\n",
"\n",
" return dates.reshape(num_dates.shape)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Definitions for time conversion"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"_NS_PER_TIME_DELTA = OrderedDict([\n",
" ('us', 1e3),\n",
" ('ms', 1e6),\n",
" ('s', 1e9),\n",
" ('m', 1e9 * 60),\n",
" ('h', 1e9 * 60 * 60),\n",
" ('D', 1e9 * 60 * 60 * 24)])\n",
"\n",
"time_unit_short_to_long_str = {\n",
" 'us': 'microseconds',\n",
" 'ms': 'milliseconds',\n",
" 's': 'seconds',\n",
" 'm': 'minutes',\n",
" 'h': 'hours',\n",
" 'D': 'days'\n",
"}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Max and min of int64"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"int_min = np.iinfo(np.int64).min\n",
"int_max = np.iinfo(np.int64).max"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# A function to produce floats with smallest increments around a certain float\n",
"This is used to study the behavior of `decode_cf_datetime()` for values close to the region where np.datetime64['ns'] overflows"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def float_array_with_smallest_increments_around_starting_point(initial_float, N_points_in_one_direction): \n",
" floats_upward = [initial_float, ]\n",
" floats_downward = [initial_float, ]\n",
" for i in range(N_points_in_one_direction):\n",
" floats_upward.append(np.nextafter(floats_upward[-1] , int_max))\n",
" floats_downward.append(np.nextafter(floats_downward[-1] , int_min)) \n",
" return np.array(floats_downward[::-1] + floats_upward[1:])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Test it"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"3.1399999999999956834528802573913708329201\n",
"3.1399999999999961275420901074539870023727\n",
"3.1399999999999965716312999575166031718254\n",
"3.1399999999999970157205098075792193412781\n",
"3.1399999999999974598097196576418355107307\n",
"3.1399999999999979038989295077044516801834\n",
"3.1399999999999983479881393577670678496361\n",
"3.1399999999999987920773492078296840190887\n",
"3.1399999999999992361665590578923001885414\n",
"3.1399999999999996802557689079549163579941\n",
"3.1400000000000001243449787580175325274467\n",
"3.1400000000000005684341886080801486968994\n",
"3.1400000000000010125233984581427648663521\n",
"3.1400000000000014566126083082053810358047\n",
"3.1400000000000019007018181582679972052574\n",
"3.1400000000000023447910280083306133747101\n",
"3.1400000000000027888802378583932295441628\n",
"3.1400000000000032329694477084558457136154\n",
"3.1400000000000036770586575585184618830681\n",
"3.1400000000000041211478674085810780525208\n",
"3.1400000000000045652370772586436942219734\n"
]
}
],
"source": [
"foo = float_array_with_smallest_increments_around_starting_point(3.14, 10)\n",
"for x in np.nditer(foo):\n",
" print '%2.40f' % x"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Produce an array of floats close to the milliseconds value that overflows"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ms_as_float ns_as_int64\n",
"9223372036854.7558593750 9223372036854756352\n",
"9223372036854.7578125000 9223372036854757376\n",
"9223372036854.7597656250 9223372036854759424\n",
"9223372036854.7617187500 9223372036854761472\n",
"9223372036854.7636718750 9223372036854763520\n",
"9223372036854.7656250000 9223372036854765568\n",
"9223372036854.7675781250 9223372036854767616\n",
"9223372036854.7695312500 9223372036854769664\n",
"9223372036854.7714843750 9223372036854771712\n",
"9223372036854.7734375000 9223372036854773760\n",
"9223372036854.7753906250 -9223372036854775808\n",
"9223372036854.7773437500 -9223372036854775808\n",
"9223372036854.7792968750 -9223372036854775808\n",
"9223372036854.7812500000 -9223372036854775808\n",
"9223372036854.7832031250 -9223372036854775808\n",
"9223372036854.7851562500 -9223372036854775808\n",
"9223372036854.7871093750 -9223372036854775808\n",
"9223372036854.7890625000 -9223372036854775808\n",
"9223372036854.7910156250 -9223372036854775808\n",
"9223372036854.7929687500 -9223372036854775808\n",
"9223372036854.7949218750 -9223372036854775808\n"
]
}
],
"source": [
"N = 10\n",
"unit = 'ms'\n",
"floats = float_array_with_smallest_increments_around_starting_point(int_max / _NS_PER_TIME_DELTA[unit], N)\n",
"\n",
"print 'ms_as_float ns_as_int64'\n",
"for x in np.nditer(floats):\n",
" print '%2.10f %d' % (x, (x * _NS_PER_TIME_DELTA[unit]).astype('int64'))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Test how `decode_cf_datetime` reacts to these float values in milliseconds"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": true,
"scrolled": false
},
"outputs": [],
"source": [
"new = decode_cf_datetime_new(floats, units=time_unit_short_to_long_str[unit] + ' since 1970-01-01')\n",
"old = decode_cf_datetime_old(floats, units=time_unit_short_to_long_str[unit] + ' since 1970-01-01')"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Float at overflow point: 9223372036854.7754\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>input</th>\n",
" <th>new</th>\n",
" <th>new_as_int</th>\n",
" <th>old</th>\n",
" <th>old_as_int</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>9223372036854.7559</td>\n",
" <td>2262-04-11 23:47:16.854756352</td>\n",
" <td>9223372036854756352</td>\n",
" <td>2262-04-11 23:47:16.854756000</td>\n",
" <td>9223372036854756000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>9223372036854.7578</td>\n",
" <td>2262-04-11 23:47:16.854757376</td>\n",
" <td>9223372036854757376</td>\n",
" <td>2262-04-11 23:47:16.854758000</td>\n",
" <td>9223372036854758000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>9223372036854.7598</td>\n",
" <td>2262-04-11 23:47:16.854759424</td>\n",
" <td>9223372036854759424</td>\n",
" <td>2262-04-11 23:47:16.854760000</td>\n",
" <td>9223372036854760000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>9223372036854.7617</td>\n",
" <td>2262-04-11 23:47:16.854761472</td>\n",
" <td>9223372036854761472</td>\n",
" <td>2262-04-11 23:47:16.854762000</td>\n",
" <td>9223372036854762000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>9223372036854.7637</td>\n",
" <td>2262-04-11 23:47:16.854763520</td>\n",
" <td>9223372036854763520</td>\n",
" <td>2262-04-11 23:47:16.854764000</td>\n",
" <td>9223372036854764000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>9223372036854.7656</td>\n",
" <td>2262-04-11 23:47:16.854765568</td>\n",
" <td>9223372036854765568</td>\n",
" <td>2262-04-11 23:47:16.854766000</td>\n",
" <td>9223372036854766000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>9223372036854.7676</td>\n",
" <td>2262-04-11 23:47:16.854767616</td>\n",
" <td>9223372036854767616</td>\n",
" <td>2262-04-11 23:47:16.854768000</td>\n",
" <td>9223372036854768000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>9223372036854.7695</td>\n",
" <td>2262-04-11 23:47:16.854769664</td>\n",
" <td>9223372036854769664</td>\n",
" <td>2262-04-11 23:47:16.854770000</td>\n",
" <td>9223372036854770000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>9223372036854.7715</td>\n",
" <td>2262-04-11 23:47:16.854771712</td>\n",
" <td>9223372036854771712</td>\n",
" <td>2262-04-11 23:47:16.854771000</td>\n",
" <td>9223372036854771000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>9223372036854.7734</td>\n",
" <td>2262-04-11 23:47:16.854773760</td>\n",
" <td>9223372036854773760</td>\n",
" <td>2262-04-11 23:47:16.854773000</td>\n",
" <td>9223372036854773000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>9223372036854.7754</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>2262-04-11 23:47:16.854775000</td>\n",
" <td>9223372036854775000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>9223372036854.7773</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145225384</td>\n",
" <td>-9223372036854774616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>9223372036854.7793</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145227384</td>\n",
" <td>-9223372036854772616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>9223372036854.7812</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145229384</td>\n",
" <td>-9223372036854770616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>9223372036854.7832</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145231384</td>\n",
" <td>-9223372036854768616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>9223372036854.7852</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145233384</td>\n",
" <td>-9223372036854766616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>9223372036854.7871</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145235384</td>\n",
" <td>-9223372036854764616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>9223372036854.7891</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145237384</td>\n",
" <td>-9223372036854762616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>9223372036854.7910</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145239384</td>\n",
" <td>-9223372036854760616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>9223372036854.7930</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145241384</td>\n",
" <td>-9223372036854758616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>9223372036854.7949</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145243384</td>\n",
" <td>-9223372036854756616</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" input new new_as_int \\\n",
"0 9223372036854.7559 2262-04-11 23:47:16.854756352 9223372036854756352 \n",
"1 9223372036854.7578 2262-04-11 23:47:16.854757376 9223372036854757376 \n",
"2 9223372036854.7598 2262-04-11 23:47:16.854759424 9223372036854759424 \n",
"3 9223372036854.7617 2262-04-11 23:47:16.854761472 9223372036854761472 \n",
"4 9223372036854.7637 2262-04-11 23:47:16.854763520 9223372036854763520 \n",
"5 9223372036854.7656 2262-04-11 23:47:16.854765568 9223372036854765568 \n",
"6 9223372036854.7676 2262-04-11 23:47:16.854767616 9223372036854767616 \n",
"7 9223372036854.7695 2262-04-11 23:47:16.854769664 9223372036854769664 \n",
"8 9223372036854.7715 2262-04-11 23:47:16.854771712 9223372036854771712 \n",
"9 9223372036854.7734 2262-04-11 23:47:16.854773760 9223372036854773760 \n",
"10 9223372036854.7754 NaT -9223372036854775808 \n",
"11 9223372036854.7773 NaT -9223372036854775808 \n",
"12 9223372036854.7793 NaT -9223372036854775808 \n",
"13 9223372036854.7812 NaT -9223372036854775808 \n",
"14 9223372036854.7832 NaT -9223372036854775808 \n",
"15 9223372036854.7852 NaT -9223372036854775808 \n",
"16 9223372036854.7871 NaT -9223372036854775808 \n",
"17 9223372036854.7891 NaT -9223372036854775808 \n",
"18 9223372036854.7910 NaT -9223372036854775808 \n",
"19 9223372036854.7930 NaT -9223372036854775808 \n",
"20 9223372036854.7949 NaT -9223372036854775808 \n",
"\n",
" old old_as_int \n",
"0 2262-04-11 23:47:16.854756000 9223372036854756000 \n",
"1 2262-04-11 23:47:16.854758000 9223372036854758000 \n",
"2 2262-04-11 23:47:16.854760000 9223372036854760000 \n",
"3 2262-04-11 23:47:16.854762000 9223372036854762000 \n",
"4 2262-04-11 23:47:16.854764000 9223372036854764000 \n",
"5 2262-04-11 23:47:16.854766000 9223372036854766000 \n",
"6 2262-04-11 23:47:16.854768000 9223372036854768000 \n",
"7 2262-04-11 23:47:16.854770000 9223372036854770000 \n",
"8 2262-04-11 23:47:16.854771000 9223372036854771000 \n",
"9 2262-04-11 23:47:16.854773000 9223372036854773000 \n",
"10 2262-04-11 23:47:16.854775000 9223372036854775000 \n",
"11 1677-09-21 00:12:43.145225384 -9223372036854774616 \n",
"12 1677-09-21 00:12:43.145227384 -9223372036854772616 \n",
"13 1677-09-21 00:12:43.145229384 -9223372036854770616 \n",
"14 1677-09-21 00:12:43.145231384 -9223372036854768616 \n",
"15 1677-09-21 00:12:43.145233384 -9223372036854766616 \n",
"16 1677-09-21 00:12:43.145235384 -9223372036854764616 \n",
"17 1677-09-21 00:12:43.145237384 -9223372036854762616 \n",
"18 1677-09-21 00:12:43.145239384 -9223372036854760616 \n",
"19 1677-09-21 00:12:43.145241384 -9223372036854758616 \n",
"20 1677-09-21 00:12:43.145243384 -9223372036854756616 "
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"print 'Float at overflow point: %2.4f' % (int_max / _NS_PER_TIME_DELTA[unit])\n",
"\n",
"df = pd.DataFrame({'input': floats, \n",
" 'new': new, \n",
" 'old': old,\n",
" 'new_as_int': new.astype('int64'), \n",
" 'old_as_int': old.astype('int64')})\n",
"df['input'] = df['input'].map('{:.4f}'.format)\n",
"df "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**The old code seems to handle the overflow incorrectly. Without warning it generates values on the opposite end of the np.datetime64 range**"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Interestingly `print` shows different time-strings than pandas\n",
"In the `old` version the negative values after the overflow (e.g. -9223372036854758616) are `1677-09-21...`, while the `print` statement gives `2262-04-10...` which is actually a tad smaller than the maximum `2262-04-11T23:47:16.854773...`."
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['2262-04-11T23:47:16.854756000' '2262-04-11T23:47:16.854758000'\n",
" '2262-04-11T23:47:16.854760000' '2262-04-11T23:47:16.854762000'\n",
" '2262-04-11T23:47:16.854764000' '2262-04-11T23:47:16.854766000'\n",
" '2262-04-11T23:47:16.854768000' '2262-04-11T23:47:16.854770000'\n",
" '2262-04-11T23:47:16.854771000' '2262-04-11T23:47:16.854773000'\n",
" '2262-04-11T23:47:16.854775000' '2262-04-10T00:12:43.145225384'\n",
" '2262-04-10T00:12:43.145227384' '2262-04-10T00:12:43.145229384'\n",
" '2262-04-10T00:12:43.145231384' '2262-04-10T00:12:43.145233384'\n",
" '2262-04-10T00:12:43.145235384' '2262-04-10T00:12:43.145237384'\n",
" '2262-04-10T00:12:43.145239384' '2262-04-10T00:12:43.145241384'\n",
" '2262-04-10T00:12:43.145243384']\n"
]
}
],
"source": [
"print old"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['2262-04-11T23:47:16.854756352' '2262-04-11T23:47:16.854757376'\n",
" '2262-04-11T23:47:16.854759424' '2262-04-11T23:47:16.854761472'\n",
" '2262-04-11T23:47:16.854763520' '2262-04-11T23:47:16.854765568'\n",
" '2262-04-11T23:47:16.854767616' '2262-04-11T23:47:16.854769664'\n",
" '2262-04-11T23:47:16.854771712' '2262-04-11T23:47:16.854773760' 'NaT'\n",
" 'NaT' 'NaT' 'NaT' 'NaT' 'NaT' 'NaT' 'NaT' 'NaT' 'NaT' 'NaT']\n"
]
}
],
"source": [
"print new"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Same problem for seconds "
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Float at overflow point: 9223372036.85477638244628906250\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>input</th>\n",
" <th>new</th>\n",
" <th>new_as_int</th>\n",
" <th>old</th>\n",
" <th>old_as_int</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>9223372036.85475730895996093750</td>\n",
" <td>2262-04-11 23:47:16.854757376</td>\n",
" <td>9223372036854757376</td>\n",
" <td>2262-04-11 23:47:16.854757000</td>\n",
" <td>9223372036854757000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>9223372036.85475921630859375000</td>\n",
" <td>2262-04-11 23:47:16.854759424</td>\n",
" <td>9223372036854759424</td>\n",
" <td>2262-04-11 23:47:16.854759000</td>\n",
" <td>9223372036854759000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>9223372036.85476112365722656250</td>\n",
" <td>2262-04-11 23:47:16.854761472</td>\n",
" <td>9223372036854761472</td>\n",
" <td>2262-04-11 23:47:16.854761000</td>\n",
" <td>9223372036854761000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>9223372036.85476303100585937500</td>\n",
" <td>2262-04-11 23:47:16.854763520</td>\n",
" <td>9223372036854763520</td>\n",
" <td>2262-04-11 23:47:16.854763000</td>\n",
" <td>9223372036854763000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>9223372036.85476493835449218750</td>\n",
" <td>2262-04-11 23:47:16.854764544</td>\n",
" <td>9223372036854764544</td>\n",
" <td>2262-04-11 23:47:16.854765000</td>\n",
" <td>9223372036854765000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>9223372036.85476684570312500000</td>\n",
" <td>2262-04-11 23:47:16.854766592</td>\n",
" <td>9223372036854766592</td>\n",
" <td>2262-04-11 23:47:16.854767000</td>\n",
" <td>9223372036854767000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>9223372036.85476875305175781250</td>\n",
" <td>2262-04-11 23:47:16.854768640</td>\n",
" <td>9223372036854768640</td>\n",
" <td>2262-04-11 23:47:16.854769000</td>\n",
" <td>9223372036854769000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>9223372036.85477066040039062500</td>\n",
" <td>2262-04-11 23:47:16.854770688</td>\n",
" <td>9223372036854770688</td>\n",
" <td>2262-04-11 23:47:16.854771000</td>\n",
" <td>9223372036854771000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>9223372036.85477256774902343750</td>\n",
" <td>2262-04-11 23:47:16.854772736</td>\n",
" <td>9223372036854772736</td>\n",
" <td>2262-04-11 23:47:16.854773000</td>\n",
" <td>9223372036854773000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>9223372036.85477447509765625000</td>\n",
" <td>2262-04-11 23:47:16.854774784</td>\n",
" <td>9223372036854774784</td>\n",
" <td>2262-04-11 23:47:16.854774000</td>\n",
" <td>9223372036854774000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>9223372036.85477638244628906250</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145224384</td>\n",
" <td>-9223372036854775616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>9223372036.85477828979492187500</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145226384</td>\n",
" <td>-9223372036854773616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>9223372036.85478019714355468750</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145228384</td>\n",
" <td>-9223372036854771616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>9223372036.85478210449218750000</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145230384</td>\n",
" <td>-9223372036854769616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>9223372036.85478401184082031250</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145232384</td>\n",
" <td>-9223372036854767616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>9223372036.85478591918945312500</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145234384</td>\n",
" <td>-9223372036854765616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>9223372036.85478782653808593750</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145236384</td>\n",
" <td>-9223372036854763616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>9223372036.85478973388671875000</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145238384</td>\n",
" <td>-9223372036854761616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>9223372036.85479164123535156250</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145240384</td>\n",
" <td>-9223372036854759616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>9223372036.85479354858398437500</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145242384</td>\n",
" <td>-9223372036854757616</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>9223372036.85479545593261718750</td>\n",
" <td>NaT</td>\n",
" <td>-9223372036854775808</td>\n",
" <td>1677-09-21 00:12:43.145243384</td>\n",
" <td>-9223372036854756616</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" input new \\\n",
"0 9223372036.85475730895996093750 2262-04-11 23:47:16.854757376 \n",
"1 9223372036.85475921630859375000 2262-04-11 23:47:16.854759424 \n",
"2 9223372036.85476112365722656250 2262-04-11 23:47:16.854761472 \n",
"3 9223372036.85476303100585937500 2262-04-11 23:47:16.854763520 \n",
"4 9223372036.85476493835449218750 2262-04-11 23:47:16.854764544 \n",
"5 9223372036.85476684570312500000 2262-04-11 23:47:16.854766592 \n",
"6 9223372036.85476875305175781250 2262-04-11 23:47:16.854768640 \n",
"7 9223372036.85477066040039062500 2262-04-11 23:47:16.854770688 \n",
"8 9223372036.85477256774902343750 2262-04-11 23:47:16.854772736 \n",
"9 9223372036.85477447509765625000 2262-04-11 23:47:16.854774784 \n",
"10 9223372036.85477638244628906250 NaT \n",
"11 9223372036.85477828979492187500 NaT \n",
"12 9223372036.85478019714355468750 NaT \n",
"13 9223372036.85478210449218750000 NaT \n",
"14 9223372036.85478401184082031250 NaT \n",
"15 9223372036.85478591918945312500 NaT \n",
"16 9223372036.85478782653808593750 NaT \n",
"17 9223372036.85478973388671875000 NaT \n",
"18 9223372036.85479164123535156250 NaT \n",
"19 9223372036.85479354858398437500 NaT \n",
"20 9223372036.85479545593261718750 NaT \n",
"\n",
" new_as_int old old_as_int \n",
"0 9223372036854757376 2262-04-11 23:47:16.854757000 9223372036854757000 \n",
"1 9223372036854759424 2262-04-11 23:47:16.854759000 9223372036854759000 \n",
"2 9223372036854761472 2262-04-11 23:47:16.854761000 9223372036854761000 \n",
"3 9223372036854763520 2262-04-11 23:47:16.854763000 9223372036854763000 \n",
"4 9223372036854764544 2262-04-11 23:47:16.854765000 9223372036854765000 \n",
"5 9223372036854766592 2262-04-11 23:47:16.854767000 9223372036854767000 \n",
"6 9223372036854768640 2262-04-11 23:47:16.854769000 9223372036854769000 \n",
"7 9223372036854770688 2262-04-11 23:47:16.854771000 9223372036854771000 \n",
"8 9223372036854772736 2262-04-11 23:47:16.854773000 9223372036854773000 \n",
"9 9223372036854774784 2262-04-11 23:47:16.854774000 9223372036854774000 \n",
"10 -9223372036854775808 1677-09-21 00:12:43.145224384 -9223372036854775616 \n",
"11 -9223372036854775808 1677-09-21 00:12:43.145226384 -9223372036854773616 \n",
"12 -9223372036854775808 1677-09-21 00:12:43.145228384 -9223372036854771616 \n",
"13 -9223372036854775808 1677-09-21 00:12:43.145230384 -9223372036854769616 \n",
"14 -9223372036854775808 1677-09-21 00:12:43.145232384 -9223372036854767616 \n",
"15 -9223372036854775808 1677-09-21 00:12:43.145234384 -9223372036854765616 \n",
"16 -9223372036854775808 1677-09-21 00:12:43.145236384 -9223372036854763616 \n",
"17 -9223372036854775808 1677-09-21 00:12:43.145238384 -9223372036854761616 \n",
"18 -9223372036854775808 1677-09-21 00:12:43.145240384 -9223372036854759616 \n",
"19 -9223372036854775808 1677-09-21 00:12:43.145242384 -9223372036854757616 \n",
"20 -9223372036854775808 1677-09-21 00:12:43.145243384 -9223372036854756616 "
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"N = 10\n",
"unit = 's'\n",
"floats = float_array_with_smallest_increments_around_starting_point(int_max / _NS_PER_TIME_DELTA[unit], N)\n",
"\n",
"new = decode_cf_datetime_new(floats, units=time_unit_short_to_long_str[unit] + ' since 1970-01-01')\n",
"old = decode_cf_datetime_old(floats, units=time_unit_short_to_long_str[unit] + ' since 1970-01-01')\n",
"\n",
"print 'Float at overflow point: %2.20f' % (int_max / _NS_PER_TIME_DELTA[unit])\n",
"\n",
"df = pd.DataFrame({'input': floats, \n",
" 'new': new, \n",
" 'old': old,\n",
" 'new_as_int': new.astype('int64'), \n",
" 'old_as_int': old.astype('int64')})\n",
"df['input'] = df['input'].map('{:.20f}'.format)\n",
"df "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# For microseconds the` _decode_datetime_with_netcdf4` seems to kick in\n",
"When a potential overflow is detected in `decode_cf_datetime` it falls back to netcdf for converting time values using the function `_decode_datetime_with_netcdf4` which returns `datetime` instead of `np.datetime64`. Hence the below code throws an error when trying to cast to int"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Float at overflow point: 9223372036854776.0000\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/chwala-c/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:5: RuntimeWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using dummy netCDF4.datetime objects instead, reason: dates out of range\n",
"/Users/chwala-c/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:6: RuntimeWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using dummy netCDF4.datetime objects instead, reason: dates out of range\n"
]
},
{
"ename": "TypeError",
"evalue": "long() argument must be a string or a number, not 'datetime.datetime'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-15-b63c3dcaf8dd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;34m'new'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnew\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m'old'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mold\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0;34m'new_as_int'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnew\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'int64'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m 'old_as_int': old.astype('int64')})\n\u001b[1;32m 15\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'input'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'input'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{:.4f}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mTypeError\u001b[0m: long() argument must be a string or a number, not 'datetime.datetime'"
]
}
],
"source": [
"N = 10\n",
"unit = 'us'\n",
"floats = float_array_with_smallest_increments_around_starting_point(int_max / _NS_PER_TIME_DELTA[unit], N)\n",
"\n",
"new = decode_cf_datetime_new(floats, units=time_unit_short_to_long_str[unit] + ' since 1970-01-01')\n",
"old = decode_cf_datetime_old(floats, units=time_unit_short_to_long_str[unit] + ' since 1970-01-01')\n",
"\n",
"print 'Float at overflow point: %2.4f' % (int_max / _NS_PER_TIME_DELTA[unit])\n",
"\n",
"df = pd.DataFrame({'input': floats, \n",
" 'new': new, \n",
" 'old': old,\n",
" 'new_as_int': new.astype('int64'), \n",
" 'old_as_int': old.astype('int64')})\n",
"df['input'] = df['input'].map('{:.4f}'.format)\n",
"df "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Again for microseconds, now with output without cast to int\n",
"This seems to looks okay."
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Float at overflow point: 9223372036854776.0000\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/chwala-c/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:5: RuntimeWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using dummy netCDF4.datetime objects instead, reason: dates out of range\n",
"/Users/chwala-c/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:6: RuntimeWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using dummy netCDF4.datetime objects instead, reason: dates out of range\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>input</th>\n",
" <th>new</th>\n",
" <th>old</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>9223372036854756.0000</td>\n",
" <td>2262-04-11 23:47:16.854756</td>\n",
" <td>2262-04-11 23:47:16.854756</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>9223372036854758.0000</td>\n",
" <td>2262-04-11 23:47:16.854758</td>\n",
" <td>2262-04-11 23:47:16.854758</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>9223372036854760.0000</td>\n",
" <td>2262-04-11 23:47:16.854760</td>\n",
" <td>2262-04-11 23:47:16.854760</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>9223372036854762.0000</td>\n",
" <td>2262-04-11 23:47:16.854762</td>\n",
" <td>2262-04-11 23:47:16.854762</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>9223372036854764.0000</td>\n",
" <td>2262-04-11 23:47:16.854764</td>\n",
" <td>2262-04-11 23:47:16.854764</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>9223372036854766.0000</td>\n",
" <td>2262-04-11 23:47:16.854766</td>\n",
" <td>2262-04-11 23:47:16.854766</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>9223372036854768.0000</td>\n",
" <td>2262-04-11 23:47:16.854768</td>\n",
" <td>2262-04-11 23:47:16.854768</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>9223372036854770.0000</td>\n",
" <td>2262-04-11 23:47:16.854770</td>\n",
" <td>2262-04-11 23:47:16.854770</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>9223372036854772.0000</td>\n",
" <td>2262-04-11 23:47:16.854772</td>\n",
" <td>2262-04-11 23:47:16.854772</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>9223372036854774.0000</td>\n",
" <td>2262-04-11 23:47:16.854774</td>\n",
" <td>2262-04-11 23:47:16.854774</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>9223372036854776.0000</td>\n",
" <td>2262-04-11 23:47:16.854776</td>\n",
" <td>2262-04-11 23:47:16.854776</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>9223372036854778.0000</td>\n",
" <td>2262-04-11 23:47:16.854778</td>\n",
" <td>2262-04-11 23:47:16.854778</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>9223372036854780.0000</td>\n",
" <td>2262-04-11 23:47:16.854780</td>\n",
" <td>2262-04-11 23:47:16.854780</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>9223372036854782.0000</td>\n",
" <td>2262-04-11 23:47:16.854782</td>\n",
" <td>2262-04-11 23:47:16.854782</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>9223372036854784.0000</td>\n",
" <td>2262-04-11 23:47:16.854784</td>\n",
" <td>2262-04-11 23:47:16.854784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>9223372036854786.0000</td>\n",
" <td>2262-04-11 23:47:16.854786</td>\n",
" <td>2262-04-11 23:47:16.854786</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>9223372036854788.0000</td>\n",
" <td>2262-04-11 23:47:16.854788</td>\n",
" <td>2262-04-11 23:47:16.854788</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>9223372036854790.0000</td>\n",
" <td>2262-04-11 23:47:16.854790</td>\n",
" <td>2262-04-11 23:47:16.854790</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>9223372036854792.0000</td>\n",
" <td>2262-04-11 23:47:16.854792</td>\n",
" <td>2262-04-11 23:47:16.854792</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>9223372036854794.0000</td>\n",
" <td>2262-04-11 23:47:16.854794</td>\n",
" <td>2262-04-11 23:47:16.854794</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>9223372036854796.0000</td>\n",
" <td>2262-04-11 23:47:16.854796</td>\n",
" <td>2262-04-11 23:47:16.854796</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" input new \\\n",
"0 9223372036854756.0000 2262-04-11 23:47:16.854756 \n",
"1 9223372036854758.0000 2262-04-11 23:47:16.854758 \n",
"2 9223372036854760.0000 2262-04-11 23:47:16.854760 \n",
"3 9223372036854762.0000 2262-04-11 23:47:16.854762 \n",
"4 9223372036854764.0000 2262-04-11 23:47:16.854764 \n",
"5 9223372036854766.0000 2262-04-11 23:47:16.854766 \n",
"6 9223372036854768.0000 2262-04-11 23:47:16.854768 \n",
"7 9223372036854770.0000 2262-04-11 23:47:16.854770 \n",
"8 9223372036854772.0000 2262-04-11 23:47:16.854772 \n",
"9 9223372036854774.0000 2262-04-11 23:47:16.854774 \n",
"10 9223372036854776.0000 2262-04-11 23:47:16.854776 \n",
"11 9223372036854778.0000 2262-04-11 23:47:16.854778 \n",
"12 9223372036854780.0000 2262-04-11 23:47:16.854780 \n",
"13 9223372036854782.0000 2262-04-11 23:47:16.854782 \n",
"14 9223372036854784.0000 2262-04-11 23:47:16.854784 \n",
"15 9223372036854786.0000 2262-04-11 23:47:16.854786 \n",
"16 9223372036854788.0000 2262-04-11 23:47:16.854788 \n",
"17 9223372036854790.0000 2262-04-11 23:47:16.854790 \n",
"18 9223372036854792.0000 2262-04-11 23:47:16.854792 \n",
"19 9223372036854794.0000 2262-04-11 23:47:16.854794 \n",
"20 9223372036854796.0000 2262-04-11 23:47:16.854796 \n",
"\n",
" old \n",
"0 2262-04-11 23:47:16.854756 \n",
"1 2262-04-11 23:47:16.854758 \n",
"2 2262-04-11 23:47:16.854760 \n",
"3 2262-04-11 23:47:16.854762 \n",
"4 2262-04-11 23:47:16.854764 \n",
"5 2262-04-11 23:47:16.854766 \n",
"6 2262-04-11 23:47:16.854768 \n",
"7 2262-04-11 23:47:16.854770 \n",
"8 2262-04-11 23:47:16.854772 \n",
"9 2262-04-11 23:47:16.854774 \n",
"10 2262-04-11 23:47:16.854776 \n",
"11 2262-04-11 23:47:16.854778 \n",
"12 2262-04-11 23:47:16.854780 \n",
"13 2262-04-11 23:47:16.854782 \n",
"14 2262-04-11 23:47:16.854784 \n",
"15 2262-04-11 23:47:16.854786 \n",
"16 2262-04-11 23:47:16.854788 \n",
"17 2262-04-11 23:47:16.854790 \n",
"18 2262-04-11 23:47:16.854792 \n",
"19 2262-04-11 23:47:16.854794 \n",
"20 2262-04-11 23:47:16.854796 "
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"N = 10\n",
"unit = 'us'\n",
"floats = float_array_with_smallest_increments_around_starting_point(int_max / _NS_PER_TIME_DELTA[unit], N)\n",
"\n",
"new = decode_cf_datetime_new(floats, units=time_unit_short_to_long_str[unit] + ' since 1970-01-01')\n",
"old = decode_cf_datetime_old(floats, units=time_unit_short_to_long_str[unit] + ' since 1970-01-01')\n",
"\n",
"print 'Float at overflow point: %2.4f' % (int_max / _NS_PER_TIME_DELTA[unit])\n",
"\n",
"df = pd.DataFrame({'input': floats, \n",
" 'new': new, \n",
" 'old': old})\n",
"df['input'] = df['input'].map('{:.4f}'.format)\n",
"df "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment