Skip to content

Instantly share code, notes, and snippets.

@mdengler

mdengler/pd.ipynb

Created October 26, 2015 19:35
Show Gist options
  • Save mdengler/245e94287e343972a789 to your computer and use it in GitHub Desktop.
Save mdengler/245e94287e343972a789 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
pd.ipynb
{
"metadata": {
"name": "",
"signature": "sha256:28407aa6899e8f4c5d2aeeb9e83990f8eb9d6f43b103a9080cf3b03afaf32d52"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "code",
"collapsed": false,
"input": [
"import pandas as pd, numpy as np"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"df = pd.DataFrame(index=pd.date_range(\"2015-03-12\", periods=6 * 24 *5, freq=\"10min\"))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 33
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"df[\"S1\"] = np.random.randn(len(df.index))\n",
"df[\"S2\"] = np.random.randn(len(df.index))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 34
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"df.head(), df.tail()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 35,
"text": [
"( S1 S2\n",
" 2015-03-12 00:00:00 -0.579566 -1.362629\n",
" 2015-03-12 00:10:00 -0.381633 0.573208\n",
" 2015-03-12 00:20:00 0.235525 0.589369\n",
" 2015-03-12 00:30:00 1.754687 1.811558\n",
" 2015-03-12 00:40:00 0.505616 1.413038,\n",
" S1 S2\n",
" 2015-03-16 23:10:00 0.927262 -1.371792\n",
" 2015-03-16 23:20:00 0.115658 1.361716\n",
" 2015-03-16 23:30:00 2.485742 0.629385\n",
" 2015-03-16 23:40:00 0.579433 0.701071\n",
" 2015-03-16 23:50:00 -1.383280 0.610668)"
]
}
],
"prompt_number": 35
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"s1 = pd.Series([(pd.Timestamp(\"2015-03-12 10:15:00\"), pd.Timestamp(\"2015-03-12 15:23:35\")),\n",
" (pd.Timestamp(\"2015-03-13 10:05:00\"), pd.Timestamp(\"2015-03-13 17:23:35\")),\n",
" (pd.Timestamp(\"2015-03-14 09:45:00\"), pd.Timestamp(\"2015-03-14 16:23:35\")),\n",
" ],\n",
" name=\"S1\")\n",
"s2 = s1.copy()\n",
"s2.name = \"S2\"\n",
"mask_extents = pd.DataFrame([s1, s2]).T"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 36
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"mask_extents"
],
"language": "python",
"metadata": {},
"outputs": [
{
"html": [
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>S1</th>\n",
" <th>S2</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td> (2015-03-12 10:15:00, 2015-03-12 15:23:35)</td>\n",
" <td> (2015-03-12 10:15:00, 2015-03-12 15:23:35)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td> (2015-03-13 10:05:00, 2015-03-13 17:23:35)</td>\n",
" <td> (2015-03-13 10:05:00, 2015-03-13 17:23:35)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td> (2015-03-14 09:45:00, 2015-03-14 16:23:35)</td>\n",
" <td> (2015-03-14 09:45:00, 2015-03-14 16:23:35)</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"metadata": {},
"output_type": "pyout",
"prompt_number": 37,
"text": [
" S1 \\\n",
"0 (2015-03-12 10:15:00, 2015-03-12 15:23:35) \n",
"1 (2015-03-13 10:05:00, 2015-03-13 17:23:35) \n",
"2 (2015-03-14 09:45:00, 2015-03-14 16:23:35) \n",
"\n",
" S2 \n",
"0 (2015-03-12 10:15:00, 2015-03-12 15:23:35) \n",
"1 (2015-03-13 10:05:00, 2015-03-13 17:23:35) \n",
"2 (2015-03-14 09:45:00, 2015-03-14 16:23:35) "
]
}
],
"prompt_number": 37
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"mask_extents[\"S1\"][0]"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 40,
"text": [
"(Timestamp('2015-03-12 10:15:00'), Timestamp('2015-03-12 15:23:35'))"
]
}
],
"prompt_number": 40
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"mask_extents.loc[0,\"S1\"][0]"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 49,
"text": [
"Timestamp('2015-03-12 10:15:00')"
]
}
],
"prompt_number": 49
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"df2 = df.copy()\n",
"#df2.loc[mask_extents.loc[0, \"S1\"][0], \"S1\"]\n",
"# don't know how to do this:\n",
"# masks_s1 = reduce(pd.DataFrame.__ror__, [(df2.index < start) & (df2.index > end) for start, end in mask_extents[\"S1\"]])\n",
"# df2[\"S1\"][~masks_s1] = np.nan\n",
"# ...so instead, based on:\n",
"# df2.query(\"'2015-03-12 10:15:00' < index < '2015-03-12 15:23:35' | '2015-03-13 10:05:00' < index < '2015-03-13 17:23:35'\")\n",
"# ...try this:\n",
"masks_s1_query = \" | \".join(\"'{}' < index < '{}'\".format(start, end) for start_end in mask_extents[\"S1\"])\n",
"df2.query(\"~ ({})\".format(masks_s1_query))\n",
"# how do I set those cells to np.nan?"
],
"language": "python",
"metadata": {},
"outputs": [
{
"html": [
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>S1</th>\n",
" <th>S2</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2015-03-12 00:00:00</th>\n",
" <td>-0.579566</td>\n",
" <td>-1.362629</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 00:10:00</th>\n",
" <td>-0.381633</td>\n",
" <td> 0.573208</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 00:20:00</th>\n",
" <td> 0.235525</td>\n",
" <td> 0.589369</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 00:30:00</th>\n",
" <td> 1.754687</td>\n",
" <td> 1.811558</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 00:40:00</th>\n",
" <td> 0.505616</td>\n",
" <td> 1.413038</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 00:50:00</th>\n",
" <td>-1.175593</td>\n",
" <td>-0.638982</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 01:00:00</th>\n",
" <td>-0.642313</td>\n",
" <td>-1.725921</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 01:10:00</th>\n",
" <td>-0.298308</td>\n",
" <td> 0.408128</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 01:20:00</th>\n",
" <td>-0.182880</td>\n",
" <td> 2.317932</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 01:30:00</th>\n",
" <td>-0.658728</td>\n",
" <td> 1.829225</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 01:40:00</th>\n",
" <td> 0.336450</td>\n",
" <td>-0.576847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 01:50:00</th>\n",
" <td> 1.174043</td>\n",
" <td>-0.018968</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 02:00:00</th>\n",
" <td> 1.306239</td>\n",
" <td> 1.043028</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 02:10:00</th>\n",
" <td>-1.855183</td>\n",
" <td> 0.601130</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 02:20:00</th>\n",
" <td>-0.241508</td>\n",
" <td> 1.227976</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 02:30:00</th>\n",
" <td>-1.518354</td>\n",
" <td>-0.499627</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 02:40:00</th>\n",
" <td> 0.167399</td>\n",
" <td>-1.358354</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 02:50:00</th>\n",
" <td> 2.562058</td>\n",
" <td>-1.095234</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 03:00:00</th>\n",
" <td> 1.737065</td>\n",
" <td> 0.308141</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 03:10:00</th>\n",
" <td>-0.332737</td>\n",
" <td>-0.883051</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 03:20:00</th>\n",
" <td>-0.466392</td>\n",
" <td>-0.082634</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 03:30:00</th>\n",
" <td>-2.454961</td>\n",
" <td>-1.262187</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 03:40:00</th>\n",
" <td> 0.419272</td>\n",
" <td> 0.370745</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 03:50:00</th>\n",
" <td>-1.267411</td>\n",
" <td>-1.055182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 04:00:00</th>\n",
" <td>-0.166380</td>\n",
" <td>-0.065995</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 04:10:00</th>\n",
" <td>-1.774168</td>\n",
" <td> 0.451581</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 04:20:00</th>\n",
" <td>-1.898200</td>\n",
" <td> 0.045828</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 04:30:00</th>\n",
" <td>-0.116386</td>\n",
" <td> 0.085524</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 04:40:00</th>\n",
" <td> 0.426295</td>\n",
" <td> 0.461055</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-12 04:50:00</th>\n",
" <td>-0.408411</td>\n",
" <td>-0.694797</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 19:00:00</th>\n",
" <td>-0.345654</td>\n",
" <td> 0.252860</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 19:10:00</th>\n",
" <td> 1.221992</td>\n",
" <td> 0.140411</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 19:20:00</th>\n",
" <td>-0.746366</td>\n",
" <td>-0.728046</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 19:30:00</th>\n",
" <td> 0.367208</td>\n",
" <td> 2.348574</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 19:40:00</th>\n",
" <td> 1.200294</td>\n",
" <td>-0.446141</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 19:50:00</th>\n",
" <td>-0.736154</td>\n",
" <td>-1.907979</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 20:00:00</th>\n",
" <td>-1.088274</td>\n",
" <td>-1.708835</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 20:10:00</th>\n",
" <td>-0.598324</td>\n",
" <td>-0.553277</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 20:20:00</th>\n",
" <td> 0.151677</td>\n",
" <td> 0.536430</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 20:30:00</th>\n",
" <td>-1.980867</td>\n",
" <td>-0.193359</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 20:40:00</th>\n",
" <td>-1.482832</td>\n",
" <td>-0.305577</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 20:50:00</th>\n",
" <td>-0.956331</td>\n",
" <td>-1.199016</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 21:00:00</th>\n",
" <td>-0.625324</td>\n",
" <td> 0.330678</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 21:10:00</th>\n",
" <td> 0.540265</td>\n",
" <td>-1.651197</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 21:20:00</th>\n",
" <td>-1.140503</td>\n",
" <td>-0.350532</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 21:30:00</th>\n",
" <td> 0.283272</td>\n",
" <td> 0.207816</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 21:40:00</th>\n",
" <td> 1.079753</td>\n",
" <td>-0.334234</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 21:50:00</th>\n",
" <td> 0.171345</td>\n",
" <td> 1.441366</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 22:00:00</th>\n",
" <td>-0.531493</td>\n",
" <td>-0.050047</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 22:10:00</th>\n",
" <td> 0.970737</td>\n",
" <td> 0.185797</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 22:20:00</th>\n",
" <td> 1.584863</td>\n",
" <td>-1.480487</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 22:30:00</th>\n",
" <td> 0.282121</td>\n",
" <td>-1.276703</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 22:40:00</th>\n",
" <td>-0.838282</td>\n",
" <td>-2.325543</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 22:50:00</th>\n",
" <td>-0.708904</td>\n",
" <td> 0.832618</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 23:00:00</th>\n",
" <td> 0.652657</td>\n",
" <td> 2.449541</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 23:10:00</th>\n",
" <td> 0.927262</td>\n",
" <td>-1.371792</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 23:20:00</th>\n",
" <td> 0.115658</td>\n",
" <td> 1.361716</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 23:30:00</th>\n",
" <td> 2.485742</td>\n",
" <td> 0.629385</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 23:40:00</th>\n",
" <td> 0.579433</td>\n",
" <td> 0.701071</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2015-03-16 23:50:00</th>\n",
" <td>-1.383280</td>\n",
" <td> 0.610668</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>680 rows \u00d7 2 columns</p>\n",
"</div>"
],
"metadata": {},
"output_type": "pyout",
"prompt_number": 81,
"text": [
" S1 S2\n",
"2015-03-12 00:00:00 -0.579566 -1.362629\n",
"2015-03-12 00:10:00 -0.381633 0.573208\n",
"2015-03-12 00:20:00 0.235525 0.589369\n",
"2015-03-12 00:30:00 1.754687 1.811558\n",
"2015-03-12 00:40:00 0.505616 1.413038\n",
"2015-03-12 00:50:00 -1.175593 -0.638982\n",
"2015-03-12 01:00:00 -0.642313 -1.725921\n",
"2015-03-12 01:10:00 -0.298308 0.408128\n",
"2015-03-12 01:20:00 -0.182880 2.317932\n",
"2015-03-12 01:30:00 -0.658728 1.829225\n",
"2015-03-12 01:40:00 0.336450 -0.576847\n",
"2015-03-12 01:50:00 1.174043 -0.018968\n",
"2015-03-12 02:00:00 1.306239 1.043028\n",
"2015-03-12 02:10:00 -1.855183 0.601130\n",
"2015-03-12 02:20:00 -0.241508 1.227976\n",
"2015-03-12 02:30:00 -1.518354 -0.499627\n",
"2015-03-12 02:40:00 0.167399 -1.358354\n",
"2015-03-12 02:50:00 2.562058 -1.095234\n",
"2015-03-12 03:00:00 1.737065 0.308141\n",
"2015-03-12 03:10:00 -0.332737 -0.883051\n",
"2015-03-12 03:20:00 -0.466392 -0.082634\n",
"2015-03-12 03:30:00 -2.454961 -1.262187\n",
"2015-03-12 03:40:00 0.419272 0.370745\n",
"2015-03-12 03:50:00 -1.267411 -1.055182\n",
"2015-03-12 04:00:00 -0.166380 -0.065995\n",
"2015-03-12 04:10:00 -1.774168 0.451581\n",
"2015-03-12 04:20:00 -1.898200 0.045828\n",
"2015-03-12 04:30:00 -0.116386 0.085524\n",
"2015-03-12 04:40:00 0.426295 0.461055\n",
"2015-03-12 04:50:00 -0.408411 -0.694797\n",
"... ... ...\n",
"2015-03-16 19:00:00 -0.345654 0.252860\n",
"2015-03-16 19:10:00 1.221992 0.140411\n",
"2015-03-16 19:20:00 -0.746366 -0.728046\n",
"2015-03-16 19:30:00 0.367208 2.348574\n",
"2015-03-16 19:40:00 1.200294 -0.446141\n",
"2015-03-16 19:50:00 -0.736154 -1.907979\n",
"2015-03-16 20:00:00 -1.088274 -1.708835\n",
"2015-03-16 20:10:00 -0.598324 -0.553277\n",
"2015-03-16 20:20:00 0.151677 0.536430\n",
"2015-03-16 20:30:00 -1.980867 -0.193359\n",
"2015-03-16 20:40:00 -1.482832 -0.305577\n",
"2015-03-16 20:50:00 -0.956331 -1.199016\n",
"2015-03-16 21:00:00 -0.625324 0.330678\n",
"2015-03-16 21:10:00 0.540265 -1.651197\n",
"2015-03-16 21:20:00 -1.140503 -0.350532\n",
"2015-03-16 21:30:00 0.283272 0.207816\n",
"2015-03-16 21:40:00 1.079753 -0.334234\n",
"2015-03-16 21:50:00 0.171345 1.441366\n",
"2015-03-16 22:00:00 -0.531493 -0.050047\n",
"2015-03-16 22:10:00 0.970737 0.185797\n",
"2015-03-16 22:20:00 1.584863 -1.480487\n",
"2015-03-16 22:30:00 0.282121 -1.276703\n",
"2015-03-16 22:40:00 -0.838282 -2.325543\n",
"2015-03-16 22:50:00 -0.708904 0.832618\n",
"2015-03-16 23:00:00 0.652657 2.449541\n",
"2015-03-16 23:10:00 0.927262 -1.371792\n",
"2015-03-16 23:20:00 0.115658 1.361716\n",
"2015-03-16 23:30:00 2.485742 0.629385\n",
"2015-03-16 23:40:00 0.579433 0.701071\n",
"2015-03-16 23:50:00 -1.383280 0.610668\n",
"\n",
"[680 rows x 2 columns]"
]
}
],
"prompt_number": 81
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment