profiling_ecsv_writer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from astropy.table import Table\n",
    "from astropy import units as u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext line_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created TAP+ (v1.0.1) - Connection:\n",
      "\tHost: gea.esac.esa.int\n",
      "\tUse HTTPS: False\n",
      "\tPort: 80\n",
      "\tSSL Port: 443\n",
      "Launched query: 'SELECT DISTANCE(POINT('ICRS',ra,dec),             POINT('ICRS',56.75000866428592,24.116696763158743)) AS dist, *             FROM gaiadr1.gaia_source WHERE CONTAINS(            POINT('ICRS',ra,dec),            CIRCLE('ICRS',56.75000866428592,24.116696763158743, 1.5))=1             ORDER BY dist ASC'\n",
      "Retrieving async. results...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: W35: None:5:0: W35: 'value' attribute required for INFO elements [astropy.io.votable.tree]\n",
      "WARNING: W35: None:6:0: W35: 'value' attribute required for INFO elements [astropy.io.votable.tree]\n",
      "WARNING: W35: None:7:0: W35: 'value' attribute required for INFO elements [astropy.io.votable.tree]\n",
      "WARNING: W35: None:8:0: W35: 'value' attribute required for INFO elements [astropy.io.votable.tree]\n",
      "WARNING: W35: None:10:0: W35: 'value' attribute required for INFO elements [astropy.io.votable.tree]\n",
      "WARNING: W27: None:11:0: W27: COOSYS deprecated in VOTable 1.2 [astropy.io.votable.tree]\n",
      "WARNING: W06: None:15:0: W06: Invalid UCD 'meta.id;meta.version': Primary word 'meta.version' is not valid as a secondary word [astropy.io.votable.tree]\n",
      "WARNING: W50: None:106:0: W50: Invalid unit string 'mas.yr**-1' [astropy.io.votable.tree]\n",
      "WARNING: W50: None:111:0: W50: Invalid unit string 'mas.yr**-1' [astropy.io.votable.tree]\n",
      "WARNING: W50: None:115:0: W50: Invalid unit string 'mas.yr**-1' [astropy.io.votable.tree]\n",
      "WARNING: W50: None:120:0: W50: Invalid unit string 'mas.yr**-1' [astropy.io.votable.tree]\n",
      "WARNING: W50: None:336:0: W50: Invalid unit string 'mas**-2' [astropy.io.votable.tree]\n",
      "WARNING: W50: None:347:0: W50: Invalid unit string 'mas**-2' [astropy.io.votable.tree]\n",
      "WARNING: W50: None:620:0: W50: Invalid unit string ''electron'.s**-1' [astropy.io.votable.tree]\n",
      "WARNING: W50: None:623:0: W50: Invalid unit string ''electron'.s**-1' [astropy.io.votable.tree]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query finished.\n"
     ]
    }
   ],
   "source": [
    "from astroquery import gaia\n",
    "tab = gaia.Gaia.query_object_async('Messier 45', 90*u.arcmin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fn =  '/Users/erik/projects/saga/erik/external_catalogs/astrometric/Phoenix_2_gaia.ecsv'\n",
    "#tab = Table.read(fn)\n",
    "#len(tab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove stuff until fast "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "testtab = tab[:100].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: AstropyDeprecationWarning: test.ecsv already exists. Automatically overwriting ASCII files is deprecated. Use the argument 'overwrite=True' in the future. [astropy.io.ascii.ui]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 390 ms, sys: 4.75 ms, total: 395 ms\n",
      "Wall time: 393 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "testtab.write('test.ecsv', format='ascii.ecsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deleting phot_variable_flag\n"
     ]
    }
   ],
   "source": [
    "for cnm in tab.columns:\n",
    "    if tab[cnm].dtype.char == 'O':\n",
    "        print('deleting', cnm)\n",
    "        del testtab[cnm]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: AstropyDeprecationWarning: test.ecsv already exists. Automatically overwriting ASCII files is deprecated. Use the argument 'overwrite=True' in the future. [astropy.io.ascii.ui]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 384 ms, sys: 5.29 ms, total: 389 ms\n",
      "Wall time: 386 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "testtab.write('test.ecsv', format='ascii.ecsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table masked=True length=100</i>\n",
       "<table id=\"table4633070112\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>dist</th><th>solution_id</th><th>source_id</th><th>random_index</th><th>ref_epoch</th><th>ra</th><th>ra_error</th><th>dec</th><th>dec_error</th><th>parallax</th><th>parallax_error</th><th>pmra</th><th>pmra_error</th><th>pmdec</th><th>pmdec_error</th><th>ra_dec_corr</th><th>ra_parallax_corr</th><th>ra_pmra_corr</th><th>ra_pmdec_corr</th><th>dec_parallax_corr</th><th>dec_pmra_corr</th><th>dec_pmdec_corr</th><th>parallax_pmra_corr</th><th>parallax_pmdec_corr</th><th>pmra_pmdec_corr</th><th>astrometric_n_obs_al</th><th>astrometric_n_obs_ac</th><th>astrometric_n_good_obs_al</th><th>astrometric_n_good_obs_ac</th><th>astrometric_n_bad_obs_al</th><th>astrometric_n_bad_obs_ac</th><th>astrometric_delta_q</th><th>astrometric_excess_noise</th><th>astrometric_excess_noise_sig</th><th>astrometric_primary_flag</th><th>astrometric_relegation_factor</th><th>astrometric_weight_al</th><th>astrometric_weight_ac</th><th>astrometric_priors_used</th><th>matched_observations</th><th>duplicated_source</th><th>scan_direction_strength_k1</th><th>scan_direction_strength_k2</th><th>scan_direction_strength_k3</th><th>scan_direction_strength_k4</th><th>scan_direction_mean_k1</th><th>scan_direction_mean_k2</th><th>scan_direction_mean_k3</th><th>scan_direction_mean_k4</th><th>phot_g_n_obs</th><th>phot_g_mean_flux</th><th>phot_g_mean_flux_error</th><th>phot_g_mean_mag</th><th>l</th><th>b</th><th>ecl_lon</th><th>ecl_lat</th></tr></thead>\n",
       "<thead><tr><th>float64</th><th>int64</th><th>int64</th><th>int64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>int32</th><th>int32</th><th>int32</th><th>int32</th><th>int32</th><th>int32</th><th>float32</th><th>float64</th><th>float64</th><th>bool</th><th>float32</th><th>float32</th><th>float32</th><th>int32</th><th>int16</th><th>bool</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>int32</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th></tr></thead>\n",
       "<tr><td>0.006104539585609355</td><td>1635378410781933568</td><td>65225920273742848</td><td>940538955</td><td>2015.0</td><td>56.756150718151616</td><td>7.469166606330692</td><td>24.119113242209913</td><td>1.333992342584129</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>0.78095</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>77</td><td>0</td><td>77</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.9069123146747043</td><td>0.301030064421654</td><td>False</td><td>3.6316462</td><td>0.047848385</td><td>--</td><td>2</td><td>13</td><td>False</td><td>0.12261821</td><td>0.54972</td><td>0.09497889</td><td>0.3434145</td><td>-41.43721</td><td>-1.7187724</td><td>-26.309181</td><td>21.00789</td><td>103</td><td>149.92142391268922</td><td>1.857249642628372</td><td>20.085110817906724</td><td>166.5733898401752</td><td>-23.515583331070918</td><td>59.89277121063544</td><td>4.087507392594524</td></tr>\n",
       "<tr><td>0.008925640602477261</td><td>1635378410781933568</td><td>65214061869072512</td><td>562311333</td><td>2015.0</td><td>56.752993251750425</td><td>0.8202020744448595</td><td>24.108197000487642</td><td>0.34379383728868756</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.208</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>85</td><td>0</td><td>85</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.9237322640079164</td><td>1.5080348270981792</td><td>False</td><td>1.1308916</td><td>0.21179168</td><td>--</td><td>2</td><td>14</td><td>False</td><td>0.12022994</td><td>0.5470629</td><td>0.17888811</td><td>0.3912407</td><td>-89.23091</td><td>2.680922</td><td>-28.3354</td><td>22.994213</td><td>111</td><td>492.6479394435697</td><td>2.7076839863330595</td><td>18.7934283872327</td><td>166.57907708027054</td><td>-23.525597147807083</td><td>59.8875591874873</td><td>4.077485297669342</td></tr>\n",
       "<tr><td>0.010525545745078283</td><td>1635378410781933568</td><td>65225812899598976</td><td>185136660</td><td>2015.0</td><td>56.74014362093697</td><td>3.3313058980134986</td><td>24.11124588209658</td><td>0.5795243273661269</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>0.83295</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>68</td><td>0</td><td>68</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.0</td><td>0.0</td><td>False</td><td>1.0</td><td>0.33290642</td><td>--</td><td>2</td><td>13</td><td>False</td><td>0.26554024</td><td>0.641134</td><td>0.2061137</td><td>0.4172788</td><td>-43.287567</td><td>3.8363693</td><td>-39.079575</td><td>15.589082</td><td>94</td><td>542.5843121312055</td><td>2.1797915337541136</td><td>18.688601981970958</td><td>166.56737679822493</td><td>-23.531234313612245</td><td>59.87675366852364</td><td>4.083025133751354</td></tr>\n",
       "<tr><td>0.013282592588353367</td><td>1635378410781933568</td><td>65225984700165248</td><td>736981994</td><td>2015.0</td><td>56.742619566900075</td><td>8.596754452497013</td><td>24.12814000981525</td><td>2.059391495353977</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>0.73165</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>57</td><td>0</td><td>57</td><td>0</td><td>0</td><td>0</td><td>--</td><td>3.163246371537079</td><td>1.863212206057955</td><td>False</td><td>2.0958343</td><td>0.024046673</td><td>--</td><td>2</td><td>9</td><td>False</td><td>0.23959842</td><td>0.67114013</td><td>0.39183876</td><td>0.34468234</td><td>-80.54777</td><td>2.6990678</td><td>-36.79564</td><td>12.142329</td><td>56</td><td>120.0853081260884</td><td>2.808640063166833</td><td>20.32604537193758</td><td>166.55680281453522</td><td>-23.517215705989642</td><td>59.88266865434032</td><td>4.099016042753339</td></tr>\n",
       "<tr><td>0.013442014080729396</td><td>1635378410781933568</td><td>65214061869065728</td><td>592780565</td><td>2015.0</td><td>56.7483814519183</td><td>0.18148264410744894</td><td>24.10333705700067</td><td>0.08425330880318127</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.36785</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>90</td><td>0</td><td>89</td><td>0</td><td>1</td><td>0</td><td>--</td><td>0.34247853631086383</td><td>6.3073491692859305</td><td>False</td><td>1.2952809</td><td>3.9423563</td><td>--</td><td>2</td><td>14</td><td>False</td><td>0.08491735</td><td>0.53421074</td><td>0.16747624</td><td>0.35715955</td><td>-172.18079</td><td>-0.0071741925</td><td>-13.29809</td><td>25.491121</td><td>114</td><td>7653.824869426284</td><td>10.225433428223235</td><td>15.815073762048502</td><td>166.57924819444543</td><td>-23.5320248479057</td><td>59.88237570312638</td><td>4.073663367263518</td></tr>\n",
       "<tr><td>0.014430809938891405</td><td>1635378410781933568</td><td>65225778539850880</td><td>1112582281</td><td>2015.0</td><td>56.74164999448887</td><td>0.535369610897651</td><td>24.104447690728243</td><td>0.2137671923960817</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.3074</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>68</td><td>0</td><td>66</td><td>0</td><td>2</td><td>0</td><td>--</td><td>0.47614991391641887</td><td>1.023873459026773</td><td>False</td><td>1.103023</td><td>0.70882154</td><td>--</td><td>2</td><td>13</td><td>False</td><td>0.1285819</td><td>0.82783115</td><td>0.2832036</td><td>0.4509531</td><td>-86.875595</td><td>11.565309</td><td>-28.006191</td><td>15.219539</td><td>94</td><td>1056.9843356556842</td><td>5.142670571663732</td><td>17.96459868589531</td><td>166.57347553022473</td><td>-23.535338120459645</td><td>59.87660819709492</td><td>4.0760908035875305</td></tr>\n",
       "<tr><td>0.014671571249819798</td><td>1635378410781933568</td><td>65225885913490048</td><td>1063825373</td><td>2015.0</td><td>56.736399552110335</td><td>2.511034545411578</td><td>24.124505549544924</td><td>0.9427138591282216</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.12635</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>60</td><td>0</td><td>60</td><td>0</td><td>0</td><td>0</td><td>--</td><td>1.5632585556234022</td><td>0.5821821668889373</td><td>False</td><td>6.1004467</td><td>0.035201944</td><td>--</td><td>2</td><td>11</td><td>False</td><td>0.20186648</td><td>0.79768914</td><td>0.34626848</td><td>0.38205397</td><td>-44.80064</td><td>12.450571</td><td>-19.595226</td><td>18.593866</td><td>74</td><td>127.81555342708906</td><td>2.2040132152891605</td><td>20.258310801808825</td><td>166.55488929652648</td><td>-23.523723818767003</td><td>59.87631826477034</td><td>4.096711173218591</td></tr>\n",
       "<tr><td>0.014831649068270168</td><td>1635378410781933568</td><td>65225920273226240</td><td>1042877071</td><td>2015.0</td><td>56.76093547367728</td><td>2.288615588036399</td><td>24.12767512063971</td><td>0.7976714706977881</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>0.01015</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>67</td><td>0</td><td>67</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.0</td><td>0.0</td><td>False</td><td>1.0</td><td>0.048763163</td><td>--</td><td>2</td><td>14</td><td>False</td><td>0.31961066</td><td>0.595308</td><td>0.2527832</td><td>0.40834948</td><td>-45.01902</td><td>7.492911</td><td>-27.902023</td><td>21.734558</td><td>93</td><td>142.69454068068328</td><td>2.5976944579164707</td><td>20.138751669204034</td><td>166.570628705492</td><td>-23.50631160553254</td><td>59.89891982459279</td><td>4.0949075594756215</td></tr>\n",
       "<tr><td>0.015807701239303758</td><td>1635378410781933568</td><td>65214061869064448</td><td>1125144790</td><td>2015.0</td><td>56.7599681870253</td><td>1.7203849050867819</td><td>24.103764508804858</td><td>0.6577836890981266</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.0702</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>75</td><td>0</td><td>75</td><td>0</td><td>0</td><td>0</td><td>--</td><td>1.0865555331106271</td><td>0.5435453290374843</td><td>False</td><td>1.1071332</td><td>0.06416486</td><td>--</td><td>2</td><td>13</td><td>False</td><td>0.23410924</td><td>0.63989824</td><td>0.29889643</td><td>0.4311024</td><td>-72.47181</td><td>7.304164</td><td>-35.923653</td><td>18.45657</td><td>100</td><td>194.69643689620312</td><td>2.535965918344298</td><td>19.801375054717873</td><td>166.58746644760595</td><td>-23.524590594369275</td><td>59.8928161360711</td><td>4.071768184049158</td></tr>\n",
       "<tr><td>0.01602634229252251</td><td>1635378410781933568</td><td>65225950338575616</td><td>935786791</td><td>2015.0</td><td>56.76450785929479</td><td>1.0051143813438201</td><td>24.12573705778495</td><td>0.40968786283155195</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.15495</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>76</td><td>0</td><td>76</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.9726849881450144</td><td>1.1977312320580187</td><td>False</td><td>1.0754759</td><td>0.16354355</td><td>--</td><td>2</td><td>13</td><td>False</td><td>0.23564242</td><td>0.6395066</td><td>0.2862787</td><td>0.43834123</td><td>-69.24557</td><td>7.9792314</td><td>-34.85538</td><td>19.125605</td><td>93</td><td>379.097448282106</td><td>3.023504373291557</td><td>19.077892910877395</td><td>166.57468072915012</td><td>-23.505550875911652</td><td>59.90168472890119</td><td>4.092303715089332</td></tr>\n",
       "<tr><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr>\n",
       "<tr><td>0.055086509599864796</td><td>1635378410781933568</td><td>65225641100899584</td><td>25647928</td><td>2015.0</td><td>56.690923349850635</td><td>0.20763502443763787</td><td>24.10547033364867</td><td>0.10073122343245781</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.3726</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>107</td><td>0</td><td>107</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.5303623140133916</td><td>14.317551745714447</td><td>False</td><td>1.735237</td><td>2.2667089</td><td>--</td><td>2</td><td>15</td><td>False</td><td>0.02462394</td><td>0.5165231</td><td>0.12767982</td><td>0.4065953</td><td>-76.47918</td><td>0.827975</td><td>-9.040386</td><td>25.43461</td><td>131</td><td>6639.045489984301</td><td>6.822484058530934</td><td>15.969505952856812</td><td>166.53535007740874</td><td>-23.565727889499204</td><td>59.83153691172818</td><td>4.087222075639451</td></tr>\n",
       "<tr><td>0.05514383420673688</td><td>1635378410781933568</td><td>66715380572144768</td><td>746191535</td><td>2015.0</td><td>56.8094365151014</td><td>11.299596427013256</td><td>24.126647531124238</td><td>2.3835367188970062</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>0.72035</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>49</td><td>0</td><td>49</td><td>0</td><td>0</td><td>0</td><td>--</td><td>1.9390018699059242</td><td>0.6832524885134077</td><td>False</td><td>7.5310774</td><td>0.028037133</td><td>--</td><td>2</td><td>11</td><td>False</td><td>0.23830956</td><td>0.73089564</td><td>0.13220055</td><td>0.5707727</td><td>-14.87126</td><td>5.562197</td><td>56.643936</td><td>13.273205</td><td>65</td><td>110.33279955029705</td><td>2.85880357700032</td><td>20.418008468910095</td><td>166.6070723610529</td><td>-23.477272749303662</td><td>59.94199999329443</td><td>4.084235304015575</td></tr>\n",
       "<tr><td>0.055239886036423276</td><td>1635378410781933568</td><td>65225434942451328</td><td>634084339</td><td>2015.0</td><td>56.694308615531085</td><td>0.6752784815537904</td><td>24.09509864988411</td><td>0.2763914319033248</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.261</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>85</td><td>0</td><td>85</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.975944970021728</td><td>2.558670369689624</td><td>False</td><td>1.259671</td><td>0.2976114</td><td>--</td><td>2</td><td>16</td><td>False</td><td>0.20004909</td><td>0.7364077</td><td>0.18566129</td><td>0.46686766</td><td>-41.9425</td><td>10.757115</td><td>-26.598879</td><td>17.99762</td><td>109</td><td>716.8376540590373</td><td>6.029657848488726</td><td>18.3862180391398</td><td>166.54545471389483</td><td>-23.571326222636916</td><td>59.83228271476389</td><td>4.0764254266898545</td></tr>\n",
       "<tr><td>0.05532246642471825</td><td>1635378410781933568</td><td>66715135759029632</td><td>108364721</td><td>2015.0</td><td>56.81060472277577</td><td>0.1706694573922033</td><td>24.118019374216892</td><td>0.09495069813225174</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.49435</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>87</td><td>0</td><td>87</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.46873390998554326</td><td>135.17778504732297</td><td>False</td><td>4.106383</td><td>4.2944765</td><td>--</td><td>2</td><td>13</td><td>False</td><td>0.20721799</td><td>0.5798238</td><td>0.19745104</td><td>0.5136127</td><td>-148.94356</td><td>6.2101374</td><td>-43.446705</td><td>21.144012</td><td>111</td><td>78255.55790083419</td><td>33.190243018270536</td><td>13.290982083919069</td><td>166.61426685245527</td><td>-23.482932961975543</td><td>59.941154777437205</td><td>4.075582495952151</td></tr>\n",
       "<tr><td>0.055880133449784236</td><td>1635378410781933568</td><td>65213855712464256</td><td>100890321</td><td>2015.0</td><td>56.80400810504035</td><td>0.24379830368738764</td><td>24.090372449171806</td><td>0.12921828597053436</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.19695</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>84</td><td>0</td><td>84</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.3066559160755269</td><td>1.6352275551391822</td><td>False</td><td>1.0369717</td><td>2.04205</td><td>--</td><td>2</td><td>14</td><td>False</td><td>0.2587183</td><td>0.5630577</td><td>0.33518255</td><td>0.3421976</td><td>-138.1873</td><td>0.301582</td><td>-37.057545</td><td>20.163668</td><td>107</td><td>2818.4512852857615</td><td>5.20931116535779</td><td>16.8997437318844</td><td>166.62971669724863</td><td>-23.50742481196176</td><td>59.92921287828651</td><td>4.049917016107271</td></tr>\n",
       "<tr><td>0.055986902794609696</td><td>1635378410781933568</td><td>65225434942443264</td><td>539713259</td><td>2015.0</td><td>56.69657780035412</td><td>1.623565028069362</td><td>24.089205184882637</td><td>0.6171711538578604</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.17915</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>93</td><td>0</td><td>92</td><td>0</td><td>1</td><td>0</td><td>--</td><td>3.727855550384722</td><td>79.06297343658167</td><td>False</td><td>3.4388256</td><td>0.06437046</td><td>--</td><td>2</td><td>16</td><td>False</td><td>0.16048513</td><td>0.5783716</td><td>0.077700585</td><td>0.40049085</td><td>-20.700043</td><td>2.7959976</td><td>-12.043238</td><td>21.042484</td><td>116</td><td>1150.717804027383</td><td>11.082509119785007</td><td>17.87234798235078</td><td>166.55145171689003</td><td>-23.574294697280745</td><td>59.833015329266765</td><td>4.070221388176193</td></tr>\n",
       "<tr><td>0.05604283892187599</td><td>1635378410781933568</td><td>66715449291954304</td><td>530142213</td><td>2015.0</td><td>56.80126404466091</td><td>11.74808969029164</td><td>24.14756443745898</td><td>2.552548946676413</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>0.6558</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>27</td><td>0</td><td>27</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.0</td><td>0.0</td><td>False</td><td>1.0</td><td>0.027711954</td><td>--</td><td>2</td><td>6</td><td>False</td><td>0.30323452</td><td>0.85522056</td><td>0.13543856</td><td>0.48984218</td><td>-6.4733987</td><td>4.1612687</td><td>-20.019703</td><td>6.153155</td><td>26</td><td>99.59683664539922</td><td>2.1434975429749783</td><td>20.529156201949583</td><td>166.58570617858965</td><td>-23.46683016938521</td><td>59.93928191515788</td><td>4.106275895718478</td></tr>\n",
       "<tr><td>0.056406174597839934</td><td>1635378410781933568</td><td>65213546472929920</td><td>515994489</td><td>2015.0</td><td>56.7314480715767</td><td>0.2969106804791327</td><td>24.062895701036744</td><td>0.14086673575557895</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.30725</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>86</td><td>0</td><td>85</td><td>0</td><td>1</td><td>0</td><td>--</td><td>0.4886796977870344</td><td>3.2582336366432902</td><td>False</td><td>1.2242709</td><td>1.3759085</td><td>--</td><td>2</td><td>15</td><td>False</td><td>0.11286087</td><td>0.56389964</td><td>0.18370754</td><td>0.390205</td><td>-98.15437</td><td>3.3130248</td><td>-28.188828</td><td>22.35001</td><td>112</td><td>2450.8856042880966</td><td>4.864659783046553</td><td>17.051462461542513</td><td>166.5964626545435</td><td>-23.572342019529714</td><td>59.85838576962189</td><td>4.037582093541912</td></tr>\n",
       "<tr><td>0.056450151539781376</td><td>1635378410781933568</td><td>65213378969242368</td><td>891114329</td><td>2015.0</td><td>56.75133327463575</td><td>9.164659012497653</td><td>24.06025956528964</td><td>1.7352547729500156</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>0.7988</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>58</td><td>0</td><td>58</td><td>0</td><td>0</td><td>0</td><td>--</td><td>1.4012587359426372</td><td>0.3627001304316669</td><td>False</td><td>5.4862</td><td>0.034338277</td><td>--</td><td>2</td><td>12</td><td>False</td><td>0.38748807</td><td>0.6270085</td><td>0.17859717</td><td>0.3553345</td><td>-26.346443</td><td>4.805386</td><td>-22.63188</td><td>18.150589</td><td>75</td><td>117.51218688574613</td><td>1.827142187117866</td><td>20.349562792581242</td><td>166.61304941539743</td><td>-23.562070796026838</td><td>59.87556967620063</td><td>4.031038935921223</td></tr>\n",
       "<tr><td>0.05647221195832776</td><td>1635378410781933568</td><td>65213718271634176</td><td>123825354</td><td>2015.0</td><td>56.78484605439508</td><td>0.4941490898194143</td><td>24.07003074040426</td><td>0.21977859190451357</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>-0.2372</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>--</td><td>85</td><td>0</td><td>85</td><td>0</td><td>0</td><td>0</td><td>--</td><td>0.15654035676319586</td><td>0.08840707891789251</td><td>False</td><td>1.0085016</td><td>0.5539446</td><td>--</td><td>2</td><td>15</td><td>False</td><td>0.21334967</td><td>0.6776706</td><td>0.33141023</td><td>0.47195858</td><td>-98.14788</td><td>7.942489</td><td>-39.731926</td><td>16.822542</td><td>106</td><td>782.8305905428459</td><td>4.82975088302577</td><td>18.290600593410698</td><td>166.63055340044397</td><td>-23.53424370969887</td><td>59.90764472603199</td><td>4.033886799961709</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table masked=True length=100>\n",
       "        dist             solution_id     ...      ecl_lat      \n",
       "      float64               int64        ...      float64      \n",
       "-------------------- ------------------- ... ------------------\n",
       "0.006104539585609355 1635378410781933568 ...  4.087507392594524\n",
       "0.008925640602477261 1635378410781933568 ...  4.077485297669342\n",
       "0.010525545745078283 1635378410781933568 ...  4.083025133751354\n",
       "0.013282592588353367 1635378410781933568 ...  4.099016042753339\n",
       "0.013442014080729396 1635378410781933568 ...  4.073663367263518\n",
       "0.014430809938891405 1635378410781933568 ... 4.0760908035875305\n",
       "0.014671571249819798 1635378410781933568 ...  4.096711173218591\n",
       "0.014831649068270168 1635378410781933568 ... 4.0949075594756215\n",
       "0.015807701239303758 1635378410781933568 ...  4.071768184049158\n",
       " 0.01602634229252251 1635378410781933568 ...  4.092303715089332\n",
       "                 ...                 ... ...                ...\n",
       "0.055086509599864796 1635378410781933568 ...  4.087222075639451\n",
       " 0.05514383420673688 1635378410781933568 ...  4.084235304015575\n",
       "0.055239886036423276 1635378410781933568 ... 4.0764254266898545\n",
       " 0.05532246642471825 1635378410781933568 ...  4.075582495952151\n",
       "0.055880133449784236 1635378410781933568 ...  4.049917016107271\n",
       "0.055986902794609696 1635378410781933568 ...  4.070221388176193\n",
       " 0.05604283892187599 1635378410781933568 ...  4.106275895718478\n",
       "0.056406174597839934 1635378410781933568 ...  4.037582093541912\n",
       "0.056450151539781376 1635378410781933568 ...  4.031038935921223\n",
       " 0.05647221195832776 1635378410781933568 ...  4.033886799961709"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for cnm in testtab.columns:\n",
    "    testtab[cnm].unit = None\n",
    "testtab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: AstropyDeprecationWarning: test.ecsv already exists. Automatically overwriting ASCII files is deprecated. Use the argument 'overwrite=True' in the future. [astropy.io.ascii.ui]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 386 ms, sys: 6.86 ms, total: 393 ms\n",
      "Wall time: 391 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "testtab.write('test.ecsv', format='ascii.ecsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "for cnm in testtab.columns:\n",
    "    testtab[cnm].meta = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: AstropyDeprecationWarning: test.ecsv already exists. Automatically overwriting ASCII files is deprecated. Use the argument 'overwrite=True' in the future. [astropy.io.ascii.ui]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 396 ms, sys: 9.99 ms, total: 406 ms\n",
      "Wall time: 403 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "testtab.write('test.ecsv', format='ascii.ecsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "for cnm in testtab.columns:\n",
    "    testtab[cnm].description = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: AstropyDeprecationWarning: test.ecsv already exists. Automatically overwriting ASCII files is deprecated. Use the argument 'overwrite=True' in the future. [astropy.io.ascii.ui]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 335 ms, sys: 12.8 ms, total: 348 ms\n",
      "Wall time: 346 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "testtab.write('test.ecsv', format='ascii.ecsv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# random data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for cnm in testtab.colnames:\n",
    "    data.append(np.random.rand(len(testtab)).astype(testtab[cnm].dtype))\n",
    "randtab = Table(data=data, names=testtab.colnames, masked=False)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 51.2 ms, sys: 2.72 ms, total: 53.9 ms\n",
      "Wall time: 51.6 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "randtab.write('test.ecsv', format='ascii.ecsv', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "OK now we're in business!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for cnm in testtab.colnames:\n",
    "    data.append(np.random.rand(len(testtab)).astype(testtab[cnm].dtype))\n",
    "randtab = Table(data=data, names=testtab.colnames, masked=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 285 ms, sys: 4.71 ms, total: 289 ms\n",
      "Wall time: 287 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "randtab.write('test.ecsv', format='ascii.ecsv', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DAMN YOU MASKED DATA!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 47.4 ms, sys: 3 ms, total: 50.4 ms\n",
      "Wall time: 48.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "randtab.write('test.csv', format='ascii.csv', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But it's OK in the regular CSV writer..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optional profiling on the full table (takes a long time...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "tab.write('test.ecsv', format='ascii.ecsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%prun -s cumulative -r\n",
    "\n",
    "tab.write('test.ecsv', format='ascii.ecsv', overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fullstats = _"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}