boris-arzur/play_w_gist.ipynb

## play_w_gist.ipynb
{
 "metadata": {
  "name": "play w gist"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "get_gist = \"\"\"\ndata_url_targz = \"https://gist.github.com/boris-arzur/7329335/download\"\nimport tarfile\nfrom urllib.request import urlopen\nwith tarfile.open(fileobj=urlopen(data_url_targz), mode=\"r:gz\") as tar:\n#just get the first file...\n    first_file = next(member for member in tar if member.isfile())\n    data = tar.extractfile(first_file).read().decode(\"utf8\")\n\"\"\"\n\nimport zlib, base64; print(base64.b64encode(zlib.compress(get_gist.encode('ascii'), 9)))",
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": "b'eNpVj9FqxCAQRd/9isG+KLQGGkrbQL4lmKiJS4zpqOySr98xpJT6NF7vPd5hRmc9FFyHrHE+oAe+5Lynrmlmn7KafV7KqKYYmjGiT28aj4LNZ/v+3bYfjYn3bY3acObDHjEDUZxfLXMYAxB29aNC+1NsynBZSI273did0L9+VRVRpzje+ssh/neTrxCisT3Hbj64BJ1qumMvt0Lw2dLniwXnkW4nUykGdE5lqAptt9lHFsGG0SK4iHCNfqss8O4SlE81IKQ8EbUIhcmiKI96yufrH1nSktoIqYydqKPgJbsvLtkTrDl56Q=='\n"
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "import zlib, base64\ndata_code = zlib.decompress(base64.b64decode(b\"\"\"eNpVj9FqxCAQRd/9isG+KLQGGkrbQL4lmKiJS4zpqOySr98xpJT6NF7vPd5hRmc9FFyHrHE+oAe+\n5Lynrmlmn7KafV7KqKYYmjGiT28aj4LNZ/v+3bYfjYn3bY3acObDHjEDUZxfLXMYAxB29aNC+1NsynBZSI273did0L9+VRVRpzje+ssh/neTrxCisT3Hbj64BJ1qu\nmMvt0Lw2dLniwXnkW4nUykGdE5lqAptt9lHFsGG0SK4iHCNfqss8O4SlE81IKQ8EbUIhcmiKI96yufrH1nSktoIqYydqKPgJbsvLtkTrDl56Q==\"\"\"))\nprint(data_code)\n# we you are ready \n# exec(data_code)\n# print(len(data.split(\"\\n\")), len(data))",
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": "b'\\ndata_url_targz = \"https://gist.github.com/boris-arzur/7329335/download\"\\nimport tarfile\\nfrom urllib.request import urlopen\\nwith tarfile.open(fileobj=urlopen(data_url_targz), mode=\"r:gz\") as tar:\\n#just get the first file...\\n    first_file = next(member for member in tar if member.isfile())\\n    data = tar.extractfile(first_file).read().decode(\"utf8\")\\n'\n112"
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": " 6245\n"
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "#this is short, points always to the same version\ndata_url_raw = \"https://gist.github.com/boris-arzur/7329335/raw/f0f71030707cca4a739f372c0dcb6902187b15bd/age_table_japan_20131106\"\nfrom urllib.request import urlopen\ndata = urlopen(data_url_raw).read().decode(\"utf8\")\nprint(len(data.split(\"\\n\")), len(data))\n\n#this is clean, and stays up to date\ndata_url_targz = \"https://gist.github.com/boris-arzur/7329335/download\"\nimport tarfile\nwith tarfile.open(fileobj=urlopen(data_url_targz), mode=\"r:gz\") as tar:\n    #just get the first file...\n    first_file = next(member for member in tar if member.isfile())\n    data2 = tar.extractfile(first_file).read().decode(\"utf8\")\n\nassert data == data2",
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": "112 6245\n"
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "import zlib\nimport base64\nimport marshal\nimport json\n\ndata1 = base64.b64encode(zlib.compress(data.encode(\"utf8\"), 9))\nprint(data1)\nprint(len(data1), len(data))\n\nassert zlib.decompress(base64.b64decode(data1)).decode(\"utf8\") == data\n\nline_f = [int, int, int, float, float, float, float, int, int]\ndef apply_f(format, line):\n    return [fun(cell) for fun, cell in zip(format, line)]\n\ndata3 = [line.split(\",\") for line in data.split(\"\\n\")]\ndata3_f = [apply_f(line_f, line.split(\",\")) for line in data.split(\"\\n\")[1:]]\n\ndata4 = base64.b64encode(zlib.compress(marshal.dumps(data3), 9))\ndata4_f = base64.b64encode(zlib.compress(marshal.dumps(data3_f), 9))\ndata5_f = base64.b64encode(zlib.compress(json.dumps(data3_f).encode(\"utf8\"), 9))\nprint(data5_f)\nprint(len(data4))\nprint(len(data4_f))\nprint(len(data5_f))\n\nassert marshal.loads(zlib.decompress(base64.b64decode(data4))) == data3",
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": "b'eNp9mM2SHbcNhfd8Cj1AV1cTBP+W2WeZF1CccaIq20pJsit6+3wHxJ14o8yGfTkECQIHBwD/8s+367fff/3725cPn3/+8PX3L398+uPzl69/mvvH28dv//p6vf/rw5eP396umD2fP3/+8tObVv76+cu3j798+vb9+uXTz28f3v7z77efvn387afv19dvH799+vzbxy/fP/z1zz/+Vp6rPvq7zMf13HvP7ozPc34/u81+zX33fu29nsV378/TS73O2jZDbI8WYk/8ZuzzmuueEput8d173auYftdx2Tltz3PaE7/POOc9l5Y9znfvPrw0fo+NsuuILTvL47fE7JrjXo+Wrcp373PU4vo97aotxVK5mspW7tbvpbuMoe8+nvUUKT36vmpNsZ3La46c5veyWMbdvA+vvQz99olFj9h+cvnzLj7bveIuzhatjzmsTP3GbDul8pBnv0tPu1dcpSFlfVZ7iiw0bF1pkJ0Geda79KwpZahY+/S1ytbvun8ohX+fe8X9ZRu8uAa+Drty6x+LjZ1i8d2X+S417Pq0HxoEJce6l+zWt777wjClWkz4j+3/XGPeS4bri/Nm34+3UltMtB+7264xUm5Ovjmke6keE6Crp1x/h8cZ/RrARCbv4BxwsfUqVRfuREZN162Xfi99OQOcyOigmG+MAU5qXLgRBJ5B8AL/KxjqNdq9ZfVune/xzIE9Q3Eu0dKeM+3YcjTOs3vLfL6xi+Ez535S3IHre6ym/1r6r3G/em+ZgWDje9Tpo1Qp7t4vTz172sNfI3ryD13HAdl4htmyYlLcn3r19J+nPXrax9fVN/9gWePuXUDuu5jw0vpz9dTT2zuXnLFL7tH9GhEPWpqvVkwTtlEgz2t53sjzBluu+5G7jLtDDV4JPBNejJgaiZeWfhjpB+b7hAO1DIrsY/ja6ClD1QFg/Qdyzto7YFyxOazSh+SkQEWN/yfX74DHMznbQaj0jAkZ+yWX9h//G7vf4S7FFNAi2DkPvKwdQE+51znjfeztlhkWfr66jeXQH6zCBME20g8t/TDm+9jtllqLOMDNQ3AutjUBOEbyREucjfU+9nqDVpaxfX8mqq8SE1N6vuReftvvY3/uJj0nmHAyyPJZWMaEmC7jwXL9THnmfd+i1oVFLl+z1YdkYpoA/DP9/ko+r2TEvK/bpdYAbT6ne6slFOjKE2lPS7vMtAvzPm8syTKM7xDTsl2aJnDItVLPFz+8+GJu1t4uszux4pBTe6zgZia430q/v/joxU9Qnfe7ywy4nDCda/VWmhzTQNtKXnrx7Yt/YTL3SOnL0MHbIiRHCUOZpPN+T/pvp/9gCG93l5ur9iDYjVKgSfFK0NXnCK5MEDUTRCUHuN1D9nugX8d9kGNp2knsU5N5V+b1mnm9wtxeb3Cxp4DsXB0vF7lmbhbVrCPWTEFLQVzjz40jN5XHvBor4NJCELHPVoo42MbBR8BzRJu2owCZpNqrLYXUKORraLqPq/oxKg49Aj1PhuDbiuw+lYPaBB7zKR4TGKIG6yKYUVgzCiv2bDOyxMSiVxtbfiyC0WzsWhNuK2FTEzaV27Vxb9yIpmzSWT84URNV6WznienHmn6snIWgeHQ+VGhQLi5CUJ7HXpdlwp4ra8F6VLUHS/YgxLEk6KQe64WahzILL1k7DDwzw1hmGCMaGgyldWTPqzW4uVOfYWYqKAT7gfjM8tGynDSyQYOiuBKczX0NEModUsFQz+bhjJm1gWWtYBitWXDGqOjXgBLxUtiPkgSctuecONY5qdVzcgOrrd7ydyfYWQe7QW6KMmoEkmg7VlXlFwKegk2J+e4SjCttzKhiUhMgAZMfuhnpx5Z+bNC67YgOSuR6EVUUvVa6JkLVfe7YU8BzA1fZTo0NUHzhMoM1GlbtmlAh5u0gp2dC9EyI7lochZc3ggFrcyKCmNnrFOOdE32fO8LURxDmMSCH25qyF26r1VfBTlu4JS2cO7ofAPR+ANABvZ2U2DwqH59WuSOIwPLkguTwto6Aqs0QXDolcg2uRe2GLSfuQAUb1BjDUjCTxsikoQrbPPxoRowYqxckJ/aqqtRGsr9ldpqZrSabGizHAXWAM8AP6dQCUWxhiKxxGKAmna6kV0ibU6LleZROTUUbVsXo++mcshJyNe+28q5wNqfc2Ho/sj7qUQ7PQhCRvkHHSup40o87/aiEh6AKc+4PxSxV3qMApbXE3lxdgiu7s5rdGcHOiWRUBMk4UMzCXnCOzpp7ReILQX6FQLV6BFVd0w1ywHToBddxKyskxUWJA5Nz05CcsQNEXP3soIsFWKnx9T3a02kSYEFuK+VrECMnH5jXdmBegTXn3HuysAHM2sHvxDxMuGxaDz8pN58z/VBsbcAVSRCy3NSHeJeWRb5o6hyqndDCCMcw/QR17Vo9lMpJ4DJQowSAdwAXOU+NaEsLtZOUBZIz4pPab+C86qDW4wh2rUUTJGBXAgnErnpig1wYcKj0uvxT5fTSdkg+qscL0TG3yI/9ImlFKkTCnlOA1K2m01XIYXVui7tYsgpmZkCoHnAzGaCHj+3sgKm4ikoIsApT4T5yMZJsNVQP1XWyz+zHpubHxqaCo5p659mVE5W8HpSlZ4efwOUT1Tup6ZjW5jEtXDMkqB66KUNRUVPgFz0zWAU86BY+Gfu0be05bZyp/agwMwtRExdaV3OKfvgUGfNzydED9SLmMDDEjDEpBtRhQpEoDGw4kYITEOstIhoa9a6BvJYFWpt6A3hUkI8hVQfVAsBjW6Kli6lPZQ3LBQXAemc0GeTmMiPYDmEKsyIFFNuXMlHI5bOH5zOIK1Mv0esIrqZxAHVlqX1RODeLcIRcDlf1frirEw/IuVobJUSneyaaV+yAMZqH7bdqEq0HIyE3FKxRepAhYDEKQfJooQjrpv6mnVS123P4g0QYrhiACjmCljhB9aY4RE5lc1fGnud+ZidvRMWlUcExdB4omkqlWGwW0jjxN1SlnfcdutbQjxoj9F2DeaJRVeQiTxvdQh8FS1IQYuoGW0oOPEcQquY7ozpwiBINF1mRrAPTFCK3Ea4mOfmLXifCAnaLsGBURlKhS38pcsZS+A8cAqUhubDLWqdjqPV0EASsussbrsTDIsNJHYpdwOZSM9tOJQbDRecGw7UzNuRcDyg2KUSIZUCLXYgiYovdRjzGcYW4JzT2hL40v5hS0WtNdQsU2gC20iQhqfos7MiqLn3hD68x6nEnGsa6ZeKtN4BC6SEyV60UDdQioGI5tg7xJfZtch+JEVhTDVFSKwu4WIIeXrehwn/iVhhJIyyFQu0eeoZzbENxVEdRDW566eCKWjZpeVcsP0nLTHVRu8UW5GYgQlz2okeKTk0EPmUCavSIXfgsDoeeqqSUEpuyPvVmLXywAweuoBSY6MnFR8OpODA9uAxFEoXJQEGa4y4y9wg6GDQym8EZcrrF+yRNDR6YQ7lFTxhbPQr+qMeEENU5gfDVyB4qr29h1vFFjd5iCx9E1IpkNGiCpCUVa9B869KSUhZTuV6+KKDLluuWnrO07Rge7UtjbYxbLSkc60qzQAsjFc7CAQSEz5Bp51mGjiGeZQhE5TmRT9WDgo3akKlKnZeaSUSAq5aS+kKU/tYkQvyq6iIvLySIAIVopH0RtJxJ9gokdtpvdFJ2dFHd3EW9YWA38gtMGlkR40RW7BAaO6saU2/henVTDh3ZTHc7udDnyYWicMx66yqmMnMWPbjg2YBJVw+j5WqKwnC6WcXG11CqrkUtLEvjJQ54hEeAS8TttKZcqm4Eo0JANcByRTlHIR1GwgNh38nlZJyqlpH9ipphiFZLUTL2he6CD+gXtS+7CeysBFOX/tPIygLdsGxXyGmnhuTfbAkcOF4LvUf0DBoH01Y0jdGKmRb8F0TA1eQ='\n3900 6245\nb'eNqFWV2yJrmp3EovoG+FEKCftUzMo/e/BWcm4rgf7LkOR3R9OiUKQZIkmr/+Gr9/2eD/fv+asX7/Gt+9O4MPY7yVcX3n71/7fol/7j3j8FfmGPn3719/GRe1yXdZuMvLwqgVPCQe9vm2LGx3/sq0e2hhatHwtfl8uPv5MGrlPez97aOXR/BXZqygBefiujzOeRbOfBtrhRawstcnA+sYf2TuZTQQWtx4w7wNtO/WpzGGIb+jE6+lX7nGGbSgk628sGBt4fZG6we6EN+Z9TLDELnCFMilxdjMybPwY+APQ9u/U+cNWvNce03u31pkyG9vp6f/+r+R/zGz53fqsM7dM7dN+V9hmfinI3jxwqndz8i23jzpuuWOowxeLdr9r5vfP3t8p4KkOAISZxWARq3Of9q9bu8m/NbNM+Nqd2Vj+H8P2/N8ne8ovsk/rpMH0dP2Wavxz3nDnv0dRTgPHdh5R7gseK36P2MH31mrTezNX/hqCr4WtUr8ZpvIP1BXD3hrAX7KVLLGgF98ShkwBSdZotbpPz/u/xyI3wX+lK9kfa9A6Ap/VgFy1mF0Hf6U309BMhb+lYXJ0Poae1Uu6nQ8q3cuOhIdiEkX5ncV97iM5UTuowKhswXr4z9k0oXsXcjOQNh3FTUwAH8t27FkQp5FwLPoU2SHMH4eeAr8TUcOonmNNecRJqYAGcx6NiaiQ5gd1IA7eb+r7zkDlayfVCSmUOkJS9kHCf+DDOshZWIoFk5+Ai49jnA1tTovfWsvvL1Y7QU/ez7V6J0MFRgtrAhhCpmThb4amd4JXZ1Q/ik3aF4vsxXkWnFunUMxtsWiif9tIrjpq1oyZg68mOuZkG+mKvx/TORXGBybPgUq5B2kVpW0HxOdyvXnQ8ZXABg8D7AMhiovCM1zq/DaxM/H1x8P6Z8Cdy7xk3OdKIYnN2LVRAZtonO69h8POT+5fA7bVSJ88EQmrlZDjNQmGt/r/PGQ9smJw8/l2Did6px1hZrXOX4s/IDh/vGQ43OZ2AReoMGe2DJhWmWAdlfp7J27bW3u+dRNzmI7jrPdRrXbqVUW5G5g/TTqn9bNP8X5Qu8S47F3hKvflmepHtrpmB3L3bHkn2J/IRPJFAaI90xl1LUa5LLTx/ghuh/q25ebvlDuglUcYF8fwhUxhFVG4jSufuj2h4DJ7JFfKmzO/Ebsc1JV6sqzE+Snebe//ePM4ZaST2fStfAD/hAkKsJTdjoSoyFxGxLkuvAvhR+TQZDVLAXmOpyRHGw8G+c+GzZOP/Ac81sK/WAnCkACPUE2ZFnUat2DTmsoaw1l7GdhH6F3t6opEKhbGYmhVb5sreTObhuzbTDPMT7C40IFwobjNTQT2WCU977qpq/KzoOWPWQZvfRbWnBvxt4PK14BpSpC30pssngpOdEfz3aHzc9PSamt7u0bENwq9ahVhs6q/8BG04U1XRjT4bva6Q4e0NclNmSD+NzO71hj/DRArQFqDIOv7xIcOAotJrat8kOrJmFw249GhzU6jA7AhvrIHhTZ6DxIeNkQsi6DNVsh7dMa395ZJgves9rAOrIRA3UihFGUQh4z7dNfM9rdmGc35skKdTCw3l7EuztaVZbEZrageWkjX7XtHhBmjwxKJhiYB0cjY3Am0P9SK+cmnZ/7sd9ukTZbtk1G22ex3zL67gDuqLbKlgy5yDrx8dxY533d7fnjLBa3T2DKk3obPF40LiqAXKNM8ZcWqXltjbbh0kFfykYd/CIFb17QqjO13t15NTy84eHsd/NW0WJewjLqHnOPiiW1Wme5Lx7ZW6ON0QJGL8IxDhEwwX9eSUmtSlOHP5Bmq4xolREKeCnocNYm8gUnygRTFbZF8M+JuC8YOV4wgnQ6gXOiwCUAgAKzUK0wvpe1w975ghHx8JX58JWswPl0hkcJ1djTKhhEHRLIftl9zc/bqulCNo6+XP0ZkGE4HGnYlVh6N5f0/mwb3V5Xt1cNXjMKHHOydCc2nSJ0cbRJfK/ujrN7++5uv/mNCUbnF20R2ahFUKliughSQpYN9nFYp7Pzu2l8zpqph8TKpBCvrDB1dyS/exrm1kE4HZZDBE1IaQJpKIfwHDOSegILHJKJEDzNgqPBcRsc0g6wofkN0SJnHs5nigaBe4462p1l4/T9gPX9gGk8mBAstIE2Tc48iHMxqRzY90j6FEihrMoPnMmeDU1g9+ORoHlIlwADDi90UGkcKFL2t5HPyH6dBTbiWVMIqlowIurX8pE1a5L4ER0d0aodwKGXEPOXIZvMjO2Psw8izKKwjEDGZISroaTYo18qoedJvC5jzoKBEUYkpsbbyPM0tdLqmkRtvspH2F4081GRpbYtCSioJoXVocGKTQlnaAjdmnjH1Z/+IRbfA/Nr+dFtW5Tx+CS+IZix0M64Okzcqphjr2ChLgpwtglSC81bh7ZpZHB4kxHeS1yRPT5QTb/EBfbO8XShXV2NhNQ5Msf6RPbxnkDCVOGR2+1VGf5UBYjONJ81CykLqTkUC1kYeIDkKSM0vSRh7bymvfOlZMZL0pQItKmboJ1SGez/o07DqymQL6ti1MyHxv5yM/fLDQh0yYauhFzdHTMXZkPZoC6eRpzC9crvuu8ywce7XpgabA1dSnczg2FHc3jXKnQfcOHuGS8gK6sG2aQqR2hSTAbkmO5FNueLnYBn+cExA7Wkq7kannkBU4D3piTfuvYamt7W0lkWJH/hnR9CJada15u9mtKb42Mqhh8PvIrZYQYaWwZu/aIyGNWp0fTevBF9VRjSRUcNZlXrwhAaVbikjiwW8lnMAcJ8XJz52DmD6Dni9dxSGHHzFgmdMsj4eVQSLxWjdgKMZWKJW0oSoo+y20LrQ6PIBO1ODdP+Wv318cgQ2qKyuvjd/ZFkUL88nZMuyoJGqqSGAa2UE3O+HluqmQ8q2SUngNktmYJQKx0UT2CKJdn9bkttPn0M3VcHOot/AnNoZjiURRNTZwrhzARkP1Pm74IHoX3EQUn/HnTb9NFXZDLUrMGdskDFAoaZsiDawGBdxQoir2LFg5q5ph6P6lWIcKGCNQDsLpmoYJ7zhk+zN46CZHQn8rFJADyi/40JpKLJ0ji6l/GnqsHmdZ8ANvf34DQRun6ce1ANGJSxVzRZ5WAAml91X46jVkzA13XrBZHDrhkinenSlugfXjUmCQIOkeKuPOD11IlAh2H1oFvTut2wqzRdXoXJgqldltCtAf6g5GsjclamjhqRCxUQHKwxKNiautQsQ4w3T92WYUKse2OIzqsHMDET8S3dkgdDClFrgoTmtanLQYRDr+87imNA9MU6c0rO+ifqgxYiFEEjyqdu85JSFhWisGGyK8oBeZdPYGCTAWkNl/DCoKFk8gcs0o9TRAmqHb3tnWCrPOen6yyWDbTjqgPw5ifV6qIIAk2kdMIEAwpWs/67A0ZoZnMv9eR34Xc1CjPB9lIAQn5fBfHoAQY1hX2qnGBmrafWKyySp07184XpW8fA7FKt0FPHwGDDKAe5hUOWdgsVR3fM+tRa5bJ7ncGvblXQbkKKhohGdOv6XQKAxRq7Nvu77sQUWted4A6JB7Gr6aZtLqvrbwpZNXpdgmA3CkeboCnKTICCtJvcIxUNOVS31qy6FLuUAmPzElagBqoW0jR+2KcATynB2sqyVSlVf0ZPqa6CwFZbSZA4PyaRrbk13mW59Mr6uTnK+dRG7Kc22OSYnk/Hnpo0dm2V3OaNkHaO4o24L2tLUbCP5LGklay2rdpS9+gAZL0PiJaBPV3SRZMv01NUawVS2tE+vxXkXKNytREOxdZ038EP1L6j/wykd8DVRfSYLupKB5FyfY3mVYy1ibiGCWUP6kjQWbPnZIiGN0nonfoQYWj6P/ZEVskvDKhT9m2/y+tZr/39978BFLt5Xg=='\n4608\n6044\n3900\n"
      }
     ],
     "prompt_number": 4
    }
   ],
   "metadata": {}
  }
 ]
}
	{
	"metadata": {
	"name": "play w gist"
	},
	"nbformat": 3,
	"nbformat_minor": 0,
	"worksheets": [
	{
	"cells": [
	{
	"cell_type": "code",
	"collapsed": false,
	"input": "get_gist = \"\"\"\ndata_url_targz = \"https://gist.github.com/boris-arzur/7329335/download\"\nimport tarfile\nfrom urllib.request import urlopen\nwith tarfile.open(fileobj=urlopen(data_url_targz), mode=\"r:gz\") as tar:\n#just get the first file...\n first_file = next(member for member in tar if member.isfile())\n data = tar.extractfile(first_file).read().decode(\"utf8\")\n\"\"\"\n\nimport zlib, base64; print(base64.b64encode(zlib.compress(get_gist.encode('ascii'), 9)))",
	"language": "python",
	"metadata": {},
	"outputs": [
	{
	"output_type": "stream",
	"stream": "stdout",
	"text": "b'eNpVj9FqxCAQRd/9isG+KLQGGkrbQL4lmKiJS4zpqOySr98xpJT6NF7vPd5hRmc9FFyHrHE+oAe+5Lynrmlmn7KafV7KqKYYmjGiT28aj4LNZ/v+3bYfjYn3bY3acObDHjEDUZxfLXMYAxB29aNC+1NsynBZSI273did0L9+VRVRpzje+ssh/neTrxCisT3Hbj64BJ1qumMvt0Lw2dLniwXnkW4nUykGdE5lqAptt9lHFsGG0SK4iHCNfqss8O4SlE81IKQ8EbUIhcmiKI96yufrH1nSktoIqYydqKPgJbsvLtkTrDl56Q=='\n"
	}
	],
	"prompt_number": 1
	},
	{
	"cell_type": "code",
	"collapsed": false,
	"input": "import zlib, base64\ndata_code = zlib.decompress(base64.b64decode(b\"\"\"eNpVj9FqxCAQRd/9isG+KLQGGkrbQL4lmKiJS4zpqOySr98xpJT6NF7vPd5hRmc9FFyHrHE+oAe+\n5Lynrmlmn7KafV7KqKYYmjGiT28aj4LNZ/v+3bYfjYn3bY3acObDHjEDUZxfLXMYAxB29aNC+1NsynBZSI273did0L9+VRVRpzje+ssh/neTrxCisT3Hbj64BJ1qu\nmMvt0Lw2dLniwXnkW4nUykGdE5lqAptt9lHFsGG0SK4iHCNfqss8O4SlE81IKQ8EbUIhcmiKI96yufrH1nSktoIqYydqKPgJbsvLtkTrDl56Q==\"\"\"))\nprint(data_code)\n# we you are ready \n# exec(data_code)\n# print(len(data.split(\"\\n\")), len(data))",
	"language": "python",
	"metadata": {},
	"outputs": [
	{
	"output_type": "stream",
	"stream": "stdout",
	"text": "b'\\ndata_url_targz = \"https://gist.github.com/boris-arzur/7329335/download\"\\nimport tarfile\\nfrom urllib.request import urlopen\\nwith tarfile.open(fileobj=urlopen(data_url_targz), mode=\"r:gz\") as tar:\\n#just get the first file...\\n first_file = next(member for member in tar if member.isfile())\\n data = tar.extractfile(first_file).read().decode(\"utf8\")\\n'\n112"
	},
	{
	"output_type": "stream",
	"stream": "stdout",
	"text": " 6245\n"
	}
	],
	"prompt_number": 2
	},
	{
	"cell_type": "code",
	"collapsed": false,
	"input": "#this is short, points always to the same version\ndata_url_raw = \"https://gist.github.com/boris-arzur/7329335/raw/f0f71030707cca4a739f372c0dcb6902187b15bd/age_table_japan_20131106\"\nfrom urllib.request import urlopen\ndata = urlopen(data_url_raw).read().decode(\"utf8\")\nprint(len(data.split(\"\\n\")), len(data))\n\n#this is clean, and stays up to date\ndata_url_targz = \"https://gist.github.com/boris-arzur/7329335/download\"\nimport tarfile\nwith tarfile.open(fileobj=urlopen(data_url_targz), mode=\"r:gz\") as tar:\n #just get the first file...\n first_file = next(member for member in tar if member.isfile())\n data2 = tar.extractfile(first_file).read().decode(\"utf8\")\n\nassert data == data2",
	"language": "python",
	"metadata": {},
	"outputs": [
	{
	"output_type": "stream",
	"stream": "stdout",
	"text": "112 6245\n"
	}
	],
	"prompt_number": 3
	},
	{
	"cell_type": "code",
	"collapsed": false,
	"input": "import zlib\nimport base64\nimport marshal\nimport json\n\ndata1 = base64.b64encode(zlib.compress(data.encode(\"utf8\"), 9))\nprint(data1)\nprint(len(data1), len(data))\n\nassert zlib.decompress(base64.b64decode(data1)).decode(\"utf8\") == data\n\nline_f = [int, int, int, float, float, float, float, int, int]\ndef apply_f(format, line):\n return [fun(cell) for fun, cell in zip(format, line)]\n\ndata3 = [line.split(\",\") for line in data.split(\"\\n\")]\ndata3_f = [apply_f(line_f, line.split(\",\")) for line in data.split(\"\\n\")[1:]]\n\ndata4 = base64.b64encode(zlib.compress(marshal.dumps(data3), 9))\ndata4_f = base64.b64encode(zlib.compress(marshal.dumps(data3_f), 9))\ndata5_f = base64.b64encode(zlib.compress(json.dumps(data3_f).encode(\"utf8\"), 9))\nprint(data5_f)\nprint(len(data4))\nprint(len(data4_f))\nprint(len(data5_f))\n\nassert marshal.loads(zlib.decompress(base64.b64decode(data4))) == data3",
	"language": "python",
	"metadata": {},
	"outputs": [
	{
	"output_type": "stream",
	"stream": "stdout",
	"text": "b'eNp9mM2SHbcNhfd8Cj1AV1cTBP+W2WeZF1CccaIq20pJsit6+3wHxJ14o8yGfTkECQIHBwD/8s+367fff/3725cPn3/+8PX3L398+uPzl69/mvvH28dv//p6vf/rw5eP396umD2fP3/+8tObVv76+cu3j798+vb9+uXTz28f3v7z77efvn387afv19dvH799+vzbxy/fP/z1zz/+Vp6rPvq7zMf13HvP7ozPc34/u81+zX33fu29nsV378/TS73O2jZDbI8WYk/8ZuzzmuueEput8d173auYftdx2Tltz3PaE7/POOc9l5Y9znfvPrw0fo+NsuuILTvL47fE7JrjXo+Wrcp373PU4vo97aotxVK5mspW7tbvpbuMoe8+nvUUKT36vmpNsZ3La46c5veyWMbdvA+vvQz99olFj9h+cvnzLj7bveIuzhatjzmsTP3GbDul8pBnv0tPu1dcpSFlfVZ7iiw0bF1pkJ0Geda79KwpZahY+/S1ytbvun8ohX+fe8X9ZRu8uAa+Drty6x+LjZ1i8d2X+S417Pq0HxoEJce6l+zWt777wjClWkz4j+3/XGPeS4bri/Nm34+3UltMtB+7264xUm5Ovjmke6keE6Crp1x/h8cZ/RrARCbv4BxwsfUqVRfuREZN162Xfi99OQOcyOigmG+MAU5qXLgRBJ5B8AL/KxjqNdq9ZfVune/xzIE9Q3Eu0dKeM+3YcjTOs3vLfL6xi+Ez535S3IHre6ym/1r6r3G/em+ZgWDje9Tpo1Qp7t4vTz172sNfI3ryD13HAdl4htmyYlLcn3r19J+nPXrax9fVN/9gWePuXUDuu5jw0vpz9dTT2zuXnLFL7tH9GhEPWpqvVkwTtlEgz2t53sjzBluu+5G7jLtDDV4JPBNejJgaiZeWfhjpB+b7hAO1DIrsY/ja6ClD1QFg/Qdyzto7YFyxOazSh+SkQEWN/yfX74DHMznbQaj0jAkZ+yWX9h//G7vf4S7FFNAi2DkPvKwdQE+51znjfeztlhkWfr66jeXQH6zCBME20g8t/TDm+9jtllqLOMDNQ3AutjUBOEbyREucjfU+9nqDVpaxfX8mqq8SE1N6vuReftvvY3/uJj0nmHAyyPJZWMaEmC7jwXL9THnmfd+i1oVFLl+z1YdkYpoA/DP9/ko+r2TEvK/bpdYAbT6ne6slFOjKE2lPS7vMtAvzPm8syTKM7xDTsl2aJnDItVLPFz+8+GJu1t4uszux4pBTe6zgZia430q/v/joxU9Qnfe7ywy4nDCda/VWmhzTQNtKXnrx7Yt/YTL3SOnL0MHbIiRHCUOZpPN+T/pvp/9gCG93l5ur9iDYjVKgSfFK0NXnCK5MEDUTRCUHuN1D9nugX8d9kGNp2knsU5N5V+b1mnm9wtxeb3Cxp4DsXB0vF7lmbhbVrCPWTEFLQVzjz40jN5XHvBor4NJCELHPVoo42MbBR8BzRJu2owCZpNqrLYXUKORraLqPq/oxKg49Aj1PhuDbiuw+lYPaBB7zKR4TGKIG6yKYUVgzCiv2bDOyxMSiVxtbfiyC0WzsWhNuK2FTEzaV27Vxb9yIpmzSWT84URNV6WznienHmn6snIWgeHQ+VGhQLi5CUJ7HXpdlwp4ra8F6VLUHS/YgxLEk6KQe64WahzILL1k7DDwzw1hmGCMaGgyldWTPqzW4uVOfYWYqKAT7gfjM8tGynDSyQYOiuBKczX0NEModUsFQz+bhjJm1gWWtYBitWXDGqOjXgBLxUtiPkgSctuecONY5qdVzcgOrrd7ydyfYWQe7QW6KMmoEkmg7VlXlFwKegk2J+e4SjCttzKhiUhMgAZMfuhnpx5Z+bNC67YgOSuR6EVUUvVa6JkLVfe7YU8BzA1fZTo0NUHzhMoM1GlbtmlAh5u0gp2dC9EyI7lochZc3ggFrcyKCmNnrFOOdE32fO8LURxDmMSCH25qyF26r1VfBTlu4JS2cO7ofAPR+ANABvZ2U2DwqH59WuSOIwPLkguTwto6Aqs0QXDolcg2uRe2GLSfuQAUb1BjDUjCTxsikoQrbPPxoRowYqxckJ/aqqtRGsr9ldpqZrSabGizHAXWAM8AP6dQCUWxhiKxxGKAmna6kV0ibU6LleZROTUUbVsXo++mcshJyNe+28q5wNqfc2Ho/sj7qUQ7PQhCRvkHHSup40o87/aiEh6AKc+4PxSxV3qMApbXE3lxdgiu7s5rdGcHOiWRUBMk4UMzCXnCOzpp7ReILQX6FQLV6BFVd0w1ywHToBddxKyskxUWJA5Nz05CcsQNEXP3soIsFWKnx9T3a02kSYEFuK+VrECMnH5jXdmBegTXn3HuysAHM2sHvxDxMuGxaDz8pN58z/VBsbcAVSRCy3NSHeJeWRb5o6hyqndDCCMcw/QR17Vo9lMpJ4DJQowSAdwAXOU+NaEsLtZOUBZIz4pPab+C86qDW4wh2rUUTJGBXAgnErnpig1wYcKj0uvxT5fTSdkg+qscL0TG3yI/9ImlFKkTCnlOA1K2m01XIYXVui7tYsgpmZkCoHnAzGaCHj+3sgKm4ikoIsApT4T5yMZJsNVQP1XWyz+zHpubHxqaCo5p659mVE5W8HpSlZ4efwOUT1Tup6ZjW5jEtXDMkqB66KUNRUVPgFz0zWAU86BY+Gfu0be05bZyp/agwMwtRExdaV3OKfvgUGfNzydED9SLmMDDEjDEpBtRhQpEoDGw4kYITEOstIhoa9a6BvJYFWpt6A3hUkI8hVQfVAsBjW6Kli6lPZQ3LBQXAemc0GeTmMiPYDmEKsyIFFNuXMlHI5bOH5zOIK1Mv0esIrqZxAHVlqX1RODeLcIRcDlf1frirEw/IuVobJUSneyaaV+yAMZqH7bdqEq0HIyE3FKxRepAhYDEKQfJooQjrpv6mnVS123P4g0QYrhiACjmCljhB9aY4RE5lc1fGnud+ZidvRMWlUcExdB4omkqlWGwW0jjxN1SlnfcdutbQjxoj9F2DeaJRVeQiTxvdQh8FS1IQYuoGW0oOPEcQquY7ozpwiBINF1mRrAPTFCK3Ea4mOfmLXifCAnaLsGBURlKhS38pcsZS+A8cAqUhubDLWqdjqPV0EASsussbrsTDIsNJHYpdwOZSM9tOJQbDRecGw7UzNuRcDyg2KUSIZUCLXYgiYovdRjzGcYW4JzT2hL40v5hS0WtNdQsU2gC20iQhqfos7MiqLn3hD68x6nEnGsa6ZeKtN4BC6SEyV60UDdQioGI5tg7xJfZtch+JEVhTDVFSKwu4WIIeXrehwn/iVhhJIyyFQu0eeoZzbENxVEdRDW566eCKWjZpeVcsP0nLTHVRu8UW5GYgQlz2okeKTk0EPmUCavSIXfgsDoeeqqSUEpuyPvVmLXywAweuoBSY6MnFR8OpODA9uAxFEoXJQEGa4y4y9wg6GDQym8EZcrrF+yRNDR6YQ7lFTxhbPQr+qMeEENU5gfDVyB4qr29h1vFFjd5iCx9E1IpkNGiCpCUVa9B869KSUhZTuV6+KKDLluuWnrO07Rge7UtjbYxbLSkc60qzQAsjFc7CAQSEz5Bp51mGjiGeZQhE5TmRT9WDgo3akKlKnZeaSUSAq5aS+kKU/tYkQvyq6iIvLySIAIVopH0RtJxJ9gokdtpvdFJ2dFHd3EW9YWA38gtMGlkR40RW7BAaO6saU2/henVTDh3ZTHc7udDnyYWicMx66yqmMnMWPbjg2YBJVw+j5WqKwnC6WcXG11CqrkUtLEvjJQ54hEeAS8TttKZcqm4Eo0JANcByRTlHIR1GwgNh38nlZJyqlpH9ipphiFZLUTL2he6CD+gXtS+7CeysBFOX/tPIygLdsGxXyGmnhuTfbAkcOF4LvUf0DBoH01Y0jdGKmRb8F0TA1eQ='\n3900 6245\nb'eNqFWV2yJrmp3EovoG+FEKCftUzMo/e/BWcm4rgf7LkOR3R9OiUKQZIkmr/+Gr9/2eD/fv+asX7/Gt+9O4MPY7yVcX3n71/7fol/7j3j8FfmGPn3719/GRe1yXdZuMvLwqgVPCQe9vm2LGx3/sq0e2hhatHwtfl8uPv5MGrlPez97aOXR/BXZqygBefiujzOeRbOfBtrhRawstcnA+sYf2TuZTQQWtx4w7wNtO/WpzGGIb+jE6+lX7nGGbSgk628sGBt4fZG6we6EN+Z9TLDELnCFMilxdjMybPwY+APQ9u/U+cNWvNce03u31pkyG9vp6f/+r+R/zGz53fqsM7dM7dN+V9hmfinI3jxwqndz8i23jzpuuWOowxeLdr9r5vfP3t8p4KkOAISZxWARq3Of9q9bu8m/NbNM+Nqd2Vj+H8P2/N8ne8ovsk/rpMH0dP2Wavxz3nDnv0dRTgPHdh5R7gseK36P2MH31mrTezNX/hqCr4WtUr8ZpvIP1BXD3hrAX7KVLLGgF98ShkwBSdZotbpPz/u/xyI3wX+lK9kfa9A6Ap/VgFy1mF0Hf6U309BMhb+lYXJ0Poae1Uu6nQ8q3cuOhIdiEkX5ncV97iM5UTuowKhswXr4z9k0oXsXcjOQNh3FTUwAH8t27FkQp5FwLPoU2SHMH4eeAr8TUcOonmNNecRJqYAGcx6NiaiQ5gd1IA7eb+r7zkDlayfVCSmUOkJS9kHCf+DDOshZWIoFk5+Ai49jnA1tTovfWsvvL1Y7QU/ez7V6J0MFRgtrAhhCpmThb4amd4JXZ1Q/ik3aF4vsxXkWnFunUMxtsWiif9tIrjpq1oyZg68mOuZkG+mKvx/TORXGBybPgUq5B2kVpW0HxOdyvXnQ8ZXABg8D7AMhiovCM1zq/DaxM/H1x8P6Z8Cdy7xk3OdKIYnN2LVRAZtonO69h8POT+5fA7bVSJ88EQmrlZDjNQmGt/r/PGQ9smJw8/l2Did6px1hZrXOX4s/IDh/vGQ43OZ2AReoMGe2DJhWmWAdlfp7J27bW3u+dRNzmI7jrPdRrXbqVUW5G5g/TTqn9bNP8X5Qu8S47F3hKvflmepHtrpmB3L3bHkn2J/IRPJFAaI90xl1LUa5LLTx/ghuh/q25ebvlDuglUcYF8fwhUxhFVG4jSufuj2h4DJ7JFfKmzO/Ebsc1JV6sqzE+Snebe//ePM4ZaST2fStfAD/hAkKsJTdjoSoyFxGxLkuvAvhR+TQZDVLAXmOpyRHGw8G+c+GzZOP/Ac81sK/WAnCkACPUE2ZFnUat2DTmsoaw1l7GdhH6F3t6opEKhbGYmhVb5sreTObhuzbTDPMT7C40IFwobjNTQT2WCU977qpq/KzoOWPWQZvfRbWnBvxt4PK14BpSpC30pssngpOdEfz3aHzc9PSamt7u0bENwq9ahVhs6q/8BG04U1XRjT4bva6Q4e0NclNmSD+NzO71hj/DRArQFqDIOv7xIcOAotJrat8kOrJmFw249GhzU6jA7AhvrIHhTZ6DxIeNkQsi6DNVsh7dMa395ZJgves9rAOrIRA3UihFGUQh4z7dNfM9rdmGc35skKdTCw3l7EuztaVZbEZrageWkjX7XtHhBmjwxKJhiYB0cjY3Am0P9SK+cmnZ/7sd9ukTZbtk1G22ex3zL67gDuqLbKlgy5yDrx8dxY533d7fnjLBa3T2DKk3obPF40LiqAXKNM8ZcWqXltjbbh0kFfykYd/CIFb17QqjO13t15NTy84eHsd/NW0WJewjLqHnOPiiW1Wme5Lx7ZW6ON0QJGL8IxDhEwwX9eSUmtSlOHP5Bmq4xolREKeCnocNYm8gUnygRTFbZF8M+JuC8YOV4wgnQ6gXOiwCUAgAKzUK0wvpe1w975ghHx8JX58JWswPl0hkcJ1djTKhhEHRLIftl9zc/bqulCNo6+XP0ZkGE4HGnYlVh6N5f0/mwb3V5Xt1cNXjMKHHOydCc2nSJ0cbRJfK/ujrN7++5uv/mNCUbnF20R2ahFUKliughSQpYN9nFYp7Pzu2l8zpqph8TKpBCvrDB1dyS/exrm1kE4HZZDBE1IaQJpKIfwHDOSegILHJKJEDzNgqPBcRsc0g6wofkN0SJnHs5nigaBe4462p1l4/T9gPX9gGk8mBAstIE2Tc48iHMxqRzY90j6FEihrMoPnMmeDU1g9+ORoHlIlwADDi90UGkcKFL2t5HPyH6dBTbiWVMIqlowIurX8pE1a5L4ER0d0aodwKGXEPOXIZvMjO2Psw8izKKwjEDGZISroaTYo18qoedJvC5jzoKBEUYkpsbbyPM0tdLqmkRtvspH2F4081GRpbYtCSioJoXVocGKTQlnaAjdmnjH1Z/+IRbfA/Nr+dFtW5Tx+CS+IZix0M64Okzcqphjr2ChLgpwtglSC81bh7ZpZHB4kxHeS1yRPT5QTb/EBfbO8XShXV2NhNQ5Msf6RPbxnkDCVOGR2+1VGf5UBYjONJ81CykLqTkUC1kYeIDkKSM0vSRh7bymvfOlZMZL0pQItKmboJ1SGez/o07DqymQL6ti1MyHxv5yM/fLDQh0yYauhFzdHTMXZkPZoC6eRpzC9crvuu8ywce7XpgabA1dSnczg2FHc3jXKnQfcOHuGS8gK6sG2aQqR2hSTAbkmO5FNueLnYBn+cExA7Wkq7kannkBU4D3piTfuvYamt7W0lkWJH/hnR9CJada15u9mtKb42Mqhh8PvIrZYQYaWwZu/aIyGNWp0fTevBF9VRjSRUcNZlXrwhAaVbikjiwW8lnMAcJ8XJz52DmD6Dni9dxSGHHzFgmdMsj4eVQSLxWjdgKMZWKJW0oSoo+y20LrQ6PIBO1ODdP+Wv318cgQ2qKyuvjd/ZFkUL88nZMuyoJGqqSGAa2UE3O+HluqmQ8q2SUngNktmYJQKx0UT2CKJdn9bkttPn0M3VcHOot/AnNoZjiURRNTZwrhzARkP1Pm74IHoX3EQUn/HnTb9NFXZDLUrMGdskDFAoaZsiDawGBdxQoir2LFg5q5ph6P6lWIcKGCNQDsLpmoYJ7zhk+zN46CZHQn8rFJADyi/40JpKLJ0ji6l/GnqsHmdZ8ANvf34DQRun6ce1ANGJSxVzRZ5WAAml91X46jVkzA13XrBZHDrhkinenSlugfXjUmCQIOkeKuPOD11IlAh2H1oFvTut2wqzRdXoXJgqldltCtAf6g5GsjclamjhqRCxUQHKwxKNiautQsQ4w3T92WYUKse2OIzqsHMDET8S3dkgdDClFrgoTmtanLQYRDr+87imNA9MU6c0rO+ifqgxYiFEEjyqdu85JSFhWisGGyK8oBeZdPYGCTAWkNl/DCoKFk8gcs0o9TRAmqHb3tnWCrPOen6yyWDbTjqgPw5ifV6qIIAk2kdMIEAwpWs/67A0ZoZnMv9eR34Xc1CjPB9lIAQn5fBfHoAQY1hX2qnGBmrafWKyySp07184XpW8fA7FKt0FPHwGDDKAe5hUOWdgsVR3fM+tRa5bJ7ncGvblXQbkKKhohGdOv6XQKAxRq7Nvu77sQUWted4A6JB7Gr6aZtLqvrbwpZNXpdgmA3CkeboCnKTICCtJvcIxUNOVS31qy6FLuUAmPzElagBqoW0jR+2KcATynB2sqyVSlVf0ZPqa6CwFZbSZA4PyaRrbk13mW59Mr6uTnK+dRG7Kc22OSYnk/Hnpo0dm2V3OaNkHaO4o24L2tLUbCP5LGklay2rdpS9+gAZL0PiJaBPV3SRZMv01NUawVS2tE+vxXkXKNytREOxdZ038EP1L6j/wykd8DVRfSYLupKB5FyfY3mVYy1ibiGCWUP6kjQWbPnZIiGN0nonfoQYWj6P/ZEVskvDKhT9m2/y+tZr/39978BFLt5Xg=='\n4608\n6044\n3900\n"
	}
	],
	"prompt_number": 4
	}
	],
	"metadata": {}
	}
	]
	}