Skip to content

Instantly share code, notes, and snippets.

@wassname

wassname/check_normalization.ipynb

Created December 26, 2017 08:26
Show Gist options
  • Save wassname/9fa642d28ea0cf7b89eb953bcdff0ed8 to your computer and use it in GitHub Desktop.
Save wassname/9fa642d28ea0cf7b89eb953bcdff0ed8 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
check_normalization.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2017-12-26T08:23:40.027791Z",
"start_time": "2017-12-26T08:23:39.951629Z"
},
"collapsed": true
},
"outputs": [],
"source": [
"import os\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2017-12-26T08:25:31.540166Z",
"start_time": "2017-12-26T08:23:41.168947Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'net_dir': './train_package/4/netfile', 'agent_type': 'nn', 'agent': None}\n",
" pair price volume\n",
"coin \n",
"ETH BTC_ETH 0.049546 770086.388780\n",
"reversed_USDT USDT_BTC 0.000067 520926.768114\n",
"LTC BTC_LTC 0.018270 177004.746549\n",
"XRP BTC_XRP 0.000066 126840.040619\n",
"STR BTC_STR 0.000014 72556.665086\n",
"DGB BTC_DGB 0.000004 69229.265526\n",
"BTS BTC_BTS 0.000037 64820.288225\n",
"ETC BTC_ETC 0.001959 60786.335176\n",
"STRAT BTC_STRAT 0.000974 60463.484425\n",
"DASH BTC_DASH 0.076988 54938.242397\n",
"XEM BTC_XEM 0.000064 38441.689283\n",
"WARNING:tensorflow:From /home/wassname/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/tflearn/initializations.py:119: UniformUnitScaling.__init__ (from tensorflow.python.ops.init_ops) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Use tf.initializers.variance_scaling instead with distribution=uniform to get equivalent behavior.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From /home/wassname/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/tflearn/initializations.py:119: UniformUnitScaling.__init__ (from tensorflow.python.ops.init_ops) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Use tf.initializers.variance_scaling instead with distribution=uniform to get equivalent behavior.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO:tensorflow:Restoring parameters from ./train_package/4/netfile\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:tensorflow:Restoring parameters from ./train_package/4/netfile\n"
]
},
{
"data": {
"text/plain": [
"<pgportfolio.trade.backtest.BackTest at 0x7f89240edac8>"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from main import _config_by_algo, _set_logging_by_algo\n",
"from pgportfolio.tools.shortcut import _construct_agent, BackTest\n",
"os.sys.argv='main.py --mode=backtest --algo=4'.split(' ')\n",
"algo = '4'\n",
"\n",
"config = _config_by_algo(algo) \n",
"agent, agent_type, net_dir = _construct_agent(algo)\n",
"print(dict(agent=agent, agent_type=agent_type, net_dir=net_dir))\n",
"\n",
"backtester = BackTest(config, agent=agent, agent_type=agent_type, net_dir=net_dir)\n",
"backtester"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2017-12-26T08:25:44.793048Z",
"start_time": "2017-12-26T08:25:44.789769Z"
}
},
"outputs": [],
"source": [
"data_matrices = backtester._rolling_trainer.data_matrices"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2017-12-26T08:25:44.990848Z",
"start_time": "2017-12-26T08:25:44.980049Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(array([ 5.30000023e-07, 5.30000023e-07, 5.20000015e-07], dtype=float32),\n",
" array([ 0.00438775, 0.00548773, 0.00435859], dtype=float32),\n",
" array([ 0.019593 , 0.01967152, 0.01956829], dtype=float32))"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# is the training data scaled (x~=1) or is is raw price data (x~=0)\n",
"# Let's check one sample for the min, mean, max value per feature columns\n",
"xx = data_matrices.next_batch()['X'][0]\n",
"x = xx.reshape((xx.shape[0], -1))\n",
"x.min(-1), x.mean(-1), x.max(-1)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"ExecuteTime": {
"end_time": "2017-12-26T08:25:45.129693Z",
"start_time": "2017-12-26T08:25:45.123772Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(array([ 5.30000023e-07, 5.30000023e-07, 5.20000015e-07], dtype=float32),\n",
" array([ 0.00438775, 0.00548773, 0.00435859], dtype=float32),\n",
" array([ 0.019593 , 0.01967152, 0.01956829], dtype=float32))"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# is test/backtest data scaled?\n",
"# Let's check one sample for the min, mean, max value per feature columns\n",
"x = xx.reshape((xx.shape[0], -1))\n",
"x.min(-1), x.mean(-1), x.max(-1)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2017-12-26T08:25:45.279622Z",
"start_time": "2017-12-26T08:25:45.267671Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"(array([ 7.99042881e-02, 2.57202249e-04, 1.19777704e-02,\n",
" 4.46699996e-05, 5.02000012e-06, 3.72000000e-06,\n",
" 4.29499996e-05, 3.80088994e-03, 1.67012005e-03,\n",
" 5.28794602e-02, 6.99200027e-05], dtype=float32),\n",
" array([ 8.18534866e-02, 2.63061345e-04, 1.22751649e-02,\n",
" 4.59432231e-05, 5.40774090e-06, 3.94677363e-06,\n",
" 4.52796739e-05, 3.90276243e-03, 1.84290356e-03,\n",
" 5.41356988e-02, 7.38416129e-05], dtype=float32),\n",
" array([ 8.35826918e-02, 2.68277159e-04, 1.25704296e-02,\n",
" 4.73600012e-05, 5.80000005e-06, 4.11000019e-06,\n",
" 4.75499983e-05, 4.02999995e-03, 2.04784004e-03,\n",
" 5.58691099e-02, 7.73400025e-05], dtype=float32))"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# that was mean per feature, not look at per asset\n",
"# they don't normalise test/backtest values\n",
"history = backtester.generate_history_matrix()\n",
"xx = np.transpose(history, (1,0,2))[:,0,:]\n",
"x = xx.reshape((xx.shape[0], -1))\n",
"# yup they are all around zero and quite differen't\n",
"x.min(-1), x.mean(-1), x.max(-1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-12-26T08:26:02.093678Z",
"start_time": "2017-12-26T08:26:02.086911Z"
}
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-12-26T08:26:10.228361Z",
"start_time": "2017-12-26T08:26:10.222011Z"
}
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "jupyter3",
"language": "python",
"name": "jupyter3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
},
"toc": {
"colors": {
"hover_highlight": "#DAA520",
"navigate_num": "#000000",
"navigate_text": "#333333",
"running_highlight": "#FF0000",
"selected_highlight": "#FFD700",
"sidebar_border": "#EEEEEE",
"wrapper_background": "#FFFFFF"
},
"moveMenuLeft": true,
"nav_menu": {
"height": "4px",
"width": "254px"
},
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 4,
"toc_cell": false,
"toc_section_display": "block",
"toc_window_display": false,
"widenNotebook": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment