Skip to content

Instantly share code, notes, and snippets.

@josef-pkt

josef-pkt/poisson_convergence_llnull_3533.ipynb

Created March 4, 2017 16:07
Show Gist options
  • Save josef-pkt/b45d0b5ceb3b692c17af5f77a657e639 to your computer and use it in GitHub Desktop.
Save josef-pkt/b45d0b5ceb3b692c17af5f77a657e639 to your computer and use it in GitHub Desktop.
Download ZIP
trying out convergence problems in poisson and llnull
Display the source blob
Display the rendered blob
Raw
poisson_convergence_llnull_3533.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"import statsmodels.api as sm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Optimization terminated successfully.\n",
" Current function value: nan\n",
" Iterations 2\n"
]
}
],
"source": [
"import pandas as pd\n",
"features = ['pp']\n",
"df = pd.read_csv('denadai2_sm3533_bug.csv')\n",
"X = (df[features] - df[features].mean())/df[features].std()\n",
"y = df['num'].values\n",
"X_sm = sm.add_constant(X[features].copy())\n",
"model = sm.Poisson(y, X_sm, ) #\n",
"res = model.fit()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"fit_kwds {'method': 'nm', 'disp': 0, 'maxiter': 10000, 'warn_convergence': False}\n",
" Poisson Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y No. Observations: 270\n",
"Model: Poisson Df Residuals: 268\n",
"Method: MLE Df Model: 1\n",
"Date: Sat, 04 Mar 2017 Pseudo R-squ.: nan\n",
"Time: 11:05:47 Log-Likelihood: nan\n",
"converged: True LL-Null: -2.6729e+05\n",
" LLR p-value: nan\n",
"==============================================================================\n",
" coef std err z P>|z| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"const nan nan nan nan nan nan\n",
"pp nan nan nan nan nan nan\n",
"==============================================================================\n"
]
}
],
"source": [
"print(res.summary())"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Optimization terminated successfully.\n",
" Current function value: 884.916016\n",
" Iterations: 9\n",
" Function evaluations: 30\n",
" Gradient evaluations: 19\n"
]
}
],
"source": [
"#res = model.fit(method='minimize', min_method='dogleg')\n",
"res = model.fit(method='bfgs')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"'0.17.0'"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import scipy\n",
"scipy.__version__"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"1167.7259259259258"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.endog.mean()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"fit_kwds {'method': 'nm', 'disp': 0, 'maxiter': 10000, 'warn_convergence': False}\n"
]
},
{
"data": {
"text/plain": [
"-267293.98059701471"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"res.llnull"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"## Note we need to assign to the original `._results` not to the wrapped instance\n",
"\n",
"res._results.null_fit_kwds = dict(method='bfgs', disp=1)\n",
"res._results.null_fit_kwds = dict(method=\"minimize\", min_method='dogleg', disp=1)\n",
"res._results.null_fit_kwds = dict(start_params=np.log([model.endog.mean()]), method='bfgs', \n",
" disp=1)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"-267293.980597\n",
"fit_kwds {'method': 'nm', 'disp': 0, 'maxiter': 10000, 'warn_convergence': False}\n"
]
},
{
"data": {
"text/plain": [
"-267293.98059701471"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# llnull is cached, need to clean\n",
"print(res._cache[\"llnull\"])\n",
"del res._cache[\"llnull\"]\n",
"res.llnull"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"{'disp': 1, 'method': 'bfgs', 'start_params': array([ 7.06281348])}"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"getattr(res, \"null_fit_kwds\", {})"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" Poisson Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y No. Observations: 270\n",
"Model: Poisson Df Residuals: 268\n",
"Method: MLE Df Model: 1\n",
"Date: Sat, 04 Mar 2017 Pseudo R-squ.: 0.1061\n",
"Time: 11:05:48 Log-Likelihood: -2.3893e+05\n",
"converged: True LL-Null: -2.6729e+05\n",
" LLR p-value: 0.000\n",
"==============================================================================\n",
" coef std err z P>|z| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"const 6.9683 0.002 3575.004 0.000 6.964 6.972\n",
"pp 0.4473 0.002 232.896 0.000 0.444 0.451\n",
"==============================================================================\n"
]
}
],
"source": [
"print(res.summary())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Optimization terminated successfully.\n",
" Current function value: 884.916016\n",
" Iterations: 11\n",
" Function evaluations: 12\n",
" Gradient evaluations: 11\n",
" Hessian evaluations: 10\n"
]
}
],
"source": [
"res = model.fit(method=\"minimize\", min_method='dogleg')"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"fit_kwds {'method': 'nm', 'disp': 0, 'maxiter': 10000, 'warn_convergence': False}\n",
" Poisson Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y No. Observations: 270\n",
"Model: Poisson Df Residuals: 268\n",
"Method: MLE Df Model: 1\n",
"Date: Sat, 04 Mar 2017 Pseudo R-squ.: 0.1061\n",
"Time: 11:05:48 Log-Likelihood: -2.3893e+05\n",
"converged: True LL-Null: -2.6729e+05\n",
" LLR p-value: 0.000\n",
"==============================================================================\n",
" coef std err z P>|z| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"const 6.9683 0.002 3575.004 0.000 6.964 6.972\n",
"pp 0.4473 0.002 232.896 0.000 0.444 0.451\n",
"==============================================================================\n"
]
}
],
"source": [
"print(res.summary())"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from scipy.optimize._trustregion_dogleg import _minimize_dogleg"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.4.4"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment