Skip to content

Instantly share code, notes, and snippets.

@oxinabox

oxinabox/AdaDelta Notebook

Created May 19, 2015 03:15
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save oxinabox/f1cf9253b116e885839c to your computer and use it in GitHub Desktop.
Save oxinabox/f1cf9253b116e885839c to your computer and use it in GitHub Desktop.
Download ZIP
Raw
AdaDelta Notebook
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"###Bring in all the functions needed from Optim:"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"\n",
"using Optim\n",
"\n",
"import Optim.OptimizationTrace\n",
"import Optim.MultivariateOptimizationResults\n",
"\n",
"function maxdiff(x::Array, y::Array)\n",
" res = 0.0\n",
" for i in 1:length(x)\n",
" delta = abs(x[i] - y[i])\n",
" if delta > res\n",
" res = delta\n",
" end\n",
" end\n",
" return res\n",
"end\n",
"\n",
"function assess_convergence(x::Array,\n",
" x_previous::Array,\n",
" f_x::Real,\n",
" f_x_previous::Real,\n",
" gr::Array,\n",
" xtol::Real,\n",
" ftol::Real,\n",
" grtol::Real)\n",
" x_converged, f_converged, gr_converged = false, false, false\n",
"\n",
" if maxdiff(x, x_previous) < xtol\n",
" x_converged = true\n",
" end\n",
"\n",
" # Absolute Tolerance\n",
" # if abs(f_x - f_x_previous) < ftol\n",
" # Relative Tolerance\n",
" if abs(f_x - f_x_previous) / (abs(f_x) + ftol) < ftol || nextfloat(f_x) >= f_x_previous\n",
" f_converged = true\n",
" end\n",
"\n",
" if norm(vec(gr), Inf) < grtol\n",
" gr_converged = true\n",
" end\n",
"\n",
" converged = x_converged || f_converged || gr_converged\n",
"\n",
" return x_converged, f_converged, gr_converged, converged\n",
"end\n",
"\n",
"macro gdtrace()\n",
" quote\n",
" if tracing\n",
" dt = Dict()\n",
" if extended_trace\n",
" dt[\"x\"] = copy(x)\n",
" dt[\"g(x)\"] = copy(gr)\n",
" end\n",
" grnorm = norm(gr, Inf)\n",
" update!(tr,\n",
" iteration,\n",
" f_x,\n",
" grnorm,\n",
" dt,\n",
" store_trace,\n",
" show_trace)\n",
" end\n",
" end\n",
"end\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"##AdaDelta algorithm from [this paper](http://www.matthewzeiler.com/pubs/googleTR2012/googleTR2012.pdf)\n",
"\n",
"Note:\n",
"\n",
" - Does not use a linesearch, \n",
" - all the Optim Algorithms that do use linesearch make many calls to `d.fg` per iteration\n",
" - this only does one, therefor can afford to do many more iterations (in same amount of time).\n",
" - has two hyper parameters, the decay constant: ρ and the smoothing constant ϵ\n",
" - not highly sensitive to their values.\n",
" \n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"adadelta (generic function with 1 method)"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"function adadelta{T}(d::Union(DifferentiableFunction,\n",
" TwiceDifferentiableFunction),\n",
" initial_x::Array{T};\n",
" xtol::Real = 1e-32,\n",
" ftol::Real = 1e-8,\n",
" grtol::Real = 1e-8,\n",
" iterations::Integer = 20*1000,\n",
" store_trace::Bool = false,\n",
" show_trace::Bool = false,\n",
" extended_trace::Bool = false,\n",
" ρ = 0.95, #decay constant\n",
" ϵ = 1e-6 #smoothing constant \n",
" )\n",
" \n",
" @assert 0.0<ρ<1.0\n",
" @assert 0.0<ϵ\n",
" \n",
" function rms(x²)\n",
" √(x².+ϵ)\n",
" end\n",
" \n",
" function update_running_squared_average!(avg, g)\n",
" avg[:].*=ρ\n",
" avg[:].+=(1-ρ).*(g.^2)\n",
" end\n",
"\n",
" # Maintain current state in x\n",
" x = copy(initial_x)\n",
" x_previous = copy(initial_x)\n",
"\n",
" # Track calls to function and gradient\n",
" f_calls, g_calls = 0, 0\n",
"\n",
" # Count number of parameters\n",
" n = length(x)\n",
"\n",
" # Maintain current gradient in gr\n",
" gr = similar(x)\n",
" \n",
"\n",
" #Running windows of pass gradient and delta\n",
" E_gr² = zeros(size(x))\n",
" E_Δx² = zeros(size(x))\n",
"\n",
"\n",
" # Store f(x) in f_x\n",
" f_x_previous = NaN\n",
" f_x = d.fg!(x, gr)\n",
" \n",
" f_calls+=1\n",
" g_calls+=1\n",
"\n",
" f_x_best = f_x\n",
" x_best = x\n",
" \n",
" # TODO: How should this flag be set?\n",
" mayterminate = false\n",
"\n",
" # Trace the history of states visited\n",
" tr = OptimizationTrace()\n",
" tracing = store_trace || show_trace || extended_trace\n",
" @gdtrace\n",
"\n",
" # Assess multiple types of convergence\n",
" x_converged, f_converged, gr_converged = false, false, false\n",
"\n",
" converged = false\n",
" iteration = 0\n",
" while !converged && iteration < iterations\n",
" # Increment the number of steps we've had to perform\n",
" iteration += 1\n",
" \n",
" update_running_squared_average!(E_gr², gr)\n",
" Δx = -rms(E_Δx²)./rms(E_gr²) .* gr \n",
" update_running_squared_average!(E_Δx², Δx)\n",
" # Update current position\n",
" x_previous = x\n",
" x+=Δx\n",
" \n",
" # Update the function value and gradient\n",
" f_x_previous = f_x\n",
" f_x = d.fg!(x, gr)\n",
" f_calls+=1\n",
" g_calls+=1\n",
"\n",
" if f_x<=f_x_best\n",
" f_x_best = f_x\n",
" x_best=x\n",
" end\n",
" \n",
" \n",
" x_converged,\n",
" f_converged,\n",
" gr_converged,\n",
" converged = assess_convergence(x,\n",
" x_previous,\n",
" f_x,\n",
" f_x_previous,\n",
" gr,\n",
" xtol,\n",
" ftol,\n",
" grtol)\n",
"\n",
" @gdtrace\n",
" end\n",
"\n",
" return MultivariateOptimizationResults(\"AdaDelta\",\n",
" initial_x,\n",
" x_best,\n",
" f_x_best,\n",
" iteration,\n",
" iteration == iterations,\n",
" x_converged,\n",
" xtol,\n",
" f_converged,\n",
" ftol,\n",
" gr_converged,\n",
" grtol,\n",
" tr,\n",
" f_calls,\n",
" g_calls)\n",
"end\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"##Now lets test it"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false,
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"--- Exponential ---\n",
"elapsed time: 0.77886314 seconds\n",
"Results of Optimization Algorithm\n",
" * Algorithm: AdaDelta\n",
" * Starting Point: [0.0,0.0]\n",
" * Minimum: [2.0,2.999765047042759]\n",
" * Value of Function at Minimum: 2.000000\n",
" * Iterations: 1813\n",
" * Convergence: true\n",
" * |x - x'| < 1.0e-32: false\n",
" * |f(x) - f(x')| / |f(x)| < 1.0e-08: true\n",
" * |g(x)| < 1.0e-08: false\n",
" * Exceeded Maximum Number of Iterations: false\n",
" * Objective Function Calls: 1814\n",
" * Gradient Call: 1814\n",
"\n",
"eg.solutions => [2.0,3.0]\n",
"eg.f(eg.solutions) => 2.0\n",
"res.minimum => [2.0,2.999765047042759]\n",
"res.f_minimum => 2.000000055202894\n",
"--------------------------------------------------\n",
"--- Fletcher-Powell ---\n",
"elapsed time: 0.019652169 seconds\n",
"Results of Optimization Algorithm\n",
" * Algorithm: AdaDelta\n",
" * Starting Point: [-1.0,0.0,0.0]\n",
" * Minimum: [-1.0,-6.91823692886755e-310,-7.013163e-317]\n",
" * Value of Function at Minimum: 2500.000000\n",
" * Iterations: 1\n",
" * Convergence: true\n",
" * |x - x'| < 1.0e-32: true\n",
" * |f(x) - f(x')| / |f(x)| < 1.0e-08: true\n",
" * |g(x)| < 1.0e-08: true\n",
" * Exceeded Maximum Number of Iterations: false\n",
" * Objective Function Calls: 2\n",
" * Gradient Call: 2\n",
"\n",
"eg.solutions => [1.0,0.0,0.0]\n",
"eg.f(eg.solutions) => 0.0\n",
"res.minimum => [-1.0,-6.91823692886755e-310,-7.013163e-317]\n",
"res.f_minimum => 2500.0\n",
"--------------------------------------------------\n",
"--- Rosenbrock ---\n",
"elapsed time: 0.008720374 seconds\n",
"Results of Optimization Algorithm\n",
" * Algorithm: AdaDelta\n",
" * Starting Point: [0.0,0.0]\n",
" * Minimum: [0.4200081647788573,0.1511715520070132]\n",
" * Value of Function at Minimum: 0.400073\n",
" * Iterations: 120\n",
" * Convergence: true\n",
" * |x - x'| < 1.0e-32: false\n",
" * |f(x) - f(x')| / |f(x)| < 1.0e-08: true\n",
" * |g(x)| < 1.0e-08: false\n",
" * Exceeded Maximum Number of Iterations: false\n",
" * Objective Function Calls: 121\n",
" * Gradient Call: 121\n",
"\n",
"eg.solutions => [1.0,1.0]\n",
"eg.f(eg.solutions) => 0.0\n",
"res.minimum => [0.4200081647788573,0.1511715520070132]\n",
"res.f_minimum => 0.40007259820630725\n",
"--------------------------------------------------\n",
"--- Parabola ---\n",
"elapsed time: 0.024784996 seconds\n",
"Results of Optimization Algorithm\n",
" * Algorithm: AdaDelta\n",
" * Starting Point: [0.0,0.0,0.0,0.0,0.0]\n",
" * Minimum: [1.0,2.0,3.0,5.0,7.999999992879856]\n",
" * Value of Function at Minimum: 0.000000\n",
" * Iterations: 2028\n",
" * Convergence: true\n",
" * |x - x'| < 1.0e-32: false\n",
" * |f(x) - f(x')| / |f(x)| < 1.0e-08: true\n",
" * |g(x)| < 1.0e-08: false\n",
" * Exceeded Maximum Number of Iterations: false\n",
" * Objective Function Calls: 2029\n",
" * Gradient Call: 2029\n",
"\n",
"eg.solutions => [1.0,2.0,3.0,5.0,8.0]\n",
"eg.f(eg.solutions) => 0.0\n",
"res.minimum => [1.0,2.0,3.0,5.0,7.999999992879856]\n",
"res.f_minimum => 5.069644468293596e-17\n",
"--------------------------------------------------\n",
"--- Hosaki ---\n",
"elapsed time: 0.02218739 seconds\n",
"Results of Optimization Algorithm\n",
" * Algorithm: AdaDelta\n",
" * Starting Point: [3.6,1.9]\n",
" * Minimum: [3.999783284989749,1.9999999999999998]\n",
" * Value of Function at Minimum: -2.345811\n",
" * Iterations: 222\n",
" * Convergence: true\n",
" * |x - x'| < 1.0e-32: false\n",
" * |f(x) - f(x')| / |f(x)| < 1.0e-08: true\n",
" * |g(x)| < 1.0e-08: false\n",
" * Exceeded Maximum Number of Iterations: false\n",
" * Objective Function Calls: 223\n",
" * Gradient Call: 223\n",
"\n",
"eg.solutions => [4.0,2.0]\n",
"eg.f(eg.solutions) => -2.345811576101292\n",
"res.minimum => [3.999783284989749,1.9999999999999998]\n",
"res.f_minimum => -2.3458114998375663\n",
"--------------------------------------------------\n",
"--- Large Polynomial ---\n",
"elapsed time: 1.214310908 seconds\n",
"Results of Optimization Algorithm\n",
" * Algorithm: AdaDelta\n",
" * Starting Point: [0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]\n",
" * Minimum: [1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0,51.0,52.0,53.0,54.0,55.0,56.0,57.0,58.0,59.0,60.0,61.0,62.0,63.0,64.0,65.0,66.0,67.0,68.0,69.0,70.0,71.0,72.0,73.0,74.0,75.0,76.0,77.0,78.0,79.0,80.0,81.0,82.0,83.0,84.0,85.0,86.0,87.0,88.0,89.0,90.0,91.0,92.0,93.0,94.0,95.0,96.0,97.0,98.0,99.0,100.0,101.0,102.0,103.0,104.0,105.0,106.0,107.0,108.0,109.0,110.0,111.0,112.0,113.0,114.0,115.0,116.0,117.0,118.0,119.0,120.0,121.0,122.0,123.0,124.0,125.0,126.0,127.0,128.0,129.0,130.0,131.0,132.0,133.0,134.0,135.0,136.0,137.0,138.0,139.0,140.0,141.0,142.0,143.0,144.0,145.0,146.0,147.0,148.0,149.0,150.0,151.0,152.0,153.0,154.0,155.0,156.0,157.0,158.0,159.0,160.0,161.0,162.0,163.0,164.0,165.0,166.0,167.0,168.0,169.0,170.0,171.0,172.0,173.0,174.0,175.0,176.0,177.0,178.0,179.0,180.0,181.0,182.0,183.0,184.0,185.0,186.0,187.0,188.0,189.0,190.0,191.0,192.0,193.0,194.0,195.0,196.0,197.0,198.0,199.0,200.0,201.0,202.0,203.0,204.0,205.0,206.0,207.0,208.0,209.0,210.0,211.0,212.0,213.0,214.0,215.0,216.0,217.0,218.0,219.0,220.0,221.0,222.0,223.0,224.0,225.0,226.0,227.0,228.0,229.0,230.0,231.0,232.0,232.99997412753697,233.99214207822638,234.95196363881726,235.87975557137312,236.7821084987402,237.663322230325,238.52622923086832,239.3728385132199,240.20466086103755,241.02288270539484,241.82846668184672,242.6222138200708,243.40480415589707,244.1768244501984,244.93878780700442,245.69114798433657,246.434310103234,247.16863883797214]\n",
" * Value of Function at Minimum: 35.700937\n",
" * Iterations: 20000\n",
" * Convergence: false\n",
" * |x - x'| < 1.0e-32: false\n",
" * |f(x) - f(x')| / |f(x)| < 1.0e-08: false\n",
" * |g(x)| < 1.0e-08: false\n",
" * Exceeded Maximum Number of Iterations: true\n",
" * Objective Function Calls: 20001\n",
" * Gradient Call: 20001\n",
"\n",
"eg.solutions => [1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0,51.0,52.0,53.0,54.0,55.0,56.0,57.0,58.0,59.0,60.0,61.0,62.0,63.0,64.0,65.0,66.0,67.0,68.0,69.0,70.0,71.0,72.0,73.0,74.0,75.0,76.0,77.0,78.0,79.0,80.0,81.0,82.0,83.0,84.0,85.0,86.0,87.0,88.0,89.0,90.0,91.0,92.0,93.0,94.0,95.0,96.0,97.0,98.0,99.0,100.0,101.0,102.0,103.0,104.0,105.0,106.0,107.0,108.0,109.0,110.0,111.0,112.0,113.0,114.0,115.0,116.0,117.0,118.0,119.0,120.0,121.0,122.0,123.0,124.0,125.0,126.0,127.0,128.0,129.0,130.0,131.0,132.0,133.0,134.0,135.0,136.0,137.0,138.0,139.0,140.0,141.0,142.0,143.0,144.0,145.0,146.0,147.0,148.0,149.0,150.0,151.0,152.0,153.0,154.0,155.0,156.0,157.0,158.0,159.0,160.0,161.0,162.0,163.0,164.0,165.0,166.0,167.0,168.0,169.0,170.0,171.0,172.0,173.0,174.0,175.0,176.0,177.0,178.0,179.0,180.0,181.0,182.0,183.0,184.0,185.0,186.0,187.0,188.0,189.0,190.0,191.0,192.0,193.0,194.0,195.0,196.0,197.0,198.0,199.0,200.0,201.0,202.0,203.0,204.0,205.0,206.0,207.0,208.0,209.0,210.0,211.0,212.0,213.0,214.0,215.0,216.0,217.0,218.0,219.0,220.0,221.0,222.0,223.0,224.0,225.0,226.0,227.0,228.0,229.0,230.0,231.0,232.0,233.0,234.0,235.0,236.0,237.0,238.0,239.0,240.0,241.0,242.0,243.0,244.0,245.0,246.0,247.0,248.0,249.0,250.0]\n",
"eg.f(eg.solutions) => 0.0\n",
"res.minimum => [1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0,51.0,52.0,53.0,54.0,55.0,56.0,57.0,58.0,59.0,60.0,61.0,62.0,63.0,64.0,65.0,66.0,67.0,68.0,69.0,70.0,71.0,72.0,73.0,74.0,75.0,76.0,77.0,78.0,79.0,80.0,81.0,82.0,83.0,84.0,85.0,86.0,87.0,88.0,89.0,90.0,91.0,92.0,93.0,94.0,95.0,96.0,97.0,98.0,99.0,100.0,101.0,102.0,103.0,104.0,105.0,106.0,107.0,108.0,109.0,110.0,111.0,112.0,113.0,114.0,115.0,116.0,117.0,118.0,119.0,120.0,121.0,122.0,123.0,124.0,125.0,126.0,127.0,128.0,129.0,130.0,131.0,132.0,133.0,134.0,135.0,136.0,137.0,138.0,139.0,140.0,141.0,142.0,143.0,144.0,145.0,146.0,147.0,148.0,149.0,150.0,151.0,152.0,153.0,154.0,155.0,156.0,157.0,158.0,159.0,160.0,161.0,162.0,163.0,164.0,165.0,166.0,167.0,168.0,169.0,170.0,171.0,172.0,173.0,174.0,175.0,176.0,177.0,178.0,179.0,180.0,181.0,182.0,183.0,184.0,185.0,186.0,187.0,188.0,189.0,190.0,191.0,192.0,193.0,194.0,195.0,196.0,197.0,198.0,199.0,200.0,201.0,202.0,203.0,204.0,205.0,206.0,207.0,208.0,209.0,210.0,211.0,212.0,213.0,214.0,215.0,216.0,217.0,218.0,219.0,220.0,221.0,222.0,223.0,224.0,225.0,226.0,227.0,228.0,229.0,230.0,231.0,232.0,232.99997412753697,233.99214207822638,234.95196363881726,235.87975557137312,236.7821084987402,237.663322230325,238.52622923086832,239.3728385132199,240.20466086103755,241.02288270539484,241.82846668184672,242.6222138200708,243.40480415589707,244.1768244501984,244.93878780700442,245.69114798433657,246.434310103234,247.16863883797214]\n",
"res.f_minimum => 35.700937365274285\n",
"--------------------------------------------------\n",
"--- Polynomial ---\n",
"elapsed time: 0.153339817 seconds\n",
"Results of Optimization Algorithm\n",
" * Algorithm: AdaDelta\n",
" * Starting Point: [0.0,0.0,0.0]\n",
" * Minimum: [10.0,6.997321404684813,107.99388626587705]\n",
" * Value of Function at Minimum: 0.000000\n",
" * Iterations: 20000\n",
" * Convergence: false\n",
" * |x - x'| < 1.0e-32: false\n",
" * |f(x) - f(x')| / |f(x)| < 1.0e-08: false\n",
" * |g(x)| < 1.0e-08: false\n",
" * Exceeded Maximum Number of Iterations: true\n",
" * Objective Function Calls: 20001\n",
" * Gradient Call: 20001\n",
"\n",
"eg.solutions => [10.0,7.0,108.0]\n",
"eg.f(eg.solutions) => 0.0\n",
"res.minimum => [10.0,6.997321404684813,107.99388626587705]\n",
"res.f_minimum => 1.4485746163543534e-9\n",
"--------------------------------------------------\n",
"--- Powell ---\n",
"elapsed time: 0.027589508 seconds\n",
"Results of Optimization Algorithm\n",
" * Algorithm: AdaDelta\n",
" * Starting Point: [3.0,-1.0,0.0,1.0]\n",
" * Minimum: [0.20814560669179713,-0.03037630017387952,0.09227197239584813,0.09369301302906907]\n",
" * Value of Function at Minimum: 0.013002\n",
" * Iterations: 1337\n",
" * Convergence: true\n",
" * |x - x'| < 1.0e-32: false\n",
" * |f(x) - f(x')| / |f(x)| < 1.0e-08: true\n",
" * |g(x)| < 1.0e-08: false\n",
" * Exceeded Maximum Number of Iterations: false\n",
" * Objective Function Calls: 1338\n",
" * Gradient Call: 1338\n",
"\n",
"eg.solutions => [0.0,0.0,0.0,0.0]\n",
"eg.f(eg.solutions) => 0.0\n",
"res.minimum => [0.20814560669179713,-0.03037630017387952,0.09227197239584813,0.09369301302906907]\n",
"res.f_minimum => 0.013002306671999522\n",
"--------------------------------------------------\n",
"--- Himmelblau ---\n",
"elapsed time: 0.015792959 seconds\n",
"Results of Optimization Algorithm\n",
" * Algorithm: AdaDelta\n",
" * Starting Point: [2.0,2.0]\n",
" * Minimum: [2.999999894522482,2.000000027434358]\n",
" * Value of Function at Minimum: 0.000000\n",
" * Iterations: 467\n",
" * Convergence: true\n",
" * |x - x'| < 1.0e-32: false\n",
" * |f(x) - f(x')| / |f(x)| < 1.0e-08: true\n",
" * |g(x)| < 1.0e-08: false\n",
" * Exceeded Maximum Number of Iterations: false\n",
" * Objective Function Calls: 468\n",
" * Gradient Call: 468\n",
"\n",
"eg.solutions => [3.0,2.0]\n",
"eg.f(eg.solutions) => 0.0\n",
"res.minimum => [2.999999894522482,2.000000027434358]\n",
"res.f_minimum => 3.6656452591459947e-13\n",
"--------------------------------------------------\n"
]
}
],
"source": [
"using Optim.UnconstrainedProblems\n",
"import Optim.UnconstrainedProblems.examples\n",
"\n",
"for problemName in keys(examples)\n",
" println(\"--- $problemName ---\")\n",
" eg = examples[problemName]\n",
"\n",
" f=TwiceDifferentiableFunction(eg.f,eg.g!,eg.h!)\n",
" tic()\n",
" res = adadelta(f, eg.initial_x)\n",
" timetaken = toc()\n",
" \n",
" println(res)\n",
" println()\n",
" \n",
" @show eg.solutions\n",
" @show eg.f(eg.solutions)\n",
" @show res.minimum\n",
" @show res.f_minimum\n",
" \n",
" println(\"-\"^50)\n",
"end\n",
" "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"##discussion\n",
"Compaired to [benchmarks](https://github.com/JuliaOpt/Optim.jl/blob/master/benchmarks/results.tsv) from the other algorithms is does awefully.\n",
"Honestly not sure why. Suspect it needs linesearch.\n",
"\n",
"Could be that it is just that it is better for problems with very large dimentionality and complex functions/gredients such as machine learning problems (like MNIST)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Julia 0.3.8",
"language": "julia",
"name": "julia-0.3"
},
"language_info": {
"name": "julia",
"version": "0.3.9"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment