A sample of iterative selection of parameters using OACIS.

README.md

Iterative Trial of Candidate Parameters

Some simulations, such as convergence calculations, need trials and errors for selecting appropriate parameters until an expected result is obtained.

In this sample, we demonstrate how to automate this kind of iterations. This sample tries candidate parameters, which are given by us in advance, one by one until an expected result is obtained.

Prerequisites

Register simulator as follows.

• Name: "sequential_trial_test"
• Parameter Definitions:
  • "p1", String, "foo"
  • "p2", Float, 1.0
  • "p3", Float, 2.0
• Command:
  • ruby -r json -e 'res=(rand<0.5)?1:0; puts({"result"=>res}.to_json)' > _output.json
• Input type: JSON
• Executable_on : localhost

The following command will register this simulator in your OACIS.

oacis_ruby prepare_simulator.rb

What does this sample code do?

For each parameter "p1", we try several set of values "p2" and "p3" until we found an expected results. In this sample, we find a result satisfactory when the result=1 is obtained from the simulator.

The candidates of ("p2","p3")-pair are given in YAML in the following format.

-
  base: {p1: "foo"}
  candidates:
    - {p2: 1.0, p3: 1.0}
    - {p2: 1.5, p3: 2.0}
    - {p2: 2.0, p3: 2.0}
    - {p2: 2.5, p3: 3.0}
-
  base: {p1: "bar"}
  candidates:
    - {p2: 10.0, p3: 0.0}
    - {p2: 11.0, p3: 1.0}
    - {p2: 12.0, p3: 2.0}
    - {p2: 13.0, p3: 3.0}
-
  base: {p1: "baz"}
  candidates:
    - {p2: 5.0, p3: -1.0}
    - {p2: 4.0, p3: -2.0}
    - {p2: 3.0, p3: -3.0}
    - {p2: 2.0, p3: -4.0}

For each "p1", we try the first candidates. If they are not satisfactory, find the next candidate and executes the job. The iteration continues until we found satisfactory results or no futher candidate is found.

How to run

There are two scripts written in Ruby and Python. You can use either one of them.

oacis_ruby try_candidates.rb candidates.yml

or

oacis_python try_candidates.py candidates.yml

candidates.yml

-
  base: {p1: "foo"}
  candidates:
    - {p2: 1.0, p3: 1.0}
    - {p2: 1.5, p3: 2.0}
    - {p2: 2.0, p3: 2.0}
    - {p2: 2.5, p3: 3.0}
-
  base: {p1: "bar"}
  candidates:
    - {p2: 10.0, p3: 0.0}
    - {p2: 11.0, p3: 1.0}
    - {p2: 12.0, p3: 2.0}
    - {p2: 13.0, p3: 3.0}
-
  base: {p1: "baz"}
  candidates:
    - {p2: 5.0, p3: -1.0}
    - {p2: 4.0, p3: -2.0}
    - {p2: 3.0, p3: -3.0}
    - {p2: 2.0, p3: -4.0}

prepare_simulator.rb

if sim = Simulator.where(name: "sequential_trial_test").first
  $stderr.puts "already Simulator '#{sim.name}' exists. Deleting this."
  sim.discard
end
sim = Simulator.create!(
  name: "sequential_trial_test",
  parameter_definitions: [
    ParameterDefinition.new(key: "p1", type: "String", default: "foo"),
    ParameterDefinition.new(key: "p2", type: "Float", default: 1.0),
    ParameterDefinition.new(key: "p3", type: "Float", default: 2.0)
  ],
  command: "ruby -r json -e 'res=(rand<0.5)?1:0; puts({\"result\"=>res}.to_json)' > _output.json",
  executable_on: [Host.where(name: "localhost").first]
)
$stderr.puts "A new simulator #{sim.id} is created."

try_candidates.py

import oacis

class CandidatesProvider():

    def __init__(self, base_param, candidate_params, watcher):
        self.watcher = watcher
        self.base = base_param
        self.candidates = candidate_params

        self.sim = oacis.Simulator.where(name="sequential_trial_test").first()
        self.host = oacis.Host.where(name="localhost").first()
        self.host_param = self.host.default_host_parameters()

    def initial_parameter(self):
        param = self.base.copy()
        param.update( self.candidates[0] )
        return param

    def create_ps_and_run(self, param):
        ps = self.sim.find_or_create_parameter_set( param )
        ps.find_or_create_runs_upto(1, submitted_to=self.host, host_param=self.host_param)
        print("Created a new PS: %s" % repr(ps.v()) )

        def on_ps_finished(ps):
            if self.need_another_trial( ps ):
                print("ParameterSet: %s needs another trial. Creating a next run." % repr(ps.v()) )
                self.create_next_ps_and_run( ps )
            else:
                print("ParameterSet: %s does not need another trial." % repr(ps.v()) )
        self.watcher.watch_ps( ps, on_ps_finished )

    def need_another_trial(self, ps):
        return (ps.runs().first().result()["result"] == 0)

    def create_next_ps_and_run(self, ps):
        next_param = self.find_next_candidate( ps.v() )
        if next_param:
            self.create_ps_and_run( next_param )

    def find_next_candidate(self, current_param):
        found_idx = None
        for idx,cand in enumerate(self.candidates):
            if all( current_param[key]==val for key,val in cand.items() ):
                found_idx = idx
                break
        assert isinstance( found_idx, int )
        next_idx = found_idx + 1
        if next_idx >= len(self.candidates):
            return None
        else:
            param = self.base.copy()
            param.update( self.candidates[next_idx] )
            return param

if __name__ == "__main__":
    import yaml,sys
    def main():
        watcher = oacis.OacisWatcher()
        f = open(sys.argv[1])
        y = yaml.load(f)
        f.close()

        for cand in y:
            print(repr(cand))
            c = CandidatesProvider( cand['base'], cand['candidates'], watcher )
            c.create_ps_and_run( c.initial_parameter() )
        watcher.loop()

    main()

try_candidates.rb

require 'yaml'

class CandidatesProvider

  def initialize(base_param, candidate_params, watcher)
    @watcher = watcher
    @base = base_param
    @candidates = candidate_params

    @sim = Simulator.where(name: "sequential_trial_test").first
    @host = Host.where(name:"localhost").first
    @host_param = @host.default_host_parameters
  end

  def initial_parameter
    @base.merge( @candidates.first )
  end

  def create_ps_and_run( param )
    ps = @sim.find_or_create_parameter_set( param )
    ps.find_or_create_runs_upto(1, submitted_to: @host, host_param: @host_param)
    @watcher.logger.info "Created a new PS: #{ps.v}"

    @watcher.watch_ps( ps ) do |completed|
      if need_another_trial?( completed )
        @watcher.logger.info "ParameterSet: #{completed.v} needs another trial. Creating a next run."
        create_next_ps_and_run( completed )
      else
        @watcher.logger.info "ParameterSet: #{completed.v} does not need another trial."
      end
    end

  end

  def need_another_trial?( ps )
    ps.runs.first.result["result"] == 0
  end

  def create_next_ps_and_run( ps )
    next_param = find_next_candidate( ps.v )
    create_ps_and_run( next_param ) if next_param
  end

  def find_next_candidate( current_param )
    current_idx = @candidates.index do |cand_param|
      cand_param.each_pair.all? {|key,val| current_param[key]==val }
    end
    next_idx = current_idx+1

    @candidates[next_idx] ? @base.merge(@candidates[next_idx]) : nil
  end
end

OacisWatcher::start do |w|
  y = YAML.load( File.open(ARGV.first) )
  y.each do |cand|
    c = CandidatesProvider.new( cand["base"], cand["candidates"], w )
    c.create_ps_and_run( c.initial_parameter )
  end
end