Adaptive Simulation with Repast Simphony and Swift supplemental materials (PADABS 2014 Workshop paper)

createUpfs.sh

#!/bin/bash

params=$1
vals=$2
num=$3
outDir=$4

cPath="lib/*"
parameters_file="scenario.rs/parameters.xml"

java -Xmx512m -cp "$cPath" simanneal.params.UPFCreator \
        -pfile "$parameters_file" \
        -params "$params" \
        -vals "$vals" \
        -num "$num" \
        -out "$outDir"

input.txt

zombieStep humanStep zombie_count human_count upref
0.7 1.1 3 80 exp0000
0.7 1.1 3 120 exp0001
0.7 1.1 7 80 exp0010
0.7 1.1 7 120 exp0011
0.7 2.2 3 80 exp0100
0.7 2.2 3 120 exp0101
0.7 2.2 7 80 exp0110
0.7 2.2 7 120 exp0111
1.3 1.1 3 80 exp1000
1.3 1.1 3 120 exp1001
1.3 1.1 7 80 exp1010
1.3 1.1 7 120 exp1011
1.3 2.2 3 80 exp1100
1.3 2.2 3 120 exp1101
1.3 2.2 7 80 exp1110
1.3 2.2 7 120 exp1111

mapper.sh

#!/bin/bash
find $2 -type f | awk '{printf("[%d] %s\n", NR-1, $0)}'

math.swift

(float result) sin(float x)
{
   result = java("java.lang.Math", "sin", x);
}

(float result) exp(float x)
{
   result = java("java.lang.Math", "exp", x);
}

(float result) jlog(float x)
{
   result = java("java.lang.Math", "log", x);
}

(float result) log10(float x)
{
   result = java("java.lang.Math", "log10", x);
}

(float result) ceil(float x)
{
   result = java("java.lang.Math", "ceil", x);
}

(float result) floor (float x)
{
   result = java("java.lang.Math", "floor", x);
}

(float result) min (float a, float b)
{
   if ( a < b ) {
       result = a;
   }
   else {
       result = b;
   }
   //tracef("math/min: result=%f\n",result);
}

(int result) minI (int a, int b)
{
   
   if ( a < b ) {
       result = a;
   }
   else {
       result = b;
   }
   //tracef("math/minI: result=%i\n",result);
}

(float result) max (float a, float b)
{
   if ( a > b ) {
       result = a;
   }
   else {
       result = b;
   }
   //tracef("math/max: result=%f\n",result);
}

(int result) maxI (int a, int b)
{
   
   if ( a > b ) {
       result = a;
   }
   else {
       result = b;
   }
   //tracef("math/maxI: result=%i\n",result);
}

(float result) pow (float x, float y)
{
   result = java("java.lang.Math", "pow", x, y);
}

(float result) random ()
{
  result = java("java.lang.Math","random");
}

(int result) round (float x)
{
    result = java("java.lang.Math","round",x);
}

obj.sh

#!/bin/bash

param_file=$1
result_file=$2

#echo "param_file: $param_file"
instanceDir="instance_xxx"

mkdir $instanceDir
cd $instanceDir
ln -s "../data" data

cPath="../lib/*"
#echo $cPath
pxml="../scenario.rs/batch_params.xml"
#echo $pxml
scenario="../scenario.rs"
#echo $scenario
java -Xmx512m -cp "$cPath" repast.simphony.batch.InstanceRunner \
        -pxml "$pxml" \
        -scenario "$scenario" \
        -id 1 \
        -pinput "../$param_file"

result=$(grep ModelOutput.*_??.txt -e "^1,150" | cut -d "," -f3)
result=$((-$result))
#echo "result: $result"
cd ..
mkdir -p "$(dirname "$result_file")" && touch "$result_file"
echo $result > $result_file

sa.swift

import "math";

type file;

global file cUpfsScript <"createUpfs.sh">;                      # parameter creation script
global file scenario[] <filesys_mapper;location="scenario.rs">; # scenario directory
global file libs[] <filesys_mapper;location="lib">;             # jars in the lib directory
global file pf <"MessageCenter.log4j.properties">;              # properties file
global file cpf <"config.props">;                               # config.props file
global file obj_script <"obj.sh">;                              # obj script
global file summarize_script <"summarize.sh">;                  # summarize script
global file data[] <ext;exec="mapper.sh",folder="data">;        # data directory

/**
 *  Call to wrapper script that creates the stochastic variation parameter files    
 */
app (file [] _unrolledParamFiles, file so, file se) createUpfs (
            string _params, string _vals, int _num, file _cUpfsScript,
            file [] _scenario, file [] _libs, string _outDir)
{
    sh @_cUpfsScript _params _vals toString(_num) _outDir stdout=@so stderr=@se;
}


/**
 *  Calculates the summarized objective function for the specified parameters "params"
 *  with the specified values "vs", where all other model parameters are taken from 
 *  the parameters.xml file.
 */
(file sum_result) sum_obj (string params, string vs, int num, int iteration, string _upref)
{

    string outDir = strcat(_upref,"_out",toString(iteration));
    file fs[] <simple_mapper; location=outDir, prefix="upf_", suffix=".txt">;
    tracef("\nTrying params: %s, vs: %s, num: %i \n", params, vs, num);
    string outDirSim = strcat(_upref,"_sim",outDir);
    string outDirErr = strcat(_upref,"_err",outDir);
    file sim <concurrent_mapper;prefix=outDirSim>;
    file err <concurrent_mapper;prefix=outDirErr>;
    (fs, sim, err) = createUpfs(params, vs, num, cUpfsScript, scenario, libs, outDir);
    float fresults[];
    
    foreach ff,i in fs{
        string outDirRes = strcat(_upref,"_result_",toString(iteration),"_",toString(i));
        file result <concurrent_mapper;prefix=outDirRes>; # concurrent mapper
        string outDirStdo = strcat(_upref,"_stdo_",toString(iteration),"_",toString(i));
        string outDirStde = strcat(_upref,"_stde_",toString(iteration),"_",toString(i));
        file stdo <concurrent_mapper;prefix=outDirStdo>;
        file stde <concurrent_mapper;prefix=outDirStde>;
        (result,stdo,stde) = obj(obj_script,libs,ff,cpf,scenario,data,pf);
        string res_string = readData(result);
        fresults[i] = toFloat(res_string);
        
    }
    
    tracef("for iteration %i: results are %s\n", iteration, strjoin(fresults,","));
    
    string outDirStdo_sr = strcat(_upref,"_stdo_sr_",toString(iteration));
    string outDirStde_sr = strcat(_upref,"_stde_sr",toString(iteration));
    file stdo_sr <concurrent_mapper;prefix=outDirStdo_sr>;
    file stde_sr <concurrent_mapper;prefix=outDirStde_sr>;
    
    (sum_result, stdo_sr, stde_sr) = summarize(summarize_script,fresults);
    
}


/**

    Inputs:
        file _obj_script:   obj.sh script
        file [] _libs:      repast libraries (jar files) in lib folder
        file _upf:          unrolled parameters file
        file _cpf:          config.props file
        file [] _scenario:  contents of scenario.rs folder
        file [] _data:          contents of data folder
        file _pf:           repast MessageCenter properties file
    
    Outputs:
        file _result:   file with result
        file std_out:   file with standard out output
        file std_err:   file with standard error output

 */
app (file _result , file _o, file _e) obj (file _obj_script, file [] _libs,
        file _upf, file _cpf, file [] _scenario, file [] _data, file _pf){
    sh @_obj_script @_upf @_result stdout=@_o stderr=@_e;
}

app (file _result, file _o, file _e) summarize (file _summarize_script, float [] _f) {
    sh @_summarize_script strjoin(_f," ") @_result stdout=@_o stderr=@_e;// summarization call 
}

(boolean a) accept (float cur, int cy, float prev, int cycles)
{
    float T_start = 25.0;
    float T_end   = 0.1;
    float t =  T_start*exp( toFloat( cy ) * ( jlog(T_end) - jlog(T_start) ) / toFloat(cycles) );
    tracef("******** Temperature for cycle %i is %f\n",cy,t);
    if (cur < prev) {
        a = true;
    }
    else {        
        float ratio = min( 1.0, exp( - (cur-prev) / t ) );
        float r = (random()); // a random float between [0.0-1.0]
        if (r > ratio) { // Reject new parameter
            a = false;
        }
        else {
            a = true;
        }
    }
}

( float nx ) newx_float ( float x, float dx , float _min, float _max)
{
  float r = (random());

  if (r > 0.66) {
    nx = max(min(_max,x + dx),_min);
  }
  else {
    if (r > 0.33) {
        nx = max(min(_max,x - dx),_min);
    }
    else {
        nx = x;
    }   
  }
  #tracef("newx_float(%f,%f)=%f\n",x,dx,nx);
}

( int nx ) newx_int ( int x, int dx , int _min, int _max)
{
  float r = (random());

  if (r > 0.66) {
    nx = maxI(minI(_max,x + dx),_min);
  }
  else {
    if (r > 0.33){
        nx = maxI(minI(_max,x - dx),_min);
    }
    else {
        nx = x;
    }
  }
  #tracef("newx_int(%i,%i)=%i\n",x,dx,nx);
}

(string result) createStringArray (float [] _fs, int [] _is)
{
    string fs = strjoin(_fs,",");
    string is = strjoin(_is,",");
    result = strcat(fs,",",is);

}

main(string _params, float [] _initValsf, int [] _initValsi, int max_iterations, int variations, string _upref)
{
  # optimizer_sweep();

  float a[][];      # new candidate a's
  float aa[][];     # latest accepted set of a's
  float a_best[][]; # best set of a's
  
  int b[][];        # new candidate b's
  int bb[][];       # latest accepted set of b's
  int b_best[][];   # best set of b's
  
  float y[];
  float y_best[];
  
  a[0] = _initValsf;
  b[0] = _initValsi;
  string ab = createStringArray(a[0],b[0]);

  
  string u_sr = strcat(_upref,"_sr_");
  file sr <concurrent_mapper;prefix=u_sr>;
  sr = sum_obj(_params,ab,variations,0,_upref);
  string sum_result_string = readData(sr);
  
  y[0] = toFloat(sum_result_string);
  y_best[0] = y[0];
  aa[0] = a[0];
  a_best[0] = a[0];
  bb[0] = b[0];
  b_best[0] = b[0];


  iterate i {
    int c = i + 1;
    a[c] = [ newx_float(aa[i][0],0.1,0.1,2.0), newx_float(aa[i][1],0.1,1.0,3.0)];
    b[c] = [ newx_int(bb[i][0],1,1,10), newx_int(bb[i][1],10,50,150) ];
    string abc = createStringArray(a[c],b[c]);
    #tracef("the abc string is %s\n",abc);
    string u_sri = strcat(_upref,"_sri_",toString(i));
    file sri <concurrent_mapper;prefix=u_sri>;
    sri = sum_obj(_params,abc,variations,c,_upref);
    string srs = readData(sri);
    float ny = toFloat(srs);

    if ( accept(ny,c,y[c-1], max_iterations) ) {
      y[c] = ny;
      aa[c] = a[c];
      bb[c] = b[c];
    }
    else {
      y[c] = y[c-1];
      aa[c] = aa[c-1];
      bb[c] = bb[c-1];
    }

    if (ny < y_best[c-1]){ # Check for best solution
        y_best[c] = ny;
        a_best[c] = a[c];
        b_best[c] = b[c];
    }
    else {
        y_best[c] = y_best[c-1];
        a_best[c] = a_best[c-1];
        b_best[c] = b_best[c-1];
    }
    trace(c,y[c],y_best[c]);
  } until (i==max_iterations);

    tracef("\nThe final parameter values for %s are \nparameters: %s\na_best[]: %s b_best[]: %s\n",
            _upref, _params, toString(a_best[max_iterations]),toString(b_best[max_iterations]));

    string f_output_string = strcat(_upref,"_output.txt");
    file f_output <single_file_mapper;file=f_output_string>;
    Output output[];
    foreach yb,i in y_best{
        output[i].y_best = y_best[i];
        output[i].zombieStep = a_best[i][0];
        output[i].humanStep = a_best[i][1];
        output[i].zombie_count = b_best[i][0];
        output[i].human_count = b_best[i][1];
    }
    f_output=writeData(output);
}

type Input{
    float zombieStep;
    float humanStep;
    int zombie_count;
    int human_count;
    string upref;
}

type Output{
    float y_best;
    float zombieStep;
    float humanStep;
    int zombie_count;
    int human_count; 
}

Input inputs[] = readData(arg("i","input.txt")); # optional -i input file command line option 

foreach input in inputs {
    string parameters_to_vary = "zombieStep,humanStep,zombie_count,human_count";
    float [] initValsf = [input.zombieStep,input.humanStep];    # initial float parameters
    int [] initValsi = [input.zombie_count,input.human_count];  # initial int parameters
    string upref = input.upref;                                 # unique prefix for inputs
    
    int stochastic_variations = toInt(arg("sv","3"));           # optional -sv command line option
    int annealing_loops = toInt(arg("al","30"));                # optional -al command line option
    main(parameters_to_vary,initValsf,initValsi,
        annealing_loops,stochastic_variations,
        upref);
}

summarize.sh

#!/bin/bash

array=$1
result_file=$2

#echo "array: $array"

mkdir -p "$(dirname "$result_file")" && touch "$result_file"

result=$(echo $array | awk '{sum = 0; n=0; for (i=1; i <=NF; i++) {sum+=$i; ++n} print sum/n}')

#echo "result: $result"

echo $result > $result_file

swift.properties

#
# Local sites
#
site.local {
   jobManager=local
   tasksPerWorker=4
   filesystem=local
   workdir=$RUNDIRECTORY
   taskThrottle=11
}
site=midway
app.local.sh=/bin/sh
app.midway.sh=/bin/bash
#file.gc.enabled=false

#
# Midway sites
#

site.midway {
   jobManager=slurm
   jobQueue=sandyb
   maxJobs=16
   jobGranularity=1
   maxNodesPerJob=1
   tasksPerWorker=16
   taskThrottle=512
   initialScore=10000
   jobWalltime=01:00:00
   taskWalltime=00:02:00
   filesystem=local
   workdir=/scratch/midway/$USER/work
}

UPFCreator.java

package simanneal.params;

import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;

import repast.simphony.data2.BatchRunDataSource;
import repast.simphony.parameter.ParameterSchema;
import repast.simphony.parameter.Parameters;
import repast.simphony.parameter.ParametersParser;
import repast.simphony.parameter.Schema;

/**
 * Class to be called from an external script to generate individual unrolled
 * parameter files.
 * 
 * @author jozik
 * 
 */
public class UPFCreator {

	private Options options;

	private Parameters parameters;
	private String paramsNames;
	private String paramsValues;
	private int num;
	private String outputPath;

	private static final String PFILE = "pfile";
	private static final String PARAMS = "params";
	private static final String VALS = "vals";
	private static final String NUM = "num";
	private static final String OUT = "out";

	private static final String RANDOMSEED = "randomSeed";
	private static final String ONE = "1";

	public UPFCreator() {
		initOptions();
	}

	@SuppressWarnings("static-access")
	private void initOptions() {
		options = new Options();
		Option help = new Option("help", "print this message");
		options.addOption(help);

		Option pfile = OptionBuilder.withArgName("file").hasArg()
				.withDescription("use given parameter xml file").create(PFILE);
		options.addOption(pfile);

		Option params = OptionBuilder.withArgName("value").hasArg()
				.withDescription("assign parameter names").create(PARAMS);
		options.addOption(params);

		Option vals = OptionBuilder.withArgName("value").hasArg()
				.withDescription("use given values for parameter values").create(VALS);
		options.addOption(vals);

		Option onum = OptionBuilder.withArgName("value").hasArg()
				.withDescription("use given number for variations").create(NUM);
		options.addOption(onum);

		Option outDir = OptionBuilder.withArgName("value").hasArg()
				.withDescription("use given directory for output").create(OUT);
		options.addOption(outDir);
	}

	public static void main(String[] args) {
		try {
			UPFCreator creator = new UPFCreator();
			creator.configure(args);
			creator.create();
		} catch (Throwable ex) {
			ex.printStackTrace();
		}
	}

	public void create() {
		Map<String, Object> pvs = parseParamsValues();

		// set params with pvs
		for (String key : pvs.keySet()) {
			Object obj = pvs.get(key);
			parameters.setValue(key, obj);
		}
		Charset charset = Charset.forName("UTF-8");
		List<String> paramNames = new ArrayList<>();
		for (String pName : parameters.getSchema().parameterNames()) {
			paramNames.add(pName);
		}

		for (int i = 0; i < num; i++) {
			parameters.setValue(RANDOMSEED, i);
			String fileName = String.format("upf_%04d.txt", i);
			Path path = Paths.get(outputPath,fileName);
			try (BufferedWriter writer = Files.newBufferedWriter(path, charset)) {
				
				writer.write(ONE);
				writer.write("\t");

				boolean addComma = false;
				for (String pName : paramNames) {
					if (addComma)
						writer.write(",");
					writer.write(pName);
					writer.append("\t");
					writer.append(parameters.getValueAsString(pName));
					addComma = true;
				}
				writer.append("\n");
				writer.flush();
			} catch (IOException x) {
				System.err.format("IOException: %s%n", x);
			}

		}
	}

	protected Map<String, Object> parseParamsValues() {
		Map<String, Object> pvs = new LinkedHashMap<String, Object>();
		Schema schema = parameters.getSchema();
		String[] ps = paramsNames.split(",");
		String[] vals = paramsValues.split(",");

		int counter = 0;
		for (String p : ps) {
			ParameterSchema pSchema = schema.getDetails(p);
			Object obj = pSchema.fromString(vals[counter]);
			pvs.put(p, obj);
			counter++;
		}
		return pvs;
	}

	public void configure(String[] args) throws Exception {
		CommandLineParser parser = new GnuParser();
		try {
			CommandLine line = parser.parse(options, args);

			if (line.hasOption(PFILE)) {
				String paramsFilePath = line.getOptionValue(PFILE);
				File paramsFile = new File(paramsFilePath);
				ParametersParser pp = new ParametersParser(paramsFile);
				parameters = pp.getParameters();
			} else {
				throw new Exception("Command line is missing required -pfile option");
			}

			if (line.hasOption(PARAMS)) {
				paramsNames = line.getOptionValue(PARAMS);
			} else {
				throw new Exception("Command line is missing required -params option");
			}

			if (line.hasOption(VALS)) {
				paramsValues = line.getOptionValue(VALS);
			} else {
				throw new Exception("Command line is missing required -vals option");
			}

			if (line.hasOption(NUM)) {
				String numString = line.getOptionValue(NUM);

				try {
					num = Integer.parseInt(numString);
				} catch (NumberFormatException e) {
					throw e;
				}
			} else {
				throw new Exception("Command line is missing required -num option");
			}

			if (line.hasOption(OUT)) {
				outputPath = line.getOptionValue(OUT);
			} else {
				outputPath = ".";
			}

		} catch (ParseException ex) {
			throw new Exception("Error while parsing command line args", ex);
		}

	}
}